From 3e88668c48122505e71740d046b0eb015f653f92 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:05:37 -0400 Subject: [PATCH 1/7] npu: q4nx_w4g32.h, the large-M int4 prefill image contract, and its layout ctest The w4g32 image keeps a z = -8s q4_1 pack's own int4 codes and carries each per-32 signed scale as an int8 magnitude times one float per column. MAGMAX(K) keeps the device's int32 C exact. Covered: the b4 tile order the npu-kernels G32H kernel reads, factor() from an assembled B and S (refusing anything that is not c*s), pack(), the A pack, and an int64 reference. tests/npu_w4g32_layout_test restates the layout from layout_w4.py's own definition. With W4G32_VEC set, it also reproduces C_ref from gate vector sets the NPU matched bit-exact (U, D edge and QKV14336 checked). Four header mutants (block order, mag slot order, A row order, the integer check) each fail it. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 7 + npu/forward/include/q4nx_w4g32.h | 197 ++++++++++++++++++++++++ tests/npu_w4g32_layout_test.cpp | 252 +++++++++++++++++++++++++++++++ tests/npu_w4g32_layout_test.sh | 31 ++++ 4 files changed, 487 insertions(+) create mode 100644 npu/forward/include/q4nx_w4g32.h create mode 100644 tests/npu_w4g32_layout_test.cpp create mode 100755 tests/npu_w4g32_layout_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 1727a99b2..8a50c8e02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1834,6 +1834,13 @@ add_test(NAME npu_pb16i_layout COMMAND ${CMAKE_SOURCE_DIR}/tests/npu_pb16i_layout_test.sh ${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/tests/npu_pb16i_layout_test.cpp) +# The w4g32 weight image (q4nx_w4g32.h): the large-M int4-weight prefill tier's contract -- the b4 tile +# order, the per-32 signed magnitudes, the A pack and the int64 reference. No device; set +# W4G32_VEC=" M K N" to also check it against a gate vector set the NPU reproduced bit-exact. +add_test(NAME npu_w4g32_layout + COMMAND ${CMAKE_SOURCE_DIR}/tests/npu_w4g32_layout_test.sh + ${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/npu/forward/include + ${CMAKE_SOURCE_DIR}/tests/npu_w4g32_layout_test.cpp) # The t2s weight stream (q4nx_weight_layout "t2s"): the same integer-exact weights at 2.125 b/w, # expanded on the core. Its host-side contract is the file size, so the test pins the tile against # the numbers tools/q4nx_t2s.py measured on a real image (50,135,040 -> 23,674,880 B for gate/up). diff --git a/npu/forward/include/q4nx_w4g32.h b/npu/forward/include/q4nx_w4g32.h new file mode 100644 index 000000000..4d6f36093 --- /dev/null +++ b/npu/forward/include/q4nx_w4g32.h @@ -0,0 +1,197 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// q4nx_w4g32.h -- the "w4g32" weight image: the large-M int4-weight prefill GEMM's contract. +// +// What it carries. A q4_1 tile grid whose zero-points are all -8 * scale (every 2-D projection +// of bonsai2-27b-NPU, checked bit-exact on 2026-11-15) is exactly +// +// w[k][n] = c[k][n] * s[k/32][n], c = code - 8 in [-8, 7], s a signed bf16 per 32 K. +// +// The image keeps c exactly and carries s as a signed int8 magnitude per (32-K group, column) times +// one float per column: +// +// s[g][n] ~= mag[g][n] * cs[n], cs[n] = max_g |s[g][n]| / MAGMAX(K), mag = rint(s / cs) +// +// The device sums sum_g mag[g][n] * sum_{k in g} a[m][k] * c[k][n] exactly (int8 x int4 into acc32 per +// 32-K group, |P| <= 32512 < 2^15, then int16 x int16 into acc32, then int32 C), and the host applies +// C_f[m][n] = C[m][n] * ascale[m] * cs[n] -- the int8 path's own dequantisation with cs as the column +// scale. MAGMAX(K) keeps |C| <= K * 127 * 8 * MAGMAX < 2^31: 127 up to K = 6144 (6,605 levels short of +// overflow at 5120), 121 at K = 17408. The only rounding before the host's is rint(s / cs): the worst +// element error is |c| * cs / 2 <= 8 * max|s| / (2 * MAGMAX), the same bound as the int8 per-column image +// (round(w * 127 / amax), amax = 8 * max|s|) when MAGMAX = 127, and the RMS error is about half of it +// (measured 0.45-0.48x on bonsai2-27b layers 0 and 3). +// +// Image geometry (npu-kernels npu/arena/GU-w4/layout_w4.py, the b4 layout, k tile 128). K % 128 == 0 and +// N % 128 == 0 (the design additionally needs N % 1024 == 0). One tile per (nt = n/128, ki = k/128), +// at byte (nt * K/128 + ki) * 8704: +// +// [0, 8192) int4 codes, [kb 8][nb 8][16 r][16 cc]: element (k, n) with kb = (k%128)/16, r = k%16, +// nb = (n%128)/16, cc = n%16; e = r*16 + cc of its 16x16 block, byte block*128 + e/2, +// even e in the low nibble, two's complement +// [8192, 8704) int8 mag, [nb 8][g 4][16 cc], g = (k%128)/32 +// +// Activations are int8 with one scale per row, host-packed per 32-row tile (the coalesced design's A): +// A_packed[rt][ki][mb][kb][r][cc] = A[rt*32 + mb*4 + r][ki*128 + kb*16 + cc] +// (rt = m/32, mb = 4-row block, r = row in it, kb = 16-K block, cc = k in it). C is int32 row-major M x N. +// +// This header is the contract between the packer, the A pack, the host dequantisation and the kernel's +// tap (mm_g32h.cc: codes at w + (kb*8+nb)*128, mags at w + 8192 + (nb*4+g)*16, A at (mb*8+kb)*64). +// tests/npu_w4g32_layout_test.cpp restates it independently and pins it without a device. +#pragma once + +#include +#include +#include +#include +#include + +namespace q4nx_w4g32 { + +constexpr int KT = 128; // K rows per tile (one kernel call) +constexpr int NT = 128; // N columns per tile +constexpr int GROUP = 32; // K rows per magnitude +constexpr int CODE_BYTES = KT * NT / 2; // 8192 +constexpr int MAG_BYTES = (KT / GROUP) * NT; // 512 +constexpr int TILE = CODE_BYTES + MAG_BYTES; // 8704 +constexpr int MT = 32; // activation rows per A tile (per core row) + +inline bool fits(int K, int N) { return K > 0 && N > 0 && K % KT == 0 && N % NT == 0; } +inline size_t w_bytes(int K, int N) { return (size_t)(K / KT) * (size_t)(N / NT) * TILE; } + +// Largest |mag| that keeps the device's int32 C exact for this K: K * 127 * 8 * MAGMAX < 2^31. +inline int magmax(int K) { + if (K <= 0) return 0; + const long long m = (long long)2147483647 / ((long long)K * 127 * 8); + return m >= 127 ? 127 : (int)m; +} + +struct offsets { + size_t code; // byte holding the element's nibble + bool high; // the HIGH nibble of that byte + size_t mag; // byte of its int8 magnitude +}; + +// Where weight element (k, n) and its group magnitude live. Undefined unless fits(K, N). +inline offsets index(int K, int N, int k, int n) { + (void)N; + const size_t tile = ((size_t)(n / NT) * (size_t)(K / KT) + (size_t)(k / KT)) * TILE; + const int kk = k % KT, nn = n % NT; + const int kb = kk / 16, r = kk % 16, nb = nn / 16, cc = nn % 16, g = kk / GROUP; + const int e = r * 16 + cc; + offsets o; + o.code = tile + (size_t)(kb * 8 + nb) * 128 + (size_t)(e / 2); + o.high = (e & 1) != 0; + o.mag = tile + CODE_BYTES + (size_t)(nb * 4 + g) * 16 + (size_t)cc; + return o; +} + +// Recover c and s from an ASSEMBLED design matrix: B[k][n] (= c * s exactly, the q4_1 dequant of a +// z = -8 s pack) and S[k][n] (the same scatter of each element's group scale). Writes code[k][n] and +// scale[g][n]. Returns false -- and the caller keeps the int8 path -- unless S is constant over every +// 32-K group of a column and B / S is an integer in [-8, 7] wherever S != 0 (B must be 0 where S == 0). +inline bool factor(const float* B, const float* S, int K, int N, int8_t* code, float* scale) { + if (!B || !S || !code || !scale || K % GROUP || N <= 0) return false; + for (int g = 0; g < K / GROUP; g++) + for (int n = 0; n < N; n++) { + const float s = S[(size_t)g * GROUP * N + n]; + scale[(size_t)g * N + n] = s; + for (int kk = 0; kk < GROUP; kk++) { + const size_t i = (size_t)(g * GROUP + kk) * N + n; + if (S[i] != s) return false; + const float b = B[i]; + if (s == 0.0f) { + if (b != 0.0f) return false; + code[i] = 0; + continue; + } + const float q = b / s; + if (!(q >= -8.0f && q <= 7.0f) || q != std::nearbyint(q) || (float)q * s != b) return false; + code[i] = (int8_t)q; + } + } + return true; +} + +// Build the image and the column scales from code[k][n] (in [-8, 7]) and scale[g][n]. `img` must hold +// w_bytes(K, N). Returns false on a bad geometry or a code out of range. +inline bool pack(const int8_t* code, const float* scale, int K, int N, uint8_t* img, float* cs) { + if (!fits(K, N) || !code || !scale || !img || !cs) return false; + const int mm = magmax(K); + std::memset(img, 0, w_bytes(K, N)); + for (int n = 0; n < N; n++) { + float amax = 0.0f; + for (int g = 0; g < K / GROUP; g++) amax = std::fmax(amax, std::fabs(scale[(size_t)g * N + n])); + cs[n] = amax > 0.0f ? amax / (float)mm : 1.0f; + } + for (int g = 0; g < K / GROUP; g++) + for (int n = 0; n < N; n++) { + long q = std::lrint(scale[(size_t)g * N + n] / cs[n]); + if (q > mm) q = mm; + if (q < -mm) q = -mm; + img[index(K, N, g * GROUP, n).mag] = (uint8_t)(int8_t)q; + } + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + const int c = code[(size_t)k * N + n]; + if (c < -8 || c > 7) return false; + const offsets o = index(K, N, k, n); + const uint8_t nib = (uint8_t)(c & 0x0F); + img[o.code] = o.high ? (uint8_t)((img[o.code] & 0x0F) | (nib << 4)) : (uint8_t)((img[o.code] & 0xF0) | nib); + } + return true; +} + +// Read element (k, n) back as (code, mag). +inline void read(const uint8_t* img, int K, int N, int k, int n, int* c, int* mag) { + const offsets o = index(K, N, k, n); + const int nib = o.high ? (img[o.code] >> 4) : (img[o.code] & 0x0F); + *c = nib >= 8 ? nib - 16 : nib; + *mag = (int)(int8_t)img[o.mag]; +} + +// The design's A: int8 rows a[m][ld] (row-major, the int8 path's own quantised rows) -> the packed tile +// order above. M % 32 == 0, K % 128 == 0; `out` holds M*K bytes. +inline bool pack_a(const int8_t* a, int ld, int M, int K, int8_t* out) { + if (!a || !out || M % MT || K % KT || ld < K) return false; + size_t o = 0; + for (int rt = 0; rt < M / MT; rt++) + for (int ki = 0; ki < K / KT; ki++) + for (int mb = 0; mb < MT / 4; mb++) + for (int kb = 0; kb < KT / 16; kb++) + for (int r = 0; r < 4; r++) { + std::memcpy(out + o, a + (size_t)(rt * MT + mb * 4 + r) * ld + (size_t)(ki * KT + kb * 16), 16); + o += 16; + } + return true; +} + +// The device's integer result for one (m, n), from the image: the CPU reference the gate compares against. +inline int64_t ref_dot(const int8_t* a_row, const uint8_t* img, int K, int N, int n) { + int64_t acc = 0; + for (int g = 0; g < K / GROUP; g++) { + int64_t p = 0; + int mag = 0; + for (int kk = 0; kk < GROUP; kk++) { + int c; + read(img, K, N, g * GROUP + kk, n, &c, &mag); + p += (int64_t)a_row[g * GROUP + kk] * c; + } + acc += p * mag; + } + return acc; +} + +} // namespace q4nx_w4g32 diff --git a/tests/npu_w4g32_layout_test.cpp b/tests/npu_w4g32_layout_test.cpp new file mode 100644 index 000000000..0f0e8e9c4 --- /dev/null +++ b/tests/npu_w4g32_layout_test.cpp @@ -0,0 +1,252 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// npu_w4g32_layout_test -- the w4g32 weight image contract (q4nx_w4g32.h), checked without a device. +// +// The packed image goes straight into the weight BO, so a header that disagrees with the kernel's tap +// multiplies the wrong pairs and the output is silently garbage. This test restates the contract +// from the b4 layout's own definition (npu-kernels layout_w4.py: codes [nt][ki][kb][nb][r][cc] from +// [ki][kb][r][nt][nb][cc]; mags [nt][ki][nb][g][cc]) rather than from the header: +// +// 1. the image size, MAGMAX(K) at the model's K values, and the int32 bound MAGMAX guarantees; +// 2. index() is a bijection: every code nibble is claimed once, every magnitude byte by exactly 32 +// elements of one column, and both agree with the restated layout element for element; +// 3. factor() recovers (c, s) from an assembled B = c * s and S, and refuses each way the input can +// fail to be that (S not constant over a group, B/S not an integer, |c| out of int4, B != 0 at S == 0); +// 4. pack()/read() round-trip the codes, and the magnitudes are rint(s / cs) with cs = max|s| / MAGMAX; +// 5. pack_a() is the restated A order; +// 6. ref_dot() equals the dense int64 sum over (code, mag); +// 7. optional, when W4G32_VEC=" M K N" names a gate vector set (W.bin, packed A.bin, C_ref.bin +// from npu-kernels W4-g128/gen_q4.py, the set the NPU reproduced bit-exact): ref_dot() on that W +// and A reproduces C_ref, so this header reads the device's own bytes the way the device does. +#include "q4nx_w4g32.h" + +#include +#include +#include +#include +#include +#include + +namespace w = q4nx_w4g32; + +static int fails = 0; +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + std::printf("FAIL %s:%d: ", __FILE__, __LINE__); \ + std::printf(__VA_ARGS__); \ + std::printf("\n"); \ + ++fails; \ + } \ + } while (0) + +// layout_w4.py, restated: byte of the code of (k, n) and byte of its magnitude. +static size_t restated_code_byte(int K, int k, int n, bool* high) { + const int KTt = K / 128; + const int ki = k / 128, kb = (k % 128) / 16, r = k % 16, nt = n / 128, nb = (n % 128) / 16, cc = n % 16; + // flat index in [nt][ki][kb][nb][r][cc] over the code region of a tile, then the tile base + const size_t e = ((((size_t)kb * 8 + nb) * 16 + r) * 16 + cc); + *high = (e & 1) != 0; + return ((size_t)nt * KTt + ki) * 8704 + e / 2; +} +static size_t restated_mag_byte(int K, int k, int n) { + const int KTt = K / 128; + const int ki = k / 128, g = (k % 128) / 32, nt = n / 128, nb = (n % 128) / 16, cc = n % 16; + return ((size_t)nt * KTt + ki) * 8704 + 8192 + ((size_t)nb * 4 + g) * 16 + cc; +} + +static void test_sizes() { + CHECK(w::TILE == 8704 && w::CODE_BYTES == 8192, "tile %d", w::TILE); + CHECK(w::w_bytes(5120, 17408) == 47349760u, "U image %zu", w::w_bytes(5120, 17408)); // = gen_q4's W + CHECK(w::magmax(5120) == 127 && w::magmax(6144) == 127, "magmax 5120/6144"); + CHECK(w::magmax(17408) == 121, "magmax(17408) = %d", w::magmax(17408)); + for (int K = 128; K <= 65536; K += 128) { + const long long worst = (long long)K * 127 * 8 * w::magmax(K); + CHECK(worst <= 2147483647LL, "K=%d worst |C| %lld", K, worst); + CHECK(w::magmax(K) == 127 || (long long)K * 127 * 8 * (w::magmax(K) + 1) > 2147483647LL, "K=%d not tight", K); + } + CHECK(!w::fits(5120, 100) && !w::fits(100, 1024) && w::fits(256, 128), "fits"); +} + +static void test_bijection() { + const int K = 384, N = 256; + std::vector nib(w::w_bytes(K, N) * 2, 0), magc(w::w_bytes(K, N), 0), magcol(w::w_bytes(K, N), -1); + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + const w::offsets o = w::index(K, N, k, n); + bool hi; + const size_t rb = restated_code_byte(K, k, n, &hi); + CHECK(o.code == rb && o.high == hi, "code (%d,%d): %zu/%d vs %zu/%d", k, n, o.code, o.high, rb, hi); + CHECK(o.mag == restated_mag_byte(K, k, n), "mag (%d,%d)", k, n); + CHECK(o.code % w::TILE < (size_t)w::CODE_BYTES && o.mag % w::TILE >= (size_t)w::CODE_BYTES, "regions"); + nib[o.code * 2 + o.high]++; + magc[o.mag]++; + CHECK(magcol[o.mag] < 0 || magcol[o.mag] == n, "mag byte shared by two columns"); + magcol[o.mag] = n; + } + size_t codes = 0, mags = 0; + for (size_t b = 0; b < w::w_bytes(K, N); b++) { + if (b % w::TILE < (size_t)w::CODE_BYTES) { + CHECK(nib[b * 2] == 1 && nib[b * 2 + 1] == 1, "byte %zu nibbles %d %d", b, nib[b * 2], nib[b * 2 + 1]); + codes++; + } else { + CHECK(magc[b] == 32, "mag byte %zu claimed %d times", b, magc[b]); + mags++; + } + } + CHECK(codes * 2 == (size_t)K * N && mags * 32 == (size_t)K * N, "coverage"); +} + +static void test_factor_pack() { + const int K = 256, N = 128; + std::mt19937 rng(7); + std::vector c(K * N); + std::vector s((K / 32) * N), B(K * N), S(K * N); + for (auto& x : c) x = (int8_t)((int)(rng() % 16) - 8); + for (auto& x : s) x = std::ldexp((float)((int)(rng() % 255) - 127), -14); // signed, exact in bf16 + s[3] = 0.0f; // an all-zero group + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + const float sg = s[(k / 32) * N + n]; + S[k * N + n] = sg; + B[k * N + n] = sg == 0.0f ? 0.0f : (float)c[k * N + n] * sg; + } + std::vector c2(K * N); + std::vector s2((K / 32) * N); + CHECK(w::factor(B.data(), S.data(), K, N, c2.data(), s2.data()), "factor refused a valid input"); + for (int i = 0; i < K * N; i++) { + const bool zero_group = S[i] == 0.0f; + CHECK(c2[i] == (zero_group ? 0 : c[i]), "factor code %d: %d vs %d", i, c2[i], c[i]); + } + CHECK(s2 == s, "factor scales"); + // refusals + auto refused = [&](std::vector b, std::vector sm, const char* what) { + CHECK(!w::factor(b.data(), sm.data(), K, N, c2.data(), s2.data()), "factor accepted: %s", what); + }; + { auto b = B; auto sm = S; sm[5 * N + 9] *= 2.0f; refused(b, sm, "S not constant in a group"); } + { auto b = B; size_t i = 40 * N + 1; while (S[i] == 0.0f) i++; b[i] = 0.5f * S[i]; refused(b, S, "B/S = 0.5"); } + { auto b = B; size_t i = 41 * N + 2; while (S[i] == 0.0f) i++; b[i] = 8.0f * S[i]; refused(b, S, "code +8"); } + { auto b = B; b[3] = 1.0f; refused(b, S, "B != 0 where S == 0"); } + + std::vector img(w::w_bytes(K, N)); + std::vector cs(N); + CHECK(w::pack(c2.data(), s2.data(), K, N, img.data(), cs.data()), "pack"); + const int mm = w::magmax(K); + for (int n = 0; n < N; n++) { + float amax = 0; + for (int g = 0; g < K / 32; g++) amax = std::fmax(amax, std::fabs(s2[g * N + n])); + CHECK(cs[n] == (amax > 0 ? amax / mm : 1.0f), "cs[%d]", n); + } + int maxmag = 0; + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + int cc, mg; + w::read(img.data(), K, N, k, n, &cc, &mg); + CHECK(cc == c2[k * N + n], "read code (%d,%d)", k, n); + const long want = std::lrint(s2[(k / 32) * N + n] / cs[n]); + CHECK(mg == want, "mag (%d,%d): %d vs %ld", k, n, mg, want); + maxmag = std::max(maxmag, std::abs(mg)); + } + CHECK(maxmag == mm, "the column max reaches MAGMAX (%d vs %d)", maxmag, mm); + std::vector bad = c2; + bad[17] = 8; + CHECK(!w::pack(bad.data(), s2.data(), K, N, img.data(), cs.data()), "pack accepted code 8"); +} + +static void test_a_and_ref() { + const int M = 64, K = 256, N = 128, ld = K + 32; + std::mt19937 rng(11); + std::vector a(M * ld), ap(M * K), c(K * N); + for (auto& x : a) x = (int8_t)((int)(rng() % 255) - 127); + for (auto& x : c) x = (int8_t)((int)(rng() % 16) - 8); + CHECK(w::pack_a(a.data(), ld, M, K, ap.data()), "pack_a"); + for (int m = 0; m < M; m++) + for (int k = 0; k < K; k++) { + // A.reshape(M/32, 8, 4, K/128, 8, 16).transpose(0, 3, 1, 4, 2, 5) + const int rt = m / 32, mb = (m % 32) / 4, r = m % 4, ki = k / 128, kb = (k % 128) / 16, cc = k % 16; + const size_t o = (((((size_t)rt * (K / 128) + ki) * 8 + mb) * 8 + kb) * 4 + r) * 16 + cc; + CHECK(ap[o] == a[(size_t)m * ld + k], "pack_a (%d,%d)", m, k); + } + CHECK(!w::pack_a(a.data(), ld, 48, K, ap.data()), "pack_a accepted M=48"); + std::vector s((K / 32) * N); + for (auto& x : s) x = (float)((int)(rng() % 255) - 127); + std::vector img(w::w_bytes(K, N)); + std::vector cs(N); + CHECK(w::pack(c.data(), s.data(), K, N, img.data(), cs.data()), "pack"); + for (int m = 0; m < M; m += 7) + for (int n = 0; n < N; n += 5) { + int64_t dense = 0; + for (int k = 0; k < K; k++) { + int cc, mg; + w::read(img.data(), K, N, k, n, &cc, &mg); + dense += (int64_t)a[(size_t)m * ld + k] * cc * mg; + } + CHECK(w::ref_dot(a.data() + (size_t)m * ld, img.data(), K, N, n) == dense, "ref_dot (%d,%d)", m, n); + } +} + +static bool slurp(const std::string& p, std::vector& v) { + std::ifstream f(p, std::ios::binary); + if (!f) return false; + v.assign(std::istreambuf_iterator(f), {}); + return true; +} + +static void test_device_vectors() { + const char* e = std::getenv("W4G32_VEC"); + if (!e || !*e) { std::printf(" (device-vector leg skipped: W4G32_VEC unset)\n"); return; } + char dir[4096]; + int M, K, N; + if (std::sscanf(e, "%4095s %d %d %d", dir, &M, &K, &N) != 4) { CHECK(false, "W4G32_VEC=\"dir M K N\""); return; } + std::vector W, A, C; + if (!slurp(std::string(dir) + "/W.bin", W) || !slurp(std::string(dir) + "/A.bin", A) || !slurp(std::string(dir) + "/C_ref.bin", C)) { + CHECK(false, "cannot read %s/{W,A,C_ref}.bin", dir); + return; + } + CHECK(W.size() == w::w_bytes(K, N) && A.size() == (size_t)M * K && C.size() == (size_t)M * N * 4, "vector sizes"); + if (fails) return; + // un-pack A with the restated order, check pack_a gives the file back, then sample C_ref + std::vector a((size_t)M * K), ap((size_t)M * K); + size_t o = 0; + for (int rt = 0; rt < M / 32; rt++) + for (int ki = 0; ki < K / 128; ki++) + for (int mb = 0; mb < 8; mb++) + for (int kb = 0; kb < 8; kb++) + for (int r = 0; r < 4; r++) + for (int cc = 0; cc < 16; cc++) a[(size_t)(rt * 32 + mb * 4 + r) * K + ki * 128 + kb * 16 + cc] = (int8_t)A[o++]; + w::pack_a(a.data(), K, M, K, ap.data()); + CHECK(std::memcmp(ap.data(), A.data(), A.size()) == 0, "pack_a does not reproduce the device's A.bin"); + const int32_t* cr = reinterpret_cast(C.data()); + int checked = 0; + for (int m = 0; m < M; m += 61) + for (int n = 0; n < N; n += 97) { + const int64_t v = w::ref_dot(a.data() + (size_t)m * K, reinterpret_cast(W.data()), K, N, n); + CHECK(v == cr[(size_t)m * N + n], "C_ref (%d,%d): %lld vs %d", m, n, (long long)v, cr[(size_t)m * N + n]); + ++checked; + } + std::printf(" device-vector leg: %s M=%d K=%d N=%d, %d C_ref entries reproduced\n", dir, M, K, N, checked); +} + +int main() { + test_sizes(); + test_bijection(); + test_factor_pack(); + test_a_and_ref(); + test_device_vectors(); + if (fails) { std::printf("npu_w4g32_layout_test: %d failure(s)\n", fails); return 1; } + std::printf("npu_w4g32_layout_test: OK\n"); + return 0; +} diff --git a/tests/npu_w4g32_layout_test.sh b/tests/npu_w4g32_layout_test.sh new file mode 100755 index 000000000..c4a4c1800 --- /dev/null +++ b/tests/npu_w4g32_layout_test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# npu_w4g32_layout_test.sh +# +# The w4g32 weight image contract (q4nx_w4g32.h, the large-M int4 prefill tier) is pure index math: no +# runtime, no device, no NPU toolchain. Same reason as npu_blocked_layout_test.sh: +# the check compiles itself with the build's own compiler at test time, so it cannot +# become "Not Run" by being absent from CI's explicit target list. +set -u +CXX=${1:?usage: npu_w4g32_layout_test.sh } +INC=${2:?} +SRC=${3:?} +work=$(mktemp -d "${TMPDIR:-/tmp}/npu-w4g32-layout.XXXXXX") +trap 'rm -rf "$work"' EXIT +"$CXX" -std=c++17 -O1 -Wall -Wextra -I "$INC" "$SRC" -o "$work/t" || { + echo "npu_w4g32_layout_test: compile failed"; exit 1; } +"$work/t" From 8ff0fa36576c32f62204799c6e07b3a8ba0d0b1b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:10:48 -0400 Subject: [PATCH 2/7] npu/w4g32: factor() from the assembled B alone (no scale stream) Each 32-K group's scale is the first s = +-max|B|/q (q = 8..1) under which every element is an exact int4 multiple. So the target can build the image from the Bmat it already packs on a weight-cache miss, for any pack whose dequant is code * per-32 scale (q4_1 with z = -8s, symmetric q4_0, ternary). Anything else is refused. On bonsai2-27b-NPU through the production q4nx_dequant_tensor: all 27 2-D projections of layers 0/3/31/63 factor, with the same error as the container's own scales (rel-RMS 0.0039-0.0067). Co-Authored-By: Claude Opus 5.5 --- npu/forward/include/q4nx_w4g32.h | 66 +++++++++++++++++++++----------- tests/npu_w4g32_layout_test.cpp | 37 ++++++++++++------ 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/npu/forward/include/q4nx_w4g32.h b/npu/forward/include/q4nx_w4g32.h index 4d6f36093..93fd04597 100644 --- a/npu/forward/include/q4nx_w4g32.h +++ b/npu/forward/include/q4nx_w4g32.h @@ -47,6 +47,9 @@ // A_packed[rt][ki][mb][kb][r][cc] = A[rt*32 + mb*4 + r][ki*128 + kb*16 + cc] // (rt = m/32, mb = 4-row block, r = row in it, kb = 16-K block, cc = k in it). C is int32 row-major M x N. // +// The packer's input is the int8 path's own assembled float B (no second scale stream): factor() finds an exact +// code x per-32 scale form of it or refuses. +// // This header is the contract between the packer, the A pack, the host dequantisation and the kernel's // tap (mm_g32h.cc: codes at w + (kb*8+nb)*128, mags at w + 8192 + (nb*4+g)*16, A at (mb*8+kb)*64). // tests/npu_w4g32_layout_test.cpp restates it independently and pins it without a device. @@ -98,30 +101,49 @@ inline offsets index(int K, int N, int k, int n) { return o; } -// Recover c and s from an ASSEMBLED design matrix: B[k][n] (= c * s exactly, the q4_1 dequant of a -// z = -8 s pack) and S[k][n] (the same scatter of each element's group scale). Writes code[k][n] and -// scale[g][n]. Returns false -- and the caller keeps the int8 path -- unless S is constant over every -// 32-K group of a column and B / S is an integer in [-8, 7] wherever S != 0 (B must be 0 where S == 0). -inline bool factor(const float* B, const float* S, int K, int N, int8_t* code, float* scale) { - if (!B || !S || !code || !scale || K % GROUP || N <= 0) return false; - for (int g = 0; g < K / GROUP; g++) - for (int n = 0; n < N; n++) { - const float s = S[(size_t)g * GROUP * N + n]; - scale[(size_t)g * N + n] = s; - for (int kk = 0; kk < GROUP; kk++) { - const size_t i = (size_t)(g * GROUP + kk) * N + n; - if (S[i] != s) return false; - const float b = B[i]; - if (s == 0.0f) { - if (b != 0.0f) return false; - code[i] = 0; - continue; - } - const float q = b / s; - if (!(q >= -8.0f && q <= 7.0f) || q != std::nearbyint(q) || (float)q * s != b) return false; - code[i] = (int8_t)q; +// Factor an ASSEMBLED design matrix B[k][n] (the float the int8 path packs) into code[k][n] in [-8, 7] and +// one scale per (32-K group, column): B = code * scale exactly, element for element. The representation the +// kernel needs is any exact one, not the container's own: for each group with a = max |B| the candidates are +// s = +-a/q for q = 8 .. 1, and the first under which every element is an exact in-range multiple is taken (a +// q4_1 z = -8s pack, a symmetric q4_0 pack and a ternary pack all have one; a/q is exact whenever the true +// scale is a/q, because the correctly rounded quotient of q*s by q is s). An all-zero group gets scale 0. +// Returns false -- and the tier must not run on this weight -- when some group has no such scale. +inline bool factor_group(const float* B, size_t stride, int8_t* code, float* scale_out) { + float a = 0.0f; + for (int i = 0; i < GROUP; i++) { + const float v = B[(size_t)i * stride]; + if (!std::isfinite(v)) return false; + a = std::fmax(a, std::fabs(v)); + } + if (a == 0.0f) { + for (int i = 0; i < GROUP; i++) code[(size_t)i * stride] = 0; + *scale_out = 0.0f; + return true; + } + for (int q = 8; q >= 1; q--) + for (int sg = 0; sg < 2; sg++) { + const float s = sg ? -a / (float)q : a / (float)q; + bool ok = true; + for (int i = 0; i < GROUP && ok; i++) { + const float v = B[(size_t)i * stride]; + const float c = std::nearbyint(v / s); + ok = c >= -8.0f && c <= 7.0f && c * s == v; } + if (!ok) continue; + for (int i = 0; i < GROUP; i++) code[(size_t)i * stride] = (int8_t)std::nearbyint(B[(size_t)i * stride] / s); + *scale_out = s; + return true; } + return false; +} + +inline bool factor(const float* B, int K, int N, int8_t* code, float* scale) { + if (!B || !code || !scale || K % GROUP || N <= 0) return false; + for (int g = 0; g < K / GROUP; g++) + for (int n = 0; n < N; n++) + if (!factor_group(B + (size_t)g * GROUP * N + n, (size_t)N, code + (size_t)g * GROUP * N + n, + scale + (size_t)g * N + n)) + return false; return true; } diff --git a/tests/npu_w4g32_layout_test.cpp b/tests/npu_w4g32_layout_test.cpp index 0f0e8e9c4..428ef4edf 100644 --- a/tests/npu_w4g32_layout_test.cpp +++ b/tests/npu_w4g32_layout_test.cpp @@ -126,20 +126,33 @@ static void test_factor_pack() { } std::vector c2(K * N); std::vector s2((K / 32) * N); - CHECK(w::factor(B.data(), S.data(), K, N, c2.data(), s2.data()), "factor refused a valid input"); - for (int i = 0; i < K * N; i++) { - const bool zero_group = S[i] == 0.0f; - CHECK(c2[i] == (zero_group ? 0 : c[i]), "factor code %d: %d vs %d", i, c2[i], c[i]); - } - CHECK(s2 == s, "factor scales"); + CHECK(w::factor(B.data(), K, N, c2.data(), s2.data()), "factor refused a valid input"); + // any exact form is acceptable: check B = c2 * s2 element for element, and |c2| fits int4 + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + const float sg = s2[(k / 32) * N + n]; + CHECK(c2[k * N + n] >= -8 && c2[k * N + n] <= 7, "code range"); + CHECK((float)c2[k * N + n] * sg == B[k * N + n], "factor (%d,%d) not exact", k, n); + } + // and where the group holds a -8 code the container's own (c, s) is the one found (q = 8 tried first) + int same = 0, groups = 0; + for (int g = 0; g < K / 32; g++) + for (int n = 0; n < N; n++) { + bool has8 = false; + for (int kk = 0; kk < 32; kk++) has8 |= c[(g * 32 + kk) * N + n] == -8; + if (!has8 || s[g * N + n] == 0.0f) continue; + ++groups; + same += s2[g * N + n] == s[g * N + n]; + } + CHECK(groups > 0 && same == groups, "own scale recovered in %d of %d groups with a -8", same, groups); + // a group of only even codes may be represented at twice the scale: still exact, checked above // refusals - auto refused = [&](std::vector b, std::vector sm, const char* what) { - CHECK(!w::factor(b.data(), sm.data(), K, N, c2.data(), s2.data()), "factor accepted: %s", what); + auto refused = [&](std::vector b, const char* what) { + CHECK(!w::factor(b.data(), K, N, c2.data(), s2.data()), "factor accepted: %s", what); }; - { auto b = B; auto sm = S; sm[5 * N + 9] *= 2.0f; refused(b, sm, "S not constant in a group"); } - { auto b = B; size_t i = 40 * N + 1; while (S[i] == 0.0f) i++; b[i] = 0.5f * S[i]; refused(b, S, "B/S = 0.5"); } - { auto b = B; size_t i = 41 * N + 2; while (S[i] == 0.0f) i++; b[i] = 8.0f * S[i]; refused(b, S, "code +8"); } - { auto b = B; b[3] = 1.0f; refused(b, S, "B != 0 where S == 0"); } + { auto b = B; size_t i = 40 * N + 1; while (S[i] == 0.0f) i++; b[i] = 0.37f * S[i]; refused(b, "a non-multiple element"); } + { auto b = B; b[50 * N + 7] = std::nanf(""); refused(b, "NaN"); } + { auto b = B; for (int kk = 0; kk < 32; kk++) b[(64 + kk) * N + 3] = (float)(kk * 9 - 140); refused(b, "a 32-level ramp (needs > 4 bits)"); } std::vector img(w::w_bytes(K, N)); std::vector cs(N); From 0e7c21ffb5a06a09d9e6bea7c179979a345654dc Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:12:32 -0400 Subject: [PATCH 3/7] npu/w4g32: parallel factor() and tile-ordered parallel pack(), byte-identical Each pack() iteration owns one tile's bytes, so the schedule cannot change the image. On 13 bonsai2-27b tensors (layers 0 and 63), a hash of the image plus cs is equal to the serial header's. 5120x17408 drops from 0.72 s to 0.20 s with 8 threads. OpenMP sits behind _OPENMP, so the self-compiling layout ctest stays warning-free. Co-Authored-By: Claude Opus 5.5 --- npu/forward/include/q4nx_w4g32.h | 79 +++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/npu/forward/include/q4nx_w4g32.h b/npu/forward/include/q4nx_w4g32.h index 93fd04597..06b7a3981 100644 --- a/npu/forward/include/q4nx_w4g32.h +++ b/npu/forward/include/q4nx_w4g32.h @@ -61,6 +61,12 @@ #include #include +#if defined(_OPENMP) +#define Q4NX_W4G32_PAR _Pragma("omp parallel for schedule(static)") +#else +#define Q4NX_W4G32_PAR +#endif + namespace q4nx_w4g32 { constexpr int KT = 128; // K rows per tile (one kernel call) @@ -139,41 +145,70 @@ inline bool factor_group(const float* B, size_t stride, int8_t* code, float* sca inline bool factor(const float* B, int K, int N, int8_t* code, float* scale) { if (!B || !code || !scale || K % GROUP || N <= 0) return false; - for (int g = 0; g < K / GROUP; g++) + int bad = 0; + Q4NX_W4G32_PAR + for (int g = 0; g < K / GROUP; g++) { + int b = 0; for (int n = 0; n < N; n++) - if (!factor_group(B + (size_t)g * GROUP * N + n, (size_t)N, code + (size_t)g * GROUP * N + n, - scale + (size_t)g * N + n)) - return false; - return true; + b |= !factor_group(B + (size_t)g * GROUP * N + n, (size_t)N, code + (size_t)g * GROUP * N + n, + scale + (size_t)g * N + n); + if (b) { +#if defined(_OPENMP) +#pragma omp atomic write +#endif + bad = 1; + } + } + return bad == 0; } // Build the image and the column scales from code[k][n] (in [-8, 7]) and scale[g][n]. `img` must hold // w_bytes(K, N). Returns false on a bad geometry or a code out of range. inline bool pack(const int8_t* code, const float* scale, int K, int N, uint8_t* img, float* cs) { if (!fits(K, N) || !code || !scale || !img || !cs) return false; - const int mm = magmax(K); - std::memset(img, 0, w_bytes(K, N)); + const int mm = magmax(K), G = K / GROUP, KTn = K / KT, NTn = N / NT; + Q4NX_W4G32_PAR for (int n = 0; n < N; n++) { float amax = 0.0f; - for (int g = 0; g < K / GROUP; g++) amax = std::fmax(amax, std::fabs(scale[(size_t)g * N + n])); + for (int g = 0; g < G; g++) amax = std::fmax(amax, std::fabs(scale[(size_t)g * N + n])); cs[n] = amax > 0.0f ? amax / (float)mm : 1.0f; } - for (int g = 0; g < K / GROUP; g++) - for (int n = 0; n < N; n++) { - long q = std::lrint(scale[(size_t)g * N + n] / cs[n]); - if (q > mm) q = mm; - if (q < -mm) q = -mm; - img[index(K, N, g * GROUP, n).mag] = (uint8_t)(int8_t)q; + int bad = 0; + // One tile per iteration: tile (nt, ki) owns bytes [(nt*KTn + ki) * TILE, +TILE), so iterations never share + // a byte and the image does not depend on the schedule. + Q4NX_W4G32_PAR + for (int t = 0; t < NTn * KTn; t++) { + const int nt = t / KTn, ki = t % KTn; + uint8_t* tile = img + (size_t)t * TILE; + std::memset(tile, 0, TILE); + int b = 0; + for (int g = 0; g < KT / GROUP; g++) + for (int nn = 0; nn < NT; nn++) { + const int n = nt * NT + nn; + long q = std::lrint(scale[(size_t)(ki * (KT / GROUP) + g) * N + n] / cs[n]); + if (q > mm) q = mm; + if (q < -mm) q = -mm; + tile[CODE_BYTES + (nn / 16 * 4 + g) * 16 + nn % 16] = (uint8_t)(int8_t)q; + } + for (int kk = 0; kk < KT; kk++) { + const int8_t* row = code + (size_t)(ki * KT + kk) * N + (size_t)nt * NT; + const int kb = kk / 16, r = kk % 16; + for (int nn = 0; nn < NT; nn++) { + const int c = row[nn]; + b |= (c < -8 || c > 7); + const int e = r * 16 + nn % 16; + uint8_t& byte = tile[(kb * 8 + nn / 16) * 128 + e / 2]; + byte |= (e & 1) ? (uint8_t)((c & 0x0F) << 4) : (uint8_t)(c & 0x0F); + } } - for (int k = 0; k < K; k++) - for (int n = 0; n < N; n++) { - const int c = code[(size_t)k * N + n]; - if (c < -8 || c > 7) return false; - const offsets o = index(K, N, k, n); - const uint8_t nib = (uint8_t)(c & 0x0F); - img[o.code] = o.high ? (uint8_t)((img[o.code] & 0x0F) | (nib << 4)) : (uint8_t)((img[o.code] & 0xF0) | nib); + if (b) { +#if defined(_OPENMP) +#pragma omp atomic write +#endif + bad = 1; } - return true; + } + return bad == 0; } // Read element (k, n) back as (code, mag). From 79f0ae38dae77c98abf00bafe0c1c06543f4444d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:24:27 -0400 Subject: [PATCH 4/7] npu: the w4g32 prefill tier behind NPU_W4_PREFILL, plus its device gate ctest - XdnaTarget::open_prefill: with NPU_W4_PREFILL=1, each int8 prefill role opens _w4g32_m.elf from the kernels directory. All roles or none, at one M. Otherwise it logs why and stays int8. Not for the pb16i/t2s/b8 layouts. The plan's prefill_m check accepts the tier's M. - I8Ctx gets a w4g32 flag: - weights: packB/packB_into factor + pack the same assembled B the int8 image comes from; group_scales carries cs, so dequant_only_rows is unchanged. - activations: the quantisers write q4nx_quant_i8 rows into a row-major scratch, then pack_a() them into bA. - Weight cache: a fill builds the int8 image (decode twins) and the w4g32 image (prefill contexts) from the one Bmat it sees, so weight_cached() stays true for both. Each launch takes the image matching its context's layout. - tests/npu_w4g32_gate_test (CTest npu_w4g32_gate): with W4G32_GATE set to an ELF and its shape, runs a seed set and an edge set (|P| at the int16 bound, |C| at the MAGMAX int32 bound), 3 runs each with poisoned C, against the full CPU reference. Skips (77) when unset. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 13 ++ npu/forward/engine-src/npu_engine_i8ctx_inc.h | 87 +++++++++++ npu/target/xdna/xdna_target.cpp | 94 +++++++++--- npu/target/xdna/xdna_target.h | 12 ++ tests/npu_w4g32_gate_test.cpp | 139 ++++++++++++++++++ 5 files changed, 327 insertions(+), 18 deletions(-) create mode 100644 tests/npu_w4g32_gate_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a50c8e02..a9ed75674 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2364,6 +2364,19 @@ if(ONEBIT_NPU) add_test(NAME npu_dmabuf_import COMMAND npu_dmabuf_import_test "") set_tests_properties(npu_dmabuf_import PROPERTIES SKIP_RETURN_CODE 77) + # The w4g32 prefill kernel (q4nx_w4g32.h, NPU_W4_PREFILL), bit-exact on the device against the CPU reference over a + # seed set and the edge set (|P| at the int16 bound, |C| at the int32 bound MAGMAX picks). W4G32_GATE=" M K N" + # names the ELF; unset it SKIPS (77). It submits to the NPU: run it inside an npu-hold ticket. + add_executable(npu_w4g32_gate_test tests/npu_w4g32_gate_test.cpp) + target_include_directories(npu_w4g32_gate_test PRIVATE ${CMAKE_SOURCE_DIR}/npu/forward/include) + target_link_libraries(npu_w4g32_gate_test PRIVATE onebit_npu_lane) + find_package(OpenMP) + if(OpenMP_CXX_FOUND) + target_link_libraries(npu_w4g32_gate_test PRIVATE OpenMP::OpenMP_CXX) + endif() + add_test(NAME npu_w4g32_gate COMMAND npu_w4g32_gate_test) + set_tests_properties(npu_w4g32_gate PROPERTIES SKIP_RETURN_CODE 77) + # The model-generic forward (npu/forward): the GGUF-on-NPU serve route # (docs/npu.md, "Open"). q4nx_forward.cpp + its Q4NX dequant/pack/semantics # + model loader, against the engine tree's I8Ctx header. diff --git a/npu/forward/engine-src/npu_engine_i8ctx_inc.h b/npu/forward/engine-src/npu_engine_i8ctx_inc.h index 1b5709504..bf792d35c 100644 --- a/npu/forward/engine-src/npu_engine_i8ctx_inc.h +++ b/npu/forward/engine-src/npu_engine_i8ctx_inc.h @@ -35,6 +35,7 @@ #include "q4nx_weight_layout.h" // bpos(): the container's device weight layout #include "q4nx_pb16i.h" // the pb16i image: integer-exact ternary decode weights +#include "q4nx_w4g32.h" // the w4g32 image: the large-M int4-weight prefill tier #include "q4nx_t2s.h" // the t2s stream: the same weights at 2.125 bits/weight #include "q4nx_pack.h" // q4nx_elf_arg_sizes(): args_fit_elf() checks the ELF's own widths #include "quant_i8.h" // the A-row quantiser: scalar reference + an AVX2 path, byte-identical @@ -116,6 +117,14 @@ struct I8Ctx { // `_m16` ELFs are byte-identical to the one-row design), which is why the batched paths fall back // to per-row launches. Measured 2026-11-02 on the OGA 0.5B pack's five ELFs. bool gemv = false; + // Weight image "w4g32" (q4nx_w4g32.h; NPU_W4_PREFILL=1, engine HETEROMOSAIC-2): the large-M int4-weight prefill + // tier. The BO holds the container's own int4 codes plus a signed int8 magnitude per (32-K group, column), built at + // the weight-cache fill by factor() of the same assembled float B the int8 image is packed from; group_scales[l] + // carries the per-column cs, so dequant_only_rows() is unchanged. A is quantised exactly as the int8 path does + // (q4nx_quant_i8::row into a row-major scratch), then permuted into the design's per-32-row tile order. + bool w4g32 = false; + std::vector w4_rows_; // w4g32: the row-major int8 rows before pack_a() + int w4_written_rows_ = -1; // rows of w4_rows_ that may be non-zero (-1 = unknown: zero it all) uint16_t* Ag = nullptr; // gemv: the bf16 activation BO, mapped float* Cg = nullptr; // gemv: the f32 output BO, mapped @@ -181,6 +190,7 @@ struct I8Ctx { size_t w_bytes() const { if (gemv) return gemv_w_bytes(KD, ND); if (t2s) return q4nx_t2s::w_bytes(KD, ND); + if (w4g32) return q4nx_w4g32::w_bytes(KD, ND); return pb16i ? q4nx_pb16i::w_bytes(KD, ND) : (size_t)KD * (size_t)ND; } @@ -208,6 +218,11 @@ struct I8Ctx { "(K=%d N=%d)\n", KD, ND); return false; } + if (w4g32 && (pb16i || t2s || blocked_b || !q4nx_w4g32::fits(KD, ND) || ND % 1024 || MD % 128)) { + fprintf(stderr, " I8Ctx::init_elf: the w4g32 layout needs K%%128==0, N%%1024==0, M%%128==0 and no other " + "weight layout (M=%d K=%d N=%d)\n", MD, KD, ND); + return false; + } if (pb16i && !t2s && !q4nx_pb16i::fits(KD, ND)) { fprintf(stderr, " I8Ctx::init_elf: the pb16i layout needs K%%256==0 and N%%1024==0 " "(K=%d N=%d)\n", KD, ND); @@ -526,8 +541,41 @@ struct I8Ctx { } // Pack weights into an arbitrary (already-allocated) weight BO. + // w4g32: factor + pack the assembled B[K][N] (zero-padded to KD x ND) into `img`; col = the per-column cs. A weight + // that is not code x per-32 scale exactly cannot run on this tier, and there is no int8 image to fall back to inside + // this context, so it is a hard, named stop: unset NPU_W4_PREFILL for this model. + void packB_w4g32(uint8_t* img, const float* w, int K, int N, std::vector& col, double& ssum) { + const float* B = w; + std::vector pad; + if (K != KD || N != ND) { + pad.assign((size_t)KD * ND, 0.0f); + for (int k = 0; k < K; k++) memcpy(pad.data() + (size_t)k * ND, w + (size_t)k * N, (size_t)N * sizeof(float)); + B = pad.data(); + } + std::vector code((size_t)KD * ND); + std::vector sc((size_t)(KD / q4nx_w4g32::GROUP) * ND), cs((size_t)ND); + if (!q4nx_w4g32::factor(B, KD, ND, code.data(), sc.data()) || + !q4nx_w4g32::pack(code.data(), sc.data(), KD, ND, img, cs.data())) { + fprintf(stderr, " I8Ctx: NPU_W4_PREFILL: a K=%d N=%d weight is not int4 code x per-32 scale exactly, so the " + "w4g32 tier cannot carry it; unset NPU_W4_PREFILL for this model\n", K, N); + std::exit(3); + } + col.assign(cs.begin(), cs.begin() + N); + ssum = 0; + for (int n = 0; n < N; n++) ssum += (double)col[(size_t)n]; + } + void packB_into(xrt::bo& bo, const float* w, int K, int N, float& sout, std::vector& col_out) { + if (w4g32) { + std::vector col; + double ssum = 0; + packB_w4g32((uint8_t*)bo.map(), w, K, N, col, ssum); + bo.sync(XCL_BO_SYNC_BO_TO_DEVICE); + col_out = std::move(col); + sout = (float)(ssum / N); + return; + } auto* Bm = (int8_t*)bo.map(); memset(Bm, 0, (size_t)KD * ND); std::vector col(N); @@ -616,6 +664,15 @@ struct I8Ctx { // per-tensor scale packed low-magnitude columns (Qwen3 v_proj rms // ~0.007 vs q/k ~0.02-0.03) onto ~10 int8 levels -> ~5% output error // that compounds over 28 layers and flips the final token. + if (w4g32) { + std::vector col; + double ssum = 0; + packB_w4g32((uint8_t*)layerB[l]->map(), w, K, N, col, ssum); + layerB[l]->sync(XCL_BO_SYNC_BO_TO_DEVICE); + group_scales[l] = std::move(col); + sout = (float)(ssum / N); + return; + } auto* Bm = (int8_t*)layerB[l]->map(); memset(Bm, 0, (size_t)KD * ND); std::vector col(N); @@ -671,6 +728,23 @@ struct I8Ctx { // which costs one whole-BO zero on the first launch and cannot depend on the allocation or on member order. int a_written_rows_ = -1; // rows [0, a_written_rows_) may hold host data; -1 = nothing known yet + // w4g32: prepare_activation()'s zeroing rules, applied to the row-major scratch; then the whole of bA is rewritten by + // pack_a() (the tile order interleaves every row, so there is no narrow dirty range). + inline int8_t* w4_prepare(int am, int ak) { + if (w4_rows_.size() != (size_t)MD * KD) { w4_rows_.assign((size_t)MD * KD, 0); w4_written_rows_ = 0; } + if (ak != KD || w4_written_rows_ < 0) { + memset(w4_rows_.data(), 0, w4_rows_.size()); + } else if (w4_written_rows_ > am) { + memset(w4_rows_.data() + (size_t)am * KD, 0, (size_t)(w4_written_rows_ - am) * KD); + } + w4_written_rows_ = ak != KD ? -1 : am; // a short K leaves the column tail to re-zero next time too + return w4_rows_.data(); + } + inline void w4_finish() { + q4nx_w4g32::pack_a(w4_rows_.data(), KD, MD, KD, Am); + a_dirty(MD); + } + inline void prepare_activation(int am, int ak) { if (ak != KD || a_written_rows_ < 0) { // (a) a K shorter than the padded geometry leaves a column tail in rows [0, am) that the quantiser does @@ -728,6 +802,12 @@ struct I8Ctx { last_am_ = am; if (gemv) { quantize_bf16(A, am, ak); return Am; } float ais = 1.0f / ascale; + if (w4g32) { + int8_t* rows = w4_prepare(am, ak); + for (int m = 0; m < am; m++) q4nx_quant_i8::row(A + (size_t)m * ak, ak, ais, rows + (size_t)m * KD); + w4_finish(); + return Am; + } prepare_activation(am, ak); for (int m = 0; m < am; m++) q4nx_quant_i8::row(A + (size_t)m * ak, ak, ais, Am + (size_t)m * KD); @@ -755,6 +835,13 @@ struct I8Ctx { return Am; } last_am_ = am; + if (w4g32) { + int8_t* rows = w4_prepare(am, ak); + for (int m = 0; m < am; m++) + q4nx_quant_i8::row(A + (size_t)m * ak, ak, 1.0f / ascales[m], rows + (size_t)m * KD); + w4_finish(); + return Am; + } prepare_activation(am, ak); for (int m = 0; m < am; m++) q4nx_quant_i8::row(A + (size_t)m * ak, ak, 1.0f / ascales[m], Am + (size_t)m * KD); diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index b33c0c0c1..7afc222f7 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -108,6 +108,38 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err = "open_prefill: no design set (" + std::to_string(n) + " designs)"; return false; } + // NPU_W4_PREFILL=1: the large-M int4-weight prefill tier (q4nx_w4g32.h). Each int8 prefill role is opened on + // "_w4g32_m.elf" instead, from the same directory, when EVERY such role has one at the same M (all + // roles or none: max_rows() is the chunk width, and a mixed set would launch some roles past their buffers). A + // missing file is logged and the set stays int8 -- an optional speed-up never stops a model loading. Not for the + // pb16i/t2s/b8 layouts, whose images are not the int8 path's float B. + w4_rows_ = 0; + std::vector w4_elf((size_t)n); + if (const char* w4 = std::getenv("NPU_W4_PREFILL"); w4 && std::atoi(w4) != 0 && !host_gemm) { + std::string why; + int rows = 0; + if (b8 || pb16i_) why = "the container's weight layout is not row-major int8"; + for (int i = 0; why.empty() && i < n; i++) { + if (role_is_pb16i_split(i)) continue; + std::string stem(designs[i].elf_name); + if (stem.size() > 4 && stem.compare(stem.size() - 4, 4, ".elf") == 0) stem.resize(stem.size() - 4); + int got = 0; + for (int m : {512, 256, 128}) { + const std::string f = dir + "/" + stem + "_w4g32_m" + std::to_string(m) + ".elf"; + if (elf_present(f)) { got = m; w4_elf[(size_t)i] = f; break; } + } + if (!got) why = "no " + stem + "_w4g32_m<512|256|128>.elf in " + dir; + else if (rows && got != rows) why = "the roles' w4g32 kernels disagree on M (" + std::to_string(rows) + " vs " + + std::to_string(got) + ")"; + rows = got; + } + if (why.empty() && rows > 0) { + w4_rows_ = rows; + std::fprintf(stderr, " [forward] w4g32 prefill tier: every prefill role at M=%d (NPU_W4_PREFILL)\n", rows); + } else { + std::fprintf(stderr, " [forward] w4g32 prefill tier NOT adopted (%s); prefill stays int8\n", why.c_str()); + } + } for (int i = 0; i < n; i++) { const Q4nxDesignGeom* g = &designs[i]; // The two full-attention-QKV-split designs exist only as pb16i kernels (the role has no int8 @@ -115,7 +147,7 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string& // is built from the _pb16i ELF by the caller's twin pass. adopt_geometry(i, *g); if (role_is_pb16i_split(i)) continue; - const std::string elf = dir + "/" + g->elf_name; + const std::string elf = w4_rows_ ? w4_elf[(size_t)i] : dir + "/" + g->elf_name; if (host_gemm) { // Offline mode: host GEMMs are plain float matmuls over the decoded weights, so no ELF is // opened and none is required. A model whose kernel set is not built yet can still be @@ -163,16 +195,18 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err = "cannot create a context for role " + std::to_string(i); return false; } - ctx->MD = g->M; ctx->KD = g->K; ctx->ND = g->N; + const int md = w4_rows_ ? w4_rows_ : g->M; + ctx->MD = md; ctx->KD = g->K; ctx->ND = g->N; ctx->blocked_b = b8; - if (!ctx->init_elf(*dev_, elf.c_str(), g->M, g->K, g->N, 1)) { + ctx->w4g32 = w4_rows_ > 0; + if (!ctx->init_elf(*dev_, elf.c_str(), md, g->K, g->N, 1)) { // engine#2434: carry the runtime's reason, so a refused context ("code 102 ... No such device") reaches // open_with_retry's queue test and the lock-holder note instead of reading as a generic init failure. err = "I8Ctx init_elf failed for " + elf + (ctx->init_error.empty() ? "" : ": " + ctx->init_error); return false; } std::fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s\n", q4nx_design_log_tag(g->role), - g->M, g->K, g->N, elf.c_str()); + md, g->K, g->N, elf.c_str()); } return true; } @@ -208,7 +242,8 @@ bool XdnaTarget::open(const onebit::npu::plan::Plan& plan, const std::string& ke if (ctx_[i] && ctx_[i]->MD > adopted_prefill) adopted_prefill = ctx_[i]->MD; if (ctx16_[i] && ctx16_[i]->MD > adopted_decode) adopted_decode = ctx16_[i]->MD; } - if (plan_.prefill_m > 0 && adopted_prefill > 0 && plan_.prefill_m != adopted_prefill) + // The w4g32 tier is a runtime choice (NPU_W4_PREFILL) over the plan's int8 set, at its own M: not a disagreement. + if (plan_.prefill_m > 0 && adopted_prefill > 0 && plan_.prefill_m != adopted_prefill && adopted_prefill != w4_rows_) { err = "plan: prefill M=" + std::to_string(plan_.prefill_m) + " but the adopted kernels run M=" + std::to_string(adopted_prefill); return false; } if (plan_.decode_enabled && plan_.decode_m > 0 && adopted_decode > 0 && @@ -285,6 +320,30 @@ bool XdnaTarget::weight_cached(int layer, int key) const { wcache_[(size_t)layer].find(key) != wcache_[(size_t)layer].end(); } +bool XdnaTarget::fill_int8_and_w4(PackedWeight& pw, I8Ctx& kctx, int role, const float* Bmat, int K, int N, + std::string& err) { + const bool in_range = role >= 0 && role < Q4NX_MAX_DESIGNS; + I8Ctx* c8 = !kctx.w4g32 ? &kctx : (in_range && ctx16_[role] ? ctx16_[role].get() : nullptr); + I8Ctx* c4 = kctx.w4g32 ? &kctx + : (w4_rows_ > 0 && in_range && ctx_[role] && ctx_[role]->w4g32 ? ctx_[role].get() : nullptr); + float sout = 1.0f; + if (c8) { + pw.bo = c8->make_weight_bo(*dev_); + c8->packB_into(*pw.bo, Bmat, K, N, sout, pw.scales); + } + if (c4) { + pw.bo4 = c4->make_weight_bo(*dev_); + c4->packB_into(*pw.bo4, Bmat, K, N, sout, pw.scales4); + } + if (!c8 && !c4) { err = "weight cache: no context to pack this weight for"; return false; } + return true; +} + +xrt::bo* XdnaTarget::slot_bo(PackedWeight& pw, const I8Ctx& kctx) { return kctx.w4g32 ? pw.bo4.get() : pw.bo.get(); } +const std::vector* XdnaTarget::slot_scales(const PackedWeight& pw, const I8Ctx& kctx) { + return kctx.w4g32 ? &pw.scales4 : &pw.scales; +} + int XdnaTarget::max_rows() const { /* engine#1900/#1903: MIN over the loaded per-role widths, never the first populated one. This width IS the chunk * width, and the activation BO holds exactly that many rows -- an over-wide launch reads and poisons past the BO, @@ -1071,8 +1130,8 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri // `prefill` row would hide. It must be zero for a warm process. q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD); PackedWeight pw; - pw.bo = kctx.make_weight_bo(*dev_); if (kctx.gemv) { + pw.bo = kctx.make_weight_bo(*dev_); // The gemv family dequantises from the container's own tiles: upload them unchanged. if (!ar.tiles) { err = "gemv: the model has no tile stream for this container"; return; } std::vector blob; @@ -1080,15 +1139,15 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri if (!kctx.upload_tiles_into(*pw.bo, blob.data(), blob.size())) { err = "gemv: tile upload failed"; return; } - } else { - float sout = 1.0f; - kctx.packB_into(*pw.bo, Bmat, K, N, sout, pw.scales); + } else if (!fill_int8_and_w4(pw, kctx, ar.role, Bmat, K, N, err)) { + return; } fill_ms = std::chrono::duration(std::chrono::steady_clock::now() - f0).count(); it = m.emplace(wkey, std::move(pw)).first; } slot = &it->second; - gs = &slot->scales; + if (!slot_bo(*slot, kctx)) { err = "weight cache: the filled slot has no image for this context's layout"; return; } + gs = slot_scales(*slot, kctx); } else if (kctx.gemv) { err = "gemv: this family needs the weight cache (NPU_NO_WCACHE=1 is not supported with it)"; return; @@ -1101,7 +1160,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri const auto tl = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; auto issue = [&]() -> xrt::run { - return slot ? kctx.launch_async_with_bo(*slot->bo, A, 1, K, as) + return slot ? kctx.launch_async_with_bo(*slot_bo(*slot, kctx), A, 1, K, as) : kctx.sync_and_launch(0); }; xrt::run r = issue(); @@ -1136,7 +1195,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri } if (m16_check_ && &kctx != &ctx) - m16_check_gemm(ctx, kctx, A, K, N, Bmat, slot ? slot->bo.get() : nullptr, as); + m16_check_gemm(ctx, kctx, A, K, N, Bmat, slot ? slot_bo(*slot, kctx) : nullptr, as); if (prof) { const double dpk = std::chrono::duration(tl - tp).count(); @@ -1361,10 +1420,8 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD); PackedWeight pw; - pw.bo = kctx.make_weight_bo(*dev_); - float sout = 1.0f; const auto w0 = std::chrono::steady_clock::now(); // engine#1416: the 22 ms candidate - kctx.packB_into(*pw.bo, Bmat, K, N, sout, pw.scales); + if (!fill_int8_and_w4(pw, kctx, ar.role, Bmat, K, N, err)) return; const auto w1 = std::chrono::steady_clock::now(); wmiss_n_++; wmiss_ms_ += std::chrono::duration(w1 - w0).count(); @@ -1375,7 +1432,8 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: it = m.emplace(wkey, std::move(pw)).first; } slot = &it->second; - gs = &slot->scales; + if (!slot_bo(*slot, kctx)) { err = "weight cache: the filled slot has no image for this context's layout"; return; } + gs = slot_scales(*slot, kctx); } else { float sout = 1.0f; const auto w0 = std::chrono::steady_clock::now(); // engine#1416: uncached branch @@ -1390,7 +1448,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } const auto tl = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; auto issue = [&]() -> xrt::run { - return slot ? kctx.launch_async_with_bo_rows(*slot->bo, A, M, K, ascales.data()) + return slot ? kctx.launch_async_with_bo_rows(*slot_bo(*slot, kctx), A, M, K, ascales.data()) : kctx.launch_async_rows(0, A, M, K, ascales.data()); }; xrt::run r = issue(); @@ -1429,7 +1487,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: o[n] = (float)cm[n] * asc * (*gs)[(size_t)n]; } if (m16_check_ && &kctx != &ctx) - m16_check_gemm_rows(ctx, kctx, A, M, K, N, slot ? slot->bo.get() : nullptr, + m16_check_gemm_rows(ctx, kctx, A, M, K, N, slot ? slot_bo(*slot, kctx) : nullptr, ascales); if (prof) { const double dpk = std::chrono::duration(tl - tp).count(); diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 347ec43cd..faa7b59bc 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -50,6 +50,10 @@ struct PackedWeight { // pb16i: the sidecar's host side table. Empty eref = an int8 pack. std::vector eref; int pb16i_mode = 0; + // NPU_W4_PREFILL: the same weight as the w4g32 image (the prefill contexts) beside the int8 one (the decode twins). + // Both come from the one Bmat a fill sees, so weight_cached() stays true for both families at once. + std::unique_ptr bo4; + std::vector scales4; }; // Per-role dispatch accounting for NPU_PROFILE=1. @@ -153,6 +157,13 @@ class XdnaTarget final : public NpuTarget { // the message a user sees is the same one the forward produced before this moved (an empty string // means no plan, i.e. a purely derived set). bool open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err); + int w4_rows() const { return w4_rows_; } // 0 unless every prefill role runs the w4g32 tier + // A weight-cache fill for the int8 families: the int8 image for the role's int8 context (kctx itself, or its decode + // twin when kctx is the w4g32 one) and, with the w4g32 tier on, the w4g32 image for its prefill context. + bool fill_int8_and_w4(PackedWeight& pw, I8Ctx& kctx, int role, const float* Bmat, int K, int N, std::string& err); + // The image and column scales `kctx` launches on, from a filled slot (nullptr when the slot has none for it). + static xrt::bo* slot_bo(PackedWeight& pw, const I8Ctx& kctx); + static const std::vector* slot_scales(const PackedWeight& pw, const I8Ctx& kctx); // The lm-head context (engine#803 A2b, the last setup block): pb16i ternary images first, then the int8 // ladder, then the q4_1 gemv head -- the body moved verbatim out of Q4nxNpuForward::open_lm_head_context. @@ -249,6 +260,7 @@ class XdnaTarget final : public NpuTarget { std::unique_ptr ctx16_[Q4NX_MAX_DESIGNS]; std::unique_ptr ctxw_[Q4NX_MAX_DESIGNS]; // the wide pb16i/t2s family (engine#1990) int wide_rows_ = 0; // 0 = no wide family + int w4_rows_ = 0; // NPU_W4_PREFILL: the w4g32 prefill tier's M (0 = off) std::unique_ptr ctxn_[Q4NX_MAX_DESIGNS]; // the narrow pb16i/t2s family (decode) int narrow_rows_ = 0; // 0 = no narrow family std::unique_ptr ctx_head_; diff --git a/tests/npu_w4g32_gate_test.cpp b/tests/npu_w4g32_gate_test.cpp new file mode 100644 index 000000000..06dfc593a --- /dev/null +++ b/tests/npu_w4g32_gate_test.cpp @@ -0,0 +1,139 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// npu_w4g32_gate_test -- the w4g32 prefill kernel, bit-exact on the device against the CPU reference. +// +// W4G32_GATE=" M K N" (e.g. ".../full_i8_Z_K5120_N6144_w4g32_m512.elf 512 5120 6144") +// +// Unset, the test SKIPS (77): a box without the ELF or the device records a skip, not a red row. Run it inside an +// npu-hold ticket. Two weight sets, both built through q4nx_w4g32.h exactly as the engine builds an image: +// seed random codes in [-8, 7], signed random per-32 scales, random int8 activations; +// edge every code -8, every scale +max, every activation +127: each per-32 partial at |P| = 32*127*8 = 32512 (the +// int16 narrowing bound inside the kernel) and |C| at K*127*8*MAGMAX(K), the int32 bound MAGMAX is chosen for. +// Each set runs three times with C poisoned (0x5a) before every launch, and every element of every run must equal the +// int64 reference computed from the packed image (q4nx_w4g32::read), with A taken through pack_a() from row-major rows. +#include "q4nx_w4g32.h" + +#include "rt.h" // the engine's own NPU runtime (ONEBIT_NPU_RT_DIR): the same launch API the forward's I8Ctx uses + +#include +#include +#include +#include +#include +#include +#include + +namespace w = q4nx_w4g32; + +// The reference, from the image the device reads: C[m][n] = sum_g mag[g][n] * sum_{k in g} a[m][k] * code[k][n]. +static void reference(const std::vector& a, const std::vector& img, int M, int K, int N, + std::vector& C) { + std::vector code((size_t)K * N); + std::vector mag((size_t)(K / 32) * N); + for (int k = 0; k < K; k++) + for (int n = 0; n < N; n++) { + int c, m; + w::read(img.data(), K, N, k, n, &c, &m); + code[(size_t)k * N + n] = (int8_t)c; + if (k % 32 == 0) mag[(size_t)(k / 32) * N + n] = (int16_t)m; + } + C.assign((size_t)M * N, 0); +#if defined(_OPENMP) +#pragma omp parallel for schedule(static) +#endif + for (int m = 0; m < M; m++) { + std::vector acc((size_t)N, 0); + std::vector p((size_t)N); + for (int g = 0; g < K / 32; g++) { + std::fill(p.begin(), p.end(), 0); + for (int kk = 0; kk < 32; kk++) { + const int av = a[(size_t)m * K + g * 32 + kk]; + const int8_t* row = code.data() + (size_t)(g * 32 + kk) * N; + for (int n = 0; n < N; n++) p[(size_t)n] += av * row[n]; + } + const int16_t* mg = mag.data() + (size_t)g * N; + for (int n = 0; n < N; n++) acc[(size_t)n] += (int64_t)p[(size_t)n] * mg[n]; + } + for (int n = 0; n < N; n++) C[(size_t)m * N + n] = (int32_t)acc[(size_t)n]; // |acc| < 2^31 by MAGMAX + } +} + +int main() { + const char* spec = std::getenv("W4G32_GATE"); + if (!spec || !*spec) { + std::printf("npu_w4g32_gate: W4G32_GATE=\" M K N\" unset -- skipped\n"); + return 77; + } + char elf[4096]; + int M = 0, K = 0, N = 0; + if (std::sscanf(spec, "%4095s %d %d %d", elf, &M, &K, &N) != 4 || M % 128 || !w::fits(K, N) || N % 1024) { + std::printf("npu_w4g32_gate: bad W4G32_GATE \"%s\" (M %% 128, K %% 128, N %% 1024)\n", spec); + return 1; + } + int fails = 0; + try { + xrt::device dev{0u}; + xrt::elf e(elf); + xrt::hw_context hc(dev, e); + xrt::ext::kernel kern(hc, "main:seq"); + const size_t ab = (size_t)M * K, wb = w::w_bytes(K, N), cb = (size_t)M * N * 4; + xrt::ext::bo ba(dev, ab), bw(dev, wb), bc(dev, cb); + for (int set = 0; set < 2; set++) { + const bool edge = set == 1; + std::mt19937 rng(20261115u + (unsigned)set); + std::vector code((size_t)K * N), arow((size_t)M * K); + std::vector sc((size_t)(K / 32) * N); + for (auto& c : code) c = edge ? (int8_t)-8 : (int8_t)((int)(rng() % 16) - 8); + for (auto& s : sc) s = edge ? 1.0f : (float)((int)(rng() % 255) - 127) * (1.0f / 1024.0f); + for (auto& x : arow) x = edge ? (int8_t)127 : (int8_t)((int)(rng() % 255) - 127); + std::vector img(wb); + std::vector cs((size_t)N); + if (!w::pack(code.data(), sc.data(), K, N, img.data(), cs.data())) { std::printf("pack refused\n"); return 1; } + std::vector ap(ab); + w::pack_a(arow.data(), K, M, K, ap.data()); + std::vector ref; + reference(arow, img, M, K, N, ref); + int32_t rmax = 0; + for (int32_t v : ref) rmax = std::max(rmax, v < 0 ? -v : v); + std::memcpy(ba.map(), ap.data(), ab); + std::memcpy(bw.map(), img.data(), wb); + ba.sync(XCL_BO_SYNC_BO_TO_DEVICE); + bw.sync(XCL_BO_SYNC_BO_TO_DEVICE); + for (int rep = 0; rep < 3; rep++) { + std::memset(bc.map(), 0x5a, cb); + bc.sync(XCL_BO_SYNC_BO_TO_DEVICE); + xrt::run r = kern(ba, bw, bc); // (A, W, C), as I8Ctx::sync_and_launch + const auto st = r.wait(std::chrono::milliseconds(30000)); + bc.sync(XCL_BO_SYNC_BO_FROM_DEVICE); + const int32_t* c = bc.map(); + size_t bad = 0, first = 0; + for (size_t i = 0; i < (size_t)M * N; i++) + if (c[i] != ref[i]) { if (!bad) first = i; ++bad; } + std::printf(" %s rep %d state=%d mismatches=%zu/%zu max|C|=%d%s\n", edge ? "edge" : "seed", rep, (int)st, + bad, (size_t)M * N, rmax, + bad ? (" first m=" + std::to_string(first / N) + " n=" + std::to_string(first % N) + + " dev=" + std::to_string(c[first]) + " ref=" + std::to_string(ref[first])).c_str() : ""); + fails += bad != 0; + } + } + } catch (const std::exception& ex) { + std::printf("npu_w4g32_gate: device error: %s\n", ex.what()); + return 1; + } + std::printf("npu_w4g32_gate: %s (%s M=%d K=%d N=%d)\n", fails ? "FAIL" : "PASS, bit-exact x3 on seed and edge", elf, M, + K, N); + return fails ? 1 : 0; +} From 1fb749ef7c75ef5b09211ad5f69876ca26990280 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 20:18:37 -0400 Subject: [PATCH 5/7] npu/w4g32: NPU_W4_CHECK=1, a per-launch device vs float-reference diagnostic (off by default) Co-Authored-By: Claude Opus 5.5 --- npu/target/xdna/xdna_target.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 7afc222f7..d67f95c45 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1489,6 +1489,23 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: if (m16_check_ && &kctx != &ctx) m16_check_gemm_rows(ctx, kctx, A, M, K, N, slot ? slot_bo(*slot, kctx) : nullptr, ascales); + // NPU_W4_CHECK=1: the w4g32 launch against the float reference sum_k A[m][k] * Bmat[k][n] on a sample of rows and + // columns, layers 0-1 only; one line per launch with the worst relative error. A diagnostic, off by default. + if (Bmat && ar.layer >= 0 && std::getenv("NPU_W4_CHECK")) { + const int astr = (ar.a_k > 0 && ar.a_k < K) ? ar.a_k : K; + double worst = 0, refn = 0, errn = 0; + for (int m : {0, std::min(5, M - 1), std::min(15, M - 1)}) { + for (int n = 0; n < N; n += std::max(1, N / 61)) { + double ref = 0; + for (int k = 0; k < K; k++) ref += (double)A[(size_t)m * astr + k] * (double)Bmat[(size_t)k * N + n]; + const double got = out[(size_t)m * N + n]; + refn += ref * ref; errn += (got - ref) * (got - ref); + worst = std::max(worst, std::fabs(got - ref)); + } + } + fprintf(stderr, "[w4check] %s layer %d role %d M=%d K=%d N=%d rel-err %.4f worst-abs %.4g (ref rms %.4g)\n", kctx.w4g32 ? "w4" : "i8", ar.layer, + ar.role, M, K, N, std::sqrt(errn / std::max(refn, 1e-30)), worst, std::sqrt(refn / 183)); + } if (prof) { const double dpk = std::chrono::duration(tl - tp).count(); const double dql = std::chrono::duration(tq - tl).count(); From 22f2c6c9c0bdf9f795e06894ad9ffd1e39acf114 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 20:42:12 -0400 Subject: [PATCH 6/7] npu/w4g32: the LM head stays on an int8 context under NPU_W4_PREFILL The batched head picked pick_ctx(widest K=H role, M), which is the w4g32 prefill context under the tier, and launched the head's int8 tile images on it. Every row's logits were garbage (0/511 next-token), while all 368 layer GEMMs checked correct against float A*Bmat. The head now takes the role's int8 decode twin. With none, the batched head is skipped. The single-row head context refuses by name if it would be w4g32. 27B 512-token prompt: 0/511 -> 372/511 next-token top-1, no zero rows. Co-Authored-By: Claude Opus 5.5 --- npu/forward/src/q4nx_forward.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c1ff4d17d..343f5949e 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -6672,6 +6672,12 @@ bool Q4nxNpuForward::run_lm_head_rows(const std::vector& h_rows, int M, const int T = i_wide >= 0 ? designs_[i_wide].N : 0; const int ntiles = T > 0 ? (NV + T - 1) / T : 0; I8Ctx* hctx = i_wide >= 0 ? xd_->pick_ctx(i_wide, M) : nullptr; + // NPU_W4_PREFILL: the role's prefill context is a w4g32 one, whose kernel reads a w4g32 image -- and the head's + // tiles (lm_head_packed_) are int8 images. Launching them there multiplies the wrong bytes (every row's logits + // were garbage while all 368 layer GEMMs checked correct). The head keeps the int8 decode twin, in chunks of its + // width; with no twin the batched head is skipped and the per-row fallback below runs on the head context. + if (hctx && hctx->w4g32) hctx = xd_->pick_ctx(i_wide, M, /*decode_twin=*/true); + if (hctx && hctx->w4g32) hctx = nullptr; // A verify batch may be wider than the twin's row capacity (mtp_acceptance measures up to 128 rows). // The launch is then made in CHUNKS of at most MD rows rather than falling back to one launch per // row: the weights are already resident, so a chunk costs one launch, and M=128 becomes 8 launches @@ -6956,7 +6962,13 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector // N, same per-launch cost. // The head keeps the int8 family even in a pb16i container: lm_head is not a // PTQ1_0 ternary tensor and has no image, and packB below is the int8 packer. - I8Ctx& lctx = xd_->head_ctx() ? *xd_->head_ctx() : *xd_->pick_ctx(i_qkv, 1); + I8Ctx& lctx = xd_->head_ctx() ? *xd_->head_ctx() : *xd_->pick_ctx(i_qkv, 1, /*decode_twin=*/xd_->w4_rows() > 0); + // The head packs and launches int8 images: it must never sit on a w4g32 (NPU_W4_PREFILL) prefill context. + if (lctx.w4g32) { + err_ = "lm head: NPU_W4_PREFILL needs an int8 head context (an LMHEAD ELF or the _m16 decode twins); this pack " + "has neither, so unset NPU_W4_PREFILL"; + return false; + } // Belt for the gemv family: packB_into writes a KD*ND int8 image, which is larger than that // family's tile-stream BO. The head context above handles this family; reaching here means a // container with no head ELF, and a loud refusal beats a heap overrun (measured: it aborted a From a38396404c806737180ea977abc33009dd92c606 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 22:37:45 -0400 Subject: [PATCH 7/7] npu/w4g32: keep this PR to the tier; the twin test arm moves to its own PR Review at 1769d39 asked for one owner per PR. The elf_twin.h refactor and the npu_elf_twin ctest (#2611's test arm) move to a separate PR from main. This branch keeps main's twin code byte for byte: #2611's static same_artifact and its inline twin path. The tier only wraps it in 'if (!w4_rows_)'. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 5 -- npu/target/xdna/elf_twin.h | 58 ---------------------- npu/target/xdna/xdna_target.cpp | 29 +++++++++-- tests/npu_elf_twin_test.cpp | 86 --------------------------------- 4 files changed, 26 insertions(+), 152 deletions(-) delete mode 100644 npu/target/xdna/elf_twin.h delete mode 100644 tests/npu_elf_twin_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9cc3d9a16..a9ed75674 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1841,11 +1841,6 @@ add_test(NAME npu_w4g32_layout COMMAND ${CMAKE_SOURCE_DIR}/tests/npu_w4g32_layout_test.sh ${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/tests/npu_w4g32_layout_test.cpp) -# engine#2599's decision (npu/target/xdna/elf_twin.h): a prefill ELF runs at 16 rows only when it is byte-identical to -# the _m16 twin of its own stem. Every arm, on files it writes itself; no device. -add_executable(npu_elf_twin_test tests/npu_elf_twin_test.cpp) -target_include_directories(npu_elf_twin_test PRIVATE ${CMAKE_SOURCE_DIR}/npu) -add_test(NAME npu_elf_twin COMMAND npu_elf_twin_test) # The t2s weight stream (q4nx_weight_layout "t2s"): the same integer-exact weights at 2.125 b/w, # expanded on the core. Its host-side contract is the file size, so the test pins the tile against # the numbers tools/q4nx_t2s.py measured on a real image (50,135,040 -> 23,674,880 B for gate/up). diff --git a/npu/target/xdna/elf_twin.h b/npu/target/xdna/elf_twin.h deleted file mode 100644 index 81c2cd415..000000000 --- a/npu/target/xdna/elf_twin.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2026 bong-water-water-bong -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// elf_twin.h -- engine#2599: is a prefill ELF the very same artifact as its own M=16 decode twin? -// -// bonsai2-27b-NPU shipped kernels/full_i8__K*_N*.elf byte-identical to the matching _m16.elf: its "prefill" set -// was the 16-row design. The artifact cannot say how many rows it computes (its declared widths are per tile), so a -// context sized from the design geometry (M=128) computed the first 16 rows of every 128-row chunk. The evidence is -// equal size and equal bytes against the twin of the SAME stem -- two roles of one shape (G and U) are legitimately -// identical and are never compared with each other. tests/npu_elf_twin_test.cpp pins both functions. -#pragma once - -#include -#include -#include -#include -#include - -namespace onebit::npu { - -// "/_m16.elf" for an ELF named ".elf" (a name without ".elf" keeps its whole name as the stem). -inline std::string m16_twin_path(const std::string& dir, const std::string& elf_name) { - std::string stem(elf_name); - if (stem.size() > 4 && stem.compare(stem.size() - 4, 4, ".elf") == 0) stem.resize(stem.size() - 4); - return dir + "/" + stem + "_m16.elf"; -} - -// True only when both files open, are the same non-zero size and hold the same bytes. -inline bool same_artifact(const std::string& a, const std::string& b) { - std::ifstream fa(a, std::ios::binary | std::ios::ate), fb(b, std::ios::binary | std::ios::ate); - if (!fa || !fb) return false; - const std::streamoff na = fa.tellg(), nb = fb.tellg(); - if (na <= 0 || na != nb) return false; - fa.seekg(0); - fb.seekg(0); - std::vector ba(1 << 16), bb(1 << 16); - for (std::streamoff left = na; left > 0;) { - const std::streamsize n = (std::streamsize)std::min(left, (std::streamoff)ba.size()); - if (!fa.read(ba.data(), n) || !fb.read(bb.data(), n)) return false; - if (std::memcmp(ba.data(), bb.data(), (size_t)n) != 0) return false; - left -= n; - } - return true; -} - -} // namespace onebit::npu diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 6a618a138..9e6f125f6 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -20,7 +20,6 @@ // // docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes. #include "target/xdna/xdna_target.h" -#include "target/xdna/elf_twin.h" // engine#2599: a prefill ELF that is its own _m16 twin #include "target/xdna/launch_fit.h" // engine#1783: refuse a launch its context cannot hold #include "batch_width.h" #include "file_create.h" // dumps are created 0600, never through a symlink @@ -84,6 +83,28 @@ static const char* role_tag(Q4nxDesign r) { // The forward's own probe, kept identical (exists and non-empty): an ELF that is present but zero bytes // is the failure this catches, and a target that decided differently from its caller would be a second // definition of "the kernel is there". +// engine#2599: is the prefill ELF the very same artifact as its M=16 decode twin? bonsai2-27b-NPU ships +// kernels/full_i8__K*_N*.elf byte-identical (md5) to the matching _m16.elf, i.e. its "prefill" set IS the +// 16-row design. Nothing in the artifact says how many rows it computes (its declared widths are per-tile, and a +// narrower declared width than the buffer is legal), so a context sized from the design geometry (M=128) chunked the +// prompt at 128 rows and the kernel computed the first 16 of each: 112 rows of every chunk came back as 0. Equal size +// and equal bytes is the evidence; anything else (a real M=128 build, or no twin in the directory) changes nothing. +static bool same_artifact(const std::string& a, const std::string& b) { + std::ifstream fa(a, std::ios::binary | std::ios::ate), fb(b, std::ios::binary | std::ios::ate); + if (!fa || !fb) return false; + const std::streamoff na = fa.tellg(), nb = fb.tellg(); + if (na <= 0 || na != nb) return false; + fa.seekg(0); fb.seekg(0); + std::vector ba(1 << 16), bb(1 << 16); + for (std::streamoff left = na; left > 0;) { + const std::streamsize n = (std::streamsize)std::min(left, (std::streamoff)ba.size()); + if (!fa.read(ba.data(), n) || !fb.read(bb.data(), n)) return false; + if (std::memcmp(ba.data(), bb.data(), (size_t)n) != 0) return false; + left -= n; + } + return true; +} + static bool elf_present(const std::string& p) { FILE* f = std::fopen(p.c_str(), "rb"); if (!f) return false; @@ -199,8 +220,10 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string& int md = w4_rows_ ? w4_rows_ : g->M; if (!w4_rows_) { // engine#2599 (the w4g32 tier opens its own M=512 ELF instead, so this does not apply to it) - const std::string twin = onebit::npu::m16_twin_path(dir, g->elf_name); - if (md > Q4NX_DESIGN_M_DECODE && onebit::npu::same_artifact(elf, twin)) { + std::string twin(g->elf_name); + if (twin.size() > 4 && twin.compare(twin.size() - 4, 4, ".elf") == 0) twin.resize(twin.size() - 4); + twin = dir + "/" + twin + "_m16.elf"; + if (md > Q4NX_DESIGN_M_DECODE && same_artifact(elf, twin)) { md = Q4NX_DESIGN_M_DECODE; prefill_is_decode_design_ = true; std::fprintf(stderr, " [forward] %-3s %s is byte-identical to its _m16 twin: it is the %d-row design, so " diff --git a/tests/npu_elf_twin_test.cpp b/tests/npu_elf_twin_test.cpp deleted file mode 100644 index 6c03608bc..000000000 --- a/tests/npu_elf_twin_test.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2026 bong-water-water-bong -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// npu_elf_twin_test -- engine#2599's decision, pinned without a device: open_prefill() runs a role at 16 rows only when -// its prefill ELF is byte-identical to the _m16 twin of the SAME stem. Every arm of that decision: -// 1. the twin path is "/_m16.elf"; -// 2. identical bytes -> twin (the bonsai2-27b-NPU case); -// 3. one byte different, same size -> not a twin (a real M=128 build of equal size); -// 4. a different size -> not a twin; -// 5. no twin on disk, or an empty file -> not a twin (nothing changes); -// 6. two ROLES of one shape (G and U) are identical by design, and the rule never compares them: G's twin is G_m16. -#include "target/xdna/elf_twin.h" - -#include -#include -#include -#include -#include -#include - -static int fails = 0; -#define CHECK(c, m) do { if (!(c)) { std::printf("FAIL: %s\n", m); ++fails; } } while (0) - -static void put(const std::string& p, const std::string& bytes) { std::ofstream(p, std::ios::binary) << bytes; } - -int main() { - char tmpl[] = "/tmp/npu-elf-twin.XXXXXX"; - const char* tmp = std::getenv("TMPDIR"); - std::string base = tmp && *tmp ? std::string(tmp) + "/npu-elf-twin.XXXXXX" : std::string(tmpl); - std::vector buf(base.begin(), base.end()); - buf.push_back('\0'); - if (!mkdtemp(buf.data())) { std::printf("mkdtemp failed\n"); return 1; } - const std::string d(buf.data()); - using onebit::npu::m16_twin_path; - using onebit::npu::same_artifact; - - CHECK(m16_twin_path(d, "full_i8_U_K5120_N17408.elf") == d + "/full_i8_U_K5120_N17408_m16.elf", "1. twin path"); - CHECK(m16_twin_path(d, "noext") == d + "/noext_m16.elf", "1. twin path without .elf"); - - const std::string big(200000, 'x'); // spans several 64 KiB compare blocks - put(d + "/U.elf", big); - put(d + "/U_m16.elf", big); - CHECK(same_artifact(d + "/U.elf", m16_twin_path(d, "U.elf")), "2. identical bytes are a twin"); - - std::string last = big; last.back() = 'y'; - put(d + "/D.elf", big); - put(d + "/D_m16.elf", last); - CHECK(!same_artifact(d + "/D.elf", m16_twin_path(d, "D.elf")), "3. a last-byte difference is not a twin"); - std::string mid = big; mid[70000] = 'z'; - put(d + "/O_m16.elf", mid); - put(d + "/O.elf", big); - CHECK(!same_artifact(d + "/O.elf", m16_twin_path(d, "O.elf")), "3. a difference in the second block is not a twin"); - - put(d + "/Z.elf", big); - put(d + "/Z_m16.elf", big + "x"); - CHECK(!same_artifact(d + "/Z.elf", m16_twin_path(d, "Z.elf")), "4. a size difference is not a twin"); - - put(d + "/Q.elf", big); - CHECK(!same_artifact(d + "/Q.elf", m16_twin_path(d, "Q.elf")), "5. no twin on disk is not a twin"); - put(d + "/E.elf", ""); - put(d + "/E_m16.elf", ""); - CHECK(!same_artifact(d + "/E.elf", m16_twin_path(d, "E.elf")), "5. two empty files are not a twin"); - - // 6. G and U: same shape, same bytes, no _m16 for G here -> G is not reported as a twin of U. - put(d + "/G.elf", big); - CHECK(!same_artifact(d + "/G.elf", m16_twin_path(d, "G.elf")), "6. G identical to U is not G's twin"); - - for (const char* f : {"U", "U_m16", "D", "D_m16", "O", "O_m16", "Z", "Z_m16", "Q", "E", "E_m16", "G"}) - std::remove((d + "/" + f + ".elf").c_str()); - rmdir(d.c_str()); - if (fails) { std::printf("npu_elf_twin_test: %d failure(s)\n", fails); return 1; } - std::printf("npu_elf_twin_test: OK\n"); - return 0; -}