From 1516de589d1914bb5a4bf6ae9e18710145113c7b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:04:21 -0400 Subject: [PATCH 1/3] npu: the integer arm's operands come from the engine's packer, not a private re-quantisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The arm was merged in #1809 with a quantisation rule of its own: per 128-K group, int4 codes in [-8, 7]. The image the device reads is `q4nx_pack_B_int8_blocked()`'s: ONE multiplier per weight column, int8 codes in [-127, 127]. Both start from the same f32 matrix, so the arm computed a DIFFERENT MODEL -- measured on the Mistral-7B 1.9 container: it answered "ikt" to "The capital of France is" and then locked onto one token, while a per-GEMM A/B read cos 0.9999 against f32 on the same tensors. So the image is now built by that same packer (`blocked=1`, the layout the device reads and the fast path -- measured 174.5 s of warm-up blocked=0 versus 32-35 s blocked) and the MAC walks it with `q4nx_layout::blocked_index()`. Measured after, on current main: warm-up 35.2 s and the answer to "The capital of France is" is **'Paris,▁Question:'**, the same first token the device produces; before it was '%%%%...' / '▁/******/'. With the layer-aware cache key already in main (engine#1830 review), the arm now computes the same model as the device to int8 precision, which is what an integer reference has to do before any family can be gated against it. `npu_pack_test` (the ctest that pins the integer forms at K=130) builds and passes; copyright rc 0. --- npu/forward/include/q4nx_pack.h | 1 - npu/forward/src/q4nx_forward.cpp | 46 +++++++++++++++----------------- npu/forward/src/q4nx_pack.c | 1 + 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..1a36053b3 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -191,7 +191,6 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out); - /* Architecture dims a design set is derived from. */ typedef struct { int H; /* hidden_size */ diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5a6b4b36a..c1d689e72 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3044,7 +3044,6 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; - } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } - } - } + t.d.assign((size_t)N, 0.0f); + // THE OPERANDS MUST BE THE ENGINE'S, NOT A PRIVATE RE-QUANTISATION (engine#1808/#1830). This arm exists to be + // the integer reference a kernel family is gated against, and the device's weight image comes from + // q4nx_pack_B_int8_blocked(): ONE multiplier per column (amax over the whole K), int8 codes in [-127, 127]. + // Quantising here with a different rule -- per 128-K group, int4 [-8, 7] -- computes a different model: measured + // 2026-11-10, the arm answered "ikt" to "The capital of France is" and then locked onto one token, while the + // per-GEMM A/B read cos 0.9999 against f32 on the same tensors. Same packer, same bytes, same model. + double ssum = 0.0; + // blocked=1: the image is written in the blocked microtile order, which is the fast path AND the layout the + // device actually reads, so the MAC below walks it with q4nx_layout::blocked_index(). (blocked=0 was measured + // at 174.5 s of warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.) + q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum); if (!cacheable) { scratch = std::move(t); cached = &scratch; @@ -3109,8 +3099,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } - q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, - out + (size_t)m * (size_t)N); + float* dst = out + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * + (int)a8[(size_t)k]); + dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + } } } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..c021cca95 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -161,6 +161,7 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ + void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out) { const int G = 128; From 7904dfa2a07b05abcadd7566655e12041a25a013 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:47:15 -0400 Subject: [PATCH 2/3] npu: keyed dumps settle it -- the device's weight scales are BIT-IDENTICAL to the arm's (384/384) Dumps were paired by index first and that was meaningless: the two sides reach the GEMM a different number of times (1024 arm dumps against 384 device dumps on the same request). Keyed by (layer, N, K, ordinal) they pair exactly. Result, one token, Mistral-7B 1.9 container, one npu-hold ticket: arm dumps 1024, device dumps 384, paired 384 exactly equal weight-scale arrays: 384/384 worst relative difference: 0.000e+00 Together with the earlier measurements this closes the GEMM question: the activation scale is the same formula on both sides (`amax/127`, device at xdna_target.cpp:982), the epilogue's two multiplication orders were measured to differ by 0 (the probe returned 0 on every call), the int32 accumulators were identical for the first token's whole pass, and now the per-column weight scales are exactly equal. So the device's GEMMs are bit-exact against this arm. What that leaves is outside the GEMM: the arm and the device still differ by 0.30 max|d| in logits, and the non-GEMM work (RoPE, attention, norms) is shared host code, so the remaining candidates are the fused/selected kernel paths (fused_gu splits GU differently from the arm's one GEMM) and the head path. That is a different question from "are the kernels right", and the answer to that one is yes. Env-gated, off by default (`NPU_I8_GS_DUMP`); nothing in the shipped path changes. --- npu/forward/src/q4nx_forward.cpp | 9 +++++++++ npu/target/xdna/xdna_target.cpp | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c1d689e72..4755c4d24 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3083,6 +3083,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, cached = &it->second; } const HostI8Tensor& t = *cached; + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/arm_L%d_N%d_K%d_%03d.bin", gdir, cur_layer_, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + std::fwrite(t.d.data(), sizeof(float), std::min((size_t)N, t.d.size()), f); + std::fclose(f); + } + } std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe4c4ecae..4c45789d5 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1149,6 +1149,18 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; out.resize((size_t)M * N); + // engine#1808: the device's per-column weight scales, KEYED by (layer, N, K) so it can be paired with the host + // arm's dump exactly. Index pairing was tried first and is meaningless -- the two sides reach the GEMM a + // different number of times (384 device dumps against 1024 arm dumps on the same request). + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/dev_L%d_N%d_K%d_%03d.bin", gdir, ar.layer, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + if (gs && !gs->empty()) std::fwrite(gs->data(), sizeof(float), (size_t)N, f); + std::fclose(f); + } + } for (int m = 0; m < M; m++) { const int32_t* cm = kctx.Cm + (size_t)m * kctx.ND; const float asc = ascales[(size_t)m]; From d5f7d48760eada1e9a422948934aa1d485a1e888 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:13:40 -0400 Subject: [PATCH 3/3] npu: the integer head, and the gate it unlocks -- device logits BIT-EXACT over four steps The head was the last place the host arm and the device computed the same tensor differently: in host mode `run_lm_head()` is a plain f32 matvec ("No I8Ctx is touched") while the device reads a packed quantised table (`lm_head_packed_`, `run_lm_head_gemv`). That was the whole of the remaining `max|d| = 0.30` once every body GEMM had been shown bit-exact (weight scales 384/384 identical, accumulators identical, epilogue order gap 0). The device packs its head tiles with `lctx.packB_into(...)` -> `packB_blocked`, so this builds them with the SAME rule through `q4nx_pack_B_int8_blocked()` -- one multiplier per column, int8 codes -- chunked by vocabulary block (8192 columns; a whole head image is 622 MB at this vocabulary) and cached per block. The activation row is quantised with `amax/127` after the same `1/embed_scale` factor the device applies, and the MAC is exact int32 walked with `q4nx_layout::blocked_index()`, then `d[n] * (ascale * acc)`. Measured, one npu-hold ticket, device vs this arm, logit dumps compared per step: step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 step 1: argmax 278 == 278 cos 1.000000 max|d| 0 step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 That is TNPU-ALL section 4's bar met literally for this family rather than read down to "at the accumulator": the device's arithmetic and the integer reference agree to the last bit, on every step, for a full request. --- npu/forward/include/q4nx_forward.h | 1 + npu/forward/src/q4nx_forward.cpp | 49 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 8586c0cea..c64be7992 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -248,6 +248,7 @@ class Q4nxNpuForward { // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. std::map, HostI8Tensor> host_i8_cache_; + std::vector lm_head_i8_; // integer arm's head image, one per vocabulary block void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 4755c4d24..c43ea283c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -5215,6 +5215,55 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector if (npu::model::run_lm_head(*xd_, h.data(), via_target.data())) { logits = std::move(via_target); return true; } } if (!host_gemm_ && xd_ && xd_->head_ctx() && xd_->head_ctx()->gemv) return run_lm_head_gemv(h, logits); + if (host_gemm_ && host_gemm_i8_) { + // engine#1808: the head's integer twin. Without this the arm's head is a plain f32 matvec ("No I8Ctx is + // touched" below) while the device's head reads a packed quantised table, which is exactly the 0.30 max|d| + // that was left once every body GEMM had been shown bit-exact. The device packs its head tiles with + // lctx.packB_into(...) -> packB_blocked, so the arm builds them with the same rule through + // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a + // whole head image is H x V bytes (622 MB at this vocabulary). + const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; + const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); + logits.assign((size_t)NV, 0.0f); + std::vector a8v((size_t)H); + float amax = 0.0f; + for (int k = 0; k < H; k++) { + const float v = std::fabs(h[(size_t)k] * inv_es); + if (v > amax) amax = v; + } + const float ascale = amax > 0 ? amax / 127.0f : 1.0f; + for (int k = 0; k < H; k++) { + int q = (int)std::round(h[(size_t)k] * inv_es / ascale); + a8v[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + std::vector bt((size_t)H * (size_t)TB); + std::vector a8i((size_t)H); + for (size_t i = 0; i < a8v.size(); i++) a8i[i] = a8v[i]; + for (int t = 0; t < (int)lm_head_i8_.size(); t++) { + const int n0 = t * TB, n1 = std::min(NV, n0 + TB), Tn = n1 - n0; + HostI8Tensor& c = lm_head_i8_[(size_t)t]; + if (c.codes.empty()) { + for (int n = n0; n < n1; n++) + for (int k = 0; k < H; k++) bt[(size_t)k * Tn + (n - n0)] = lm_at(n, k); + c.N = Tn; + c.codes.assign((size_t)H * (size_t)Tn, 0); + c.d.assign((size_t)Tn, 0.0f); + double ssum = 0.0; + q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, /*blocked=*/1, c.d.data(), &ssum); + } + for (int n = 0; n < Tn; n++) { + int32_t acc = 0; + for (int k = 0; k < H; k++) + acc += (int32_t)((int)c.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, H, Tn)] * + (int)a8i[(size_t)k]); + float v = c.d[(size_t)n] * (ascale * (float)acc); + if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); + logits[(size_t)(n0 + n)] = v; + } + } + return true; + } if (host_gemm_) { // Offline mode: plain float matvec against the (possibly dequantized) // tied table, matching the Python reference. No I8Ctx is touched.