From 69e18f72e25beb2c95c948b1ffb31f2bae6f0c2e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 04:25:30 -0400 Subject: [PATCH] npu: the integer host arm's weight cache carries the layer, and never caches routed experts Main's arm (`b6db79e9`, #1809) caches its packed weight image under `host_i8_cache_.find(wkey)` -- `wkey` ALONE. `wkey` is a per-ROLE slot (0 QKV, 1 O/gate, 2 GU, 3 up, 4 down), identical in every layer, and `Bmat` is the one scratch buffer `load_layer()` decodes each layer into, so neither identifies a layer. Every layer after 0 therefore ran on LAYER 0's weights: layer 0 at cos 0.99923, layer 1 at 0.37, then 31 layers of wrong residual updates compounding into the "explosion, not drift" the per-layer dump shows. The device cache this arm mirrors states the rule in its own comment (one weight BO per (layer, wkey)) and its launch descriptor carries `a.layer = cur_layer_` (:3125, :3178). The key is now `(cur_layer_, wkey, K, N)` -- the layer, and the shape the older branch version already had. `wkey < 0` is the routed-expert case: those weights vary per TOKEN, which is why the device path passes -1. Caching them would serve every token's experts from the first token's, so that case packs a throwaway image for the call and caches nothing. Verified: `onebit_npu_forward_syntax` -- the target that compiles `q4nx_forward.cpp` -- builds clean at this head. The end-to-end confirmation is the 7B rerun (#1830): the per-layer dump should hold cos ~0.999 past layer 1 and the degenerate answer should go away. Not run here, not claimed. Refs #1830 --- npu/forward/include/q4nx_forward.h | 5 ++++- npu/forward/src/q4nx_forward.cpp | 27 +++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index bdbd6a346..8586c0cea 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -244,7 +244,10 @@ class Q4nxNpuForward { bool host_gemm_i8_ = false; // Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809). struct HostI8Tensor { int groups = 0; int N = 0; std::vector codes; std::vector d; }; - std::map host_i8_cache_; + // Keyed on (LAYER, wkey, K, N): `wkey` alone is a per-role slot that repeats in every layer and `Bmat` is the one + // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer + // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. + std::map, HostI8Tensor> host_i8_cache_; void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 95ec35e94..9c0d4817f 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3042,8 +3042,20 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector= 0; + const auto key = std::make_tuple(cur_layer_, wkey, K, N); + HostI8Tensor scratch; + const HostI8Tensor* cached = nullptr; + auto it = cacheable ? host_i8_cache_.find(key) : host_i8_cache_.end(); + if (!cacheable || it == host_i8_cache_.end()) { HostI8Tensor t; t.groups = groups; t.N = N; t.codes.assign((size_t)K * (size_t)N, 0); @@ -3068,9 +3080,16 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, } } } - it = host_i8_cache_.emplace(wkey, std::move(t)).first; + if (!cacheable) { + scratch = std::move(t); + cached = &scratch; + } else { + cached = &host_i8_cache_.emplace(key, std::move(t)).first->second; + } + } else { + cached = &it->second; } - const HostI8Tensor& t = it->second; + const HostI8Tensor& t = *cached; std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k;