diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index bdbd6a346..8586c0cea 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -244,7 +244,10 @@ class Q4nxNpuForward { bool host_gemm_i8_ = false; // Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809). struct HostI8Tensor { int groups = 0; int N = 0; std::vector codes; std::vector d; }; - std::map host_i8_cache_; + // Keyed on (LAYER, wkey, K, N): `wkey` alone is a per-role slot that repeats in every layer and `Bmat` is the one + // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer + // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. + std::map, HostI8Tensor> host_i8_cache_; void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 95ec35e94..9c0d4817f 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3042,8 +3042,20 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector= 0; + const auto key = std::make_tuple(cur_layer_, wkey, K, N); + HostI8Tensor scratch; + const HostI8Tensor* cached = nullptr; + auto it = cacheable ? host_i8_cache_.find(key) : host_i8_cache_.end(); + if (!cacheable || it == host_i8_cache_.end()) { HostI8Tensor t; t.groups = groups; t.N = N; t.codes.assign((size_t)K * (size_t)N, 0); @@ -3068,9 +3080,16 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, } } } - it = host_i8_cache_.emplace(wkey, std::move(t)).first; + if (!cacheable) { + scratch = std::move(t); + cached = &scratch; + } else { + cached = &host_i8_cache_.emplace(key, std::move(t)).first->second; + } + } else { + cached = &it->second; } - const HostI8Tensor& t = it->second; + const HostI8Tensor& t = *cached; std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k;