Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,10 @@ class Q4nxNpuForward {
bool host_gemm_i8_ = false;
// Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809).
struct HostI8Tensor { int groups = 0; int N = 0; std::vector<int8_t> codes; std::vector<float> d; };
std::map<int, HostI8Tensor> host_i8_cache_;
// Keyed on (LAYER, wkey, K, N): `wkey` alone is a per-role slot that repeats in every layer and `Bmat` is the one
// scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer
// (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token.
std::map<std::tuple<int, int, int, int>, HostI8Tensor> host_i8_cache_;
void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out);
bool load_layer(int l);
// `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or
Expand Down
27 changes: 23 additions & 4 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3042,8 +3042,20 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector<float>& h, std::vector<f
void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat,
int wkey, float* out) {
const int G = 128, groups = (K + G - 1) / G;
auto it = host_i8_cache_.find(wkey);
if (it == host_i8_cache_.end()) {
// THE KEY MUST CARRY THE LAYER AND THE SHAPE (engine#1830 review). `wkey` is a per-ROLE slot -- 0 QKV, 1 O/gate,
// 2 GU, 3 up, 4 down -- so it is the same number in every layer, and `Bmat` is the one scratch buffer that
// `load_layer()` decodes each layer into, so neither identifies a layer. Keyed on `wkey` alone, every layer after 0
// ran on LAYER 0's weights: layer 0 scored cos 0.99923, layer 1 collapsed to 0.37, and 31 layers of wrong residual
// updates compounded into the "explosion, not drift" the per-layer dump shows. The device cache this arm mirrors
// says the rule in its own comment -- one weight BO per (layer, wkey) -- and carries `a.layer = cur_layer_`.
// `wkey < 0` is the routed-expert case: those weights vary per TOKEN, which is why the device path passes -1; caching
// them would serve every token's experts from the first token's, so that case packs a throwaway image instead.
const bool cacheable = wkey >= 0;
const auto key = std::make_tuple(cur_layer_, wkey, K, N);
HostI8Tensor scratch;
const HostI8Tensor* cached = nullptr;
auto it = cacheable ? host_i8_cache_.find(key) : host_i8_cache_.end();
if (!cacheable || it == host_i8_cache_.end()) {
HostI8Tensor t;
t.groups = groups; t.N = N;
t.codes.assign((size_t)K * (size_t)N, 0);
Expand All @@ -3068,9 +3080,16 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N,
}
}
}
it = host_i8_cache_.emplace(wkey, std::move(t)).first;
if (!cacheable) {
scratch = std::move(t);
cached = &scratch;
} else {
cached = &host_i8_cache_.emplace(key, std::move(t)).first->second;
}
} else {
cached = &it->second;
}
const HostI8Tensor& t = it->second;
const HostI8Tensor& t = *cached;
std::vector<int8_t> a8((size_t)K);
for (int m = 0; m < M; m++) {
const float* a = A + (size_t)m * (size_t)a_k;
Expand Down
Loading