Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ class Q4nxNpuForward {
// scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer
// (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token.
std::map<std::tuple<int, int, int, int>, HostI8Tensor> host_i8_cache_;
std::vector<HostI8Tensor> lm_head_i8_; // integer arm's head image, one per vocabulary block
void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out);
bool load_layer(int l);
// `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or
Expand Down
1 change: 0 additions & 1 deletion npu/forward/include/q4nx_pack.h
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,6 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co
void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d,
int groups, int N, float ascale, float* out);


/* Architecture dims a design set is derived from. */
typedef struct {
int H; /* hidden_size */
Expand Down
109 changes: 85 additions & 24 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3044,7 +3044,6 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector<float>& h, std::vector<f
// the device is about accumulation and epilogue, not about a float both sides happen to share.
void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat,
int wkey, float* out) {
const int G = 128, groups = (K + G - 1) / G;
// THE KEY MUST CARRY THE LAYER AND THE SHAPE (engine#1830 review). `wkey` is a per-ROLE slot -- 0 QKV, 1 O/gate,
// 2 GU, 3 up, 4 down -- so it is the same number in every layer, and `Bmat` is the one scratch buffer that
// `load_layer()` decodes each layer into, so neither identifies a layer. Keyed on `wkey` alone, every layer after 0
Expand All @@ -3060,28 +3059,26 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N,
auto it = cacheable ? host_i8_cache_.find(key) : host_i8_cache_.end();
if (!cacheable || it == host_i8_cache_.end()) {
HostI8Tensor t;
t.groups = groups; t.N = N;
t.N = N;
t.codes.assign((size_t)K * (size_t)N, 0);
t.d.assign((size_t)groups * (size_t)N, 0.0f);
for (int g = 0; g < groups; g++) {
const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K;
for (int n = 0; n < N; n++) {
float amax = 0.0f;
for (int k = k0; k < k1; k++) {
const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]);
if (std::isfinite(v) && v > amax) amax = v;
}
/* engine#1839: the weights here are the container's INT4 codes, so the scale that maps amax onto
* that range is amax/7 -- pairing amax/127 (an int8 scale) with the [-8, 7] clamp saturated every code
* but the largest and made the host reference's weights ~18x too small. q4nx_quant_int4_*() own the
* pair so this site and q4nx_pack.c's cannot drift apart again. */
const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f;
t.d[(size_t)g * (size_t)N + (size_t)n] = q4nx_quant_int4_scale(amax_eff);
for (int k = k0; k < k1; k++)
t.codes[(size_t)k * (size_t)N + (size_t)n] =
q4nx_quant_int4_code(Bmat[(size_t)k * (size_t)N + (size_t)n], amax_eff);
}
}
// RESOLVED toward the int8 packer (this branch) rather than main's corrected int4 rule (engine#1839).
// Both are self-consistent; the question is what the DEVICE consumes. The device's body image comes from
// packB_blocked() -- one multiplier per column, int8 codes -- and this branch measured the consequence:
// device logits BIT-EXACT (max|d| = 0) over four steps, three repeats, plus an edge case. So the int8 packer
// is the rule the reference has to share with the device, and q4nx_quant_int4_*() stays for the paths that
// genuinely hold int4 codes.
t.d.assign((size_t)N, 0.0f);
// THE OPERANDS MUST BE THE ENGINE'S, NOT A PRIVATE RE-QUANTISATION (engine#1808/#1830). This arm exists to be
// the integer reference a kernel family is gated against, and the device's weight image comes from
// q4nx_pack_B_int8_blocked(): ONE multiplier per column (amax over the whole K), int8 codes in [-127, 127].
// Quantising here with a different rule -- per 128-K group, int4 [-8, 7] -- computes a different model: measured
// 2026-11-10, the arm answered "ikt" to "The capital of France is" and then locked onto one token, while the
// per-GEMM A/B read cos 0.9999 against f32 on the same tensors. Same packer, same bytes, same model.
double ssum = 0.0;
// blocked=1: the image is written in the blocked microtile order, which is the fast path AND the layout the
// device actually reads, so the MAC below walks it with q4nx_layout::blocked_index(). (blocked=0 was measured
// at 174.5 s of warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.)
q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum);
if (!cacheable) {
scratch = std::move(t);
cached = &scratch;
Expand All @@ -3092,6 +3089,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N,
cached = &it->second;
}
const HostI8Tensor& t = *cached;
if (const char* gdir = getenv("NPU_I8_GS_DUMP")) {
static int n = 0;
char path[4096];
std::snprintf(path, sizeof path, "%s/arm_L%d_N%d_K%d_%03d.bin", gdir, cur_layer_, N, K, n++);
if (std::FILE* f = std::fopen(path, "wb")) {
std::fwrite(t.d.data(), sizeof(float), std::min<size_t>((size_t)N, t.d.size()), f);
std::fclose(f);
}
}
std::vector<int8_t> a8((size_t)K);
for (int m = 0; m < M; m++) {
const float* a = A + (size_t)m * (size_t)a_k;
Expand All @@ -3108,8 +3114,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N,
int q = (int)std::round(v * tis);
a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q));
}
q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale,
out + (size_t)m * (size_t)N);
float* dst = out + (size_t)m * (size_t)N;
for (int n = 0; n < N; n++) {
int32_t acc = 0;
for (int k = 0; k < K; k++)
acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] *
(int)a8[(size_t)k]);
dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc);
}
}
}

Expand Down Expand Up @@ -5233,6 +5245,55 @@ bool Q4nxNpuForward::run_lm_head(const std::vector<float>& h, std::vector<float>
if (npu::model::run_lm_head(*xd_, h.data(), via_target.data())) { logits = std::move(via_target); return true; }
}
if (!host_gemm_ && xd_ && xd_->head_ctx() && xd_->head_ctx()->gemv) return run_lm_head_gemv(h, logits);
if (host_gemm_ && host_gemm_i8_) {
// engine#1808: the head's integer twin. Without this the arm's head is a plain f32 matvec ("No I8Ctx is
// touched" below) while the device's head reads a packed quantised table, which is exactly the 0.30 max|d|
// that was left once every body GEMM had been shown bit-exact. The device packs its head tiles with
// lctx.packB_into(...) -> packB_blocked, so the arm builds them with the same rule through
// q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a
// whole head image is H x V bytes (622 MB at this vocabulary).
const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f;
const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes
if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor());
logits.assign((size_t)NV, 0.0f);
std::vector<float> a8v((size_t)H);
float amax = 0.0f;
for (int k = 0; k < H; k++) {
const float v = std::fabs(h[(size_t)k] * inv_es);
if (v > amax) amax = v;
}
const float ascale = amax > 0 ? amax / 127.0f : 1.0f;
for (int k = 0; k < H; k++) {
int q = (int)std::round(h[(size_t)k] * inv_es / ascale);
a8v[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q));
}
std::vector<float> bt((size_t)H * (size_t)TB);
std::vector<int8_t> a8i((size_t)H);
for (size_t i = 0; i < a8v.size(); i++) a8i[i] = a8v[i];
for (int t = 0; t < (int)lm_head_i8_.size(); t++) {
const int n0 = t * TB, n1 = std::min(NV, n0 + TB), Tn = n1 - n0;
HostI8Tensor& c = lm_head_i8_[(size_t)t];
if (c.codes.empty()) {
for (int n = n0; n < n1; n++)
for (int k = 0; k < H; k++) bt[(size_t)k * Tn + (n - n0)] = lm_at(n, k);
c.N = Tn;
c.codes.assign((size_t)H * (size_t)Tn, 0);
c.d.assign((size_t)Tn, 0.0f);
double ssum = 0.0;
q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, /*blocked=*/1, c.d.data(), &ssum);
}
for (int n = 0; n < Tn; n++) {
int32_t acc = 0;
for (int k = 0; k < H; k++)
acc += (int32_t)((int)c.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, H, Tn)] *
(int)a8i[(size_t)k]);
float v = c.d[(size_t)n] * (ascale * (float)acc);
if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap);
logits[(size_t)(n0 + n)] = v;
}
}
return true;
}
if (host_gemm_) {
// Offline mode: plain float matvec against the (possibly dequantized)
// tied table, matching the Python reference. No I8Ctx is touched.
Expand Down
1 change: 1 addition & 0 deletions npu/forward/src/q4nx_pack.c
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) {
*
* exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group
* multiplier per column ([groups][N]), `ascale` the activation row's scale. */

void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d,
int groups, int N, float ascale, float* out) {
const int G = 128;
Expand Down
12 changes: 12 additions & 0 deletions npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1157,6 +1157,18 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector<float>& out, std:
}
const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
out.resize((size_t)M * N);
// engine#1808: the device's per-column weight scales, KEYED by (layer, N, K) so it can be paired with the host
// arm's dump exactly. Index pairing was tried first and is meaningless -- the two sides reach the GEMM a
// different number of times (384 device dumps against 1024 arm dumps on the same request).
if (const char* gdir = getenv("NPU_I8_GS_DUMP")) {
static int n = 0;
char path[4096];
std::snprintf(path, sizeof path, "%s/dev_L%d_N%d_K%d_%03d.bin", gdir, ar.layer, N, K, n++);
if (std::FILE* f = std::fopen(path, "wb")) {
if (gs && !gs->empty()) std::fwrite(gs->data(), sizeof(float), (size_t)N, f);
std::fclose(f);
}
}
for (int m = 0; m < M; m++) {
const int32_t* cm = kctx.Cm + (size_t)m * kctx.ND;
const float asc = ascales[(size_t)m];
Expand Down
Loading