diff --git a/docs/npu.md b/docs/npu.md index 054ca1c39..963acc958 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1993,8 +1993,39 @@ again, the known-answer prompt answered `*,,,,,,,`; with the plain form it answe form, and gemma2 keeps `(1 + w)` (not measured here). `tests/q4nx_gemma3_semantics_test.c` pins each rule with Medgemma's own keys. -A prefill runs batched (`step_batch`) with the same three extra norms; only Gemma4 still falls back to one -`step()` per prompt token. `batch_check_state_test` on the Medgemma pack: batched equals sequential, rel-L1 0. +A prefill runs batched (`step_batch`) with the same three extra norms. `batch_check_state_test` on the Medgemma +pack: batched equals sequential, rel-L1 0. + +**Gemma4 runs batched too (engine#1780).** It used to fall back to one `step()` per row, so verifying k drafted +tokens cost k weight streams, and a DFlash pair on a Gemma4 target could not beat plain decode. The batched body +now does what `step()` does for each row: +- it builds the row's per-layer inputs; +- it runs the PLE gate and projection as M-row GEMMs, then the layer scalar; +- on a KV-shared layer it reads the owner layer's cache and appends nothing (the owner already appended every row + of the batch earlier in the same pass); +- a shared layer's double-wide MLP uses `GU2`/`D2`. + +A batch with injected embeddings is still refused on a PLE model, because the per-layer inputs need token ids. + +The KV cache's row width is now per layer, from `kv_widths()`. A model with two attention geometries holds more +floats per position on its full layers: on Gemma4-E4B, 2 × 512 against 2 × 256 on the sliding layers. Snapshot, +restore, a verify's rollback and `extend_context` used to count every layer at `NKV * HD`. That reported twice the +positions and cut the full layers at half on a snapshot or a partial accept, which corrupts the prefix cache and +any verify on such a model. + +`batch_check_state_test` on `Nemotron-3.5-Bolt-E4B` (host GEMM) checks all of this: +- chunks [0,3), [3,5), [5,6): every chunk's argmax matches; the last chunk's logits equal the sequential ones + (rel-L1 0); the positions held are exact after each chunk; +- a verify arm: a 4-row `step_batch_all` is bit-identical to 4 sequential steps (the oracle reports + `identical=4`); after `commit_verify_rows(2)` exactly 5 positions are held, and the next step equals a forward + that saw only those rows. + +Before the width fix, the first arm failed 4 of 4. With the old single width for the rollback, the verify arm fails. +`q4nx_state_planes_test` holds the per-layer cut to Gemma4-E4B's 42-layer geometry. + +The verify arm also runs on the CI's fabricated containers. There it found that `step_batch_all()` on a model that +still falls back to one `step()` per row (gpt-oss, a MoE with a leading dense layer) returned no per-row logits: the +fallback filled only the last row and left `logits_all` as it found it. The fallback now appends every step's row. The serve also needs the BOS. Medgemma's `config.json` states neither `bos_token_id` nor `add_bos_token`, and its `tokenizer_config.json` states both (``, true), as HF reads them. forward-serve now takes what `config.json` diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 4c38db9aa..ecb098caa 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -124,6 +124,9 @@ class Q4nxNpuForward { // Copy the state of the first `tokens` positions out. False when the forward is not ready or // `tokens` exceeds what it holds. bool snapshot_state(int tokens, StateSnapshot& out) const; + // Floats one position adds to each layer's K (and V) cache: NKV * HD, or the full layers' own width on a model + // with two attention geometries (engine#1780). + std::vector kv_widths() const; // Replace the state with the first `tokens` positions of `in`: the caller then continues at // position `tokens`. Restoring fewer positions than the snapshot holds is the normal case (a // request resumes to just short of its own prompt and re-runs the last token), and restoring @@ -323,6 +326,9 @@ class Q4nxNpuForward { // The same elementwise gate*up on caller memory, so a batched pass can read its ACT row and write its HH // row where they already are instead of staging both through a vector (a copy on each side). bool mlp(const float* gu, float* h, int im); + // engine#1780: the end of a Gemma-family layer for M rows in the batched body, as step() runs it for one: the + // per-layer embedding block (when the model has one; `ple_rows` is [M][NL * ple_dim]) and then layer_scalar. + bool gemma4_tail_rows(int l, int M, const std::vector& ple_rows, std::vector& X); // `tag` is the NPU_INFER_DUMP_OPS label (ops-dump.h); the per-head q/k norms pass their own. float rmsnorm(const float* x, const float* w, int n, float* out, const char* tag = "rmsnorm") const; float rmsnorm_noweight(const float* x, int n, float* out) const; diff --git a/npu/forward/include/q4nx_state_planes.h b/npu/forward/include/q4nx_state_planes.h index 740dcfc6f..7d46afe57 100644 --- a/npu/forward/include/q4nx_state_planes.h +++ b/npu/forward/include/q4nx_state_planes.h @@ -80,6 +80,57 @@ inline bool give_whole(const std::vector>& src, std::vector& per, size_t l) { + return per.empty() ? 0 : (l < per.size() ? per[l] : per.back()); +} + +// The positions the planes hold: the longest plane, counted in its own row width (a shared-KV plane holds none). +inline size_t held_positions(const std::vector>& src, const std::vector& per) { + size_t held = 0; + for (size_t l = 0; l < src.size(); l++) { + const size_t w = per_of(per, l); + if (w) held = std::max(held, src[l].size() / w); + } + return held; +} + +inline void take_per_pos(const std::vector>& src, std::vector>& dst, + const std::vector& per, size_t tokens) { + dst.resize(src.size()); + for (size_t l = 0; l < src.size(); l++) { + const size_t n = std::min(tokens * per_of(per, l), src[l].size()); + dst[l].assign(src[l].begin(), src[l].begin() + long(n)); + } +} + +inline bool give_per_pos(const std::vector>& src, std::vector>& dst, + const std::vector& per, size_t tokens) { + if (dst.size() < src.size()) return false; + for (size_t l = 0; l < src.size(); l++) { + if (src[l].empty()) { + dst[l].clear(); + continue; + } + const size_t want = tokens * per_of(per, l); + if (src[l].size() < want) return false; + dst[l].assign(src[l].begin(), src[l].begin() + long(want)); + } + return true; +} + +// Keep the first `tokens` positions of every non-empty plane (a verify's rollback): planes are cut, never grown. +inline void keep_positions(std::vector>& planes, const std::vector& per, size_t tokens) { + for (size_t l = 0; l < planes.size(); l++) { + const size_t keep = tokens * per_of(per, l); + if (!planes[l].empty() && planes[l].size() > keep) planes[l].resize(keep); + } +} + } // namespace q4nx_state #endif // Q4NX_STATE_PLANES_H diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 6fc4e52a9..6434ba206 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -440,21 +440,37 @@ size_t Q4nxNpuForward::StateSnapshot::bytes() const { return n; } +// engine#1780: the floats one position adds to layer l's K (and V) cache. attention() appends n_kv_heads * head_dim +// of the layer's own geometry, which on a model with two attention geometries (Gemma4: QKV2 on the full layers, +// 2 x 512, against 2 x 256 on the sliding ones) is not NKV * HD on every layer. Every place that turns positions into +// cache floats -- snapshot, restore, a verify's rollback, extend_context -- goes through this. +std::vector Q4nxNpuForward::kv_widths() const { + const size_t base = size_t(c_.NKV) * size_t(c_.HD); + std::vector w((size_t)std::max(c_.NL, 0), base); + const int i_qkv2 = idx_of(Q4NX_DESIGN_QKV2); + if (i_qkv2 < 0 || designs_[i_qkv2].k_rows <= 0) return w; + for (int l = 0; l < c_.NL; l++) { + const int lt = (l < sem_.n_layer_types) ? sem_.layer_types[l] : Q4NX_LAYER_FULL; + if (lt != Q4NX_LAYER_SLIDING) w[(size_t)l] = (size_t)designs_[i_qkv2].k_rows; // as step(): QKV2 on a full layer + } + return w; +} + bool Q4nxNpuForward::snapshot_state(int tokens, StateSnapshot& out) const { if (!ready_ || tokens <= 0) return false; const size_t per = size_t(c_.NKV) * size_t(c_.HD); - size_t held = 0; // positions the KV caches hold (a layer that shares its KV holds none) - for (const auto& layer : kcache_) held = std::max(held, layer.size() / per); + const std::vector widths = kv_widths(); + // positions the KV caches hold, each plane counted in its own row width (a layer that shares its KV holds none) + const size_t held = q4nx_state::held_positions(kcache_, widths); if (size_t(tokens) > held) return false; out.tokens = tokens; out.per_pos = per; - const size_t want = size_t(tokens) * per; // KV is per position, so a snapshot of the first `tokens` positions keeps exactly that many; the GDN conv // window and recurrent state are fixed-size blocks updated per position, so they go in whole. Cutting them // with `want` truncated them (23 % of the recurrent plane at 180 tokens) and made restore_state refuse the // snapshot -- the prefix cache's `slot has no usable state`, which cost a full re-prefill on every request. - q4nx_state::take_per_pos(kcache_, out.k, want); - q4nx_state::take_per_pos(vcache_, out.v, want); + q4nx_state::take_per_pos(kcache_, out.k, widths, size_t(tokens)); + q4nx_state::take_per_pos(vcache_, out.v, widths, size_t(tokens)); q4nx_state::take_whole(conv_state_, out.conv); q4nx_state::take_whole(rec_state_, out.rec); return true; @@ -464,12 +480,12 @@ bool Q4nxNpuForward::restore_state(const StateSnapshot& in, int tokens) { if (!ready_ || tokens <= 0 || tokens > in.tokens) return false; const size_t per = size_t(c_.NKV) * size_t(c_.HD); if (in.per_pos != per) return false; - const size_t want = size_t(tokens) * per; + const std::vector widths = kv_widths(); // Same split as snapshot_state: KV is restored per position (a layer whose KV is shared carries none of its // own, so an empty plane leaves the destination empty rather than filling it with the owner's), and the GDN // planes are restored whole -- refusing a truncated one instead of installing it short. - if (!q4nx_state::give_per_pos(in.k, kcache_, want)) return false; - if (!q4nx_state::give_per_pos(in.v, vcache_, want)) return false; + if (!q4nx_state::give_per_pos(in.k, kcache_, widths, size_t(tokens))) return false; + if (!q4nx_state::give_per_pos(in.v, vcache_, widths, size_t(tokens))) return false; if (!q4nx_state::give_whole(in.conv, conv_state_)) return false; if (!q4nx_state::give_whole(in.rec, rec_state_)) return false; return true; @@ -490,8 +506,9 @@ void Q4nxNpuForward::reset() { bool Q4nxNpuForward::extend_context(int n_tokens) { if (!ready_ || n_tokens <= 0) return false; if (kcache_.empty() || vcache_.empty()) return false; - const size_t per = (size_t)c_.NKV * (size_t)c_.HD; // floats per layer per token + const std::vector widths = kv_widths(); // floats per token, per layer (engine#1780) for (int l = 0; l < c_.NL; l++) { + const size_t per = widths[(size_t)l]; kcache_[l].reserve(kcache_[l].size() + (size_t)n_tokens * per); vcache_[l].reserve(vcache_[l].size() + (size_t)n_tokens * per); kcache_[l].insert(kcache_[l].end(), (size_t)n_tokens * per, 0.0f); @@ -2967,12 +2984,10 @@ bool Q4nxNpuForward::commit_verify_rows(int accepted) { expect_rec = rec_state_; expect_conv = conv_state_; } - const size_t per = (size_t)c_.NKV * (size_t)c_.HD; - const size_t keep = (size_t)(verify_pos_ + accepted) * per; - for (size_t l = 0; l < kcache_.size(); l++) { - if (!kcache_[l].empty() && kcache_[l].size() > keep) kcache_[l].resize(keep); - if (!vcache_[l].empty() && vcache_[l].size() > keep) vcache_[l].resize(keep); - } + // Each plane in its own row width (engine#1780): one width cut a two-geometry model's wide planes at half. + const std::vector widths = kv_widths(); + q4nx_state::keep_positions(kcache_, widths, (size_t)(verify_pos_ + accepted)); + q4nx_state::keep_positions(vcache_, widths, (size_t)(verify_pos_ + accepted)); // A full accept keeps the state the batch's own recurrence left: that is the state the restore + replay would // rebuild (NPU_REPLAY_CHECK holds the two equal), so the restore and the replay are skipped. On the served 27B // they are 11 ms + 7 ms per row, 67 ms of an 8-row round. NPU_COMMIT_FULL_REPLAY=1 keeps them, for an A/B. @@ -4294,6 +4309,82 @@ bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int sta return step_batch(placeholder, start_pos, logits); } +bool Q4nxNpuForward::gemma4_tail_rows(int l, int M, const std::vector& ple_rows, std::vector& X) { + const int H = c_.H; + if (ple_dim_ > 0) { + // x = x + post_per_layer_input_norm(per_layer_projection(act(per_layer_input_gate(x)) * per_layer_input[l])) + const int i_pg = idx_of(Q4NX_DESIGN_PLEGATE); + const int i_pp = idx_of(Q4NX_DESIGN_PLEPROJ); + if (i_pg < 0 || i_pp < 0) { err_ = "batch: the PLE gate/projection designs are missing"; return false; } + const Q4nxDesignGeom* gg = &designs_[i_pg]; + const Q4nxDesignGeom* gp = &designs_[i_pp]; + const size_t w = (size_t)c_.NL * (size_t)ple_dim_; + if (ple_rows.size() < (size_t)M * w) { err_ = "batch: per-layer inputs short"; return false; } + // The gate reads the residual stream; the activation rows go at the stride npu_gemm_rows walks. + const int sg = onebit::npu::row_stride(gg->real_k, gg->K); + if (sg < H) { err_ = "batch: the PLE gate's K is narrower than the hidden size"; return false; } + std::vector a_g((size_t)M * (size_t)sg, 0.0f), pg, pp; + for (int m = 0; m < M; m++) + std::copy(X.begin() + (ptrdiff_t)((size_t)m * H), X.begin() + (ptrdiff_t)((size_t)(m + 1) * H), + a_g.begin() + (ptrdiff_t)((size_t)m * sg)); + { + Q4nxProjections p{}; + p.ple = inp_gate_[l].data(); + if (!weight_cached(onebit::npu::kWkeyPleGate) && q4nx_assemble_B(gg, &p, B_.data()) != 0) { + err_ = "assemble plegate"; return false; + } + npu_gemm_rows(i_pg, a_g.data(), M, gg->K, gg->N, B_.data(), pg, onebit::npu::kWkeyPleGate, "ple_gate", + gg->real_k); + } + if (pg.size() < (size_t)M * (size_t)gg->N || !err_.empty()) { + if (err_.empty()) err_ = "batch: the PLE gate returned short"; + return false; + } + const int sp = onebit::npu::row_stride(gp->real_k, gp->K); + if (sp < ple_dim_ || gg->N < ple_dim_) { err_ = "batch: the PLE widths do not fit the designs"; return false; } + std::vector a_p((size_t)M * (size_t)sp, 0.0f); + const int gelu = (sem_.mlp_act == Q4NX_ACT_GELU); + for (int m = 0; m < M; m++) { + const float* g = pg.data() + (size_t)m * gg->N; + const float* in = ple_rows.data() + (size_t)m * w + (size_t)l * ple_dim_; + float* dst = a_p.data() + (size_t)m * sp; + for (int i = 0; i < ple_dim_; i++) { + float a; + if (gelu) { + const float cc = 0.7978845608028654f; + a = 0.5f * g[i] * (1.0f + std::tanh(cc * (g[i] + 0.044715f * g[i] * g[i] * g[i]))); + } else { + a = g[i] / (1.0f + std::exp(-g[i])); + } + dst[i] = a * in[i]; + } + } + { + Q4nxProjections p{}; + p.ple = ple_proj_[l].data(); + if (!weight_cached(onebit::npu::kWkeyPleProj) && q4nx_assemble_B(gp, &p, B_.data()) != 0) { + err_ = "assemble pleproj"; return false; + } + npu_gemm_rows(i_pp, a_p.data(), M, gp->K, gp->N, B_.data(), pp, onebit::npu::kWkeyPleProj, "ple_proj", + gp->real_k); + } + if (pp.size() < (size_t)M * (size_t)gp->N || gp->N < H || !err_.empty()) { + if (err_.empty()) err_ = "batch: the PLE projection returned short"; + return false; + } + std::vector xn((size_t)H); + for (int m = 0; m < M; m++) { + rmsnorm(pp.data() + (size_t)m * gp->N, post_ln_[l].data(), H, xn.data()); + for (int k = 0; k < H; k++) X[(size_t)m * H + k] += xn[(size_t)k]; + } + } + // layer_scalar (layer_output_scale) closes the layer. + const float ls = (l < (int)layer_scale_.size()) ? layer_scale_[l] : 1.0f; + if (ls != 1.0f) + for (size_t i = 0; i < (size_t)M * (size_t)H; i++) X[i] *= ls; + return true; +} + bool Q4nxNpuForward::step_batch_all(const std::vector& tokens, int start_pos, std::vector& logits_all) { // A noncausal pack's full-attention layers read the whole prompt, so in a batch every row attends to @@ -4378,12 +4469,15 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p // recurrence), so lin_kd_ no longer disqualifies a model, and attn_output_gate // is handled in the attention block below. MLA, per-layer embeddings, Gemma4 // and shared-KV stacks still fall back. - // Gemma3 (extra_layer_norms 2) is Gemma4's sandwich without PLE or KV sharing, so the batched body - // below runs it with the three extra norms; Gemma4 itself still falls back. - const bool sandwich = gemma4_layer_ && ple_dim_ == 0 && sem_.extra_layer_norms == 2; + // Gemma3 (extra_layer_norms 2) is Gemma4's sandwich without PLE or KV sharing. engine#1780: Gemma4 itself runs + // here too -- its per-layer inputs are built per row, its PLE gate and projection are M-row GEMMs, a KV-shared + // layer reads (and does not append to) its owner's cache, which the owner filled for every row of this batch + // earlier in the same pass, and a shared layer's double-wide MLP uses the GU2/D2 designs, all as step() does. A + // verify of k drafted rows is then one weight stream instead of k. + const bool sandwich = gemma4_layer_; // gpt_oss runs its MoE, sinks and o bias in step() only, so a batched prefill takes the per-token path const bool simple_dense = - !mla_ && ple_dim_ == 0 && (!gemma4_layer_ || sandwich) && kv_self && !sem_.gpt_oss && + !mla_ && (ple_dim_ == 0 || !inject_embeds_) && (kv_self || gemma4_layer_) && !sem_.gpt_oss && !(n_expert_ > 0 && leading_dense_ > 0); // a leading dense layer of a non-MLA MoE runs in step() only if (!simple_dense || getenv("NPU_NO_BATCH")) { if (inject_embeds_) { @@ -4397,9 +4491,15 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p "gate=%d kv_self=%d no_batch=%d\n", M, mla_, lin_kd_, ple_dim_, (int)gemma4_layer_, n_expert_, attn_gate_, (int)kv_self, (int)(getenv("NPU_NO_BATCH") != nullptr)); + // step_batch_all() asks for every row: the per-token path owes it each step's logits, not only the last + // one's (it used to leave logits_all as it found it, so a verifier on a fallback model read no rows, or a + // previous batch's -- engine#1780, found by batch_check_state's verify arm on hy_v3 and gpt-oss). std::vector one; - for (int m = 0; m < M; m++) + if (batch_all_out_) batch_all_out_->clear(); + for (int m = 0; m < M; m++) { if (!step(tokens[m], start_pos + m, one)) return false; + if (batch_all_out_) batch_all_out_->insert(batch_all_out_->end(), one.begin(), one.end()); + } logits = one; return true; } @@ -4507,6 +4607,18 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p X[size_t(r) * (size_t)H + k] += add[size_t(r - lo) * (size_t)H + k]; }; + // Gemma4: every row's per-layer inputs, [M][NL * ple_dim], built as step() builds one token's (engine#1780). + std::vector ple_rows; + if (ple_dim_ > 0) { + const size_t w = (size_t)c_.NL * (size_t)ple_dim_; + ple_rows.resize((size_t)M * w); + for (int m = 0; m < M; m++) { + build_ple_inputs(tokens[m]); + if (ple_input_.size() < w) { err_ = "batch: per-layer inputs were not built"; return false; } + std::copy(ple_input_.begin(), ple_input_.begin() + (ptrdiff_t)w, ple_rows.begin() + (ptrdiff_t)((size_t)m * w)); + } + } + std::vector QKV, CTX, O, ACT, HH, D, qkv_row, ctx_row; for (int l = 0; l < c_.NL; l++) { // engine#1780: the stream as this layer receives it is the previous layer's output. Taken here and not at @@ -4519,7 +4631,11 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p // norm) live outside the weight cache and are per layer, so load_gdn_layer // must run every time; it skips its own big deqs when the BOs are cached. const bool gdn_l = (lin_kd_ > 0 && lt0 == Q4NX_LAYER_LINEAR); - if ((gdn_l || !cached) && !load_layer(l)) { + // A Gemma4 layer's PLE gate and projection have their own weight keys; reload when either is missing, as + // step() does (its dense_weights_cached). + const bool ple_cached = ple_dim_ == 0 || (weight_cached(onebit::npu::kWkeyPleGate) && + weight_cached(onebit::npu::kWkeyPleProj)); + if ((gdn_l || !cached || !ple_cached) && !load_layer(l)) { if (err_.empty()) err_ = "dequant failed"; return false; } @@ -4532,6 +4648,14 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p const int l_win = l_full ? 0 : sem_.sliding_window; const int i_qkv_l = (two_widths && l_full) ? i_qkv2 : i_qkv; const int i_o_l = (two_widths && l_full) ? i_o2 : i_o; + // KV sharing (Gemma4), as step(): a shared layer reads its owner's cache and appends nothing. The owner ran + // earlier in this loop and appended every row of the batch, so a causal row still sees exactly its prefix. + const bool kv_shared = (l < (int)kv_owner_.size() && kv_owner_[l] != l); + const int kv_layer = (l < (int)kv_owner_.size()) ? kv_owner_[l] : l; + const bool wide = kv_shared && sem_.double_wide_mlp; + const int i_gu_l = wide ? idx_of(Q4NX_DESIGN_GU2) : i_gu; + const int i_d_l = wide ? idx_of(Q4NX_DESIGN_D2) : i_d; + const int im_l = wide ? 2 * im : im; // ---- attention ---- (a GDN layer goes through gdn_layer_batch instead) ---- for (int m = 0; m < M; m++) @@ -4546,6 +4670,7 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p const Q4nxDesignGeom* g = &designs_[i_qkv_l]; Q4nxProjections p{}; p.q = wq_.data(); p.k = wk_.data(); p.v = wv_.data(); + if (kv_shared) { p.k = wq_.data(); p.v = wq_.data(); } // no k/v weights: that half is discarded (step()) if (!xd_->pb16i() && !cached && q4nx_assemble_B(g, &p, B_.data()) != 0) { err_ = "assemble qkv"; return false; } @@ -4659,7 +4784,7 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p } if (!attention(*in, start_pos + m, pos3_ ? trio : nullptr, dst ? ctx_row : ctx_scratch, l_hd, l_nkv, l_theta, - l_rot, l_prop, l_win, l, append, kvlen, q_keep)) { + l_rot, l_prop, l_win, kv_layer, append && !kv_shared, kvlen, q_keep)) { err_ = "attention failed"; return false; } if (q_keep) return true; // prepare only: q is in *q_keep, k/v are in the cache @@ -4710,7 +4835,7 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p : 1.0f / std::sqrt(sem_.query_pre_attn_scalar > 0 ? sem_.query_pre_attn_scalar : (float)l_hd); const int nt = ((long)M * (start_pos + M) * l_hd >= 8192) ? omp_heads_threads() : 1; - q4nx_attn::rows_context(qall.data(), kcache_[l].data(), vcache_[l].data(), M, c_.NH, l_nkv, + q4nx_attn::rows_context(qall.data(), kcache_[kv_layer].data(), vcache_[kv_layer].data(), M, c_.NH, l_nkv, l_hd, Tm.data(), t0m.data(), scale, CTX.data(), nt); if (attn_gate_) for (int m = 0; m < M; m++) { @@ -4804,16 +4929,20 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p // design's N); the split branch packs its own buffer at the same width rather than at 2*im, so // there is one layout and one stride -- reading a real-width buffer at the design stride is // what the review of #885 caught (latent until a container has im % 256 != 0, then an OOB read). - const size_t act_w = (size_t)q4nx_pad_n(2 * im); - if (i_gu >= 0) { - const Q4nxDesignGeom* g = &designs_[i_gu]; + const size_t act_w = (size_t)q4nx_pad_n(2 * im_l); + if (i_gu_l >= 0) { + const Q4nxDesignGeom* g = &designs_[i_gu_l]; Q4nxProjections p{}; p.gate = wg_.data(); p.up = wu_.data(); if (!xd_->pb16i() && !cached && q4nx_assemble_B(g, &p, B_.data()) != 0) { err_ = "assemble gu"; return false; } - npu_gemm_rows(i_gu, XN.data(), M, g->K, g->N, B_.data(), ACT, 2, + npu_gemm_rows(i_gu_l, XN.data(), M, g->K, g->N, B_.data(), ACT, 2, "ffn_gate_up", g->real_k); + if ((size_t)g->N != act_w) { err_ = "batch: the gate/up design's N is not the padded 2 x intermediate"; return false; } + } else if (wide) { + err_ = "batch: a double-wide MLP layer has no fused GU2 design"; + return false; } else if (i_g >= 0 && i_u >= 0) { std::vector gv, uv, av((size_t)M * act_w, 0.0f); const Q4nxDesignGeom* gg = &designs_[i_g]; @@ -4843,25 +4972,25 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p err_ = "no gate/up design"; return false; } - HH.assign((size_t)M * (size_t)im, 0.0f); + HH.assign((size_t)M * (size_t)im_l, 0.0f); { // act_w is the stride both branches above leave ACT at; only the first 2*im columns of a // row are real (the rest is the kernel's zero column pad), so the elementwise gate*up reads // that row in place and writes HH's row in place: no staging copy on either side. The two // copies this replaces were 12.75 MiB of a 15.38 MiB row (NPU_COPY_PROF=1, 2026-11-05). for (int m = 0; m < M; m++) - if (!mlp(ACT.data() + (size_t)m * act_w, HH.data() + (size_t)m * im, im)) { + if (!mlp(ACT.data() + (size_t)m * act_w, HH.data() + (size_t)m * im_l, im_l)) { err_ = "mlp failed"; return false; } } { - const Q4nxDesignGeom* g = &designs_[i_d]; + const Q4nxDesignGeom* g = &designs_[i_d_l]; Q4nxProjections p{}; p.down = wd_.data(); if (!xd_->pb16i() && !cached && q4nx_assemble_B(g, &p, B_.data()) != 0) { err_ = "assemble d"; return false; } - npu_gemm_rows(i_d, HH.data(), M, g->K, g->N, B_.data(), D, 4, + npu_gemm_rows(i_d_l, HH.data(), M, g->K, g->N, B_.data(), D, 4, "ffn_down", g->real_k); } { @@ -4880,6 +5009,7 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p X[(size_t)m * H + k] += D[(size_t)m * nrow + k] * residual_scale_; } } + if (sandwich && !gemma4_tail_rows(l, M, ple_rows, X)) return false; // Same post-residual hidden-state dump as step() emits as [L], one row per // position, so the two paths can be compared layer by layer. diff --git a/tests/batch_check_state_test.cpp b/tests/batch_check_state_test.cpp index 8150c7f3f..afcab66e2 100644 --- a/tests/batch_check_state_test.cpp +++ b/tests/batch_check_state_test.cpp @@ -135,6 +135,44 @@ int main() { argmax(logits), argmax(ref)); check(argmax(logits) == argmax(ref) && rel == 0.0, msg); + // engine#1780: the verify a speculative decoder runs. A prefix, then k drafted rows in one step_batch_all(): every + // row's logits must be the sequential step's at that position, bit for bit; then a partial accept rolls the cache + // back to the accepted rows, and the next step must continue exactly as a forward that only ever saw those rows. + // The rollback is where a model with two attention geometries (Gemma4) lost half its full layers' positions. + { + const std::vector prefix = {1, 2, 3}, drafts = {4, 5, 6, 7}; + const int next_tok = 8, accepted = 2; + const int P = (int)prefix.size(), K = (int)drafts.size(); + fwd.reset(); + std::vector seq, rows_ref; + for (int i = 0; i < P; i++) + if (!fwd.step(prefix[(size_t)i], i, seq)) { std::printf("FAIL prefix step: %s\n", fwd.last_error()); return 1; } + for (int i = 0; i < K; i++) { + if (!fwd.step(drafts[(size_t)i], P + i, seq)) { std::printf("FAIL draft step: %s\n", fwd.last_error()); return 1; } + rows_ref.insert(rows_ref.end(), seq.begin(), seq.end()); + } + // The forward that only ever saw the accepted rows, then the next token. + fwd.reset(); + std::vector next_ref; + for (int i = 0; i < P; i++) fwd.step(prefix[(size_t)i], i, seq); + for (int i = 0; i < accepted; i++) fwd.step(drafts[(size_t)i], P + i, seq); + if (!fwd.step(next_tok, P + accepted, next_ref)) { std::printf("FAIL reference step: %s\n", fwd.last_error()); return 1; } + + fwd.reset(); + for (int i = 0; i < P; i++) fwd.step(prefix[(size_t)i], i, seq); + std::vector rows; + fwd.begin_verify(K, P); + if (!fwd.step_batch_all(drafts, P, rows)) { std::printf("FAIL step_batch_all: %s\n", fwd.last_error()); return 1; } + check(rows.size() == rows_ref.size() && rows == rows_ref, + "verify: every drafted row's logits are the sequential step's, bit for bit (" + std::to_string(K) + " rows)"); + if (!fwd.commit_verify_rows(accepted)) { std::printf("FAIL commit_verify_rows: %s\n", fwd.last_error()); return 1; } + check(holds_exactly(fwd, P + accepted), + "verify: a partial accept leaves exactly prefix + accepted positions (" + std::to_string(P + accepted) + ")"); + std::vector next; + if (!fwd.step(next_tok, P + accepted, next)) { std::printf("FAIL step after commit: %s\n", fwd.last_error()); return 1; } + check(next == next_ref, "verify: the step after a partial accept equals the forward that saw only those rows"); + } + std::printf("%s (%d failed)\n", fails == 0 ? "ALL PASS" : "FAILED", fails); return fails == 0 ? 0 : 1; } diff --git a/tests/q4nx_state_planes_test.cpp b/tests/q4nx_state_planes_test.cpp index e438b16d2..7d6494aaa 100644 --- a/tests/q4nx_state_planes_test.cpp +++ b/tests/q4nx_state_planes_test.cpp @@ -32,6 +32,7 @@ #include "npu_prefix_cache.h" #include "q4nx_state_planes.h" +#include #include #include #include @@ -155,6 +156,51 @@ void test_kv_planes_still_per_position() { check(dst[3].empty(), "shared-KV layer is left empty"); } +// engine#1780: Gemma4-E4B's geometry. 42 layers, sliding ones 2 x 256 = 512 floats per position and every sixth (the +// full layers) 2 x 512 = 1024; layers 24.. share an owner's KV and hold nothing. One width (NKV * HD = 512) counted the +// full layers as holding twice the positions and cut them at half on snapshot, restore and rollback. +void test_two_width_planes() { + using q4nx_state::give_per_pos; + using q4nx_state::held_positions; + using q4nx_state::keep_positions; + using q4nx_state::take_per_pos; + constexpr size_t kL = 42, kShared = 24, kNarrow = 512, kWide = 1024, kPos = 6; + std::vector per(kL, kNarrow); + for (size_t l = 5; l < kL; l += 6) per[l] = kWide; + std::vector> kv(kL); + for (size_t l = 0; l < kShared; l++) { + kv[l].resize(kPos * per[l]); + for (size_t i = 0; i < kv[l].size(); i++) kv[l][i] = float(l * 100000 + i); + } + // The old count: the longest plane at the one width says 12 positions for a 6-position cache. + size_t old_held = 0; + for (const auto& p : kv) old_held = std::max(old_held, p.size() / kNarrow); + check(old_held == 2 * kPos, "the one-width count reads a full layer's 6 positions as 12 (the defect)"); + check(held_positions(kv, per) == kPos, "the per-layer count holds exactly the 6 positions"); + + std::vector> old_saved, saved; + take_per_pos(kv, old_saved, 4 * kNarrow); + check(old_saved[5].size() == 2 * kWide, "a one-width snapshot of 4 positions keeps 2 of a full layer's (the defect)"); + take_per_pos(kv, saved, per, 4); + check(saved[5].size() == 4 * kWide && saved[0].size() == 4 * kNarrow, + "a per-layer snapshot keeps 4 positions of every layer, wide and narrow"); + check(saved[5].back() == kv[5][4 * kWide - 1], "...and they are the first 4 positions, not another cut"); + check(saved[30].empty(), "a shared-KV layer's empty plane stays empty"); + + std::vector> dst(kL); + check(give_per_pos(saved, dst, per, 3), "restore of 3 of the 4 positions is accepted"); + check(dst[5].size() == 3 * kWide && dst[11].size() == 3 * kWide && dst[4].size() == 3 * kNarrow, + "...and every layer resumes at 3 positions of its own width"); + check(!give_per_pos(saved, dst, per, 5), "restore refuses positions the snapshot does not hold"); + + auto rolled = kv; + keep_positions(rolled, per, 2); + check(rolled[5].size() == 2 * kWide && rolled[0].size() == 2 * kNarrow && rolled[30].empty(), + "a verify rollback to 2 positions keeps 2 of every layer"); + keep_positions(rolled, per, 5); + check(rolled[5].size() == 2 * kWide, "a rollback never grows a plane"); +} + void test_truncated_whole_plane_is_refused() { // The safety property: a snapshot whose GDN plane is short must be refused (the caller re-prefills), never // installed short, which would leave the GDN step reading past the end of the plane. @@ -195,6 +241,7 @@ int main() { test_old_planes_were_truncated(); test_whole_planes_survive(); test_kv_planes_still_per_position(); + test_two_width_planes(); test_truncated_whole_plane_is_refused(); test_resume_point_is_one_token_short(); if (failures) {