diff --git a/CMakeLists.txt b/CMakeLists.txt index e73fafab1..a36b963f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1975,6 +1975,12 @@ endif() if(OpenMP_CXX_FOUND) target_link_libraries(gdn_row_bench PRIVATE OpenMP::OpenMP_CXX) endif() + # A DFlash draft scored against its target, both on the host (engine#1780): the target's tapped hidden states + # into the draft's reference forward, proposals checked against the target's greedy tokens. A tool, not a ctest: + # it needs two packages and minutes of host time. + add_executable(dflash_acceptance tools/dflash_acceptance.cpp npu/dflash_forward.cpp npu/dflash_draft.cpp) + target_include_directories(dflash_acceptance PRIVATE npu npu/forward/include) + target_link_libraries(dflash_acceptance PRIVATE onebit_npu_forward onebit_npu) add_executable(mtp_acceptance tools/mtp_acceptance.cpp npu/mtp/mtp_head.cpp) target_include_directories(mtp_acceptance PRIVATE npu npu/forward/include) target_link_libraries(mtp_acceptance PRIVATE onebit_npu_forward) diff --git a/docs/npu.md b/docs/npu.md index ce6e200af..776bd7911 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2903,6 +2903,14 @@ sink, `taps x H` wide, the listed layers side by side. It works for `step()` and a DFlash draft's `target_layer_ids` are read with (docs/amd-drafters.md). Layer `l`'s block is the stream after layer `l`, the same point `layer_.f32` records; `layer_taps_model_test` holds the two to each other bit for bit. +**A Gemma4 (per-layer-embedding) pack no longer re-reads its layer weights every token.** `step()` skips the +per-token Q4NX dequant once a dense layer's projections are packed on the device; a per-layer-embedding model was +excluded from that, although its gate and projection are read once at init and are packed like the rest. Measured on +`Nemotron-3.5-Bolt-E4B` (42 layers), one ticket, one binary, arms old / new / old / new (`NPU_PLE_RELOAD=1` is the old +behaviour): 3377 and 3410 ms per step with the reload, 2486 and 1914 ms without, same reply in all four. What is left +is the device: on this per-op int8 family a 16-row launch streams the whole weight image (`GU` 21.5 ms, `D` 13.6 ms, +`QKV` 4.0 ms per launch), which is about 1.9 s per token over 42 layers. + The files these dumps write (`NPU_INFER_DUMP_DIR`'s `layer_.f32`, `NPU_I8_GS_DUMP`'s weight-scale pairs) are created owner-only (0600) and never through a symlink at the final path component, whatever the umask, and a dump file an older build left at 0666 is narrowed to 0600 when it is rewritten; the head-image cache's files are at most 0644. Both go through `npu/forward/include/file_create.h`, so no write site uses `fopen(…, "w")`, diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c55d73522..7eef6f0be 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -5132,9 +5132,17 @@ bool Q4nxNpuForward::step_impl(int token, int pos, std::vector& logits, i cur_layer_ = l; // Dense models: once this layer's projections are packed (after the // first token), the f32 weights are no longer read, so skip the per-token - // Q4NX dequant. GDN/MLA/PLE load members used outside the GEMMs, so they + // Q4NX dequant. GDN/MLA load members used outside the GEMMs, so they // keep loading every token. - const bool dense_weights_cached = lin_kd_ == 0 && !mla_ && ple_dim_ == 0 && n_expert_ == 0 && weight_cached(0); + // A per-layer-embedding (Gemma4) model is dense in this sense too: its gate and projection are read once at + // init and are GEMM weights like the rest, so once THEIR images are packed as well nothing here reads the + // layer's f32 weights. It used to be excluded, and every token dequantised all 42 layers again: 3.5 s per + // token on an E4B with the device idle most of that time. + // NPU_PLE_RELOAD=1 keeps the per-token reload, for an A/B. + static const bool ple_reload = getenv("NPU_PLE_RELOAD") != nullptr; + const bool ple_cached = ple_dim_ == 0 || (!ple_reload && weight_cached(onebit::npu::kWkeyPleGate) && + weight_cached(onebit::npu::kWkeyPleProj)); + const bool dense_weights_cached = lin_kd_ == 0 && !mla_ && ple_cached && n_expert_ == 0 && weight_cached(0); { LhScope _tlh(LH_LOAD); if (!dense_weights_cached && !load_layer(l)) { if (err_.empty()) err_ = "dequant failed"; return false; } diff --git a/tests/dflash_forward_test.cpp b/tests/dflash_forward_test.cpp index 94f0b8128..938b632cd 100644 --- a/tests/dflash_forward_test.cpp +++ b/tests/dflash_forward_test.cpp @@ -96,8 +96,11 @@ DraftWeights tiny_model() { w.hidden_norm = {0.5f, 3.0f}; w.norm = {1.0f, 1.0f}; DraftLayerWeights L; - L.q = L.k = L.o = L.down = identity(2); - L.v = {2.0f, 0.0f, 0.0f, 3.0f}; + L.q = L.k = L.o = identity(2); + // v and down each have one entry off the diagonal, so a projection applied transposed gives another answer + // (review of #2174: with every matrix diagonal, W and W^T were the same thing). + L.v = {2.0f, 1.0f, 0.0f, 3.0f}; // v(x) = (2 x0 + x1, 3 x1) + L.down = {1.0f, 0.5f, 0.0f, 1.0f}; // down(m) = (m0 + 0.5 m1, m1) // gate and up differ, and gate drives its first output negative where up is positive, so silu(gate) * up and // silu(up) * gate are different numbers: which of the two is activated is part of the answer. L.gate = {-1.0f, 0.0f, 0.0f, 2.0f}; @@ -125,8 +128,10 @@ void tiny_tail(double x0, double x1, double a0, double a1, double& o0, double& o double y0 = x0 + a0, y1 = x1 + a1; // residual (o_proj is the identity) double p0 = y0, p1 = y1; rms2(p0, p1); // post_norm - y0 += silu(-1.0 * p0) * (3.0 * p0); // silu(gate(p)) * up(p); down is the identity - y1 += silu(2.0 * p1) * (0.5 * p1); + const double m0 = silu(-1.0 * p0) * (3.0 * p0); // silu(gate(p)) * up(p) + const double m1 = silu(2.0 * p1) * (0.5 * p1); + y0 += m0 + 0.5 * m1; // down + y1 += m1; rms2(y0, y1); // the final norm o0 = y0; o1 = y1; } @@ -138,7 +143,7 @@ void hand_cases() { double h0 = 3.0, h1 = 4.0; rmsg(h0, h1, 2.0, 0.5); // input_norm double e0, e1; - tiny_tail(3.0, 4.0, 2.0 * h0, 3.0 * h1, e0, e1); + tiny_tail(3.0, 4.0, 2.0 * h0 + h1, 3.0 * h1, e0, e1); const std::vector out = draft_forward(w, nullptr, 0, noise, 1); check(out.size() == 2 && std::fabs(out[0] - e0) < 1e-5 && std::fabs(out[1] - e1) < 1e-5, "one row, no context: the output is the hand-computed one"); @@ -165,7 +170,7 @@ void hand_cases() { rot(khn0, khn1, 6.0, kh0, kh1); const double sc = (q0 * kc0 + q1 * kc1) / std::sqrt(2.0), sh = (q0 * kh0 + q1 * kh1) / std::sqrt(2.0); const double wc = std::exp(sc) / (std::exp(sc) + std::exp(sh)), wh = 1.0 - wc; - const double a0 = wc * 2.0 * c0 + wh * 2.0 * h0, a1 = wc * 3.0 * c1 + wh * 3.0 * h1; // v = diag(2, 3) + const double a0 = wc * (2.0 * c0 + c1) + wh * (2.0 * h0 + h1), a1 = wc * 3.0 * c1 + wh * 3.0 * h1; // v as above double e0, e1; tiny_tail(-2.0, 1.0, a0, a1, e0, e1); const std::vector out = draft_forward(w, tap, 1, noise, 1, 5); diff --git a/tools/dflash_acceptance.cpp b/tools/dflash_acceptance.cpp new file mode 100644 index 000000000..d0967c2ab --- /dev/null +++ b/tools/dflash_acceptance.cpp @@ -0,0 +1,207 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// dflash_acceptance.cpp -- a DFlash draft scored against its target model, both on the host (engine#1780). +// +// dflash_acceptance --model --draft --prompt-ids [--tokens 32] [--window 0] +// [--all-positions] +// +// holds one prompt per line as token ids (the tokenizer stays out of the harness; ids for a chat-formatted +// prompt are made once with the pack's tokenizer.json). +// +// What it does, per prompt: +// 1. the target runs the prompt and then `--tokens` greedy tokens through Q4nxNpuForward::step(), with +// set_layer_taps() on the draft's target_layer_ids, so every position has its tapped row; +// 2. the draft is then asked, at a position s, for the block after token s: context = the tapped rows of +// positions 0..s-1, block = the target's embeddings of [token s, mask, mask, ...] times the draft's +// embedding_scale, and each block row after the first goes through the target's own head; +// 3. a proposal is accepted while it equals the target's greedy token at that position. The run walks the +// positions the way a speculative loop would (after a pass that accepted a, the next pass starts a + 1 later), +// or every position with --all-positions. +// +// Reported: passes, accepted tokens per pass, the first proposal's hit rate, and the tokens a pass would emit +// (accepted + 1, the target's own next token). This is the acceptance half of the issue's measured row; it says +// nothing about tok/s -- both models run on the host here, one thread, as references. +// +// Host-GEMM only: it sets NPU_INFER_HOST_GEMM and never opens the device. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dflash_draft.h" +#include "dflash_forward.h" +#include "q4nx_forward.h" + +namespace { + +using namespace onebit::npu::dflash; + +double now_s() { + return std::chrono::duration(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +int argmax(const std::vector& v) { + return int(std::max_element(v.begin(), v.end()) - v.begin()); +} + +} // namespace + +int main(int argc, char** argv) { + std::string model_dir, draft_dir, ids_path; + int tokens = 32, window = 0; + bool all_positions = false; + for (int i = 1; i < argc; i++) { + const std::string a = argv[i]; + auto next = [&]() -> std::string { return (i + 1 < argc) ? argv[++i] : std::string(); }; + if (a == "--model") model_dir = next(); + else if (a == "--draft") draft_dir = next(); + else if (a == "--prompt-ids") ids_path = next(); + else if (a == "--tokens") tokens = std::atoi(next().c_str()); + else if (a == "--window") window = std::atoi(next().c_str()); + else if (a == "--all-positions") all_positions = true; + else { std::fprintf(stderr, "unknown argument %s\n", a.c_str()); return 2; } + } + if (model_dir.empty() || draft_dir.empty() || ids_path.empty() || tokens < 2) { + std::fprintf(stderr, "usage: dflash_acceptance --model --draft --prompt-ids " + "[--tokens 32] [--window 0] [--all-positions]\n"); + return 2; + } + + std::vector> prompts; + { + std::ifstream in(ids_path); + std::string line; + while (std::getline(in, line)) { + std::istringstream ls(line); + std::vector ids; + int v; + while (ls >> v) ids.push_back(v); + if (!ids.empty()) prompts.push_back(ids); + } + } + if (prompts.empty()) { std::fprintf(stderr, "%s holds no prompt\n", ids_path.c_str()); return 2; } + + DraftWeights w; + DraftConfig cfg; + try { + const DraftModel m = load_draft(draft_dir); + cfg = m.cfg; + w = decode_draft(m); + } catch (const std::exception& e) { + std::fprintf(stderr, "draft: %s\n", e.what()); + return 1; + } + const int block = cfg.block_size, T = cfg.taps(), H = cfg.hidden; + + setenv("NPU_INFER_HOST_GEMM", "1", 1); + Q4nxNpuForward fwd; + if (!fwd.init_host((model_dir + "/model.q4nx").c_str(), model_dir + "/kernels")) { + std::fprintf(stderr, "target: %s\n", fwd.last_error()); + return 1; + } + std::vector row; + fwd.embed_row(0, row); + if (int(row.size()) != H) { + std::fprintf(stderr, "the target is %zu wide and the draft %d: not a pair\n", row.size(), H); + return 1; + } + std::vector taps; + if (!fwd.set_layer_taps(cfg.target_layer_ids, &taps)) { + std::fprintf(stderr, "the draft's target layers do not fit a %d-layer target\n", fwd.layer_count()); + return 1; + } + std::printf("pair: target %d layers, H %d; draft %zu layers, block %d, taps", fwd.layer_count(), H, w.layers.size(), block); + for (int l : cfg.target_layer_ids) std::printf(" %d", l); + std::printf(", mask %lld, embedding scale %.5f, window %d\n", (long long)cfg.mask_token_id, cfg.embedding_scale, window); + + long passes = 0, accepted = 0, first_hits = 0, proposals = 0; + std::vector hist(size_t(block), 0); // hist[a] = passes that accepted exactly a + double t_target = 0.0, t_draft = 0.0; + for (size_t p = 0; p < prompts.size(); p++) { + std::vector seq = prompts[p]; + const int P = int(seq.size()); + fwd.reset(); + taps.clear(); + std::vector logits; + const double t0 = now_s(); + for (int pos = 0; pos < P + tokens; pos++) { + if (!fwd.step(seq[size_t(pos)], pos, logits)) { + std::fprintf(stderr, "target step at %d: %s\n", pos, fwd.last_error()); + return 1; + } + if (pos >= P - 1) seq.push_back(argmax(logits)); // greedy: the token for position pos + 1 + } + t_target += now_s() - t0; + if (taps.size() != size_t(P + tokens) * size_t(T) * size_t(H)) { + std::fprintf(stderr, "the taps hold %zu floats, expected %zu\n", taps.size(), size_t(P + tokens) * size_t(T) * size_t(H)); + return 1; + } + std::printf("prompt %zu: %d prompt tokens, greedy:", p, P); + for (int i = P; i < P + std::min(tokens, 12); i++) std::printf(" %d", seq[size_t(i)]); + std::printf(" ...\n"); + + // seq[s] is known for s <= P + tokens; a block after s needs seq[s + 1 .. s + block - 1] to score against. + const int last_start = P + tokens - (block - 1); + const double t1 = now_s(); + long pa = 0, pp = 0; + for (int s = P; s <= last_start;) { + std::vector noise(size_t(block) * size_t(H)); + for (int i = 0; i < block; i++) { + fwd.embed_row(i == 0 ? seq[size_t(s)] : int(cfg.mask_token_id), row); + for (int k = 0; k < H; k++) noise[size_t(i) * size_t(H) + size_t(k)] = row[size_t(k)] * cfg.embedding_scale; + } + std::vector out; + try { + out = draft_forward(w, taps.data(), s, noise.data(), block, 0, window); + } catch (const std::exception& e) { + std::fprintf(stderr, "draft at %d: %s\n", s, e.what()); + return 1; + } + int a = 0; + bool open = true; + for (int i = 1; i < block; i++) { + std::vector h(out.begin() + ptrdiff_t(size_t(i) * size_t(H)), out.begin() + ptrdiff_t(size_t(i + 1) * size_t(H))); + std::vector lg; + if (!fwd.lm_head_logits(h, lg)) { std::fprintf(stderr, "head: %s\n", fwd.last_error()); return 1; } + const bool hit = argmax(lg) == seq[size_t(s + i)]; + if (i == 1) { proposals++; first_hits += hit ? 1 : 0; } + if (open && hit) a++; + else open = false; + if (!open && i >= 1) break; + } + passes++; pp++; + accepted += a; pa += a; + hist[size_t(a)]++; + s += all_positions ? 1 : a + 1; + } + t_draft += now_s() - t1; + std::printf("prompt %zu: %ld passes, %ld accepted, %.2f accepted per pass\n", p, pp, pa, pp ? double(pa) / double(pp) : 0.0); + std::fflush(stdout); + } + std::printf("TOTAL %s: %ld passes, %ld accepted, %.3f accepted per pass, %.3f tokens per pass, first proposal right %ld of %ld (%.1f %%)\n", + all_positions ? "every position" : "speculative walk", passes, accepted, + passes ? double(accepted) / double(passes) : 0.0, passes ? double(accepted + passes) / double(passes) : 0.0, + first_hits, proposals, proposals ? 100.0 * double(first_hits) / double(proposals) : 0.0); + std::printf("accepted-per-pass histogram:"); + for (int a = 0; a < block; a++) std::printf(" %d:%ld", a, hist[size_t(a)]); + std::printf("\nhost time: target %.1f s, draft %.1f s (references, one thread; not a speed result)\n", t_target, t_draft); + return 0; +}