From 7dd323f8beb8240637013490e7eda8bf3e1dd14c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 13:59:42 -0400 Subject: [PATCH 1/3] npu: count and time the host's own row copies (`NPU_COPY_PROF=1`) The `[ph]`/`[bs]` phase lines attribute *device* time. A host row copy therefore lands in their `host=` residual with nothing to separate it, which is why the copy audit had to derive its per-token figure from the model's geometry instead of reading it off a run -- and got it wrong twice (a member list from a different model, then a "hard bound" that omitted two member classes). `NPU_COPY_PROF=1` adds a `[cp]` line per decode step and per batched pass: bytes, count and milliseconds for each host copy site (`qkv_split`, `attn_gate`, `ctx_row`, `mlp_in`, `mlp_out`), a total, and `KiB/row` for a batched pass. Same shape and the same additive contract as the phases beside it: with the variable unset every `CpScope` is a no-op and the memcpy is the same call in the same place, so nothing is counted, timed or printed. Measured on the 27B shipped route (bonsai2-27b-t2s2-serve, driver 349FDC99, one ticket, 251 s held, amdxdma errors 0), identity gated across an unpatched control, the branch binary with the knob unset, and the branch binary counting twice -- emitted token ids and all top-5 logprobs equal in every arm, and the two counting arms byte-identical: per prefill row: 15 744 KiB = 15.38 MiB, 0.32-0.67 ms (mlp_in 8.5 + mlp_out 4.25 + qkv_split 1.75 + attn_gate 0.5 + ctx_row 0.375) per decode token: 1 408 KiB = 1.375 MiB, 0.0285 ms (qkv_split + attn_gate only; the MLP rows are batch-path-only) That is 0.8-1.3 % of an M=16 prefill pass and 0.009 % of a 313 ms token, and it is 83 % two batch-only members -- so the removal candidate is prefill-only and worth ~0.4 ms/row. No compute path changes here: this is the instrument those numbers came from. Part of #698. --- docs/npu.md | 5 ++ npu/forward/src/q4nx_forward.cpp | 131 +++++++++++++++++++++++++++---- 2 files changed, 120 insertions(+), 16 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index c1252c9bd..ed2947edc 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -344,6 +344,11 @@ the same fill is 0.47 s. A first request on a fresh serve therefore costs one fi cold) on top of its prefill, and every request after it on the same serve costs none. Do not read a per-serve benchmark's request wall as a decode number: `[ph]`/`[lh]` are the per-token instruments, and `[bs]` is the same partition for one batched pass (a verify batch), whose interior a per-token line cannot see. +`NPU_COPY_PROF=1` adds a `[cp]` line to both: the **bytes and milliseconds the host moves at its own copy +sites** (`qkv_split`, `attn_gate`, `ctx_row`, `mlp_in`, `mlp_out`), one figure per site and a `KiB/row` total +for a batched pass. The phase lines above attribute device time only, so a host row copy lands in their `host=` +residual with nothing to separate it; `[cp]` is the line that says how much is left to remove rather than +time it as part of the layer's wall. ### From a GGUF diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 3cf958a45..55d3bdbdd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2871,6 +2871,58 @@ inline double ph_ms(std::chrono::steady_clock::time_point t) { return std::chrono::duration( std::chrono::steady_clock::now() - t).count(); } +// Host-copy accounting (NPU_COPY_PROF=1). The [ph]/[bs] lines say what the *device* did; these say what +// the host moved into and out of its own vectors on the same pass, per site, in bytes and milliseconds. +// That is the figure an audit of "what is left to remove" needs and a phase residual cannot give: the +// sites are host-side row gathers whose cost is invisible inside gemm/wait/readback. Additive +// measurement only, exactly like the phases above: nothing in the compute path reads it, and with the +// variable unset nothing is counted and nothing is printed. +enum CopySite { CP_QKV, CP_GATE, CP_CTX, CP_MLP_IN, CP_MLP_OUT, CP_N }; +const char* const kCopySiteName[CP_N] = {"qkv_split", "attn_gate", "ctx_row", "mlp_in", "mlp_out"}; +struct CopySites { + double b[CP_N]{}; + double ms[CP_N]{}; + long n[CP_N]{}; + int rows = 0; // rows a batched pass accounted, so bytes/row is read, not divided +}; +CopySites g_cp; +inline bool cpprof_on() { + static const bool on = getenv("NPU_COPY_PROF") != nullptr; + return on; +} +inline void cp_count(int site, size_t bytes) { + if (!cpprof_on()) return; + g_cp.b[site] += (double)bytes; g_cp.n[site]++; +} +inline void cp_reset() { + if (!cpprof_on()) return; + g_cp = CopySites{}; +} +// Times one site's region; its bytes are counted with cp_count() inside the same scope, so the site's +// ms and its bytes describe the same work. +struct CpScope { + int site; + bool on; + std::chrono::steady_clock::time_point t0; + explicit CpScope(int s) + : site(s), on(cpprof_on()), + t0(on ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}) {} + ~CpScope() { if (on) g_cp.ms[site] += ph_ms(t0); } +}; +void cp_print(const char* what) { + if (!cpprof_on()) return; + double tb = 0, tm = 0; + fprintf(stderr, "[cp] %s host copies:", what); + for (int i = 0; i < CP_N; i++) { + if (!g_cp.n[i]) continue; + fprintf(stderr, " %s=%.1fKiB/%ld/%.3fms", kCopySiteName[i], g_cp.b[i] / 1024.0, g_cp.n[i], + g_cp.ms[i]); + tb += g_cp.b[i]; tm += g_cp.ms[i]; + } + fprintf(stderr, " total=%.1fKiB %.3fms", tb / 1024.0, tm); + if (g_cp.rows > 1) fprintf(stderr, " (%.1fKiB/row)", tb / 1024.0 / (double)g_cp.rows); + fprintf(stderr, "\n"); +} struct BatchPhaseScope { // g_in_batch for the batched body, cleared on every return path bool prev; explicit BatchPhaseScope(bool on) : prev(g_in_batch) { if (on) g_in_batch = true; } @@ -3709,6 +3761,8 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, g_ph = StepPhases{}; for (int i = 0; i < LH_LAYER_N; i++) { g_lh[i] = 0.0; g_lh_n[i] = 0; } } + cp_reset(); + if (cpprof_on()) g_cp.rows = M; BatchPhases bph; // measurement only (NPU_PHASE_PROF=1); prints this call's own delta at the end const auto tp_bs = phb ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -3795,6 +3849,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, // Each of the four GEMMs wrote M rows of *its* design's N (padded, engine#854), so the // reads go through the design's stride -- the same class as O/D/gate-up below. const size_t sq = (size_t)gq->N, skv = (size_t)gk->N; + CpScope _cp(CP_QKV); for (int m = 0; m < M; m++) { float* d = QKV.data() + (size_t)m * n_on; memcpy(d, qh0.data() + (size_t)m * sq, (size_t)nhd * sizeof(float)); @@ -3803,6 +3858,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, memcpy(d + 2 * nhd + nkv, kvv.data() + (size_t)m * skv, (size_t)nkv * sizeof(float)); } + cp_count(CP_QKV, (size_t)M * n_on * sizeof(float)); } else { npu_gemm_rows(*ctx_[i_qkv_l], XN.data(), M, g->K, g->N, B_.data(), QKV, 0, "attn_qkv", g->real_k); @@ -3843,8 +3899,12 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, // positions to read (0 = up to and including `pos`, i.e. causal), and `dst` receives the // context -- null when the pass only fills the cache (the noncausal first pass below). auto row_attn = [&](int m, float* dst, bool append, int kvlen) -> bool { - qkv_row.assign(QKV.begin() + (size_t)m * g->N, - QKV.begin() + (size_t)(m + 1) * g->N); + { + CpScope _cp(CP_QKV); + qkv_row.assign(QKV.begin() + (size_t)m * g->N, + QKV.begin() + (size_t)(m + 1) * g->N); + cp_count(CP_QKV, (size_t)g->N * sizeof(float)); + } // attn_output_gate: the fused QKV is [q | gate | k | v], so attention() // must not see the gate half; the gate multiplies the context after. const float* gate_ptr = nullptr; @@ -3852,9 +3912,13 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, if (attn_gate_) { const int kvsz = g->k_rows + g->v_rows; qkv_use.assign((size_t)nhd + (size_t)kvsz, 0.0f); - memcpy(qkv_use.data(), qkv_row.data(), (size_t)nhd * sizeof(float)); - memcpy(qkv_use.data() + nhd, qkv_row.data() + 2 * nhd, - (size_t)kvsz * sizeof(float)); + { + CpScope _cp(CP_GATE); + memcpy(qkv_use.data(), qkv_row.data(), (size_t)nhd * sizeof(float)); + memcpy(qkv_use.data() + nhd, qkv_row.data() + 2 * nhd, + (size_t)kvsz * sizeof(float)); + cp_count(CP_GATE, ((size_t)nhd + (size_t)kvsz) * sizeof(float)); + } gate_ptr = qkv_row.data() + nhd; in = &qkv_use; } @@ -3867,7 +3931,11 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, if (gate_ptr) for (int i = 0; i < nhd; i++) ctx_row[(size_t)i] *= 1.0f / (1.0f + expf(-gate_ptr[i])); - memcpy(dst, ctx_row.data(), (size_t)ctx_w * sizeof(float)); + { + CpScope _cp(CP_CTX); + memcpy(dst, ctx_row.data(), (size_t)ctx_w * sizeof(float)); + cp_count(CP_CTX, (size_t)ctx_w * sizeof(float)); + } return true; }; // A noncausal pack: every full-attention layer reads the whole prompt, so the cache has to @@ -3972,10 +4040,18 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, // act_w is the stride both branches above leave ACT at; only the first 2*im columns of a // row are real (the rest is the kernel's zero column pad). for (int m = 0; m < M; m++) { - act_row.assign(ACT.begin() + (size_t)m * act_w, - ACT.begin() + (size_t)m * act_w + (size_t)2 * im); + { + CpScope _cp(CP_MLP_IN); + act_row.assign(ACT.begin() + (size_t)m * act_w, + ACT.begin() + (size_t)m * act_w + (size_t)2 * im); + cp_count(CP_MLP_IN, (size_t)2 * im * sizeof(float)); + } if (!mlp(act_row, hh, im)) { err_ = "mlp failed"; return false; } - memcpy(HH.data() + (size_t)m * im, hh.data(), (size_t)im * sizeof(float)); + { + CpScope _cp(CP_MLP_OUT); + memcpy(HH.data() + (size_t)m * im, hh.data(), (size_t)im * sizeof(float)); + cp_count(CP_MLP_OUT, (size_t)im * sizeof(float)); + } } } { @@ -4064,6 +4140,11 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, M, start_pos, c_.NL, (int)two_widths); } bph.print(M); + { + char cp_what[48]; + snprintf(cp_what, sizeof cp_what, "batch M=%d start=%d", M, start_pos); + cp_print(cp_what); + } if (phb) { // epilogue = final norm + the head rows; head= is run_lm_head_rows' own timer, so // epilogue - head is the per-row final norm. tail must be ~0: it is what the partition misses. @@ -4711,6 +4792,7 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { g_ph = StepPhases{}; for (int i = 0; i < LH_LAYER_N; i++) { g_lh[i] = 0.0; g_lh_n[i] = 0; } } + cp_reset(); const int H = c_.H; if (token < 0 || token >= c_.NV) { err_ = "token out of range"; return false; } @@ -4881,10 +4963,14 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { return false; } qkv.assign((size_t)2 * nhd + (size_t)2 * nkv, 0.0f); - memcpy(qkv.data(), qh0.data(), (size_t)nhd * sizeof(float)); - memcpy(qkv.data() + nhd, qh1.data(), (size_t)nhd * sizeof(float)); - memcpy(qkv.data() + 2 * nhd, kvk.data(), (size_t)nkv * sizeof(float)); - memcpy(qkv.data() + 2 * nhd + nkv, kvv.data(), (size_t)nkv * sizeof(float)); + { + CpScope _cp(CP_QKV); + memcpy(qkv.data(), qh0.data(), (size_t)nhd * sizeof(float)); + memcpy(qkv.data() + nhd, qh1.data(), (size_t)nhd * sizeof(float)); + memcpy(qkv.data() + 2 * nhd, kvk.data(), (size_t)nkv * sizeof(float)); + memcpy(qkv.data() + 2 * nhd + nkv, kvv.data(), (size_t)nkv * sizeof(float)); + cp_count(CP_QKV, ((size_t)2 * nhd + (size_t)2 * nkv) * sizeof(float)); + } } else { npu_gemm(*ctx_[i_qkv_l], xn.data(), g->K, g->N, B_.data(), qkv, 0, "attn_qkv", g->real_k); @@ -4897,12 +4983,18 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { if (attn_gate_) { const int kvsz = g->k_rows + g->v_rows; qkv_use.resize((size_t)nhd + (size_t)kvsz); - memcpy(qkv_use.data(), qkv.data(), (size_t)nhd * sizeof(float)); - memcpy(qkv_use.data() + nhd, qkv.data() + 2 * nhd, - (size_t)kvsz * sizeof(float)); + { + CpScope _cp(CP_GATE); + memcpy(qkv_use.data(), qkv.data(), (size_t)nhd * sizeof(float)); + memcpy(qkv_use.data() + nhd, qkv.data() + 2 * nhd, + (size_t)kvsz * sizeof(float)); + cp_count(CP_GATE, ((size_t)nhd + (size_t)kvsz) * sizeof(float)); + } gate_ptr = qkv.data() + nhd; } else { + CpScope _cp(CP_GATE); qkv_use = qkv; + cp_count(CP_GATE, qkv.size() * sizeof(float)); } { LhScope _tlh(LH_ATTN); @@ -5140,6 +5232,13 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { snprintf(lhwhat, sizeof lhwhat, "tok %d", token); lh_dump_layer_host(lhwhat, "token"); } + // Independent of NPU_PHASE_PROF: the copy accounting answers a different question and must be readable + // with only NPU_COPY_PROF set. + { + char cp_what[48]; + snprintf(cp_what, sizeof cp_what, "tok %d pos %d", token, pos); + cp_print(cp_what); + } return true; } From 2b89cc20663a05f5bce88ee2adcbaaaf89deaaa7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 14:29:15 -0400 Subject: [PATCH 2/3] npu: remove the two MLP row copies the instrument counted (12.75 of 15.38 MiB per row) `NPU_COPY_PROF=1` named the members of the prefill row's host-copy class and showed that two of them were 83 % of it: `act_row.assign(ACT row, 2*im)` (8.5 MiB/row) and `memcpy(HH row, hh, im)` (4.25 MiB/row), both staging the elementwise gate*up through a vector. Neither is needed -- ACT's rows are already at `act_w` with the first 2*im columns real, and HH's rows are already the destination: mlp() gains a caller-memory overload (the vector form now forwards to it, so the decode path is unchanged), and step_batch() runs it on ACT.data() + m*act_w into HH.data() + m*im. Same arithmetic, same order, same values; act_row and hh are gone. Measured, one NPU ticket (queued 228 s, held 184 s, amdxdma errors 0), four arms in the same window -- unpatched control, the pre-removal binary, and the removal twice -- with emitted token ids and all top-5 logprobs equal in every arm: per prefill row: 15 744 KiB -> 2 688 KiB (15.38 -> 2.63 MiB, -83 %) copy time / row: 0.649 ms -> 0.107 ms (median over the M>1 passes, 6.1x) per decode token: 1 408 KiB, 0.0285 ms (unchanged: this path has no MLP row copy) What is NOT claimed: the pass wall. The prefill pass is weight-bound and this window was contended (M=16 walls 1.7 s against 0.85 s in the 14:00 window), so the ~7 ms a pass saves is not separable from the noise here; the figures above are the instrument's own deltas. Part of #698. --- docs/npu.md | 9 +++++---- npu/forward/include/q4nx_forward.h | 3 +++ npu/forward/src/q4nx_forward.cpp | 32 +++++++++++++----------------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index ed2947edc..8bd4ac734 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -345,10 +345,11 @@ cold) on top of its prefill, and every request after it on the same serve costs per-serve benchmark's request wall as a decode number: `[ph]`/`[lh]` are the per-token instruments, and `[bs]` is the same partition for one batched pass (a verify batch), whose interior a per-token line cannot see. `NPU_COPY_PROF=1` adds a `[cp]` line to both: the **bytes and milliseconds the host moves at its own copy -sites** (`qkv_split`, `attn_gate`, `ctx_row`, `mlp_in`, `mlp_out`), one figure per site and a `KiB/row` total -for a batched pass. The phase lines above attribute device time only, so a host row copy lands in their `host=` -residual with nothing to separate it; `[cp]` is the line that says how much is left to remove rather than -time it as part of the layer's wall. +sites** (`qkv_split`, `attn_gate`, `ctx_row`), one figure per site and a `KiB/row` total for a batched pass. +The phase lines above attribute device time only, so a host row copy lands in their `host=` residual with +nothing to separate it; `[cp]` is the line that says how much is left to remove rather than time it as part of +the layer's wall. The MLP row copies this line first counted (`mlp_in`, `mlp_out` — 12.75 MiB of a 15.38 MiB +prefill row) are gone: the batched pass now runs `mlp()` on the ACT row and into the HH row in place. ### From a GGUF diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 36fd00f18..c1d5888f7 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -318,6 +318,9 @@ class Q4nxNpuForward { bool proportional, int window, int kv_layer, bool append_kv, int kv_len = 0); bool mlp(const std::vector& gu, std::vector& h, int im); + // The same elementwise gate*up on caller memory, so a batched pass can read its ACT row and write its HH + // row where they already are instead of staging both through a vector (a copy on each side). + bool mlp(const float* gu, float* h, int im); float rmsnorm(const float* x, const float* w, int n, float* out) const; float rmsnorm_noweight(const float* x, int n, float* out) const; bool run_lm_head(const std::vector& h, std::vector& logits); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 55d3bdbdd..64be2f23a 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2877,8 +2877,8 @@ inline double ph_ms(std::chrono::steady_clock::time_point t) { // sites are host-side row gathers whose cost is invisible inside gemm/wait/readback. Additive // measurement only, exactly like the phases above: nothing in the compute path reads it, and with the // variable unset nothing is counted and nothing is printed. -enum CopySite { CP_QKV, CP_GATE, CP_CTX, CP_MLP_IN, CP_MLP_OUT, CP_N }; -const char* const kCopySiteName[CP_N] = {"qkv_split", "attn_gate", "ctx_row", "mlp_in", "mlp_out"}; +enum CopySite { CP_QKV, CP_GATE, CP_CTX, CP_N }; +const char* const kCopySiteName[CP_N] = {"qkv_split", "attn_gate", "ctx_row"}; struct CopySites { double b[CP_N]{}; double ms[CP_N]{}; @@ -3787,7 +3787,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, X[(size_t)m * H + k] = embed_at(tokens[m], k) * embed_scale_; if (phb) { g_ph.embed = ph_ms(tp_bs); tp_lap = std::chrono::steady_clock::now(); } - std::vector QKV, CTX, O, ACT, HH, D, qkv_row, ctx_row, act_row, hh; + std::vector QKV, CTX, O, ACT, HH, D, qkv_row, ctx_row; for (int l = 0; l < c_.NL; l++) { cur_layer_ = l; const bool cached = weight_cached(0); @@ -4038,21 +4038,13 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, HH.assign((size_t)M * (size_t)im, 0.0f); { // act_w is the stride both branches above leave ACT at; only the first 2*im columns of a - // row are real (the rest is the kernel's zero column pad). - for (int m = 0; m < M; m++) { - { - CpScope _cp(CP_MLP_IN); - act_row.assign(ACT.begin() + (size_t)m * act_w, - ACT.begin() + (size_t)m * act_w + (size_t)2 * im); - cp_count(CP_MLP_IN, (size_t)2 * im * sizeof(float)); - } - if (!mlp(act_row, hh, im)) { err_ = "mlp failed"; return false; } - { - CpScope _cp(CP_MLP_OUT); - memcpy(HH.data() + (size_t)m * im, hh.data(), (size_t)im * sizeof(float)); - cp_count(CP_MLP_OUT, (size_t)im * sizeof(float)); + // row are real (the rest is the kernel's zero column pad), so the elementwise gate*up reads + // that row in place and writes HH's row in place: no staging copy on either side. The two + // copies this replaces were 12.75 MiB of a 15.38 MiB row (NPU_COPY_PROF=1, 2026-11-05). + for (int m = 0; m < M; m++) + if (!mlp(ACT.data() + (size_t)m * act_w, HH.data() + (size_t)m * im, im)) { + err_ = "mlp failed"; return false; } - } } { const Q4nxDesignGeom* g = &designs_[i_d]; @@ -4748,8 +4740,12 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, // SiLU(gate) * up from either a fused GU result or separate G and U results. bool Q4nxNpuForward::mlp(const std::vector& gu, std::vector& h, int im) { - LhScope _tlh(LH_MLP); h.resize(im); + return mlp(gu.data(), h.data(), im); +} + +bool Q4nxNpuForward::mlp(const float* gu, float* h, int im) { + LhScope _tlh(LH_MLP); const int gelu = (sem_.mlp_act == Q4NX_ACT_GELU); for (int i = 0; i < im; i++) { const float g = gu[i]; From ed1d3b2eb352197e712e57a69f7604c0e195c1ce Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 15:05:35 -0400 Subject: [PATCH 3/3] npu: state the counters' threading contract, which is single-writer by caller The review's nit was that `g_cp` is a plain global mutated from whatever threads a pass runs on, so `--npu-streams 2` would make the byte counts a data race. It would, if `--npu-streams` were threads -- it is not: it is N serve processes sharing the device by flock (`app/npu_device_lock.h`, `LOCK_SH`), each with its own copy of the struct. In-process there is one pass in flight by construction: every serve-side entry into step() and step_batch() holds the engine mutex (`app/forward_serve.cpp:258`, `:698`; the four call sites are the only ones outside the tests). So the counters need no atomics and no per-pass objects to be correct, and paying an atomic on a path that exists to measure memcpy would be the wrong trade. What was missing was the statement of that contract, which the review was right to ask for -- it is now a paragraph in `docs/npu.md` and a comment on the globals, naming the mutex and the escalation if a caller ever parallelises the layer loop (then `g_cp` and `g_ph` both need per-pass objects). No behaviour change; the TU compiles with the same 23 `-Wall` warnings as `main` under the reviewer's flags. --- docs/npu.md | 5 +++++ npu/forward/src/q4nx_forward.cpp | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 8bd4ac734..1552e9790 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -351,6 +351,11 @@ nothing to separate it; `[cp]` is the line that says how much is left to remove the layer's wall. The MLP row copies this line first counted (`mlp_in`, `mlp_out` — 12.75 MiB of a 15.38 MiB prefill row) are gone: the batched pass now runs `mlp()` on the ACT row and into the HH row in place. +The counters are per **process**, single writer, like `[ph]`: a serve enters `step()`/`step_batch()` under +its engine mutex (`app/forward_serve.cpp:258`, `:698`), so one pass is in flight, and `--npu-streams N` is +N processes sharing the device by flock (`app/npu_device_lock.h`), each counting its own. A caller that +parallelised the layer loop would need per-pass counters instead. + ### From a GGUF `tools/gguf_to_q4nx.py pack --tokenizer ` writes a model diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 64be2f23a..46bffcad9 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2877,6 +2877,14 @@ inline double ph_ms(std::chrono::steady_clock::time_point t) { // sites are host-side row gathers whose cost is invisible inside gemm/wait/readback. Additive // measurement only, exactly like the phases above: nothing in the compute path reads it, and with the // variable unset nothing is counted and nothing is printed. +// +// Single writer, like `g_ph` beside it, and that is a property of the callers rather than of these +// globals: every serve-side entry into step()/step_batch() holds the engine's mutex +// (app/forward_serve.cpp:258 and :698), so one pass is in flight per process, and `--npu-streams N` +// is N processes sharing the device by flock (app/npu_device_lock.h), each with its own copy of this +// struct -- not N threads. A caller that ever parallelises the layer loop would make cp_count() and +// the CpScope timings a data race and cp_reset() clear another pass's counts; the counters would +// then have to become per-pass objects, as would `g_ph`. enum CopySite { CP_QKV, CP_GATE, CP_CTX, CP_N }; const char* const kCopySiteName[CP_N] = {"qkv_split", "attn_gate", "ctx_row"}; struct CopySites {