From f3bb331febf6f49e944a99946094e20c2b8f73be Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 19:31:03 -0400 Subject: [PATCH 1/2] npu: count the batch pass's launches in the phase line (it reported 0 for 224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[bs]` and `[ph]` both print `g_ph`, and `prof_add()` only added to it when the accumulator was the DECODE one. A batched pass launches through `prof_pre_`, so its 224 launches were never counted and their device wait fell into the line's `host=` residual: [bs] M=219 … (gemm pack=0.0 ql=0.00 wait=0.0 rb=0.0 sum=0.0 host=93113.2 n=0) That reads exactly like "the prefill ran on the host" — and it is how engine #1582 came to be filed: a 93 s host prefill for a pass that had launched 224 times through the blocked device path. The batched body already sets `g_in_batch` for its whole duration (`BatchPhaseScope`), so accepting `prof_pre_` while that flag is set counts those launches and leaves a decode-only measurement decode-only. Same run, same inputs, after the change (`NPU_PHASE_PROF=1 NPU_PROFILE=1`): [bs] M=219 wall=25411.0 | layers=20219.6 (gemm pack=4167.9 ql=265.03 wait=7972.0 rb=7.7 sum=12412.6 host=7807.0 n=224) n=224 is 28 layers x 8 projections, which is exactly what the trace shows, and the numbers behind it are the ones worth looking at: of 12.4 s of layer GEMM, **4.2 s is weight packing** and 8.0 s is launch wait. Note this only shows up with `NPU_PROFILE=1`: without it `prof_` is false, `prof_add()` is never called for either accumulator, and the gemm columns print as zeros in both lines. Nothing in the compute path depends on any of it. --- npu/target/xdna/xdna_target.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index def6a0534..cd9127b99 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -447,7 +447,14 @@ int64_t XdnaTarget::m16_check_gemm_rows(I8Ctx& ref, I8Ctx& fast, const float* A, void XdnaTarget::prof_add(ProfAcc* acc, int design, double pack, double quant, double wait, double rb) { - if (q4nx_lh::phprof_on() && acc == prof_dec_ && !q4nx_lh::g_in_head) { + // g_ph is what the [bs] (batch) and [ph] (decode step) lines print, and both read it. Only the DECODE + // accumulator used to be counted, so a 219-row prefill -- which launches through prof_pre_ -- reported + // "n=0 launches, host=93113 ms" for 224 launches and 93 s of device wait: the wait fell into the line's + // host= residual and the launch count stayed zero, which reads exactly like a host fallback and sent one + // lane (engine #1582) looking for a fallback that does not exist. The batched body sets g_in_batch for + // its whole duration, so accepting prof_pre_ there counts those launches and keeps a decode-only + // measurement decode-only. + if (q4nx_lh::phprof_on() && (acc == prof_dec_ || q4nx_lh::g_in_batch) && !q4nx_lh::g_in_head) { q4nx_lh::g_ph.g_pack += pack; q4nx_lh::g_ph.g_ql += quant; q4nx_lh::g_ph.g_wait += wait; q4nx_lh::g_ph.g_rb += rb; q4nx_lh::g_ph.launches++; From 3255ca1a602959fd0b41f09fba9aacd242859f45 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 20:14:52 -0400 Subject: [PATCH 2/2] npu: the batch phase line does not also count the head's launches Review finding, and it was right: `(acc == prof_dec_ || g_in_batch) && !g_in_head` admits **any** accumulator while `g_in_batch` is set, and `g_in_batch` stays set for the whole of `step_batch` -- which includes the batched head. `run_lm_head_rows` reaches `note_head` -> `prof_add(&prof_head_, ...)` inside that window, and only the SINGLE-row head sets `g_in_head`. So on every pass that runs the all-rows head, the head's pack/quant/wait/rb went into `g_ph.g_*` as well as into the epilogue line's own `h_*` slices, and `[bs]` computes `host = layers - gsum`. The head's device time was therefore counted twice and host was understated by it -- able to go negative -- on exactly the multi-row passes the line exists for. The gate now names the three admissible cases instead of accepting a flag's whole window: decode accumulator always pre accumulator only inside a batch head accumulator never (it keeps its own h_* accounting on the epilogue line) `g_in_head` stays in the condition as well, so the single-row head's exclusion is unchanged. Compiles (`ninja 1bit`). Not device-verified: that needs a verify-shaped batch under a ticket with `NPU_PHASE_PROF=1 NPU_PROFILE=1`, and the check is the one thing this change can be judged by -- `host` on the `[bs]` line must stop being negative and `n=` must equal the layer launches, with `epilogue ... head=` unchanged. I did not want to report a host figure from a run I have not made. --- npu/target/xdna/xdna_target.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index cd9127b99..9e2e5a1f3 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -454,7 +454,14 @@ void XdnaTarget::prof_add(ProfAcc* acc, int design, double pack, double quant, // lane (engine #1582) looking for a fallback that does not exist. The batched body sets g_in_batch for // its whole duration, so accepting prof_pre_ there counts those launches and keeps a decode-only // measurement decode-only. - if (q4nx_lh::phprof_on() && (acc == prof_dec_ || q4nx_lh::g_in_batch) && !q4nx_lh::g_in_head) { + // The head is NEVER counted here, and that is the second half of this gate. `g_in_batch` stays set for the whole of + // step_batch, and the batched head (run_lm_head_rows -> note_head -> prof_add(&prof_head_, ...)) runs inside that + // window without setting g_in_head -- which only the single-row head does. So accepting "any accumulator while in a + // batch" put the head's device time into g_ph as well, where [bs] computes host = layers - gsum: the same time was + // then reported twice, once in gsum and once on the epilogue line's own h_* slices, and host was understated by it + // (it can go negative) on exactly the multi-row passes this line exists for. The head keeps its own accounting. + if (q4nx_lh::phprof_on() && acc != &prof_head_ && + (acc == prof_dec_ || (q4nx_lh::g_in_batch && acc == prof_pre_)) && !q4nx_lh::g_in_head) { q4nx_lh::g_ph.g_pack += pack; q4nx_lh::g_ph.g_ql += quant; q4nx_lh::g_ph.g_wait += wait; q4nx_lh::g_ph.g_rb += rb; q4nx_lh::g_ph.launches++;