From 377d2eaf03c32ec39e6f084ca70f0659cb0ac2c9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 19:43:39 -0400 Subject: [PATCH] npu: the phase rows say host_gemm and rows_max, so a zero-launch row cannot be read as a device row (refs #1582) engine#1582 reports a prefill row with `n=0` and 93 s of host time next to a decode row with a device number, in one process, and reads that as either a host prefill or a mis-attributed device wait. The code decides it before either candidate: `npu_gemm_rows()` opens with `if (host_gemm_) { host GEMM; return; }` and `host_gemm_` is written in exactly two places -- `init_host()` and the `NPU_INFER_HOST_GEMM` env var. So a row with `n=0` on host_gemm=1 is a host pass by construction. The other candidate cannot be it either: `max_batch_rows() == 0` does not skip the launch, it skips the *blocked* branch and falls through to an unblocked GEMM wider than the activation BO, which is the memory-poisoning path documented above that code (engine#1372) and would show launches, not their absence. Neither fact was visible in the row, which is why the issue could not be settled from it. Both lines now carry them: [bs] M=219 start=0 wall=... | host_gemm=0 rows_max=128 | embed=... (prefill) [ph] tok 1 pos 219 wall=... | host_gemm=0 | embed=... (token) `rows_max=` is on the batch line only, because that is the line where a batch wider than the kernels matters (it is walked in blocks). Nothing in the compute path reads either value; the phase partition, the counters and every existing field are unchanged, so old and new rows compare field by field. docs/npu.md gets the reading rule next to the table that quotes these lines, including the trap: a published row whose prefill says host_gemm=1 is a host prefill, and mixing it with a device-arm decode number produces exactly the contradiction #1582 describes. Compile-checked with the target built for it: `onebit_npu_forward_syntax` rc 0 (`-DONEBIT_NPU=ON` against the private runtime). Not run against a model: the format/argument pairs are validated at compile time by -Wformat, and a run needs a ticket plus a pack, which is the issue's own next step rather than this diff's. --- docs/npu.md | 8 ++++++++ npu/forward/src/q4nx_forward.cpp | 13 +++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 8e6a5fdec..ca561188f 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -488,6 +488,14 @@ a pb16i ELF would read 4,608-byte tiles out of 2,176-byte images). `tools/npu_pl #### What the stream is worth at the model level (measured) +**Reading a `[ph]`/`[bs]` row: the first field after the identifiers is `host_gemm`.** A row with +`host_gemm=1` is a pass that never asked the device — `npu_gemm_rows()` returns to the host GEMM on that +flag *before* `max_batch_rows()` or the blocked-launch path is consulted, and the flag is set only by +`init_host()` or `NPU_INFER_HOST_GEMM` — so `n=0` on such a row is a host pass by construction, not a +device launch that went uncounted (engine#1582, where a prefill row with `n=0` and a decode row with a +device number were read as a contradiction in one process). `[bs]` also prints `rows_max=`, the widest +launch the target's kernels take, because a batch wider than it is walked in blocks rather than skipped. + The phase numbers above are the kernel's. The number a user feels is a decode step, and it is the one that decided P3: same container file and same engine binary, two weight paths, adjacent windows on one quiet box (2026-11-02, three `npu-hold` tickets on `:18099`, `amdxdna` counter 0 in each, seven steady-state tokens per diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index a0af1fb3e..bccc111ae 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3629,10 +3629,15 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, const double gsum = g_ph.g_pack + g_ph.g_ql + g_ph.g_wait + g_ph.g_rb; const double hsum = g_ph.h_memcpy + g_ph.h_pack + g_ph.h_ql + g_ph.h_wait + g_ph.h_rb; fprintf(stderr, - "[bs] M=%2d start=%4d wall=%7.1f | embed=%5.1f | layers=%7.1f (gemm pack=%6.1f " + // host_gemm and rows_max are here because a [bs] line with n=0 is ambiguous without them: + // npu_gemm_rows() returns to the host GEMM on host_gemm_ BEFORE max_batch_rows() or the + // blocked path is consulted (q4nx_forward.cpp:2979), and the flag is set only by + // init_host() (:407) and NPU_INFER_HOST_GEMM (:413). So a row that says host_gemm=1 is a + // host prefill by construction, not a device launch that went uncounted (engine#1582). + "[bs] M=%2d start=%4d wall=%7.1f | host_gemm=%d rows_max=%d | embed=%5.1f | layers=%7.1f (gemm pack=%6.1f " "ql=%6.2f wait=%7.1f rb=%6.1f sum=%7.1f host=%7.1f n=%ld) | epilogue=%6.1f " "(head=%6.1f sum=%6.1f host=%5.1f) | tail=%5.1f\n", - M, start_pos, total, g_ph.embed, g_ph.layer, g_ph.g_pack, g_ph.g_ql, g_ph.g_wait, + M, start_pos, total, (int)host_gemm_, max_batch_rows(), g_ph.embed, g_ph.layer, g_ph.g_pack, g_ph.g_ql, g_ph.g_wait, g_ph.g_rb, gsum, g_ph.layer - gsum, g_ph.launches, epi, g_ph.h_total, hsum, g_ph.h_total - hsum, total - g_ph.embed - g_ph.layer - epi); char what[48]; @@ -4720,11 +4725,11 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits, int ca const double gsum = g_ph.g_pack + g_ph.g_ql + g_ph.g_wait + g_ph.g_rb; const double hsum = g_ph.h_memcpy + g_ph.h_pack + g_ph.h_ql + g_ph.h_wait + g_ph.h_rb; fprintf(stderr, - "[ph] tok %d pos %d wall=%.1f | embed=%.1f | layers=%.1f (gemm pack=%.1f " + "[ph] tok %d pos %d wall=%.1f | host_gemm=%d | embed=%.1f | layers=%.1f (gemm pack=%.1f " "ql=%.1f wait=%.1f rb=%.1f sum=%.1f host=%.1f n=%ld) | ffnorm=%.1f | " "head=%.1f (memcpy=%.1f pack=%.1f ql=%.1f wait=%.1f rb=%.1f sum=%.1f " "host=%.1f) | tail=%.1f\n", - token, pos, total, g_ph.embed, g_ph.layer, g_ph.g_pack, g_ph.g_ql, g_ph.g_wait, + token, pos, total, (int)host_gemm_, g_ph.embed, g_ph.layer, g_ph.g_pack, g_ph.g_ql, g_ph.g_wait, g_ph.g_rb, gsum, g_ph.layer - gsum, g_ph.launches, g_ph.ffnorm, g_ph.head, g_ph.h_memcpy, g_ph.h_pack, g_ph.h_ql, g_ph.h_wait, g_ph.h_rb, hsum, g_ph.head - hsum, tail);