From 1917f7cae281f9f2843f6b118650bc24b8b6fdfa Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 19:43:03 -0400 Subject: [PATCH] npu: say the prefill's block width out loud, once, when it is narrower than the prefill designs engine#1582 read a `[bs] M=219` phase line as "0 device launches, 93 s host" and could not tell which of two things had happened. Part of that was the counter (prof_add only feeds g_ph.launches from the decode accumulator), and part is mine: npu_gemm_rows() blocks a wide prefill at max_batch_rows(), which is the LOADED context's MD -- 128 for a prefill set, 16 for a decode-only (_m16) set, 1 when no context is loaded. A 219-row prefill at 16 rows/launch is 14 launches per projection instead of 2, and the cost lands in per-launch host packing, i.e. exactly what a phase profile reports as "host". The width is now said once per process, in the log line that was misread: [batch] prefill M=219 blocked at 16 rows/launch (14 launch(es) per projection): the loaded context set accepts 16 rows, not the 128 the prefill designs are built for. The family's 128-row prefill ELFs would make it 2 (engine#1582) The note is q4nx_block_note(rows, block, prefill_rows) in q4nx_semantics.c -- pure, so the existing device-free ctest pins it: silent at the prefill width, silent inside the block, and the two counts (14 and 2) plus the 219-at-one-row pathological case named when it does fire. Verified: npu_rmsnorm_offset_test ok (17 assertions before, 21 now); with the helper's guard removed the new assertions fail (2), so the test is load-bearing; `strings 1bit` carries the sentence. No device, no ticket: this changes a diagnostic, not a launch. --- npu/forward/include/q4nx_semantics.h | 13 +++++++++++++ npu/forward/src/q4nx_forward.cpp | 7 +++++++ npu/forward/src/q4nx_semantics.c | 15 +++++++++++++++ tests/npu_rmsnorm_offset_test.c | 12 ++++++++++++ 4 files changed, 47 insertions(+) diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index e29f53e28..77393f1b1 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -261,6 +261,19 @@ int q4nx_rmsnorm_offset_from_norm(const float* w, int n); * `why`. Exported so the rule is testable with no model and no device. */ int q4nx_attention_mode(const char* declared, int plan_loaded, int plan_noncausal, char* why, size_t why_len); +/* Why a wide prefill was blocked below the prefill design's width, or "" when it was not (engine#1582). + + * npu_gemm_rows() cannot launch more rows than the loaded activation BO holds, and that width is the + * *loaded context's* MD -- 128 for a prefill set, 16 for a decode-only ("_m16") set, 1 when no context + * is loaded at all. A 219-row prefill at 16 rows/launch is 14 launches per projection instead of 2, and + * the cost lands in per-launch host packing, which is what a phase profile then reports as "host". A + * reader looking at such a log has no way to tell "the device ran it in blocks" from "the device did not + * run it", so the block width is said out loud, once, with the launch count it implies. + + * Returns "" when the batch fits the prefill width (nothing to say), else a one-line note. Pure, so + * tests/npu_rmsnorm_offset_test.c pins it with no model and no device. */ +const char* q4nx_block_note(int rows, int block, int prefill_rows); + /* Description of the layer-type mix, for error messages. */ void q4nx_semantics_layer_mix(const Q4nxSemantics* s, char* out, size_t out_sz); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index a0af1fb3e..8e72cd386 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2997,6 +2997,13 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // recurrence) never needed the whole prompt in one launch, only the GEMMs did. const int blk = max_batch_rows(); if (blk > 0 && M > blk) { + // engine#1582: say the block width out loud, once. A log that reads "0 launches, 93 s host" cannot be told + // apart from a device that never ran the batch, and the width the batch was blocked at is the whole answer. + static bool note_said = false; + if (!note_said) { + const char* note = q4nx_block_note(M, blk, xd_ ? xd_->caps().prefill_rows : 0); + if (note && note[0]) { std::fprintf(stderr, " [batch] %s\n", note); note_said = true; } + } out.assign((size_t)M * N, 0.0f); for (int b0 = 0; b0 < M && err_.empty(); b0 += blk) { const int rows = std::min(blk, M - b0); diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index bfb88d276..c623e660d 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -600,3 +600,18 @@ int q4nx_attention_mode(const char* declared, int plan_loaded, int plan_noncausa if (declared && *declared) return 0; /* the pack declares causal, and any plan agrees */ return plan_noncausal ? 1 : 0; /* no declaration: the plan decides */ } + +const char* q4nx_block_note(int rows, int block, int prefill_rows) { + static char note[320]; + note[0] = '\0'; + if (block <= 0 || rows <= block) return note; /* fits: nothing blocked */ + if (prefill_rows <= 0 || block >= prefill_rows) return note; /* already the prefill width */ + const int launches = (rows + block - 1) / block; + const int at_prefill = (rows + prefill_rows - 1) / prefill_rows; + snprintf(note, sizeof note, + "prefill M=%d blocked at %d rows/launch (%d launch(es) per projection): the loaded context set accepts %d " + "rows, not the %d the prefill designs are built for. The family's %d-row prefill ELFs would make it %d " + "(engine#1582)", + rows, block, launches, block, prefill_rows, prefill_rows, at_prefill); + return note; +} diff --git a/tests/npu_rmsnorm_offset_test.c b/tests/npu_rmsnorm_offset_test.c index 3d1da271a..b2735d2fe 100644 --- a/tests/npu_rmsnorm_offset_test.c +++ b/tests/npu_rmsnorm_offset_test.c @@ -100,6 +100,18 @@ int main(void) { check(q4nx_attention_mode("causal", 1, 1, why, sizeof why) == -1 && strstr(why, "stale"), "the disagreement is refused in both directions, not only the one this issue measured"); + /* The prefill block width (engine#1582): a note only when the batch was blocked BELOW the prefill width, and + * it must carry the launch counts that make the cost legible (14 launches per projection for 219 rows at 16). */ + check(strstr(q4nx_block_note(219, 128, 128), "blocked") == NULL, + "a 219-row prefill at the 128-row width says nothing (it was not blocked)"); + const char* blocked = q4nx_block_note(219, 16, 128); + check(strstr(blocked, "M=219") && strstr(blocked, "14 launch") && strstr(blocked, "make it 2"), + "a 219-row prefill at 16 rows/launch names the width, the launch count and the prefill alternative"); + check(strstr(q4nx_block_note(219, 1, 128), "219 launch") != NULL, + "at one row per launch the note says 219, which is the pathological case"); + check(q4nx_block_note(0, 16, 128)[0] == '\0' && q4nx_block_note(10, 16, 128)[0] == '\0', + "a batch inside the block, or an empty one, prints nothing"); + printf(failures ? "npu_rmsnorm_offset: %d FAILED\n" : "npu_rmsnorm_offset: ok\n", failures); return failures ? 1 : 0; }