Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions npu/forward/include/q4nx_semantics.h
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,19 @@ int q4nx_rmsnorm_offset_from_norm(const float* w, int n);
* `why`. Exported so the rule is testable with no model and no device. */
int q4nx_attention_mode(const char* declared, int plan_loaded, int plan_noncausal, char* why, size_t why_len);

/* Why a wide prefill was blocked below the prefill design's width, or "" when it was not (engine#1582).

* npu_gemm_rows() cannot launch more rows than the loaded activation BO holds, and that width is the
* *loaded context's* MD -- 128 for a prefill set, 16 for a decode-only ("_m16") set, 1 when no context
* is loaded at all. A 219-row prefill at 16 rows/launch is 14 launches per projection instead of 2, and
* the cost lands in per-launch host packing, which is what a phase profile then reports as "host". A
* reader looking at such a log has no way to tell "the device ran it in blocks" from "the device did not
* run it", so the block width is said out loud, once, with the launch count it implies.

* Returns "" when the batch fits the prefill width (nothing to say), else a one-line note. Pure, so
* tests/npu_rmsnorm_offset_test.c pins it with no model and no device. */
const char* q4nx_block_note(int rows, int block, int prefill_rows);

/* Description of the layer-type mix, for error messages. */
void q4nx_semantics_layer_mix(const Q4nxSemantics* s, char* out, size_t out_sz);

Expand Down
7 changes: 7 additions & 0 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2997,6 +2997,13 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N
// recurrence) never needed the whole prompt in one launch, only the GEMMs did.
const int blk = max_batch_rows();
if (blk > 0 && M > blk) {
// engine#1582: say the block width out loud, once. A log that reads "0 launches, 93 s host" cannot be told
// apart from a device that never ran the batch, and the width the batch was blocked at is the whole answer.
static bool note_said = false;
if (!note_said) {
const char* note = q4nx_block_note(M, blk, xd_ ? xd_->caps().prefill_rows : 0);
if (note && note[0]) { std::fprintf(stderr, " [batch] %s\n", note); note_said = true; }
}
out.assign((size_t)M * N, 0.0f);
for (int b0 = 0; b0 < M && err_.empty(); b0 += blk) {
const int rows = std::min(blk, M - b0);
Expand Down
15 changes: 15 additions & 0 deletions npu/forward/src/q4nx_semantics.c
Original file line number Diff line number Diff line change
Expand Up @@ -600,3 +600,18 @@ int q4nx_attention_mode(const char* declared, int plan_loaded, int plan_noncausa
if (declared && *declared) return 0; /* the pack declares causal, and any plan agrees */
return plan_noncausal ? 1 : 0; /* no declaration: the plan decides */
}

const char* q4nx_block_note(int rows, int block, int prefill_rows) {
static char note[320];
note[0] = '\0';
if (block <= 0 || rows <= block) return note; /* fits: nothing blocked */
if (prefill_rows <= 0 || block >= prefill_rows) return note; /* already the prefill width */
const int launches = (rows + block - 1) / block;
const int at_prefill = (rows + prefill_rows - 1) / prefill_rows;
snprintf(note, sizeof note,
"prefill M=%d blocked at %d rows/launch (%d launch(es) per projection): the loaded context set accepts %d "
"rows, not the %d the prefill designs are built for. The family's %d-row prefill ELFs would make it %d "
"(engine#1582)",
rows, block, launches, block, prefill_rows, prefill_rows, at_prefill);
return note;
}
12 changes: 12 additions & 0 deletions tests/npu_rmsnorm_offset_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,18 @@ int main(void) {
check(q4nx_attention_mode("causal", 1, 1, why, sizeof why) == -1 && strstr(why, "stale"),
"the disagreement is refused in both directions, not only the one this issue measured");

/* The prefill block width (engine#1582): a note only when the batch was blocked BELOW the prefill width, and
* it must carry the launch counts that make the cost legible (14 launches per projection for 219 rows at 16). */
check(strstr(q4nx_block_note(219, 128, 128), "blocked") == NULL,
"a 219-row prefill at the 128-row width says nothing (it was not blocked)");
const char* blocked = q4nx_block_note(219, 16, 128);
check(strstr(blocked, "M=219") && strstr(blocked, "14 launch") && strstr(blocked, "make it 2"),
"a 219-row prefill at 16 rows/launch names the width, the launch count and the prefill alternative");
check(strstr(q4nx_block_note(219, 1, 128), "219 launch") != NULL,
"at one row per launch the note says 219, which is the pathological case");
check(q4nx_block_note(0, 16, 128)[0] == '\0' && q4nx_block_note(10, 16, 128)[0] == '\0',
"a batch inside the block, or an empty one, prints nothing");

printf(failures ? "npu_rmsnorm_offset: %d FAILED\n" : "npu_rmsnorm_offset: ok\n", failures);
return failures ? 1 : 0;
}
Loading