From 06563018208aec3e8dcebd2478292eae80e7fefd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:53:39 -0400 Subject: [PATCH] npu: the per-row embedding path is chosen with the row reader's own rule, so a table it refuses cannot serve 0.0f (engine#1842) The forward took the tiled per-row embedding read on `esz % tiles == 0` alone (tiles = (NV/32)*(kpad/256), truncating). Tables with no supported chunk kind, two tiles per cell, NV off the 32-row grid or kpad off the 256-column grid all pass that test, q4nx_dequant_tile_row() then refuses every row, and embed_at() serves the refusal as 0.0f with no message. q4nx_tile_row_kind(nbytes, rows, cols) states the reader's conditions: it returns the chunk kind the reader would take, or 0. The forward asks it before setting embed_tile_; a 0 leaves the table on the whole-table decode, which reports its own failure. tests/npu_dequant_bf16_test.c: tile_row_kind_arms covers the three kinds (accepted as themselves) and four tables that the old rule took and the reader refuses (all refused). With the predicate reduced to the old modulo rule, 5 checks fail. Co-Authored-By: Claude Opus 5.5 --- npu/forward/include/q4nx_dequant.h | 5 ++++ npu/forward/src/q4nx_dequant.c | 9 +++++++ npu/forward/src/q4nx_forward.cpp | 11 +++++---- tests/npu_dequant_bf16_test.c | 38 ++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/npu/forward/include/q4nx_dequant.h b/npu/forward/include/q4nx_dequant.h index 644bcb73b..6b2968b44 100644 --- a/npu/forward/include/q4nx_dequant.h +++ b/npu/forward/include/q4nx_dequant.h @@ -90,6 +90,11 @@ int q4nx_dequant_tensor_chunked(const uint8_t* data, size_t nbytes, int rows, * refuses, plus an out-of-range row. The values are the ones that function would place at that row, bit for bit. */ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols, int chunk_bytes, int r, float* out /* cols */); +/* The chunk kind q4nx_dequant_tile_row() would accept for a [rows, cols] table of `nbytes`, or 0 when it would refuse + * every row: rows a multiple of 32, cols a multiple of 256, and nbytes exactly (rows/32)*(cols/256) tiles of one of the + * three kinds. A loader that picks the per-row path asks this first (engine#1842): choosing it on `nbytes % tiles == 0` + * alone let a table the reader refuses load fine and then serve every embedding read as 0.0f. */ +int q4nx_tile_row_kind(size_t nbytes, int rows, int cols); /* The (rows, cols) tile grid a container actually stores, when it is LARGER than the model's. * diff --git a/npu/forward/src/q4nx_dequant.c b/npu/forward/src/q4nx_dequant.c index c8652b871..98b95d212 100644 --- a/npu/forward/src/q4nx_dequant.c +++ b/npu/forward/src/q4nx_dequant.c @@ -151,6 +151,15 @@ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols return 0; } +int q4nx_tile_row_kind(size_t nbytes, int rows, int cols) { + if (rows <= 0 || cols <= 0 || rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return 0; + const size_t tiles = (size_t)(rows / Q4NX_TILE_ROWS) * (size_t)(cols / Q4NX_TILE_COLS); + const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; + for (int i = 0; i < 3; i++) + if (nbytes == tiles * (size_t)kinds[i]) return kinds[i]; + return 0; +} + int q4nx_packed_grid(size_t nbytes, int rows, int cols, int* cb_out, int* rp_out, int* cp_out) { const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; if (rows <= 0 || cols <= 0 || rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return 0; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 95ec35e94..6ddf44aa9 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -742,14 +742,17 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, * 155,200 of them), so a row is read out of the `cols/256` tiles that cover it instead of the whole * table being decoded into f32 -- which for that pack is 0.74 GiB of container becoming 4.74 GiB. * Same tile grid and ordering as q4nx_dequant_tensor_chunked(), pinned by the row test. */ - const int ntr = NV / 32, ntc = kpad_ / 256; - const size_t tiles = (size_t)ntr * (size_t)ntc; - if (tiles > 0 && esz % tiles == 0) { + /* engine#1842: the grid is checked with the row reader's own rule, not `esz % tiles == 0`. That test let a + * table with no supported chunk kind, or an NV / kpad_ off the 32 x 256 grid, take this path; the reader + * then refused every row and embed_at() served 0.0f with no message. A table it would refuse stays on + * the whole-table decode below, which reports its own failure. */ + const int cb = q4nx_tile_row_kind(esz, NV, kpad_); + if (cb != 0) { embed_tile_ = (const uint8_t*)e; embed_tile_bytes_ = esz; embed_tile_rows_ = NV; embed_tile_cols_ = kpad_; - embed_tile_cb_ = int(esz / tiles); + embed_tile_cb_ = cb; embed_row_buf_.assign((size_t)kpad_, 0.0f); embed_row_tok_ = -1; } diff --git a/tests/npu_dequant_bf16_test.c b/tests/npu_dequant_bf16_test.c index 0ae922ed1..2ab1003e8 100644 --- a/tests/npu_dequant_bf16_test.c +++ b/tests/npu_dequant_bf16_test.c @@ -155,7 +155,45 @@ static void chunked_row_arms(void) { printf(" chunked row: 3 kinds x 64 rows bit-exact + 5 refusal arms\n"); } +/* engine#1842: the forward picks the per-row path for a chunked embedding with q4nx_tile_row_kind(). It used to pick it + * on `esz % tiles == 0` alone (tiles = (NV/32)*(kpad/256), truncating), which three kinds of table pass while the reader + * refuses every row of them -- and embed_at() turns that refusal into 0.0f. Each arm below is one such table: the old + * rule accepts it, the reader refuses it, and the predicate must refuse it too, so the table stays on the whole-table + * decode, which reports its own failure. The valid kinds must still be accepted, with the kind the reader takes. */ +static int old_rule(size_t esz, int nv, int kpad) { + const size_t tiles = (size_t)(nv / 32) * (size_t)(kpad / 256); + return tiles > 0 && esz % tiles == 0; +} +static void tile_row_kind_arms(void) { + const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; + for (int k = 0; k < 3; k++) + check(q4nx_tile_row_kind((size_t)4 * (size_t)kinds[k], 64, 512) == kinds[k], + "tile-row kind: each supported chunk kind on a 2x2 grid is accepted as itself"); + struct { size_t esz; int nv, kpad; const char* what; } bad[] = { + {(size_t)4 * 1000, 64, 512, "a chunk of 1,000 B (no such kind) on an exact 2x2 grid"}, + {(size_t)8 * Q4NX_TILE_BYTES, 64, 512, "two tiles' worth per grid cell (esz / tiles = 10,240 B)"}, + {(size_t)2 * Q4NX_TILE_BYTES, 48, 512, "NV 48: NV / 32 truncates to 1 tile-row and the grid misses rows 32..47"}, + {(size_t)2 * Q4NX_TILE_BYTES, 64, 384, "kpad 384: kpad / 256 truncates to 1 tile-column"}, + }; + uint8_t* q = calloc(bad[1].esz, 1); + float* row = malloc(512 * sizeof(float)); + if (!q || !row) { check(0, "tile-row kind: the arms allocate"); free(q); free(row); return; } + for (size_t i = 0; i < sizeof bad / sizeof bad[0]; i++) { + const int cb_old = (int)(bad[i].esz / ((size_t)(bad[i].nv / 32) * (size_t)(bad[i].kpad / 256))); + char msg[256]; + snprintf(msg, sizeof msg, "tile-row kind: the old rule took %s", bad[i].what); + check(old_rule(bad[i].esz, bad[i].nv, bad[i].kpad), msg); + snprintf(msg, sizeof msg, "tile-row kind: the reader refuses %s", bad[i].what); + check(q4nx_dequant_tile_row(q, bad[i].esz, bad[i].nv, bad[i].kpad, cb_old, 0, row) == -1, msg); + snprintf(msg, sizeof msg, "tile-row kind: and so does the predicate the forward now asks (%s)", bad[i].what); + check(q4nx_tile_row_kind(bad[i].esz, bad[i].nv, bad[i].kpad) == 0, msg); + } + free(q); free(row); + printf(" tile-row kind: 3 kinds accepted, 4 tables the old rule took and the reader refuses are refused\n"); +} + int main(void) { + tile_row_kind_arms(); chunked_row_arms(); embed_row_arms(); const int rows = 64, cols = 512;