diff --git a/npu/forward/include/q4nx_dequant.h b/npu/forward/include/q4nx_dequant.h index 644bcb73b..6b2968b44 100644 --- a/npu/forward/include/q4nx_dequant.h +++ b/npu/forward/include/q4nx_dequant.h @@ -90,6 +90,11 @@ int q4nx_dequant_tensor_chunked(const uint8_t* data, size_t nbytes, int rows, * refuses, plus an out-of-range row. The values are the ones that function would place at that row, bit for bit. */ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols, int chunk_bytes, int r, float* out /* cols */); +/* The chunk kind q4nx_dequant_tile_row() would accept for a [rows, cols] table of `nbytes`, or 0 when it would refuse + * every row: rows a multiple of 32, cols a multiple of 256, and nbytes exactly (rows/32)*(cols/256) tiles of one of the + * three kinds. A loader that picks the per-row path asks this first (engine#1842): choosing it on `nbytes % tiles == 0` + * alone let a table the reader refuses load fine and then serve every embedding read as 0.0f. */ +int q4nx_tile_row_kind(size_t nbytes, int rows, int cols); /* The (rows, cols) tile grid a container actually stores, when it is LARGER than the model's. * diff --git a/npu/forward/src/q4nx_dequant.c b/npu/forward/src/q4nx_dequant.c index c8652b871..98b95d212 100644 --- a/npu/forward/src/q4nx_dequant.c +++ b/npu/forward/src/q4nx_dequant.c @@ -151,6 +151,15 @@ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols return 0; } +int q4nx_tile_row_kind(size_t nbytes, int rows, int cols) { + if (rows <= 0 || cols <= 0 || rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return 0; + const size_t tiles = (size_t)(rows / Q4NX_TILE_ROWS) * (size_t)(cols / Q4NX_TILE_COLS); + const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; + for (int i = 0; i < 3; i++) + if (nbytes == tiles * (size_t)kinds[i]) return kinds[i]; + return 0; +} + int q4nx_packed_grid(size_t nbytes, int rows, int cols, int* cb_out, int* rp_out, int* cp_out) { const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; if (rows <= 0 || cols <= 0 || rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return 0; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 9c0d4817f..5a6b4b36a 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -742,14 +742,17 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, * 155,200 of them), so a row is read out of the `cols/256` tiles that cover it instead of the whole * table being decoded into f32 -- which for that pack is 0.74 GiB of container becoming 4.74 GiB. * Same tile grid and ordering as q4nx_dequant_tensor_chunked(), pinned by the row test. */ - const int ntr = NV / 32, ntc = kpad_ / 256; - const size_t tiles = (size_t)ntr * (size_t)ntc; - if (tiles > 0 && esz % tiles == 0) { + /* engine#1842: the grid is checked with the row reader's own rule, not `esz % tiles == 0`. That test let a + * table with no supported chunk kind, or an NV / kpad_ off the 32 x 256 grid, take this path; the reader + * then refused every row and embed_at() served 0.0f with no message. A table it would refuse stays on + * the whole-table decode below, which reports its own failure. */ + const int cb = q4nx_tile_row_kind(esz, NV, kpad_); + if (cb != 0) { embed_tile_ = (const uint8_t*)e; embed_tile_bytes_ = esz; embed_tile_rows_ = NV; embed_tile_cols_ = kpad_; - embed_tile_cb_ = int(esz / tiles); + embed_tile_cb_ = cb; embed_row_buf_.assign((size_t)kpad_, 0.0f); embed_row_tok_ = -1; } diff --git a/tests/npu_dequant_bf16_test.c b/tests/npu_dequant_bf16_test.c index 0ae922ed1..2ab1003e8 100644 --- a/tests/npu_dequant_bf16_test.c +++ b/tests/npu_dequant_bf16_test.c @@ -155,7 +155,45 @@ static void chunked_row_arms(void) { printf(" chunked row: 3 kinds x 64 rows bit-exact + 5 refusal arms\n"); } +/* engine#1842: the forward picks the per-row path for a chunked embedding with q4nx_tile_row_kind(). It used to pick it + * on `esz % tiles == 0` alone (tiles = (NV/32)*(kpad/256), truncating), which three kinds of table pass while the reader + * refuses every row of them -- and embed_at() turns that refusal into 0.0f. Each arm below is one such table: the old + * rule accepts it, the reader refuses it, and the predicate must refuse it too, so the table stays on the whole-table + * decode, which reports its own failure. The valid kinds must still be accepted, with the kind the reader takes. */ +static int old_rule(size_t esz, int nv, int kpad) { + const size_t tiles = (size_t)(nv / 32) * (size_t)(kpad / 256); + return tiles > 0 && esz % tiles == 0; +} +static void tile_row_kind_arms(void) { + const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8}; + for (int k = 0; k < 3; k++) + check(q4nx_tile_row_kind((size_t)4 * (size_t)kinds[k], 64, 512) == kinds[k], + "tile-row kind: each supported chunk kind on a 2x2 grid is accepted as itself"); + struct { size_t esz; int nv, kpad; const char* what; } bad[] = { + {(size_t)4 * 1000, 64, 512, "a chunk of 1,000 B (no such kind) on an exact 2x2 grid"}, + {(size_t)8 * Q4NX_TILE_BYTES, 64, 512, "two tiles' worth per grid cell (esz / tiles = 10,240 B)"}, + {(size_t)2 * Q4NX_TILE_BYTES, 48, 512, "NV 48: NV / 32 truncates to 1 tile-row and the grid misses rows 32..47"}, + {(size_t)2 * Q4NX_TILE_BYTES, 64, 384, "kpad 384: kpad / 256 truncates to 1 tile-column"}, + }; + uint8_t* q = calloc(bad[1].esz, 1); + float* row = malloc(512 * sizeof(float)); + if (!q || !row) { check(0, "tile-row kind: the arms allocate"); free(q); free(row); return; } + for (size_t i = 0; i < sizeof bad / sizeof bad[0]; i++) { + const int cb_old = (int)(bad[i].esz / ((size_t)(bad[i].nv / 32) * (size_t)(bad[i].kpad / 256))); + char msg[256]; + snprintf(msg, sizeof msg, "tile-row kind: the old rule took %s", bad[i].what); + check(old_rule(bad[i].esz, bad[i].nv, bad[i].kpad), msg); + snprintf(msg, sizeof msg, "tile-row kind: the reader refuses %s", bad[i].what); + check(q4nx_dequant_tile_row(q, bad[i].esz, bad[i].nv, bad[i].kpad, cb_old, 0, row) == -1, msg); + snprintf(msg, sizeof msg, "tile-row kind: and so does the predicate the forward now asks (%s)", bad[i].what); + check(q4nx_tile_row_kind(bad[i].esz, bad[i].nv, bad[i].kpad) == 0, msg); + } + free(q); free(row); + printf(" tile-row kind: 3 kinds accepted, 4 tables the old rule took and the reader refuses are refused\n"); +} + int main(void) { + tile_row_kind_arms(); chunked_row_arms(); embed_row_arms(); const int rows = 64, cols = 512;