Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions npu/forward/include/q4nx_dequant.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ int q4nx_dequant_tensor_chunked(const uint8_t* data, size_t nbytes, int rows,
* refuses, plus an out-of-range row. The values are the ones that function would place at that row, bit for bit. */
int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols, int chunk_bytes, int r,
float* out /* cols */);
/* The chunk kind q4nx_dequant_tile_row() would accept for a [rows, cols] table of `nbytes`, or 0 when it would refuse
* every row: rows a multiple of 32, cols a multiple of 256, and nbytes exactly (rows/32)*(cols/256) tiles of one of the
* three kinds. A loader that picks the per-row path asks this first (engine#1842): choosing it on `nbytes % tiles == 0`
* alone let a table the reader refuses load fine and then serve every embedding read as 0.0f. */
int q4nx_tile_row_kind(size_t nbytes, int rows, int cols);

/* The (rows, cols) tile grid a container actually stores, when it is LARGER than the model's.
*
Expand Down
9 changes: 9 additions & 0 deletions npu/forward/src/q4nx_dequant.c
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,15 @@ int q4nx_dequant_tile_row(const uint8_t* data, size_t nbytes, int rows, int cols
return 0;
}

int q4nx_tile_row_kind(size_t nbytes, int rows, int cols) {
if (rows <= 0 || cols <= 0 || rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return 0;
const size_t tiles = (size_t)(rows / Q4NX_TILE_ROWS) * (size_t)(cols / Q4NX_TILE_COLS);
const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8};
for (int i = 0; i < 3; i++)
if (nbytes == tiles * (size_t)kinds[i]) return kinds[i];
return 0;
}

int q4nx_packed_grid(size_t nbytes, int rows, int cols, int* cb_out, int* rp_out, int* cp_out) {
const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8};
if (rows <= 0 || cols <= 0 || rows % Q4NX_TILE_ROWS || cols % Q4NX_TILE_COLS) return 0;
Expand Down
11 changes: 7 additions & 4 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -742,14 +742,17 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
* 155,200 of them), so a row is read out of the `cols/256` tiles that cover it instead of the whole
* table being decoded into f32 -- which for that pack is 0.74 GiB of container becoming 4.74 GiB.
* Same tile grid and ordering as q4nx_dequant_tensor_chunked(), pinned by the row test. */
const int ntr = NV / 32, ntc = kpad_ / 256;
const size_t tiles = (size_t)ntr * (size_t)ntc;
if (tiles > 0 && esz % tiles == 0) {
/* engine#1842: the grid is checked with the row reader's own rule, not `esz % tiles == 0`. That test let a
* table with no supported chunk kind, or an NV / kpad_ off the 32 x 256 grid, take this path; the reader
* then refused every row and embed_at() served 0.0f with no message. A table it would refuse stays on
* the whole-table decode below, which reports its own failure. */
const int cb = q4nx_tile_row_kind(esz, NV, kpad_);
if (cb != 0) {
embed_tile_ = (const uint8_t*)e;
embed_tile_bytes_ = esz;
embed_tile_rows_ = NV;
embed_tile_cols_ = kpad_;
embed_tile_cb_ = int(esz / tiles);
embed_tile_cb_ = cb;
embed_row_buf_.assign((size_t)kpad_, 0.0f);
embed_row_tok_ = -1;
}
Expand Down
38 changes: 38 additions & 0 deletions tests/npu_dequant_bf16_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,45 @@ static void chunked_row_arms(void) {
printf(" chunked row: 3 kinds x 64 rows bit-exact + 5 refusal arms\n");
}

/* engine#1842: the forward picks the per-row path for a chunked embedding with q4nx_tile_row_kind(). It used to pick it
* on `esz % tiles == 0` alone (tiles = (NV/32)*(kpad/256), truncating), which three kinds of table pass while the reader
* refuses every row of them -- and embed_at() turns that refusal into 0.0f. Each arm below is one such table: the old
* rule accepts it, the reader refuses it, and the predicate must refuse it too, so the table stays on the whole-table
* decode, which reports its own failure. The valid kinds must still be accepted, with the kind the reader takes. */
static int old_rule(size_t esz, int nv, int kpad) {
const size_t tiles = (size_t)(nv / 32) * (size_t)(kpad / 256);
return tiles > 0 && esz % tiles == 0;
}
static void tile_row_kind_arms(void) {
const int kinds[3] = {Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K, Q4NX_CHUNK_Q8};
for (int k = 0; k < 3; k++)
check(q4nx_tile_row_kind((size_t)4 * (size_t)kinds[k], 64, 512) == kinds[k],
"tile-row kind: each supported chunk kind on a 2x2 grid is accepted as itself");
struct { size_t esz; int nv, kpad; const char* what; } bad[] = {
{(size_t)4 * 1000, 64, 512, "a chunk of 1,000 B (no such kind) on an exact 2x2 grid"},
{(size_t)8 * Q4NX_TILE_BYTES, 64, 512, "two tiles' worth per grid cell (esz / tiles = 10,240 B)"},
{(size_t)2 * Q4NX_TILE_BYTES, 48, 512, "NV 48: NV / 32 truncates to 1 tile-row and the grid misses rows 32..47"},
{(size_t)2 * Q4NX_TILE_BYTES, 64, 384, "kpad 384: kpad / 256 truncates to 1 tile-column"},
};
uint8_t* q = calloc(bad[1].esz, 1);
float* row = malloc(512 * sizeof(float));
if (!q || !row) { check(0, "tile-row kind: the arms allocate"); free(q); free(row); return; }
for (size_t i = 0; i < sizeof bad / sizeof bad[0]; i++) {
const int cb_old = (int)(bad[i].esz / ((size_t)(bad[i].nv / 32) * (size_t)(bad[i].kpad / 256)));
char msg[256];
snprintf(msg, sizeof msg, "tile-row kind: the old rule took %s", bad[i].what);
check(old_rule(bad[i].esz, bad[i].nv, bad[i].kpad), msg);
snprintf(msg, sizeof msg, "tile-row kind: the reader refuses %s", bad[i].what);
check(q4nx_dequant_tile_row(q, bad[i].esz, bad[i].nv, bad[i].kpad, cb_old, 0, row) == -1, msg);
snprintf(msg, sizeof msg, "tile-row kind: and so does the predicate the forward now asks (%s)", bad[i].what);
check(q4nx_tile_row_kind(bad[i].esz, bad[i].nv, bad[i].kpad) == 0, msg);
}
free(q); free(row);
printf(" tile-row kind: 3 kinds accepted, 4 tables the old rule took and the reader refuses are refused\n");
}

int main(void) {
tile_row_kind_arms();
chunked_row_arms();
embed_row_arms();
const int rows = 64, cols = 512;
Expand Down
Loading