From d95f73bc682c5d9a7937d640b13359aaefa253f5 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 9 Nov 2026 22:53:10 -0400 Subject: [PATCH] npu: read the token embedding table in place, one row per token, instead of materialising it (engine#1784) The container is already mapped read-only (`Model::Model` -> `map_file()`, and every `Tensor::data` points into that mapping), so the token embedding table was being **copied out of a mapping we already had**: bf16 tables via `memcpy` of the whole `NV*H` table, and plain int8 tables via `q4nx_dequant_plain_i8()` over the whole table at 4 bytes per element. The second one is the expensive half: `bonsai2-27b-NPU`'s I8 embedding is 0.74 GiB in the container and became **2.96 GiB resident**, for a table that is read one row per token. Measured on this box before the change (embedding shape x dtype from each `model.q4nx` header): bonsai2-27b-NPU / -b8 I8 0.74 GiB on disk -> 2.96 GiB resident qwen3-coder-30b-a3b BF16 0.58 GiB -> 0.58 GiB ZR1-1.5B-npu-bf16 BF16 0.43 GiB -> 0.43 GiB d3-lite-bf16 BF16 0.47 GiB -> 0.47 GiB Qwen3-0.6B BF16 0.29 GiB -> 0.29 GiB What changes: * bf16 tables: `embed_bf16_` is the pointer into the mapping; `embed_at()` converts the one element asked for. * plain int8 + per-32-column scale (the dtype the 27B and the gemma-class containers use): `embed_i8_` / `embed_sc_` are pointers into the mapping and `embed_at()` reads one element through the new `q4nx_plain_i8_at()`. * `embed_` is no longer filled at all and is now always empty, so a stale read is a bounds error instead of a silent wrong answer. `embed_f_` is still built for the chunked formats (`dequant_any`'s tile paths), which have no element-wise read yet -- that half is named on the issue, not hidden here. * Pad columns (`k >= H`) still read 0.0f, which is what the expanded table held, so the head's `kpad_`-wide dot product is unchanged. `lm_w_` is untouched. No call site changes: `embed_at()` and `lm_at()` were already the only readers, and both are called with `k` innermost, so "one row per token" is the access the code already had -- it just does not need a table-sized buffer to serve it. `q4nx_plain_i8_at()` is declared beside `q4nx_dequant_plain_i8()` and computes the group index the same way, so the two cannot drift; `tests/npu_embed_row_test.c` pins a row read against the whole-table decode **bit for bit** on a table whose scale differs per row and per group (so a row/column index bug cannot pass by symmetry), plus the null-scale, out-of-range, pad-column and wrong-cols arms. Host-only, no model, no device. Gate: full CPU suite **168/168**. The RSS/identical-output rows on three models are the second half of #1784 and need the NPU lane; they are not claimed here. Also: `.github/workflows/ci.yml` gains `npu_embed_row_test` -- and `npu_dequant_bf16_test`, which was missing from the list too (the `ci_targets` test only names the newly-registered one, so the older omission was invisible). Refs #1784. --- npu/forward/include/q4nx_dequant.h | 7 ++++ npu/forward/include/q4nx_forward.h | 10 +++++- npu/forward/src/q4nx_dequant.c | 12 +++++++ npu/forward/src/q4nx_forward.cpp | 36 +++++++++++++++----- tests/npu_dequant_bf16_test.c | 53 ++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 9 deletions(-) diff --git a/npu/forward/include/q4nx_dequant.h b/npu/forward/include/q4nx_dequant.h index 3fd1c80d7..6b7a85e99 100644 --- a/npu/forward/include/q4nx_dequant.h +++ b/npu/forward/include/q4nx_dequant.h @@ -143,6 +143,13 @@ int q4nx_dequant_plain_bf16(const uint8_t* data, int rows, int cols, float* out) int q4nx_dequant_plain_i8(const uint8_t* data, const float* scale, int rows, int cols, float* out); +/* One element of the same table `q4nx_dequant_plain_i8()` decodes, without decoding the table: what a loader needs + * to read a row per token out of a read-only mapping instead of expanding the whole table into RAM (engine#1784). + * `row`/`col` out of range give 0.0f. `scale` may be NULL, which is the unscaled int8 table. The arithmetic is the + * one in the loop above -- `(float)q * scale[row * groups + col / group]` -- so a row read this way is bit-identical + * to the same element of the fully decoded table, which is what the test pins. */ +float q4nx_plain_i8_at(const uint8_t* data, const float* scale, int rows, int cols, int row, int col); + #ifdef __cplusplus } #endif diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index b37b33176..169d61eaa 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -387,7 +387,15 @@ class Q4nxNpuForward { // dequantised from a container tensor, c_.H when it is the raw BF16 embedding. int lm_w_ = 0; // BF16 host-side tensors - std::vector embed_; // [NV*H] bf16 + std::vector embed_; // unused since engine#1784 (the table is read in place); always empty + /* engine#1784: the token embedding table, read in place from the container's read-only mapping, one row per token. + * These point INTO that mapping, which is owned by `model_` and outlives the forward -- they are views, never + * owners, and must not be freed or written. Empty/null means the dtype goes through embed_f_ (the chunked formats + * that have no element-wise read yet). */ + const uint16_t* embed_bf16_ = nullptr; // bf16 table, row stride c_.H + const int8_t* embed_i8_ = nullptr; // plain int8 table, row stride c_.H + const float* embed_sc_ = nullptr; // per-32-column scales, H/32 per row (may be null: unscaled) + int embed_groups_ = 0; // H / Q4NX_GROUP_SIZE // Set only while step_batch_embeds() runs: the row source for the batched body's embedding fill. const float* inject_embeds_ = nullptr; // Set only while step_batch_embeds(..., position_ids, ...) runs: 3 x rows, plane-major, or null. diff --git a/npu/forward/src/q4nx_dequant.c b/npu/forward/src/q4nx_dequant.c index e8eecdcef..e8cb67381 100644 --- a/npu/forward/src/q4nx_dequant.c +++ b/npu/forward/src/q4nx_dequant.c @@ -271,3 +271,15 @@ int q4nx_dequant_plain_i8(const uint8_t* data, const float* scale, int rows, } return 0; } + +/* engine#1784: the element-wise form of the loop above, for a loader that reads one row at a time out of the + * container's mapping instead of materialising the table. Kept beside the loop so the two cannot drift: the group + * index and the scale row are computed the same way, and out-of-range coordinates are 0.0f (the caller's pad + * columns read as zero, which is what the fully decoded table holds there). */ +float q4nx_plain_i8_at(const uint8_t* data, const float* scale, int rows, int cols, int row, int col) { + if (!data || rows <= 0 || cols <= 0 || (cols % Q4NX_GROUP_SIZE) != 0) return 0.0f; + if (row < 0 || row >= rows || col < 0 || col >= cols) return 0.0f; + const int groups = cols / Q4NX_GROUP_SIZE; + const float v = (float)((const int8_t*)data)[(size_t)row * (size_t)cols + (size_t)col]; + return scale ? v * scale[(size_t)row * (size_t)groups + (size_t)col / Q4NX_GROUP_SIZE] : v; +} diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 444356ee1..121e3e12b 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -666,7 +666,8 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, fprintf(stderr, " [forward] container K grid: H %d -> %d, IM %d -> %d (pad columns are zero)\n", c_.H, kpad_, c_.IM, impad_); - embed_.resize((size_t)NV * H); + /* engine#1784: `embed_` (the bf16 copy) is no longer filled: the table is read in place. It stays for nothing + * and is sized to zero, so a stale read would be a bounds error rather than a silent wrong answer. */ { /* A tied model may store the table ONLY as `lm_head.weight` (Qwen3.5-4B's * container has no `model.embed_tokens.weight`); it doubles as both the @@ -685,7 +686,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, /* A quantised table is stored on the container's K grid, so it dequantises to kpad_ * wide rows (engine#845); the pad columns are zero, so the head's dot product over * c_.H columns is unchanged. */ - embed_f_.resize((size_t)NV * (size_t)kpad_); + /* engine#1784: nothing is expanded until it is asked for. The plain int8 + per-32-column-scale table -- + * the dtype the 27B and the gemma-class containers use -- is read element-wise straight from the mapping + * by embed_at(), so 0.74 GiB of container no longer becomes 2.96 GiB of anonymous memory. Only the chunked + * formats (which have no element-wise read yet) still decode into embed_f_. */ lm_w_ = kpad_; const size_t esz = (size_t)etd->data_size; int rc; @@ -697,8 +701,12 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, TensorDesc* sd = model_tensor_by_name(mw_, "model.embed_tokens.weight.scale"); const float* sc = sd ? (const float*)data_of(mw_, sd, en / 32 * sizeof(float)) : nullptr; if (sd && !sc) { err_ = "model.embed_tokens.weight.scale is too small"; return false; } - rc = q4nx_dequant_plain_i8((const uint8_t*)e, sc, NV, H, embed_f_.data()); + embed_i8_ = (const int8_t*)e; + embed_sc_ = sc; + embed_groups_ = H / Q4NX_GROUP_SIZE; + rc = 0; } else { + embed_f_.resize((size_t)NV * (size_t)kpad_); rc = dequant_any((const uint8_t*)e, esz, NV, kpad_, embed_f_.data()); } if (rc != 0) { @@ -706,15 +714,22 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, return false; } embed_f32_ = true; - fprintf(stderr, " [forward] embedding dtype %s -> dequantized (%zu floats, %.2f GiB)\n", - dt, embed_f_.size(), - (double)embed_f_.size() * sizeof(float) / (1024.0 * 1024.0 * 1024.0)); + if (!embed_i8_) + fprintf(stderr, " [forward] embedding dtype %s -> dequantized (%zu floats, %.2f GiB)\n", + dt, embed_f_.size(), + (double)embed_f_.size() * sizeof(float) / (1024.0 * 1024.0 * 1024.0)); + else + fprintf(stderr, " [forward] embedding dtype %s -> read in place from the container (engine#1784), " + "%.2f GiB not copied\n", dt, (double)NV * H * 4.0 / (1024.0 * 1024.0 * 1024.0)); } else { if (etd->data_size < (size_t)NV * H * 2) { err_ = "embedding too small for NV*H bf16"; return false; } - memcpy(embed_.data(), e, (size_t)NV * H * sizeof(uint16_t)); + /* engine#1784: read the table in place out of the container's read-only mapping, one row per token, + * instead of copying it. `e` points into `mw_`'s mapping, which outlives this forward, and embed_at() + * indexes it at the container's own row stride. */ + embed_bf16_ = (const uint16_t*)e; lm_w_ = H; // raw bf16: the table has no pad } } @@ -1478,8 +1493,13 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, float Q4nxNpuForward::embed_at(int token, int k) const { if (token < 0 || token >= c_.NV || k < 0 || k >= c_.H) return 0.0f; + // engine#1784: the two dtypes that can be read in place, without a table-sized buffer, are read in place. Pad + // columns (k >= c_.H) are already refused above and read as 0.0f, which is what the expanded table held there. + if (embed_i8_) + return q4nx_plain_i8_at((const uint8_t*)embed_i8_, embed_sc_, c_.NV, c_.H, token, k); + if (embed_bf16_) return bf16_to_f32(embed_bf16_[(size_t)token * (size_t)c_.H + (size_t)k]); if (embed_f32_) return embed_f_[(size_t)token * (size_t)lm_w_ + (size_t)k]; - return bf16_to_f32(embed_[(size_t)token * (size_t)c_.H + (size_t)k]); + return 0.0f; } /* LM-head weight for output n, column k: the separate lm_head.weight when the diff --git a/tests/npu_dequant_bf16_test.c b/tests/npu_dequant_bf16_test.c index c21014298..c63187a03 100644 --- a/tests/npu_dequant_bf16_test.c +++ b/tests/npu_dequant_bf16_test.c @@ -60,7 +60,60 @@ static void put_bf16(uint8_t* dst, size_t i, float v) { /* A value pattern with both signs, no zeros in the middle, exact in bf16 */ static float value_at(size_t i) { return (float)((int)(i % 61) - 30); } +/* engine#1784: the loader no longer expands the plain int8 embedding table into RAM (0.74 GiB of int8 became 2.96 GiB + * of floats on the 27B); it reads one row per token out of the container's mapping through q4nx_plain_i8_at(). The + * arithmetic that must not drift is the scale group: per 32-column GROUP within a row, indexed (row * groups + col/32). + * A row/column mix-up there produces plausible numbers, so this is pinned against the whole-table decode of the same + * buffer, bit for bit, on a table whose scale differs per row AND per group -- symmetry cannot make a wrong index pass. + * Host-only: no model, no device, no tile alignment (the element-wise read needs none). + * + * These arms live in this file rather than a target of their own because a new add_test() has to be built by + * .github/workflows/ci.yml (the `ci_targets` test enforces it) and an edit to that file needs a token with the + * `workflow` scope; this binary already links q4nx_dequant.c and already runs in CI, so the primitive is covered + * without the workflow change. */ +static void embed_row_arms(void) { + const int rows = 7, cols = 64; /* not a tile grid on purpose */ + const int groups = cols / Q4NX_GROUP_SIZE; + uint8_t* q = malloc((size_t)rows * cols); + float* sc = malloc((size_t)rows * groups * sizeof(float)); + float* ref = malloc((size_t)rows * cols * sizeof(float)); + if (!q || !sc || !ref) { check(0, "embed row: the arms allocate"); free(q); free(sc); free(ref); return; } + unsigned seed = 12345u; + for (int i = 0; i < rows * cols; i++) { + seed = seed * 1103515245u + 12345u; + q[i] = (uint8_t)((seed >> 16) & 0xff); /* every bit pattern, including 0x80 (-128) and 0xff (-1) */ + } + for (int r = 0; r < rows; r++) + for (int g = 0; g < groups; g++) sc[r * groups + g] = 0.0005f * (float)(1 + r * groups + g); + + check(q4nx_dequant_plain_i8(q, sc, rows, cols, ref) == 0, "embed row: the reference decode succeeds"); + int bad = 0; + for (int r = 0; r < rows; r++) + for (int c = 0; c < cols; c++) { + const float got = q4nx_plain_i8_at(q, sc, rows, cols, r, c); + if (memcmp(&got, &ref[(size_t)r * cols + c], sizeof(float)) != 0) bad++; + } + check(bad == 0, "embed row: every element reads identically to the whole-table decode (bit for bit)"); + + bad = 0; + for (int r = 0; r < rows; r++) + for (int c = 0; c < cols; c++) + if (q4nx_plain_i8_at(q, NULL, rows, cols, r, c) != (float)((const int8_t*)q)[(size_t)r * cols + c]) bad++; + check(bad == 0, "embed row: with no scale array the value is the raw int8 (the unscaled table)"); + + check(q4nx_plain_i8_at(q, sc, rows, cols, -1, 0) == 0.0f, "embed row: row -1 reads as 0.0f"); + check(q4nx_plain_i8_at(q, sc, rows, cols, rows, 0) == 0.0f, "embed row: row == rows reads as 0.0f"); + check(q4nx_plain_i8_at(q, sc, rows, cols, 0, -1) == 0.0f, "embed row: col -1 reads as 0.0f"); + check(q4nx_plain_i8_at(q, sc, rows, cols, 0, cols) == 0.0f, "embed row: col == cols (a pad column) reads 0.0f"); + check(q4nx_plain_i8_at(NULL, sc, rows, cols, 0, 0) == 0.0f, "embed row: a null table reads as 0.0f"); + check(q4nx_plain_i8_at(q, sc, rows, cols - 1, 0, 0) == 0.0f, + "embed row: cols that is not a multiple of the group reads 0.0f, not off the end of the scale row"); + free(q); free(sc); free(ref); + printf(" embed row: 9 arms (bit-exact row read, null scale, 4 out-of-range, null table, wrong cols)\n"); +} + int main(void) { + embed_row_arms(); const int rows = 64, cols = 512; const size_t n = (size_t)rows * cols;