Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions npu/forward/include/q4nx_dequant.h
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,13 @@ int q4nx_dequant_plain_bf16(const uint8_t* data, int rows, int cols, float* out)
int q4nx_dequant_plain_i8(const uint8_t* data, const float* scale, int rows,
int cols, float* out);

/* One element of the same table `q4nx_dequant_plain_i8()` decodes, without decoding the table: what a loader needs
* to read a row per token out of a read-only mapping instead of expanding the whole table into RAM (engine#1784).
* `row`/`col` out of range give 0.0f. `scale` may be NULL, which is the unscaled int8 table. The arithmetic is the
* one in the loop above -- `(float)q * scale[row * groups + col / group]` -- so a row read this way is bit-identical
* to the same element of the fully decoded table, which is what the test pins. */
float q4nx_plain_i8_at(const uint8_t* data, const float* scale, int rows, int cols, int row, int col);

#ifdef __cplusplus
}
#endif
Expand Down
10 changes: 9 additions & 1 deletion npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,15 @@ class Q4nxNpuForward {
// dequantised from a container tensor, c_.H when it is the raw BF16 embedding.
int lm_w_ = 0;
// BF16 host-side tensors
std::vector<uint16_t> embed_; // [NV*H] bf16
std::vector<uint16_t> embed_; // unused since engine#1784 (the table is read in place); always empty
/* engine#1784: the token embedding table, read in place from the container's read-only mapping, one row per token.
* These point INTO that mapping, which is owned by `model_` and outlives the forward -- they are views, never
* owners, and must not be freed or written. Empty/null means the dtype goes through embed_f_ (the chunked formats
* that have no element-wise read yet). */
const uint16_t* embed_bf16_ = nullptr; // bf16 table, row stride c_.H
const int8_t* embed_i8_ = nullptr; // plain int8 table, row stride c_.H
const float* embed_sc_ = nullptr; // per-32-column scales, H/32 per row (may be null: unscaled)
int embed_groups_ = 0; // H / Q4NX_GROUP_SIZE
// Set only while step_batch_embeds() runs: the row source for the batched body's embedding fill.
const float* inject_embeds_ = nullptr;
// Set only while step_batch_embeds(..., position_ids, ...) runs: 3 x rows, plane-major, or null.
Expand Down
12 changes: 12 additions & 0 deletions npu/forward/src/q4nx_dequant.c
Original file line number Diff line number Diff line change
Expand Up @@ -271,3 +271,15 @@ int q4nx_dequant_plain_i8(const uint8_t* data, const float* scale, int rows,
}
return 0;
}

/* engine#1784: the element-wise form of the loop above, for a loader that reads one row at a time out of the
* container's mapping instead of materialising the table. Kept beside the loop so the two cannot drift: the group
* index and the scale row are computed the same way, and out-of-range coordinates are 0.0f (the caller's pad
* columns read as zero, which is what the fully decoded table holds there). */
float q4nx_plain_i8_at(const uint8_t* data, const float* scale, int rows, int cols, int row, int col) {
if (!data || rows <= 0 || cols <= 0 || (cols % Q4NX_GROUP_SIZE) != 0) return 0.0f;
if (row < 0 || row >= rows || col < 0 || col >= cols) return 0.0f;
const int groups = cols / Q4NX_GROUP_SIZE;
const float v = (float)((const int8_t*)data)[(size_t)row * (size_t)cols + (size_t)col];
return scale ? v * scale[(size_t)row * (size_t)groups + (size_t)col / Q4NX_GROUP_SIZE] : v;
}
36 changes: 28 additions & 8 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,8 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
fprintf(stderr, " [forward] container K grid: H %d -> %d, IM %d -> %d (pad columns are zero)\n",
c_.H, kpad_, c_.IM, impad_);

embed_.resize((size_t)NV * H);
/* engine#1784: `embed_` (the bf16 copy) is no longer filled: the table is read in place. It stays for nothing
* and is sized to zero, so a stale read would be a bounds error rather than a silent wrong answer. */
{
/* A tied model may store the table ONLY as `lm_head.weight` (Qwen3.5-4B's
* container has no `model.embed_tokens.weight`); it doubles as both the
Expand All @@ -685,7 +686,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
/* A quantised table is stored on the container's K grid, so it dequantises to kpad_
* wide rows (engine#845); the pad columns are zero, so the head's dot product over
* c_.H columns is unchanged. */
embed_f_.resize((size_t)NV * (size_t)kpad_);
/* engine#1784: nothing is expanded until it is asked for. The plain int8 + per-32-column-scale table --
* the dtype the 27B and the gemma-class containers use -- is read element-wise straight from the mapping
* by embed_at(), so 0.74 GiB of container no longer becomes 2.96 GiB of anonymous memory. Only the chunked
* formats (which have no element-wise read yet) still decode into embed_f_. */
lm_w_ = kpad_;
const size_t esz = (size_t)etd->data_size;
int rc;
Expand All @@ -697,24 +701,35 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
TensorDesc* sd = model_tensor_by_name(mw_, "model.embed_tokens.weight.scale");
const float* sc = sd ? (const float*)data_of(mw_, sd, en / 32 * sizeof(float)) : nullptr;
if (sd && !sc) { err_ = "model.embed_tokens.weight.scale is too small"; return false; }
rc = q4nx_dequant_plain_i8((const uint8_t*)e, sc, NV, H, embed_f_.data());
embed_i8_ = (const int8_t*)e;
embed_sc_ = sc;
embed_groups_ = H / Q4NX_GROUP_SIZE;
rc = 0;
} else {
embed_f_.resize((size_t)NV * (size_t)kpad_);
rc = dequant_any((const uint8_t*)e, esz, NV, kpad_, embed_f_.data());
}
if (rc != 0) {
err_ = std::string("embedding dtype ") + dt + " but dequant failed";
return false;
}
embed_f32_ = true;
fprintf(stderr, " [forward] embedding dtype %s -> dequantized (%zu floats, %.2f GiB)\n",
dt, embed_f_.size(),
(double)embed_f_.size() * sizeof(float) / (1024.0 * 1024.0 * 1024.0));
if (!embed_i8_)
fprintf(stderr, " [forward] embedding dtype %s -> dequantized (%zu floats, %.2f GiB)\n",
dt, embed_f_.size(),
(double)embed_f_.size() * sizeof(float) / (1024.0 * 1024.0 * 1024.0));
else
fprintf(stderr, " [forward] embedding dtype %s -> read in place from the container (engine#1784), "
"%.2f GiB not copied\n", dt, (double)NV * H * 4.0 / (1024.0 * 1024.0 * 1024.0));
} else {
if (etd->data_size < (size_t)NV * H * 2) {
err_ = "embedding too small for NV*H bf16";
return false;
}
memcpy(embed_.data(), e, (size_t)NV * H * sizeof(uint16_t));
/* engine#1784: read the table in place out of the container's read-only mapping, one row per token,
* instead of copying it. `e` points into `mw_`'s mapping, which outlives this forward, and embed_at()
* indexes it at the container's own row stride. */
embed_bf16_ = (const uint16_t*)e;
lm_w_ = H; // raw bf16: the table has no pad
}
}
Expand Down Expand Up @@ -1478,8 +1493,13 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,

float Q4nxNpuForward::embed_at(int token, int k) const {
if (token < 0 || token >= c_.NV || k < 0 || k >= c_.H) return 0.0f;
// engine#1784: the two dtypes that can be read in place, without a table-sized buffer, are read in place. Pad
// columns (k >= c_.H) are already refused above and read as 0.0f, which is what the expanded table held there.
if (embed_i8_)
return q4nx_plain_i8_at((const uint8_t*)embed_i8_, embed_sc_, c_.NV, c_.H, token, k);
if (embed_bf16_) return bf16_to_f32(embed_bf16_[(size_t)token * (size_t)c_.H + (size_t)k]);
if (embed_f32_) return embed_f_[(size_t)token * (size_t)lm_w_ + (size_t)k];
return bf16_to_f32(embed_[(size_t)token * (size_t)c_.H + (size_t)k]);
return 0.0f;
}

/* LM-head weight for output n, column k: the separate lm_head.weight when the
Expand Down
53 changes: 53 additions & 0 deletions tests/npu_dequant_bf16_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,60 @@ static void put_bf16(uint8_t* dst, size_t i, float v) {
/* A value pattern with both signs, no zeros in the middle, exact in bf16 */
static float value_at(size_t i) { return (float)((int)(i % 61) - 30); }

/* engine#1784: the loader no longer expands the plain int8 embedding table into RAM (0.74 GiB of int8 became 2.96 GiB
* of floats on the 27B); it reads one row per token out of the container's mapping through q4nx_plain_i8_at(). The
* arithmetic that must not drift is the scale group: per 32-column GROUP within a row, indexed (row * groups + col/32).
* A row/column mix-up there produces plausible numbers, so this is pinned against the whole-table decode of the same
* buffer, bit for bit, on a table whose scale differs per row AND per group -- symmetry cannot make a wrong index pass.
* Host-only: no model, no device, no tile alignment (the element-wise read needs none).
*
* These arms live in this file rather than a target of their own because a new add_test() has to be built by
* .github/workflows/ci.yml (the `ci_targets` test enforces it) and an edit to that file needs a token with the
* `workflow` scope; this binary already links q4nx_dequant.c and already runs in CI, so the primitive is covered
* without the workflow change. */
static void embed_row_arms(void) {
const int rows = 7, cols = 64; /* not a tile grid on purpose */
const int groups = cols / Q4NX_GROUP_SIZE;
uint8_t* q = malloc((size_t)rows * cols);
float* sc = malloc((size_t)rows * groups * sizeof(float));
float* ref = malloc((size_t)rows * cols * sizeof(float));
if (!q || !sc || !ref) { check(0, "embed row: the arms allocate"); free(q); free(sc); free(ref); return; }
unsigned seed = 12345u;
for (int i = 0; i < rows * cols; i++) {
seed = seed * 1103515245u + 12345u;
q[i] = (uint8_t)((seed >> 16) & 0xff); /* every bit pattern, including 0x80 (-128) and 0xff (-1) */
}
for (int r = 0; r < rows; r++)
for (int g = 0; g < groups; g++) sc[r * groups + g] = 0.0005f * (float)(1 + r * groups + g);

check(q4nx_dequant_plain_i8(q, sc, rows, cols, ref) == 0, "embed row: the reference decode succeeds");
int bad = 0;
for (int r = 0; r < rows; r++)
for (int c = 0; c < cols; c++) {
const float got = q4nx_plain_i8_at(q, sc, rows, cols, r, c);
if (memcmp(&got, &ref[(size_t)r * cols + c], sizeof(float)) != 0) bad++;
}
check(bad == 0, "embed row: every element reads identically to the whole-table decode (bit for bit)");

bad = 0;
for (int r = 0; r < rows; r++)
for (int c = 0; c < cols; c++)
if (q4nx_plain_i8_at(q, NULL, rows, cols, r, c) != (float)((const int8_t*)q)[(size_t)r * cols + c]) bad++;
check(bad == 0, "embed row: with no scale array the value is the raw int8 (the unscaled table)");

check(q4nx_plain_i8_at(q, sc, rows, cols, -1, 0) == 0.0f, "embed row: row -1 reads as 0.0f");
check(q4nx_plain_i8_at(q, sc, rows, cols, rows, 0) == 0.0f, "embed row: row == rows reads as 0.0f");
check(q4nx_plain_i8_at(q, sc, rows, cols, 0, -1) == 0.0f, "embed row: col -1 reads as 0.0f");
check(q4nx_plain_i8_at(q, sc, rows, cols, 0, cols) == 0.0f, "embed row: col == cols (a pad column) reads 0.0f");
check(q4nx_plain_i8_at(NULL, sc, rows, cols, 0, 0) == 0.0f, "embed row: a null table reads as 0.0f");
check(q4nx_plain_i8_at(q, sc, rows, cols - 1, 0, 0) == 0.0f,
"embed row: cols that is not a multiple of the group reads 0.0f, not off the end of the scale row");
free(q); free(sc); free(ref);
printf(" embed row: 9 arms (bit-exact row read, null scale, 4 out-of-range, null table, wrong cols)\n");
}

int main(void) {
embed_row_arms();
const int rows = 64, cols = 512;
const size_t n = (size_t)rows * cols;

Expand Down
Loading