From 26c4d1d72f83cdca72da8816b4b383b9d05f1f36 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 04:19:22 -0400 Subject: [PATCH 1/2] npu: config.json's vocab_size is held against the embedding table it overrides Fixes #1843. When a container ships the table (or, tied, only lm_head), a table that cannot hold vocab_size rows is refused by name: more rows than the table makes the embedding read run into the next tensor (Tensor::avail is measured to the end of the file, so lane.cpp's bound misses it); fewer leaves rows unreachable. Rows are derived per layout the forward reads: BF16/F16/F32 by bytes (a flat 1-D table included), plain int8 [V, H], and chunked tiles as a 3-D grid or a 2-D chunk list, where vocab padded to a whole 32-row tile is fine. An unrecognised dtype is not evidence and stays silent. The issue's literal `rows != vocab` would have refused two layouts in use (bonsai2-27b's [155200, 5120] chunk list, read as 155,200 rows; zr1's flat 1-D table); measured over the 83 containers on strixhalo, this rule refuses none. Nine arms in npu_model_bounds_test; with the check disabled, 5 fail. Co-Authored-By: Claude Opus 5.5 --- npu/model.cpp | 53 +++++++++++++++++++++++++++++++++ tests/npu_model_bounds_test.cpp | 47 ++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/npu/model.cpp b/npu/model.cpp index d5a312b4a..1f2a0cd1e 100644 --- a/npu/model.cpp +++ b/npu/model.cpp @@ -36,6 +36,46 @@ int Tensor::tiles() const { return int(shape[0]); } +namespace { + +// engine#1843: does `t` hold exactly `vocab` rows of `hidden`, under ANY layout the forward reads an embedding table in +// (q4nx_forward.cpp's init: BF16/F16/F32 rows at stride `hidden`; I8 either plain [vocab, hidden] -- bytes == V*H -- or +// chunked tiles of 32 rows x 256 columns at one of the three chunk sizes, as a 3-D [row_tiles, col_tiles, chunk] grid or +// a 2-D [tiles, chunk] list)? A tiled table may carry vocab padded up to a whole 32-row tile. An unrecognised dtype +// returns true: it is not evidence either way, and the forward has its own refusal for it. `seen` gets what the table +// does hold, for the message. +bool table_holds_vocab(const Tensor& t, int64_t vocab, int64_t hidden, std::string* seen) { + if (vocab <= 0 || hidden <= 0) return true; + const int64_t pad_v = (vocab + 31) / 32 * 32; + const int64_t col_tiles = (hidden + 255) / 256; + const auto rows_ok = [&](int64_t r) { return r == vocab || r == pad_v; }; + int esz = 0; + if (t.dtype == "BF16" || t.dtype == "F16") esz = 2; + else if (t.dtype == "F32") esz = 4; + if (esz) { + const uint64_t row = uint64_t(hidden) * uint64_t(esz); + if (t.bytes % row != 0) { *seen = "is " + std::to_string(t.bytes) + " B, not a whole number of rows"; return false; } + const int64_t r = int64_t(t.bytes / row); + *seen = "has " + std::to_string(r) + " rows of " + std::to_string(hidden); + return r == vocab; // a plain table has no tile padding + } + if (t.dtype != "I8") return true; + if (t.bytes == uint64_t(vocab) * uint64_t(hidden)) return true; // plain int8 + per-32-column scale (Gemma4) + const auto is_chunk = [](int64_t b) { return b == 5120 || b == 4736 || b == 8704; }; // Q4NX tile / Q4_K / Q8 chunk + int64_t r = -1; + if (t.shape.size() == 3 && is_chunk(t.shape[2]) && t.shape[1] == col_tiles) r = t.shape[0] * 32; + else if (t.shape.size() == 2 && is_chunk(t.shape[1]) && t.shape[0] % col_tiles == 0) r = t.shape[0] / col_tiles * 32; + if (r < 0) { + *seen = "is neither " + std::to_string(vocab) + " x " + std::to_string(hidden) + + " plain int8 nor a tile grid " + std::to_string(col_tiles) + " tiles wide"; + return false; + } + *seen = "holds " + std::to_string(r) + " rows (" + std::to_string(r / 32) + " row tiles)"; + return rows_ok(r); +} + +} // namespace + Model::Model(const std::string& dir) { const std::string path = dir + "/model.q4nx"; std::string why; @@ -81,6 +121,19 @@ Model::Model(const std::string& dir) { // table is [7760, 4, 8704] and was reported as vocab 7760, hidden 4. if (c.contains("vocab_size")) { dims_.vocab = c.at("vocab_size"); + // engine#1843: when the container also ships the table, it is evidence, and a table that cannot hold + // vocab_size rows is refused by name. A config stating MORE rows than the table makes the embedding read run + // into whatever tensor follows (Tensor::avail is measured to the end of the FILE, so lane.cpp's bound does not + // catch it); FEWER leaves rows nothing can reach. Neither number can be trusted then. + const Tensor* emb = find("model.embed_tokens.weight"); + if (!emb && c.value("tie_word_embeddings", false)) emb = find("lm_head.weight"); + if (emb) { + std::string seen; + if (!table_holds_vocab(*emb, dims_.vocab, dims_.hidden, &seen)) + throw std::runtime_error(dir + "/config.json states vocab_size " + std::to_string(dims_.vocab) + + " but the embedding table (" + emb->dtype + ") " + seen + + "; refusing rather than reading past it or leaving rows unreachable"); + } } else { const Tensor* emb = find("model.embed_tokens.weight"); if (!emb && c.value("tie_word_embeddings", false)) emb = find("lm_head.weight"); diff --git a/tests/npu_model_bounds_test.cpp b/tests/npu_model_bounds_test.cpp index d76e7805a..3eda08c64 100644 --- a/tests/npu_model_bounds_test.cpp +++ b/tests/npu_model_bounds_test.cpp @@ -64,6 +64,7 @@ struct Entry { std::string name; std::vector shape; uint64_t bytes; + std::string dtype = "U8"; // engine#1843: the vocab check reads the table's dtype; U8 is "not evidence" }; // Writes dir/model.q4nx (tensors packed in order, zero-filled) and dir/config.json. @@ -75,7 +76,7 @@ void write_model(const fs::path& dir, const std::vector& entries, uint64_ json table = json::object(); uint64_t off = 0; for (const auto& e : entries) { - table[e.name] = {{"dtype", "U8"}, {"shape", e.shape}, {"data_offsets", {off, off + e.bytes}}}; + table[e.name] = {{"dtype", e.dtype}, {"shape", e.shape}, {"data_offsets", {off, off + e.bytes}}}; off += e.bytes; } if (!bad_name.empty()) table[bad_name]["data_offsets"] = bad_offsets; @@ -160,6 +161,50 @@ int main() { check(why.empty(), "a tiled embedding without vocab_size derives rows, not tiles" + (why.empty() ? "" : " (" + why + ")")); } + { // engine#1843: vocab_size is held against the table it overrides, under every layout the forward reads. + // hidden is 128 in this fixture, so one 256-column tile spans a row and a chunked table is 1 tile wide. + struct Case { + const char* tag; + std::vector shape; + uint64_t bytes; + const char* dtype; + int vocab; + bool loads; + const char* what; + }; + const std::vector cases = { + {"bf16-eq", {64, 128}, 64 * 128 * 2, "BF16", 64, true, "a BF16 table of exactly vocab rows loads"}, + {"bf16-flat", {64 * 128}, 64 * 128 * 2, "BF16", 64, true, + "a FLAT 1-D BF16 table (the zr1 container's shape) is rows by bytes, and loads"}, + {"bf16-short", {64, 128}, 64 * 128 * 2, "BF16", 100, false, + "a config stating MORE rows than a BF16 table holds is refused (the read would run into the next tensor)"}, + {"bf16-long", {64, 128}, 64 * 128 * 2, "BF16", 32, false, + "a config stating FEWER rows than a BF16 table holds is refused (the tail would be unreachable)"}, + {"i8-plain", {64, 128}, 64 * 128, "I8", 64, true, "plain int8 [vocab, hidden] (the Gemma4 layout) loads"}, + {"i8-grid-pad", {2, 1, 5120}, 2 * 5120, "I8", 50, true, + "a 3-D tile grid holding vocab padded up to whole 32-row tiles (50 -> 64) loads"}, + {"i8-list", {2, 5120}, 2 * 5120, "I8", 64, true, + "a 2-D chunk LIST [tiles, 5120] (the bonsai2-27b layout, which is NOT [rows, hidden]) loads"}, + {"i8-grid-short", {2, 1, 5120}, 2 * 5120, "I8", 100, false, + "a config stating more rows than a tile grid holds (100 > 64) is refused"}, + {"i8-grid-long", {4, 1, 5120}, 4 * 5120, "I8", 40, false, + "a tile grid holding more than vocab's padded rows (128 > 64) is refused"}, + }; + for (const auto& k : cases) { + std::vector e = one_layer(); + for (auto& x : e) + if (x.name == "model.embed_tokens.weight") { x.shape = k.shape; x.bytes = k.bytes; x.dtype = k.dtype; } + const fs::path d = root / (std::string("vocab-") + k.tag); + write_model(d, e, 0, nullptr, "", json{{"vocab_size", k.vocab}}); + const std::string why = thrown([&] { Model m(d.string()); }); + if (k.loads) + check(why.empty(), std::string("vocab vs table: ") + k.what + (why.empty() ? "" : " (" + why + ")")); + else + check(why.find("vocab_size " + std::to_string(k.vocab)) != std::string::npos && + why.find("embedding table") != std::string::npos, + std::string("vocab vs table: ") + k.what + ", by name (" + why + ")"); + } + } { // engine#1782: neither -> refused by name, not by "model has no tensor" std::vector e = one_layer(); e.erase(std::remove_if(e.begin(), e.end(), From 399f3678f2276cd0adaf93a91725d9bd46b03479 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 04:37:03 -0400 Subject: [PATCH 2/2] npu: the vocab check reads the forward's tile macros instead of restating them engine#1878 review: the three chunk sizes and the 32x256 tile were literals in a second file, the one drift that would turn a valid container into a refusal. model.cpp now includes forward/include/q4nx_dequant.h and uses Q4NX_TILE_ROWS, Q4NX_TILE_COLS, Q4NX_TILE_BYTES, Q4NX_CHUNK_Q4K and Q4NX_CHUNK_Q8. Also says why a plain row table gets no tile padding (no grid; all 78 measured BF16 tables exact). npu_model_bounds pass; Model() over the 82 container dirs on strixhalo: 0 refused. Co-Authored-By: Claude Opus 5.5 --- npu/model.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/npu/model.cpp b/npu/model.cpp index 1f2a0cd1e..fb8a5fed6 100644 --- a/npu/model.cpp +++ b/npu/model.cpp @@ -23,6 +23,10 @@ #include #include +// The tile geometry and chunk sizes the forward reads an embedding table with -- included, not restated, so the vocab +// check below cannot drift from the reader it stands in for (engine#1878 review). +#include "forward/include/q4nx_dequant.h" + namespace onebit::npu { int Tensor::tiles() const { @@ -46,8 +50,8 @@ namespace { // does hold, for the message. bool table_holds_vocab(const Tensor& t, int64_t vocab, int64_t hidden, std::string* seen) { if (vocab <= 0 || hidden <= 0) return true; - const int64_t pad_v = (vocab + 31) / 32 * 32; - const int64_t col_tiles = (hidden + 255) / 256; + const int64_t pad_v = (vocab + Q4NX_TILE_ROWS - 1) / Q4NX_TILE_ROWS * Q4NX_TILE_ROWS; + const int64_t col_tiles = (hidden + Q4NX_TILE_COLS - 1) / Q4NX_TILE_COLS; const auto rows_ok = [&](int64_t r) { return r == vocab || r == pad_v; }; int esz = 0; if (t.dtype == "BF16" || t.dtype == "F16") esz = 2; @@ -57,20 +61,23 @@ bool table_holds_vocab(const Tensor& t, int64_t vocab, int64_t hidden, std::stri if (t.bytes % row != 0) { *seen = "is " + std::to_string(t.bytes) + " B, not a whole number of rows"; return false; } const int64_t r = int64_t(t.bytes / row); *seen = "has " + std::to_string(r) + " rows of " + std::to_string(hidden); - return r == vocab; // a plain table has no tile padding + // A plain row table has no tile grid, so nothing pads it: every BF16 table on strixhalo (78 measured, 11-10) + // has exactly vocab_size rows, and the forward reads it at row stride `hidden` with no pad rows. + return r == vocab; } if (t.dtype != "I8") return true; if (t.bytes == uint64_t(vocab) * uint64_t(hidden)) return true; // plain int8 + per-32-column scale (Gemma4) - const auto is_chunk = [](int64_t b) { return b == 5120 || b == 4736 || b == 8704; }; // Q4NX tile / Q4_K / Q8 chunk + const auto is_chunk = [](int64_t b) { return b == Q4NX_TILE_BYTES || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8; }; int64_t r = -1; - if (t.shape.size() == 3 && is_chunk(t.shape[2]) && t.shape[1] == col_tiles) r = t.shape[0] * 32; - else if (t.shape.size() == 2 && is_chunk(t.shape[1]) && t.shape[0] % col_tiles == 0) r = t.shape[0] / col_tiles * 32; + if (t.shape.size() == 3 && is_chunk(t.shape[2]) && t.shape[1] == col_tiles) r = t.shape[0] * Q4NX_TILE_ROWS; + else if (t.shape.size() == 2 && is_chunk(t.shape[1]) && t.shape[0] % col_tiles == 0) + r = t.shape[0] / col_tiles * Q4NX_TILE_ROWS; if (r < 0) { *seen = "is neither " + std::to_string(vocab) + " x " + std::to_string(hidden) + " plain int8 nor a tile grid " + std::to_string(col_tiles) + " tiles wide"; return false; } - *seen = "holds " + std::to_string(r) + " rows (" + std::to_string(r / 32) + " row tiles)"; + *seen = "holds " + std::to_string(r) + " rows (" + std::to_string(r / Q4NX_TILE_ROWS) + " row tiles)"; return rows_ok(r); }