diff --git a/npu/model.cpp b/npu/model.cpp index d5a312b4a..fb8a5fed6 100644 --- a/npu/model.cpp +++ b/npu/model.cpp @@ -23,6 +23,10 @@ #include #include +// The tile geometry and chunk sizes the forward reads an embedding table with -- included, not restated, so the vocab +// check below cannot drift from the reader it stands in for (engine#1878 review). +#include "forward/include/q4nx_dequant.h" + namespace onebit::npu { int Tensor::tiles() const { @@ -36,6 +40,49 @@ int Tensor::tiles() const { return int(shape[0]); } +namespace { + +// engine#1843: does `t` hold exactly `vocab` rows of `hidden`, under ANY layout the forward reads an embedding table in +// (q4nx_forward.cpp's init: BF16/F16/F32 rows at stride `hidden`; I8 either plain [vocab, hidden] -- bytes == V*H -- or +// chunked tiles of 32 rows x 256 columns at one of the three chunk sizes, as a 3-D [row_tiles, col_tiles, chunk] grid or +// a 2-D [tiles, chunk] list)? A tiled table may carry vocab padded up to a whole 32-row tile. An unrecognised dtype +// returns true: it is not evidence either way, and the forward has its own refusal for it. `seen` gets what the table +// does hold, for the message. +bool table_holds_vocab(const Tensor& t, int64_t vocab, int64_t hidden, std::string* seen) { + if (vocab <= 0 || hidden <= 0) return true; + const int64_t pad_v = (vocab + Q4NX_TILE_ROWS - 1) / Q4NX_TILE_ROWS * Q4NX_TILE_ROWS; + const int64_t col_tiles = (hidden + Q4NX_TILE_COLS - 1) / Q4NX_TILE_COLS; + const auto rows_ok = [&](int64_t r) { return r == vocab || r == pad_v; }; + int esz = 0; + if (t.dtype == "BF16" || t.dtype == "F16") esz = 2; + else if (t.dtype == "F32") esz = 4; + if (esz) { + const uint64_t row = uint64_t(hidden) * uint64_t(esz); + if (t.bytes % row != 0) { *seen = "is " + std::to_string(t.bytes) + " B, not a whole number of rows"; return false; } + const int64_t r = int64_t(t.bytes / row); + *seen = "has " + std::to_string(r) + " rows of " + std::to_string(hidden); + // A plain row table has no tile grid, so nothing pads it: every BF16 table on strixhalo (78 measured, 11-10) + // has exactly vocab_size rows, and the forward reads it at row stride `hidden` with no pad rows. + return r == vocab; + } + if (t.dtype != "I8") return true; + if (t.bytes == uint64_t(vocab) * uint64_t(hidden)) return true; // plain int8 + per-32-column scale (Gemma4) + const auto is_chunk = [](int64_t b) { return b == Q4NX_TILE_BYTES || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8; }; + int64_t r = -1; + if (t.shape.size() == 3 && is_chunk(t.shape[2]) && t.shape[1] == col_tiles) r = t.shape[0] * Q4NX_TILE_ROWS; + else if (t.shape.size() == 2 && is_chunk(t.shape[1]) && t.shape[0] % col_tiles == 0) + r = t.shape[0] / col_tiles * Q4NX_TILE_ROWS; + if (r < 0) { + *seen = "is neither " + std::to_string(vocab) + " x " + std::to_string(hidden) + + " plain int8 nor a tile grid " + std::to_string(col_tiles) + " tiles wide"; + return false; + } + *seen = "holds " + std::to_string(r) + " rows (" + std::to_string(r / Q4NX_TILE_ROWS) + " row tiles)"; + return rows_ok(r); +} + +} // namespace + Model::Model(const std::string& dir) { const std::string path = dir + "/model.q4nx"; std::string why; @@ -81,6 +128,19 @@ Model::Model(const std::string& dir) { // table is [7760, 4, 8704] and was reported as vocab 7760, hidden 4. if (c.contains("vocab_size")) { dims_.vocab = c.at("vocab_size"); + // engine#1843: when the container also ships the table, it is evidence, and a table that cannot hold + // vocab_size rows is refused by name. A config stating MORE rows than the table makes the embedding read run + // into whatever tensor follows (Tensor::avail is measured to the end of the FILE, so lane.cpp's bound does not + // catch it); FEWER leaves rows nothing can reach. Neither number can be trusted then. + const Tensor* emb = find("model.embed_tokens.weight"); + if (!emb && c.value("tie_word_embeddings", false)) emb = find("lm_head.weight"); + if (emb) { + std::string seen; + if (!table_holds_vocab(*emb, dims_.vocab, dims_.hidden, &seen)) + throw std::runtime_error(dir + "/config.json states vocab_size " + std::to_string(dims_.vocab) + + " but the embedding table (" + emb->dtype + ") " + seen + + "; refusing rather than reading past it or leaving rows unreachable"); + } } else { const Tensor* emb = find("model.embed_tokens.weight"); if (!emb && c.value("tie_word_embeddings", false)) emb = find("lm_head.weight"); diff --git a/tests/npu_model_bounds_test.cpp b/tests/npu_model_bounds_test.cpp index 89834e1a9..cd4ff1921 100644 --- a/tests/npu_model_bounds_test.cpp +++ b/tests/npu_model_bounds_test.cpp @@ -64,6 +64,7 @@ struct Entry { std::string name; std::vector shape; uint64_t bytes; + std::string dtype = "U8"; // engine#1843: the vocab check reads the table's dtype; U8 is "not evidence" }; // Writes dir/model.q4nx (tensors packed in order, zero-filled) and dir/config.json. @@ -75,7 +76,7 @@ void write_model(const fs::path& dir, const std::vector& entries, uint64_ json table = json::object(); uint64_t off = 0; for (const auto& e : entries) { - table[e.name] = {{"dtype", "U8"}, {"shape", e.shape}, {"data_offsets", {off, off + e.bytes}}}; + table[e.name] = {{"dtype", e.dtype}, {"shape", e.shape}, {"data_offsets", {off, off + e.bytes}}}; off += e.bytes; } if (!bad_name.empty()) table[bad_name]["data_offsets"] = bad_offsets; @@ -160,6 +161,50 @@ int main() { check(why.empty(), "a tiled embedding without vocab_size derives rows, not tiles" + (why.empty() ? "" : " (" + why + ")")); } + { // engine#1843: vocab_size is held against the table it overrides, under every layout the forward reads. + // hidden is 128 in this fixture, so one 256-column tile spans a row and a chunked table is 1 tile wide. + struct Case { + const char* tag; + std::vector shape; + uint64_t bytes; + const char* dtype; + int vocab; + bool loads; + const char* what; + }; + const std::vector cases = { + {"bf16-eq", {64, 128}, 64 * 128 * 2, "BF16", 64, true, "a BF16 table of exactly vocab rows loads"}, + {"bf16-flat", {64 * 128}, 64 * 128 * 2, "BF16", 64, true, + "a FLAT 1-D BF16 table (the zr1 container's shape) is rows by bytes, and loads"}, + {"bf16-short", {64, 128}, 64 * 128 * 2, "BF16", 100, false, + "a config stating MORE rows than a BF16 table holds is refused (the read would run into the next tensor)"}, + {"bf16-long", {64, 128}, 64 * 128 * 2, "BF16", 32, false, + "a config stating FEWER rows than a BF16 table holds is refused (the tail would be unreachable)"}, + {"i8-plain", {64, 128}, 64 * 128, "I8", 64, true, "plain int8 [vocab, hidden] (the Gemma4 layout) loads"}, + {"i8-grid-pad", {2, 1, 5120}, 2 * 5120, "I8", 50, true, + "a 3-D tile grid holding vocab padded up to whole 32-row tiles (50 -> 64) loads"}, + {"i8-list", {2, 5120}, 2 * 5120, "I8", 64, true, + "a 2-D chunk LIST [tiles, 5120] (the bonsai2-27b layout, which is NOT [rows, hidden]) loads"}, + {"i8-grid-short", {2, 1, 5120}, 2 * 5120, "I8", 100, false, + "a config stating more rows than a tile grid holds (100 > 64) is refused"}, + {"i8-grid-long", {4, 1, 5120}, 4 * 5120, "I8", 40, false, + "a tile grid holding more than vocab's padded rows (128 > 64) is refused"}, + }; + for (const auto& k : cases) { + std::vector e = one_layer(); + for (auto& x : e) + if (x.name == "model.embed_tokens.weight") { x.shape = k.shape; x.bytes = k.bytes; x.dtype = k.dtype; } + const fs::path d = root / (std::string("vocab-") + k.tag); + write_model(d, e, 0, nullptr, "", json{{"vocab_size", k.vocab}}); + const std::string why = thrown([&] { Model m(d.string()); }); + if (k.loads) + check(why.empty(), std::string("vocab vs table: ") + k.what + (why.empty() ? "" : " (" + why + ")")); + else + check(why.find("vocab_size " + std::to_string(k.vocab)) != std::string::npos && + why.find("embedding table") != std::string::npos, + std::string("vocab vs table: ") + k.what + ", by name (" + why + ")"); + } + } { // engine#1782: neither -> refused by name, not by "model has no tensor" std::vector e = one_layer(); e.erase(std::remove_if(e.begin(), e.end(),