Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions npu/model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@
#include <fstream>
#include <stdexcept>

// The tile geometry and chunk sizes the forward reads an embedding table with -- included, not restated, so the vocab
// check below cannot drift from the reader it stands in for (engine#1878 review).
#include "forward/include/q4nx_dequant.h"

namespace onebit::npu {

int Tensor::tiles() const {
Expand All @@ -36,6 +40,49 @@ int Tensor::tiles() const {
return int(shape[0]);
}

namespace {

// engine#1843: does `t` hold exactly `vocab` rows of `hidden`, under ANY layout the forward reads an embedding table in
// (q4nx_forward.cpp's init: BF16/F16/F32 rows at stride `hidden`; I8 either plain [vocab, hidden] -- bytes == V*H -- or
// chunked tiles of 32 rows x 256 columns at one of the three chunk sizes, as a 3-D [row_tiles, col_tiles, chunk] grid or
// a 2-D [tiles, chunk] list)? A tiled table may carry vocab padded up to a whole 32-row tile. An unrecognised dtype
// returns true: it is not evidence either way, and the forward has its own refusal for it. `seen` gets what the table
// does hold, for the message.
bool table_holds_vocab(const Tensor& t, int64_t vocab, int64_t hidden, std::string* seen) {
if (vocab <= 0 || hidden <= 0) return true;
const int64_t pad_v = (vocab + Q4NX_TILE_ROWS - 1) / Q4NX_TILE_ROWS * Q4NX_TILE_ROWS;
const int64_t col_tiles = (hidden + Q4NX_TILE_COLS - 1) / Q4NX_TILE_COLS;
const auto rows_ok = [&](int64_t r) { return r == vocab || r == pad_v; };
int esz = 0;
if (t.dtype == "BF16" || t.dtype == "F16") esz = 2;
else if (t.dtype == "F32") esz = 4;
if (esz) {
const uint64_t row = uint64_t(hidden) * uint64_t(esz);
if (t.bytes % row != 0) { *seen = "is " + std::to_string(t.bytes) + " B, not a whole number of rows"; return false; }
const int64_t r = int64_t(t.bytes / row);
*seen = "has " + std::to_string(r) + " rows of " + std::to_string(hidden);
// A plain row table has no tile grid, so nothing pads it: every BF16 table on strixhalo (78 measured, 11-10)
// has exactly vocab_size rows, and the forward reads it at row stride `hidden` with no pad rows.
return r == vocab;
}
if (t.dtype != "I8") return true;
if (t.bytes == uint64_t(vocab) * uint64_t(hidden)) return true; // plain int8 + per-32-column scale (Gemma4)
const auto is_chunk = [](int64_t b) { return b == Q4NX_TILE_BYTES || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8; };
int64_t r = -1;
if (t.shape.size() == 3 && is_chunk(t.shape[2]) && t.shape[1] == col_tiles) r = t.shape[0] * Q4NX_TILE_ROWS;
else if (t.shape.size() == 2 && is_chunk(t.shape[1]) && t.shape[0] % col_tiles == 0)
r = t.shape[0] / col_tiles * Q4NX_TILE_ROWS;
if (r < 0) {
*seen = "is neither " + std::to_string(vocab) + " x " + std::to_string(hidden) +
" plain int8 nor a tile grid " + std::to_string(col_tiles) + " tiles wide";
return false;
}
*seen = "holds " + std::to_string(r) + " rows (" + std::to_string(r / Q4NX_TILE_ROWS) + " row tiles)";
return rows_ok(r);
}

} // namespace

Model::Model(const std::string& dir) {
const std::string path = dir + "/model.q4nx";
std::string why;
Expand Down Expand Up @@ -81,6 +128,19 @@ Model::Model(const std::string& dir) {
// table is [7760, 4, 8704] and was reported as vocab 7760, hidden 4.
if (c.contains("vocab_size")) {
dims_.vocab = c.at("vocab_size");
// engine#1843: when the container also ships the table, it is evidence, and a table that cannot hold
// vocab_size rows is refused by name. A config stating MORE rows than the table makes the embedding read run
// into whatever tensor follows (Tensor::avail is measured to the end of the FILE, so lane.cpp's bound does not
// catch it); FEWER leaves rows nothing can reach. Neither number can be trusted then.
const Tensor* emb = find("model.embed_tokens.weight");
if (!emb && c.value("tie_word_embeddings", false)) emb = find("lm_head.weight");
if (emb) {
std::string seen;
if (!table_holds_vocab(*emb, dims_.vocab, dims_.hidden, &seen))
throw std::runtime_error(dir + "/config.json states vocab_size " + std::to_string(dims_.vocab) +
" but the embedding table (" + emb->dtype + ") " + seen +
"; refusing rather than reading past it or leaving rows unreachable");
}
} else {
const Tensor* emb = find("model.embed_tokens.weight");
if (!emb && c.value("tie_word_embeddings", false)) emb = find("lm_head.weight");
Expand Down
47 changes: 46 additions & 1 deletion tests/npu_model_bounds_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ struct Entry {
std::string name;
std::vector<int64_t> shape;
uint64_t bytes;
std::string dtype = "U8"; // engine#1843: the vocab check reads the table's dtype; U8 is "not evidence"
};

// Writes dir/model.q4nx (tensors packed in order, zero-filled) and dir/config.json.
Expand All @@ -75,7 +76,7 @@ void write_model(const fs::path& dir, const std::vector<Entry>& entries, uint64_
json table = json::object();
uint64_t off = 0;
for (const auto& e : entries) {
table[e.name] = {{"dtype", "U8"}, {"shape", e.shape}, {"data_offsets", {off, off + e.bytes}}};
table[e.name] = {{"dtype", e.dtype}, {"shape", e.shape}, {"data_offsets", {off, off + e.bytes}}};
off += e.bytes;
}
if (!bad_name.empty()) table[bad_name]["data_offsets"] = bad_offsets;
Expand Down Expand Up @@ -160,6 +161,50 @@ int main() {
check(why.empty(), "a tiled embedding without vocab_size derives rows, not tiles" +
(why.empty() ? "" : " (" + why + ")"));
}
{ // engine#1843: vocab_size is held against the table it overrides, under every layout the forward reads.
// hidden is 128 in this fixture, so one 256-column tile spans a row and a chunked table is 1 tile wide.
struct Case {
const char* tag;
std::vector<int64_t> shape;
uint64_t bytes;
const char* dtype;
int vocab;
bool loads;
const char* what;
};
const std::vector<Case> cases = {
{"bf16-eq", {64, 128}, 64 * 128 * 2, "BF16", 64, true, "a BF16 table of exactly vocab rows loads"},
{"bf16-flat", {64 * 128}, 64 * 128 * 2, "BF16", 64, true,
"a FLAT 1-D BF16 table (the zr1 container's shape) is rows by bytes, and loads"},
{"bf16-short", {64, 128}, 64 * 128 * 2, "BF16", 100, false,
"a config stating MORE rows than a BF16 table holds is refused (the read would run into the next tensor)"},
{"bf16-long", {64, 128}, 64 * 128 * 2, "BF16", 32, false,
"a config stating FEWER rows than a BF16 table holds is refused (the tail would be unreachable)"},
{"i8-plain", {64, 128}, 64 * 128, "I8", 64, true, "plain int8 [vocab, hidden] (the Gemma4 layout) loads"},
{"i8-grid-pad", {2, 1, 5120}, 2 * 5120, "I8", 50, true,
"a 3-D tile grid holding vocab padded up to whole 32-row tiles (50 -> 64) loads"},
{"i8-list", {2, 5120}, 2 * 5120, "I8", 64, true,
"a 2-D chunk LIST [tiles, 5120] (the bonsai2-27b layout, which is NOT [rows, hidden]) loads"},
{"i8-grid-short", {2, 1, 5120}, 2 * 5120, "I8", 100, false,
"a config stating more rows than a tile grid holds (100 > 64) is refused"},
{"i8-grid-long", {4, 1, 5120}, 4 * 5120, "I8", 40, false,
"a tile grid holding more than vocab's padded rows (128 > 64) is refused"},
};
for (const auto& k : cases) {
std::vector<Entry> e = one_layer();
for (auto& x : e)
if (x.name == "model.embed_tokens.weight") { x.shape = k.shape; x.bytes = k.bytes; x.dtype = k.dtype; }
const fs::path d = root / (std::string("vocab-") + k.tag);
write_model(d, e, 0, nullptr, "", json{{"vocab_size", k.vocab}});
const std::string why = thrown([&] { Model m(d.string()); });
if (k.loads)
check(why.empty(), std::string("vocab vs table: ") + k.what + (why.empty() ? "" : " (" + why + ")"));
else
check(why.find("vocab_size " + std::to_string(k.vocab)) != std::string::npos &&
why.find("embedding table") != std::string::npos,
std::string("vocab vs table: ") + k.what + ", by name (" + why + ")");
}
}
{ // engine#1782: neither -> refused by name, not by "model has no tensor"
std::vector<Entry> e = one_layer();
e.erase(std::remove_if(e.begin(), e.end(),
Expand Down
Loading