Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions npu/lane.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,23 @@ struct Lane::Impl {
throw std::runtime_error("model.q4nx: model.norm.weight holds " + std::to_string(norm.bytes) +
" bytes, fewer than hidden x 2");
std::memcpy(fnorm->map(), norm.data, size_t(d.hidden) * 2);
if (model.tensor("model.embed_tokens.weight").bytes < size_t(d.vocab) * size_t(d.hidden) * 2)
throw std::runtime_error("model.q4nx: model.embed_tokens.weight holds fewer than vocab x hidden bf16 values");
// engine#1881 bounded this by the tensor's own span (`bytes`), which is right, but kept a **bf16** expected
// size -- and that refuses a quantised table for being smaller than an f32->bf16 table it never claimed to be
// (bonsai2-27b: 794,624,000 B of I8 chunks against 2,542,796,800 B). The expected size comes from the dtype now,
// and the comparison is made only where a row-major reading is meaningful:
// * a float table (bf16/f16/f32) of V x H must be AT LEAST bytes(V*H*width) -- the case #1881 was written for,
// a config overstating vocab, still refused;
// * an I8 table is either plain row-major (exactly V*H bytes, the gemma4 layout) or a CHUNK GRID, whose shape
// product is already a byte count (155,200 "rows" are 7,760 row tiles x 20 column tiles). Comparing those
// against V*H is the mistake this guard made, so an I8 table keeps the file-end bound (`avail`), which is a
// looser but never-wrong answer.
const Tensor& emb = model.tensor("model.embed_tokens.weight");
const uint64_t emb_need = emb.dtype == "I8" ? 0 : emb.expected_bytes(uint64_t(d.vocab) * uint64_t(d.hidden));
const uint64_t emb_have = emb_need ? emb.bytes : emb.avail;
if (emb_need ? (emb.bytes < emb_need) : (emb_have < size_t(d.vocab) * size_t(d.hidden) * 2))
throw std::runtime_error("model.q4nx: model.embed_tokens.weight holds " + std::to_string(emb_have) +
" bytes, fewer than the " + std::to_string(d.vocab) + " x " + std::to_string(d.hidden) +
" elements config.json declares" + (emb_need ? "" : " (I8 chunked table: file-end bound)"));
lmhead_w = make_bo(dev, lmhead_weight_bytes(model));
pack_lmhead_weights(model, static_cast<uint8_t*>(lmhead_w->map()));

Expand Down
14 changes: 14 additions & 0 deletions npu/model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,20 @@

namespace onebit::npu {

uint64_t Tensor::expected_bytes(uint64_t elements) const {
// Element widths of the dtypes this reader decodes. Callers pass a row-major element count, so a tiled/chunked entry
// (whose shape product is already a byte count: its last dimension is the chunk's BYTES) must not be sent here --
// npu/lane.cpp checks `shape.size() == 2` for that reason. 0 means "not understood", and the caller then keeps the
// file-end bound rather than refusing on a guess.
uint64_t per = 0;
if (dtype == "BF16" || dtype == "F16") per = 2;
else if (dtype == "F32") per = 4;
else if (dtype == "I32") per = 4;
else if (dtype == "I8" || dtype == "U8") per = 1;
if (per == 0 || elements == 0) return 0;
return per * elements;
}

int Tensor::tiles() const {
// [tile rows, tile cols, chunk bytes] (Q4NX with Q4_K or Q8 chunks, npu/q4nx.h)
if (shape.size() == 3 && shape[0] > 0 && shape[1] > 0) return int(shape[0] * shape[1]);
Expand Down
6 changes: 6 additions & 0 deletions npu/model.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ struct Tensor {

// Tiles in a projection: shape[0] rows of shape[1] bytes, in 5120-byte tiles.
int tiles() const;

// Bytes this tensor SHOULD occupy for `elements` logical elements, from its own dtype, or 0 for a dtype this reader
// does not decode. engine#1881 bounded the embedding by the tensor's own span (`bytes`) but kept a **bf16** figure as
// the expected size, which refuses a quantised table: bonsai2-27b's I8 tiled table is 794,624,000 B against that
// expression's 2,542,796,800 B. `bytes` is the right field; the expected number has to come from the dtype.
uint64_t expected_bytes(uint64_t elements) const;
};

struct Dims {
Expand Down
23 changes: 23 additions & 0 deletions tests/npu_model_bounds_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,30 @@ std::vector<Entry> one_layer(std::vector<int64_t> q_shape = {1, 5120}, int64_t g

} // namespace

// engine#1890: the widths `Tensor::expected_bytes()` uses, pinned here because the guard in npu/lane.cpp is only as right as
// this table. A quantised table is the case that broke: #1881 compared the embedding's `bytes` against a bf16 figure, so an I8
// table (0.625 B/element) was refused for being legitimately small (bonsai2-27b: 794,624,000 B against 2,542,796,800 B). The
// table below is what makes the comparison legal, and the 0 return is what keeps an unknown dtype on the file-end bound instead
// of refusing on a guess.
void expected_bytes_widths() {
auto tb = [](const char* dtype) {
Tensor t;
t.dtype = dtype;
return t;
};
check(tb("BF16").expected_bytes(1000) == 2000, "expected_bytes: BF16 is 2 bytes per element");
check(tb("F16").expected_bytes(1000) == 2000, "expected_bytes: F16 is 2 bytes per element");
check(tb("F32").expected_bytes(1000) == 4000, "expected_bytes: F32 is 4 bytes per element");
check(tb("I32").expected_bytes(1000) == 4000, "expected_bytes: I32 is 4 bytes per element");
check(tb("I8").expected_bytes(1000) == 1000, "expected_bytes: I8 is 1 byte per element");
check(tb("U8").expected_bytes(1000) == 1000, "expected_bytes: U8 is 1 byte per element");
check(tb("Q4").expected_bytes(1000) == 0, "expected_bytes: an unknown dtype answers 0, so the caller keeps `avail`");
check(tb("BF16").expected_bytes(0) == 0, "expected_bytes: zero elements is 0, not a bogus width");
std::printf(" expected_bytes: 8 arms (six widths, unknown dtype, zero elements)\n");
}

int main() {
expected_bytes_widths();
const fs::path root = fs::temp_directory_path() / ("npu_model_bounds_test." + std::to_string(::getpid()));
fs::remove_all(root);

Expand Down
Loading