Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions npu/lane.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,14 @@ struct Lane::Impl {
logits = make_bo(dev, kSmallBo);
fnorm = make_bo(dev, kSmallBo);
const Tensor& norm = model.tensor("model.norm.weight");
if (norm.avail < size_t(d.hidden) * 2) throw std::runtime_error("model.q4nx: model.norm.weight runs past the end of the file");
// engine#1881: bounded by the tensor (`bytes`), not by the end of the file (`avail`), which let a short
// tensor be read into its neighbour.
if (norm.bytes < size_t(d.hidden) * 2)
throw std::runtime_error("model.q4nx: model.norm.weight holds " + std::to_string(norm.bytes) +
" bytes, fewer than hidden x 2");
std::memcpy(fnorm->map(), norm.data, size_t(d.hidden) * 2);
if (model.tensor("model.embed_tokens.weight").avail < size_t(d.vocab) * size_t(d.hidden) * 2)
throw std::runtime_error("model.q4nx: model.embed_tokens.weight runs past the end of the file");
if (model.tensor("model.embed_tokens.weight").bytes < size_t(d.vocab) * size_t(d.hidden) * 2)
throw std::runtime_error("model.q4nx: model.embed_tokens.weight holds fewer than vocab x hidden bf16 values");
lmhead_w = make_bo(dev, lmhead_weight_bytes(model));
pack_lmhead_weights(model, static_cast<uint8_t*>(lmhead_w->map()));

Expand Down
11 changes: 8 additions & 3 deletions npu/pack.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,15 @@ size_t tiles_read(int n_tiles, int G) {
return most;
}

// A tensor's shape decides how much the packer reads; refuse one whose reads would run past the
// end of model.q4nx.
// A tensor's shape decides how much the packer reads; refuse one whose reads would run past the TENSOR. engine#1881:
// this used to test `t.avail`, which is measured to the end of the FILE, so a read that overran its own tensor into
// the next one passed -- every tensor but the last has neighbours. `t.bytes` is the tensor; `avail` stays the outer
// bound of the mapping (and is >= bytes by construction).
void need(const Tensor& t, size_t bytes, const char* what) {
if (bytes > t.avail) throw std::runtime_error(std::string("model.q4nx: ") + what + " runs past the end of the file");
if (bytes > t.bytes)
throw std::runtime_error(std::string("model.q4nx: ") + what + " needs " + std::to_string(bytes) +
" bytes but the tensor holds " + std::to_string(t.bytes) +
(bytes > t.avail ? " (and runs past the end of the file)" : " (the read would run into the next tensor)"));
}

const char* const kProj[] = {"self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight",
Expand Down
27 changes: 27 additions & 0 deletions tests/npu_model_bounds_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,33 @@ int main() {
});
check(why.find("runs past the end") != std::string::npos, "a projection larger than the file is refused");
}
{ // engine#1881: k_proj claims four tiles and holds one, with more tensors AFTER it in the file. Bounded by the end
// of the file (`avail`), the packer read three tiles of v/o/down as k_proj's; bounded by the tensor, it refuses.
std::vector<Entry> e = one_layer();
for (auto& x : e)
if (x.name == "model.layers.0.self_attn.k_proj.weight") x.shape = {4, 5120}; // bytes stay 5120
write_model(root / "k-overread", e);
const std::string why = thrown([&] {
Model m((root / "k-overread").string());
std::vector<uint8_t> w(layer_weight_bytes(m));
pack_layer_weights(m, 0, w.data());
});
check(why.find("run into the next tensor") != std::string::npos,
"a projection that over-reads into the NEXT tensor (not past the file) is refused (" + why + ")");
}
{ // engine#1881: input_layernorm holds 128 of the 256 bytes hidden x 2 needs; the next tensor follows it
std::vector<Entry> e = one_layer();
for (auto& x : e)
if (x.name == "model.layers.0.input_layernorm.weight") { x.shape = {64}; x.bytes = 128; }
write_model(root / "norm-short", e);
const std::string why = thrown([&] {
Model m((root / "norm-short").string());
std::vector<uint8_t> i5(4 * 128);
fill_i5(m, 0, i5.data());
});
check(why.find("input_layernorm") != std::string::npos && why.find("next tensor") != std::string::npos,
"a layer norm shorter than hidden x 2 is refused, not completed from its neighbour (" + why + ")");
}
{ // gate/up of one tile each: the interleave would write a whole chunk past the layer's buffer
write_model(root / "chunks", one_layer({1, 5120}, 1));
const std::string why = thrown([&] {
Expand Down
Loading