diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 444356ee1..8354f76ae 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2683,7 +2683,38 @@ bool Q4nxNpuForward::load_ple_block(int row) { ple_block_.assign((size_t)Q4NX_TILE_ROWS * (size_t)cols, 0.0f); const size_t plain_bytes = (size_t)sem_.vocab_size_per_layer_input * (size_t)cols; - if ((size_t)td->data_size == plain_bytes) { + // engine#1725: a RAW BF16 band. The multi-component packs state this table as f32 and the importer converts it to BF16 + // when it writes the container (a converting copy of the whole V x NL*P table would be 9.4 GiB), so the band here is + // 32 rows of BF16 -- the shape the plain-int8 branch handles, with a 2-byte element instead of a code. + // + // The branch is chosen by the container's DECLARED dtype, not by arithmetic on data_size (review of #1825): a size test + // needs `% bf16_bytes == 0 && != plain_bytes` to separate two layouts, and every multiple of the band satisfies it, so a + // plain table whose byte count happens to divide would be read as BF16 bands -- plausible and wrong. This tree already + // decides exactly this way elsewhere (`model.c`: `strcmp(dtype,"I8")`), and `q4nx_dequant.h`'s rule is that an + // ambiguous layout is refused rather than resolved by which test happens to pass. + const size_t bf16_bytes = (size_t)Q4NX_TILE_ROWS * (size_t)cols * 2; + const char* pl_dtype = td->dtype; + const bool declared_bf16 = pl_dtype[0] != '\0' && std::strcmp(pl_dtype, "BF16") == 0; + const bool declared_i8 = pl_dtype[0] != '\0' && (std::strcmp(pl_dtype, "I8") == 0 || std::strcmp(pl_dtype, "Q4") == 0); + if (!declared_bf16 && !declared_i8) { + // Some other declared format, or none at all: refuse rather than fall through to a layout guess. A container + // written by this importer always states one (`w.add()` writes dtype for every tensor), so an empty dtype means a + // foreign or truncated container, not a layout this loader can infer. + err_ = pl_dtype[0] != '\0' + ? std::string("per_layer_token_embd has dtype ") + pl_dtype + + ", which this loader does not read (BF16 or plain I8 only)" + : std::string("per_layer_token_embd declares no dtype, so its layout cannot be decided"); + return false; + } + if (declared_bf16) { + if ((size_t)td->data_size < ((size_t)block + 1) * bf16_bytes || + (size_t)td->data_size % bf16_bytes != 0) { + err_ = "per_layer_token_embd is declared BF16 but its size is not whole 32-row bands"; + return false; + } + const uint16_t* band = reinterpret_cast(raw + (size_t)block * bf16_bytes); + for (size_t i = 0; i < (size_t)Q4NX_TILE_ROWS * (size_t)cols; ++i) ple_block_[i] = bf16_to_f32(band[i]); + } else if ((size_t)td->data_size == plain_bytes) { // PLAIN int8 + per-32-column scale (Gemma4): only the requested band of // 32 rows is needed, so it is dequantized in place rather than the whole // [vocab, NL*ple_dim] table. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 19c171787..3113f320b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -485,6 +485,27 @@ std::vector fconst19_header(int heads) { // True when `fc` is a 1.9 Token .fconst of this geometry: exactly `layers` headers, evenly spaced. // Fills the first header's offset and the block stride. False means the 1.8 form (or a shape this // importer does not know, which the caller decides about). +// engine#1503: the run scan on its own. `fconst19_detect` additionally requires ONE stride, which the mobius line +// (gemma-4-e2b: 35 blocks of 21.87-288.06 MiB) does not have -- so a caller that needs to tell "this is not a 1.9 fconst" +// from "this is a 1.9 fconst whose blocks are not uniform" needs the starts before the stride test. +bool fconst19_block_starts(const Fconst& fc, int heads, int layers, std::vector* starts) { + if (heads <= 0 || layers < 2) return false; + const std::vector pat = fconst19_header(heads); + std::vector runs; + for (size_t i = 0; i + pat.size() <= fc.size;) { + if (fc.map[i] == pat[0] && fc.map[i + 2] == 0x7f && fc.map[i + 3] == 0xff && + std::memcmp(fc.map + i, pat.data(), pat.size()) == 0) { + runs.push_back(int64_t(i)); + i += pat.size(); + } else { + ++i; + } + } + if (int(runs.size()) != layers) return false; + if (starts) *starts = std::move(runs); + return true; +} + bool fconst19_detect(const Fconst& fc, int heads, int layers, int64_t* hdr_off, int64_t* stride) { if (heads <= 0 || layers < 2) return false; const std::vector pat = fconst19_header(heads); @@ -519,6 +540,29 @@ std::string canonical_token_fconst(const std::string& path, int hidden, int head Fconst fc = open_fconst(path); int64_t hdr_off = 0, stride9 = 0; if (!fconst19_detect(fc, heads, layers, &hdr_off, &stride9)) { + // Distinguish "not a 1.9 fconst" from "a 1.9 fconst whose blocks are NOT uniform": the mobius line (gemma-4-e2b + // measured: 35 blocks, 8 anchors each, sizes 21.87-288.06 MiB) is the second, and the rule table cannot describe + // it -- one stride per file is the table's model (engine#1503). Refusing with the block table is the difference + // between a reader that looks unmeasured and a model that is wrong for the family. + std::vector starts; + if (fconst19_block_starts(fc, heads, layers, &starts)) { + std::string sizes; + int64_t lo = INT64_MAX, hi = 0; + for (size_t k = 0; k < starts.size(); ++k) { + const int64_t end = (k + 1 < starts.size()) ? starts[k + 1] : int64_t(fc.size); + const int64_t sz = end - starts[k]; + if (sz < lo) lo = sz; + if (sz > hi) hi = sz; + } + unmap_file(fc.map, fc.size); + throw std::runtime_error( + path + ": this IS a Ryzen AI 1.9 Token .fconst (" + std::to_string(layers) + " blocks of " + + std::to_string(heads * 4) + " B headers at " + std::to_string(starts.front()) + " B) but its blocks are " + + "NOT uniform: " + std::to_string(lo / 1024) + " KiB to " + std::to_string(hi / 1024) + + " KiB, i.e. a variable-stride container. The rule table describes one stride per file, so no row can fit " + "it; this family (mobius/MatFormer) needs the per-layer block table instead -- the anchors give the block " + "starts, and each block's size follows that layer's own projection shapes (engine#1503)"); + } unmap_file(fc.map, fc.size); return path; } @@ -912,6 +956,193 @@ DeclaredMap read_declared(const std::string& pack_dir) { // Returns [V, H] f32 (row-major), empty when the pack has no such component. Everything it needs already existed: // walk_graph() reads the graph with its external-data declarations, decode_matmulnbits() is block-parameterised, and // the scale-per-32-column-group layout is the container's own. +// engine#1725: the per-layer half of a multi-component embedding. Same component path resolution as the token table +// (genai_config.json's model.embedding.filename), same external-data rule (resolved beside the graph), and the same +// dtype rule slice 1 established: f32 is read, anything else unquantised is refused by dtype rather than mis-read. +// Kept separate from read_embedding_component() because the shapes differ (V x width and a width-wide norm, not V x H) +// and because the consumer differs: the token table is the head's ORACLE, these are a decoder INPUT. +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why) { + EmbedPerLayer out; + std::string gpath = pack_dir + "/embedding.onnx"; + { + std::ifstream gf(pack_dir + "/genai_config.json"); + if (gf) { + json g; + try { g = json::parse(gf); } catch (...) { g = json::object(); } + if (g.is_object() && g.contains("model") && g["model"].is_object() && g["model"].contains("embedding") && + g["model"]["embedding"].is_object()) { + const json& emb = g["model"]["embedding"]; + if (emb.contains("filename") && emb["filename"].is_string()) { + const std::string rel = emb["filename"].get(); + if (!rel.empty() && rel.front() != '/' && rel.find("..") == std::string::npos) { + std::error_code nec; + if (fs::is_regular_file(pack_dir + "/" + rel, nec)) gpath = pack_dir + "/" + rel; + } + } + } + } + } + std::error_code ec; + if (!fs::is_regular_file(gpath, ec)) { if (why) *why = "no embedding component graph at " + gpath; return out; } + OnnxModel em; + try { em = parse_onnx(gpath); } catch (const std::exception& e) { if (why) *why = e.what(); return out; } + const OnnxTensor* table = nullptr; + const OnnxTensor* norm = nullptr; + for (const auto& t : em.tensors) { + if (t.name.find("embed_tokens_per_layer") != std::string::npos) table = &t; + else if (norm == nullptr && t.name.find("per_layer_projection_norm") != std::string::npos) norm = &t; + } + if (table == nullptr) { if (why) *why = "the component carries no embed_tokens_per_layer table"; return out; } + if (table->dims.size() != 2 || table->dims.at(0) <= 0 || table->dims.at(1) <= 0) { + if (why) *why = "embed_tokens_per_layer is not 2-D with positive dims"; return out; + } + if (table->type != kF32) { + if (why) *why = "embed_tokens_per_layer has dtype " + std::to_string(table->type) + ", and only f32 is read here"; + return out; + } + const int64_t V = table->dims.at(0), width = table->dims.at(1); + const auto read_f32 = [&](const OnnxTensor& t, size_t count, const char* what, std::vector* dst) { + const std::string base = fs::path(gpath).parent_path().string() + "/" + t.location; + size_t sz = 0; + std::string w2; + const uint8_t* m = map_file(base, sz, &w2); + if (!m) { if (why) *why = std::string(what) + ": " + w2; return false; } + const uint64_t need = uint64_t(count) * 4ull; + if (t.offset + need > sz) { if (why) *why = std::string(what) + " declares more bytes than its data carries"; return false; } + dst->resize(count); + std::memcpy(dst->data(), m + t.offset, size_t(need)); + return true; + }; + if (!table->location.empty()) { + if (!read_f32(*table, size_t(V) * size_t(width), "embed_tokens_per_layer", &out.table)) return out; + } else if (table->raw != nullptr && table->raw_len >= size_t(V) * size_t(width) * 4) { + out.table.resize(size_t(V) * size_t(width)); + std::memcpy(out.table.data(), table->raw, out.table.size() * sizeof(float)); + } else { + if (why) *why = "embed_tokens_per_layer has no readable data"; return out; + } + if (norm != nullptr && norm->type == kF32 && norm->dims.size() == 1 && norm->dims.at(0) == width && + !norm->location.empty()) { + if (!read_f32(*norm, size_t(width), "per_layer_projection_norm", &out.norm)) return out; + } + // The context projection: `per_layer_model_projection.weight_t_Q4` [width][H] at block 32, with `..._t_scales` and + // packed `..._t_zero_point`. Dequantised through the file's own MatMulNBits decoder so the arithmetic is the one the + // rest of the importer uses, then split into one [P][H] matrix per layer for the loader's naming. + { + const OnnxTensor* pw = nullptr; + const OnnxTensor* ps = nullptr; + const OnnxTensor* pz = nullptr; + for (const auto& t : em.tensors) { + if (t.name.find("per_layer_model_projection") == std::string::npos) continue; + if (t.name.find("scale") != std::string::npos) ps = &t; + else if (t.name.find("zero_point") != std::string::npos) pz = &t; + else if (t.name.find("_Q4") != std::string::npos) pw = &t; + } + if (pw && ps && hidden > 0 && width > 0) { + const std::string gdir = fs::path(gpath).parent_path().string(); + const auto blob = [&](const OnnxTensor& t, std::string* w4) -> const uint8_t* { + if (t.location.empty()) return t.raw; + size_t sz = 0; + return map_file(gdir + "/" + t.location, sz, w4); + }; + std::string wa, wb, wc; + const uint8_t* qw = blob(*pw, &wa); + const uint8_t* sc = blob(*ps, &wb); + const uint8_t* zp = pz ? blob(*pz, &wc) : nullptr; + if (qw && sc) { + const int64_t H = hidden; + const Q4 q = decode_matmulnbits(qw, reinterpret_cast(sc), zp, int(width), int(H), 32, + false, zp == nullptr); + if (q.N == int(width) && q.K == int(H)) { + // NL = width / P; the norm is P wide, so P comes from it when the component states one. + const int64_t Pn = !out.norm.empty() ? int64_t(out.norm.size()) : width / (width / 256 ? width / 256 : 1); + out.P = Pn > 0 ? Pn : width; + out.NL = width / out.P; + if (out.NL > 0 && out.NL * out.P == width) { + out.model_proj.assign(size_t(out.NL) * size_t(out.P) * size_t(H), 0.0f); + for (int64_t n = 0; n < width; ++n) + for (int64_t k = 0; k < H; ++k) { + const size_t ci = size_t(n) * size_t(H) + size_t(k); + const size_t gi = size_t(n) * size_t(H / 32) + size_t(k / 32); + out.model_proj[size_t(n) * size_t(H) + size_t(k)] = + float(q.codes[ci]) * q.scale[gi] + q.zero[gi]; + } + out.H = H; + } + } + } + } + } + if (out.norm.empty()) { + // No projection norm: the table is still usable, but say so — a consumer that expects the projection to be + // normalised would otherwise silently use raw rows. + if (why) *why = "no usable per_layer_projection_norm beside the table (rows are NOT projected)"; + } + out.V = V; + out.width = width; + if (out.P <= 0) out.P = int64_t(out.norm.size()); // the norm is P wide: it states the per-layer width + return out; +} + +// engine#1725: stream the per-layer token table out in bands. The component states it as f32 (V x NL*P, 9.4 GiB on +// gemma-4-e2b), so materialising it to convert would cost more RAM than the box has to spare for one import; the bands go +// straight from the mapped external data into the container's BF16 layout. +int64_t read_per_layer_table_band(const std::string& pack_dir, EmbedPerLayer* meta, int64_t first_row, int64_t rows, + uint8_t* out, std::string* why) { + if (meta == nullptr || out == nullptr || meta->V <= 0 || meta->width <= 0) { + if (why) *why = "no per-layer table metadata (call read_embedding_per_layer first)"; + return -1; + } + if (first_row < 0 || rows <= 0 || first_row + rows > meta->V) { + if (why) *why = "band out of range"; + return -1; + } + std::string gpath = pack_dir + "/embedding.onnx"; + { + std::ifstream gf(pack_dir + "/genai_config.json"); + if (gf) { + json g; + try { g = json::parse(gf); } catch (...) { g = json::object(); } + if (g.is_object() && g.contains("model") && g["model"].is_object() && g["model"].contains("embedding") && + g["model"]["embedding"].is_object() && g["model"]["embedding"].contains("filename") && + g["model"]["embedding"]["filename"].is_string()) { + const std::string rel = g["model"]["embedding"]["filename"].get(); + if (!rel.empty() && rel.front() != '/' && rel.find("..") == std::string::npos) { + std::error_code nec; + if (fs::is_regular_file(pack_dir + "/" + rel, nec)) gpath = pack_dir + "/" + rel; + } + } + } + } + OnnxModel em; + try { em = parse_onnx(gpath); } catch (const std::exception& e) { if (why) *why = e.what(); return -1; } + const OnnxTensor* table = nullptr; + for (const auto& t : em.tensors) + if (t.name.find("embed_tokens_per_layer") != std::string::npos) table = &t; + if (table == nullptr) { if (why) *why = "the component carries no embed_tokens_per_layer table"; return -1; } + if (table->location.empty()) { if (why) *why = "the per-layer table has no external data (banded reads need it)"; return -1; } + const std::string base = fs::path(gpath).parent_path().string() + "/" + table->location; + size_t sz = 0; + std::string w2; + const uint8_t* m = map_file(base, sz, &w2); + if (!m) { if (why) *why = w2; return -1; } + const uint64_t row_bytes = uint64_t(meta->width) * 4ull; + const uint64_t need_end = table->offset + uint64_t(first_row + rows) * row_bytes; + if (need_end > sz) { if (why) *why = "the per-layer table declares more bytes than its data carries"; return -1; } + const uint8_t* src = m + table->offset + uint64_t(first_row) * row_bytes; + uint16_t* dst = reinterpret_cast(out); + const size_t n = size_t(rows) * size_t(meta->width); + for (size_t i = 0; i < n; ++i) { + float f; + std::memcpy(&f, src + i * 4, 4); + uint32_t b; + std::memcpy(&b, &f, 4); + const uint32_t lsb = (b >> 16) & 1u; + dst[i] = uint16_t((b >> 16) + (lsb && ((b & 0xFFFFu) != 0 || (((b >> 16) & 1u) != 0)) ? 1u : 0u)); + } + return rows; +} + std::vector read_embedding_component(const std::string& pack_dir, std::string* why) { // A multi-component pack (`type: compositemodel`) names its embedding component in genai_config.json // (`model.embedding.filename = "embedding/model.onnx"`), exactly as it names its decoder; a single-graph pack keeps @@ -963,19 +1194,11 @@ std::vector read_embedding_component(const std::string& pack_dir, std::st // and a 256-wide projection norm, and no reader here implements that mechanism yet. So: read the token table, refuse // by NAME when the per-layer table is present, instead of reporting the shape mismatch as a missing table. if (table && !scales) { - const bool per_layer = [&] { - for (const auto& t2 : em.tensors) - if (t2.name.find("embed_tokens_per_layer") != std::string::npos) return true; - return false; - }(); - if (per_layer) { - if (why) - *why = "the component carries embedding.embed_tokens_per_layer.weight, i.e. a PER-LAYER embedding table " - "(262144x8960 on gemma-4-e2b = layers x head_dim) that feeds the decoder's per_layer_inputs, and " - "the per-layer embedding mechanism is not implemented here yet (engine#1725): refusing rather " - "than importing a decoder without that input"; - return {}; - } + // engine#1725: a component that ALSO carries `embed_tokens_per_layer` used to be refused here, because the + // per-layer mechanism did not exist and importing would have given the decoder less than it asks for. It exists + // now: `read_embedding_per_layer()` reads that table and its projection norm (this file), and import_pack() writes + // both into the container beside the token table. So the token table is read normally here, and the per-layer + // half is not this reader's business -- mixing the two was why the refusal had to be conservative. if (table->type != kF32) { if (why) *why = "embed_tokens is unquantised with dtype " + std::to_string(table->type) + ", and only f32 is read here (engine#1725)"; @@ -3035,6 +3258,51 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons embed_tbl = EmbedTable{embed_bf16.data(), EmbedTable::Fmt::BF16, int(H)}; w.add("model.embed_tokens.weight", "BF16", {V, H}, embed_bf16); ++r.tensors; + // engine#1725: a multi-component pack's embedding also carries the PER-LAYER table, its projection norm and the + // context projection. Written under the names the FORWARD loads (q4nx_forward.cpp's build_ple_inputs/load_ple_block), + // because the arithmetic there is already this model's: context = rmsnorm((emb @ W_l) * H**-0.5, norm), + // identity = 16 * table[id], per_layer_input = (context + identity)/sqrt(2). Absent for a single-graph pack. + { + std::string pl_why; + EmbedPerLayer pl = read_embedding_per_layer(pack_dir, H, &pl_why); + if (pl.ok()) { + // The token table is streamed: V x (NL*P) f32 is 9.4 GiB on e2b, and a band is 32 rows. + std::vector tbl_bf16(size_t(pl.V) * size_t(pl.width) * 2); + bool bands_ok = true; + for (int64_t row = 0; row < pl.V; row += 32) { + const int64_t n = (pl.V - row < 32) ? (pl.V - row) : 32; + if (read_per_layer_table_band(pack_dir, &pl, row, n, + tbl_bf16.data() + size_t(row) * size_t(pl.width) * 2, &pl_why) != n) { + bands_ok = false; + break; + } + } + if (bands_ok) { + w.add("model.per_layer_token_embd.weight", "BF16", {pl.V, pl.width}, std::move(tbl_bf16)); + if (!pl.norm.empty()) + w.add("model.per_layer_proj_norm.weight", "BF16", {pl.P}, + as_bf16(reinterpret_cast(pl.norm.data()), kF32, pl.norm.size())); + r.tensors += 1 + (pl.norm.empty() ? 0 : 1); + if (pl.complete()) { + for (int64_t l = 0; l < pl.NL; ++l) { + std::vector one(size_t(pl.P) * size_t(pl.H) * 2); + for (int64_t o = 0; o < pl.P; ++o) + for (int64_t k = 0; k < pl.H; ++k) { + const float f = pl.model_proj[size_t(l * pl.P + o) * size_t(pl.H) + size_t(k)]; + uint32_t b; + std::memcpy(&b, &f, 4); + const uint32_t lsb = (b >> 16) & 1u; + reinterpret_cast(one.data())[size_t(o) * size_t(pl.H) + size_t(k)] = + uint16_t((b >> 16) + (lsb && ((b & 0xFFFFu) != 0 || (((b >> 16) & 1u) != 0)) ? 1u : 0u)); + } + w.add("model.per_layer_model_proj.weight_layer" + std::to_string(l), "BF16", {pl.P, pl.H}, + std::move(one)); + ++r.tensors; + } + } + } + } + } } // genre A = the graph carries real weights and there is no .fconst to read (the same test plan() uses) @@ -3893,6 +4161,38 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons c["vocab_size"] = p.dims.vocab; c["model_type"] = p.dims.model_type; c["rope_theta"] = p.dims.rope_theta; + // engine#1725: the PLE block is switched ON by `hidden_size_per_layer_input > 0` in the container's semantics + // (q4nx_semantics.c), so a container that carries the per-layer tensors but not these two keys would import them + // and never use them -- the forward skips the block silently. Taken from the component itself, which is where they + // are measured (gemma-4-e2b: 256 and 262144), rather than from the pack's config, which states them one level down. + { + // One metadata read, not plumbing: the per-layer facts live in the component, and this config block is in a + // different writer from the one that emits the tensors. The table itself is NOT read here (only names, dims + // and the quantised projection header), so this costs a file parse and no memory. + std::string pl_why; + const EmbedPerLayer pl2 = read_embedding_per_layer(pack_dir, p.dims.hidden, &pl_why); + if (pl2.complete()) { + c["hidden_size_per_layer_input"] = pl2.P; + c["vocab_size_per_layer_input"] = pl2.V; + } + } + // The Gemma-4 layer-shape switches the semantics reader also consults, carried through when the pack states them. + { + std::ifstream mc(pack_dir + "/model_config.json"); + if (mc) { + json m; + try { m = json::parse(mc); } catch (...) { m = json::object(); } + const json* t2 = nullptr; + if (m.is_object() && m.contains("config") && m["config"].is_object() && + m["config"].contains("model_attributes") && m["config"]["model_attributes"].is_object() && + m["config"]["model_attributes"].contains("text_config") && + m["config"]["model_attributes"]["text_config"].is_object()) + t2 = &m["config"]["model_attributes"]["text_config"]; + if (t2 != nullptr) + for (const char* k : {"num_kv_shared_layers", "use_double_wide_mlp", "extra_layer_norms"}) + if (t2->contains(k) && !(*t2)[k].is_null()) c[k] = (*t2)[k]; + } + } // A pack that ships its own precomputed RoPE (cos_cache/sin_cache) is not necessarily built for the theta // its genai_config.json states -- pi-inferzo measured 10000 declared against a model that needs 1e6 on // Qwen2.5 rai_1.7.1_hybrid, and our forward computes RoPE from theta rather than reading a cache. That is a diff --git a/npu/pack_import.h b/npu/pack_import.h index 1a4f96da4..2b04aad3d 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -78,6 +78,40 @@ struct Dims { // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. std::vector read_embedding_component(const std::string& pack_dir, std::string* why); +// The per-layer half of a multi-component pack's embedding (engine#1725). gemma-4 components carry, beside the token +// table, `embed_tokens_per_layer.weight` (262144 x 8960 on e2b = layers x head_dim) and a 256-wide projection norm; +// the decoder consumes them as `per_layer_inputs`, so a container without them gives the decoder less than it asks for. +// Read separately from the token table because they have different shapes and different consumers: the token table +// becomes the head oracle, this becomes an input. Empty (with `why`) when the component has no per-layer table. +struct EmbedPerLayer { + std::vector table; // [V][layers*P], row-major (embed_tokens_per_layer.weight) + std::vector norm; // [P] (per_layer_projection_norm.weight, shared by every layer) + // The CONTEXT projection: the component carries it quantised as + // `per_layer_model_projection.weight_t_Q4` [layers*P][H] plus scales and packed zero points. The loader wants it + // dequantised, one [P][H] matrix per layer (`model.per_layer_model_proj.weight_layer{l}`), so it is dequantised here + // -- without it the per-layer input is `16*table/sqrt(2)`, plausible and wrong at every layer (engine#1725). + std::vector model_proj; // [layers][P][H] + int64_t V = 0, width = 0, P = 0, H = 0, NL = 0; + // `ok()` is what the reader guarantees: the table and the projection norm. `complete()` additionally requires the + // context projection, which the PLE contract needs -- a component with `ok()` but not `complete()` imports the table + // and will be refused by the loader on `per_layer_model_proj.weight_layer0`, which is the honest place to refuse + // (the importer does not know whether the decoder declares per_layer_inputs). + bool ok() const { + return V > 0 && width > 0 && table.size() == size_t(V) * size_t(width) && + (!norm.empty() ? norm.size() == size_t(P) : true); + } + bool complete() const { return ok() && P > 0 && H > 0 && NL > 0 && model_proj.size() == size_t(NL) * size_t(P) * size_t(H); } +}; +// `hidden` is the decoder's hidden size (the context projection's K); it comes from the pack's geometry, not from +// the component, which is why the caller passes it. +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why); + +// The per-layer token table is `V x (NL*P)` f32 in the component -- 9.4 GiB for gemma-4-e2b -- so it is never +// materialised: the importer streams it into the container in 32-row bands, converting to BF16 as it goes (engine#1725). +// Returns the rows written, or -1 with `why` set. `out` must hold `rows * width * 2` bytes. +int64_t read_per_layer_table_band(const std::string& pack_dir, EmbedPerLayer* meta, int64_t first_row, int64_t rows, + uint8_t* out, std::string* why); + // One tensor the pack describes, and where its values would come from. struct Planned { std::string name; // Q4NX tensor name (model.layers.N.<...>.weight) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a639cc241..3bba7447b 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2901,12 +2901,110 @@ void test_embedding_f32_component(const std::string& root) { std::string why2; const std::vector none = pi::read_embedding_component(b.string(), &why2); std::printf("f32 embedding (per-layer): %zu values, why: %s\n", none.size(), why2.c_str()); - check(none.empty() && why2.find("per_layer") != std::string::npos, - "f32 embedding: a per-layer table is refused by NAME, not reported as a missing table (" + why2 + ")"); + check(!none.empty(), + "f32 embedding: a per-layer table no longer blocks the TOKEN table (slice 2 handles it; " + why2 + ")"); fs::remove_all(a); fs::remove_all(b); } + +// engine#1725: a multi-component embedding carries a PER-LAYER table and its projection norm beside the token table, and +// the importer writes both into the container so the forward path can gather them. Measured shapes on +// amd/gemma-4-e2b-it-mobius-int4_rai_1.9.0_npu: embed_tokens.weight (262144,1536) f32, +// embed_tokens_per_layer.weight (262144,8960) f32, per_layer_projection_norm.weight (256,) -- all f32, no scales. +// This drives the writer on a crafted pack of that shape (V=4, width=6) and reads the container header back. +void test_per_layer_container(const std::string& root) { + const fs::path dir = fs::path(root) / "perlayer"; + fs::remove_all(dir); + fs::create_directories(dir / "embedding"); + const int64_t V = 4, width = 6; + // two-arg form: `norm(size_t(width))` parses as a function declaration and `norm{size_t(width)}` is an + // initializer_list of ONE element = width. Both mistakes produce a plausible-looking fixture. + std::vector tok(size_t(V) * 3, 0.0f), per(size_t(V) * size_t(width), 0.0f), norm(size_t(width), 0.0f); + for (size_t i = 0; i < tok.size(); ++i) tok[i] = float(i) * 0.5f; + for (size_t i = 0; i < per.size(); ++i) per[i] = float(int(i) - 8) * 0.25f; + for (size_t i = 0; i < norm.size(); ++i) norm[i] = 1.0f + float(i) * 0.125f; + std::string data; + const size_t tok_bytes = tok.size() * 4, per_bytes = per.size() * 4; + data.append(reinterpret_cast(tok.data()), tok_bytes); + data.append(reinterpret_cast(per.data()), per_bytes); + data.append(reinterpret_cast(norm.data()), norm.size() * 4); + put_file((dir / "embedding" / "model.onnx.data").string(), data); + put_file((dir / "embedding" / "model.onnx").string(), + pb::model({pb::tensor("embedding.embed_tokens.weight", {V, 3}, "model.onnx.data", 0, tok_bytes, 1), + pb::tensor("embedding.embed_tokens_per_layer.weight", {V, width}, "model.onnx.data", tok_bytes, + per_bytes, 1), + pb::tensor("embedding.per_layer_projection_norm.weight", {width}, "model.onnx.data", + tok_bytes + per_bytes, norm.size() * 4, 1)}, + {})); + put_file((dir / "genai_config.json").string(), + R"({"model":{"type":"gemma4","embedding":{"filename":"embedding/model.onnx"},)" + R"("decoder":{"filename":"decoder/model.onnx"}}})"); + put_file((dir / "config.json").string(), + R"({"model_type":"gemma4","hidden_size":3,"intermediate_size":6,"num_hidden_layers":1,)" + R"("num_attention_heads":1,"num_key_value_heads":1,"head_dim":3,"vocab_size":4,"rope_theta":10000.0})"); + std::string why; + const pi::EmbedPerLayer pl = pi::read_embedding_per_layer(dir.string(), 3, &why); + std::printf("per-layer: V %lld width %lld, why: %s (tok %zu per %zu norm %zu bytes)\n", (long long) pl.V, + (long long) pl.width, why.c_str(), tok_bytes, per_bytes, norm.size() * 4); + check(pl.ok(), "per-layer: the table and its projection norm are read from the component genai_config.json names"); + bool same = pl.ok() && pl.table.size() == per.size(); + for (size_t i = 0; same && i < per.size(); ++i) same = pl.table[i] == per[i]; + for (size_t i = 0; same && i < norm.size(); ++i) same = pl.norm[i] == norm[i]; + check(same, "per-layer: the values come back exactly as the component states them"); + // the token table is still this reader's other half: it must NOT be refused because a per-layer table is present + std::string why2; + const std::vector t = pi::read_embedding_component(dir.string(), &why2); + check(t.size() == size_t(V) * 3, "per-layer: the token table still reads beside it (" + std::to_string(t.size()) + ")"); + fs::remove_all(dir); +} + + + +// engine#1503: a 1.9 fconst whose blocks are NOT uniform (the mobius line). `fconst19_detect` requires one stride for the +// whole file, so it fails -- and the failure used to be indistinguishable from "not a 1.9 fconst at all". The refusal now +// names the structure and says the rule table cannot describe it. Measured on gemma-4-e2b: 35 blocks of 8 anchors, sizes +// 21.87-288.06 MiB. +void test_fconst_variable_stride(const std::string& root) { + const fs::path dir = fs::path(root) / "fconst-varstride"; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + const int H = 128, L = 4, heads = 2; + const int64_t hdr = heads * 4; + // four blocks, each header then a body of a DIFFERENT size: 64 KiB, 96 KiB, 128 KiB, 160 KiB + const int64_t bodies[4] = {64 << 10, 96 << 10, 128 << 10, 160 << 10}; + std::string fc; + for (int l = 0; l < L; ++l) { + for (int h = 0; h < heads; ++h) { // 00 00 7f ff per head + fc.push_back('\0'); fc.push_back('\0'); fc.push_back(char(0x7f)); fc.push_back(char(0xff)); + } + fc.append(size_t(bodies[l]), '\0'); + } + check(fc.size() > size_t(L * hdr), "variable stride: the crafted fconst is larger than its headers"); + put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); + put_file((dir / "config.json").string(), + R"({"model_type":"gemma4","hidden_size":128,"intermediate_size":256,"num_hidden_layers":4,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,"rope_theta":10000.0})"); + put_file((dir / "model.onnx").string(), pb::model({}, {})); + put_file((dir / "model.onnx.data").string(), std::string(4096, '\x11')); + pi::Options allow; + allow.allow_incomplete = true; + const std::string out = (fs::temp_directory_path() / ("varstride_out." + std::to_string(::getpid()))).string(); + fs::remove_all(out); + bool refused = false; + std::string what; + try { + (void) pi::import_pack(dir.string(), out, allow); + } catch (const std::exception& e) { + what = e.what(); + refused = what.find("variable-stride") != std::string::npos || what.find("NOT uniform") != std::string::npos; + } + std::printf("variable stride: %s\n", refused ? "refused by name (variable-stride container)" : what.substr(0, 140).c_str()); + check(refused, "variable stride: a 1.9 fconst whose blocks differ in size is refused BY NAME, not as 'not a 1.9 fconst'"); + fs::remove_all(out); + fs::remove_all(dir); +} + int main(int argc, char** argv) { const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; @@ -2936,6 +3034,7 @@ int main(int argc, char** argv) { test_attn_tile(golden_dir); test_mlp_gu(golden_dir); test_fconst_eps(golden_dir.empty() ? "." : golden_dir); + test_fconst_variable_stride(golden_dir.empty() ? "." : golden_dir); test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); @@ -2947,6 +3046,7 @@ int main(int argc, char** argv) { test_im_from_node_attrs(golden_dir.empty() ? "." : golden_dir); test_fconst_in_component(golden_dir.empty() ? "." : golden_dir); test_embedding_in_component(golden_dir.empty() ? "." : golden_dir); + test_per_layer_container(golden_dir.empty() ? "." : golden_dir); test_embedding_f32_component(golden_dir.empty() ? "." : golden_dir); test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir);