diff --git a/docs/npu.md b/docs/npu.md index 6862549d4..55abd24a7 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -909,6 +909,25 @@ What converts, and how faithfully: | `model.norm` (final) | `.fconst`, the tail envelope at `layers x stride` | bf16 | byte-exact | | `sin_cache_cos_cache` | `model.onnx.data` | not imported | the lane computes RoPE from `rope_theta` (`fill_rope`) | +**A multi-component pack's per-layer embedding** (engine#1725). gemma-4's embedding component +carries, beside the token table, `embed_tokens_per_layer.weight` (`vocab x layers*P` f32; 262,144 x +8,960 on e2b, 9.4 GiB), a `P`-wide `per_layer_projection_norm.weight` shared by every layer, and +the context projection `per_layer_model_projection.weight_t_Q4` quantised at block 32. The import +writes them under the names the forward loads: + +| container tensor | from | form | +|---|---|---| +| `model.per_layer_token_embd.weight` `[vocab, layers*P]` | the f32 table | BF16, round to nearest even | +| `model.per_layer_proj_norm.weight` `[P]` | the norm | BF16 | +| `model.per_layer_model_proj.weight_layer{l}` `[P, hidden]` | the projection, dequantised, one per layer | BF16 | + +The table is never held in memory, in either form: the reader points into the component's mapped +external data (its extent is checked once, when it is located), and the container writer converts +it 32 rows at a time while it writes the file. A projection with no zero-point tensor takes the +implicit zero point 8, the same rule as every other MatMulNBits here. `config.json` gets +`hidden_size_per_layer_input` and `vocab_size_per_layer_input` only when all three parts went into +the container, because those two keys are what switch the block on in the forward. + An OGA pack keeps its tensors inside the `If` branches: the top-level graph has **no initializers** and four nodes (`Shape`/`Constant`/`Equal`/`If`). The importer walks the subgraphs. The 4-bit decode is `w = (code - zp) * scale` with two codes per byte (low nibble diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index b587a1eb6..10773f32c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2770,7 +2770,38 @@ bool Q4nxNpuForward::load_ple_block(int row) { ple_block_.assign((size_t)Q4NX_TILE_ROWS * (size_t)cols, 0.0f); const size_t plain_bytes = (size_t)sem_.vocab_size_per_layer_input * (size_t)cols; - if ((size_t)td->data_size == plain_bytes) { + // engine#1725: a RAW BF16 band. The multi-component packs state this table as f32 and the importer converts it to BF16 + // when it writes the container (a converting copy of the whole V x NL*P table would be 9.4 GiB), so the band here is + // 32 rows of BF16 -- the shape the plain-int8 branch handles, with a 2-byte element instead of a code. + // + // The branch is chosen by the container's DECLARED dtype, not by arithmetic on data_size (review of #1825): a size test + // needs `% bf16_bytes == 0 && != plain_bytes` to separate two layouts, and every multiple of the band satisfies it, so a + // plain table whose byte count happens to divide would be read as BF16 bands -- plausible and wrong. This tree already + // decides exactly this way elsewhere (`model.c`: `strcmp(dtype,"I8")`), and `q4nx_dequant.h`'s rule is that an + // ambiguous layout is refused rather than resolved by which test happens to pass. + const size_t bf16_bytes = (size_t)Q4NX_TILE_ROWS * (size_t)cols * 2; + const char* pl_dtype = td->dtype; + const bool declared_bf16 = pl_dtype[0] != '\0' && std::strcmp(pl_dtype, "BF16") == 0; + const bool declared_i8 = pl_dtype[0] != '\0' && (std::strcmp(pl_dtype, "I8") == 0 || std::strcmp(pl_dtype, "Q4") == 0); + if (!declared_bf16 && !declared_i8) { + // Some other declared format, or none at all: refuse rather than fall through to a layout guess. A container + // written by this importer always states one (`w.add()` writes dtype for every tensor), so an empty dtype means a + // foreign or truncated container, not a layout this loader can infer. + err_ = pl_dtype[0] != '\0' + ? std::string("per_layer_token_embd has dtype ") + pl_dtype + + ", which this loader does not read (BF16 or plain I8 only)" + : std::string("per_layer_token_embd declares no dtype, so its layout cannot be decided"); + return false; + } + if (declared_bf16) { + if ((size_t)td->data_size < ((size_t)block + 1) * bf16_bytes || + (size_t)td->data_size % bf16_bytes != 0) { + err_ = "per_layer_token_embd is declared BF16 but its size is not whole 32-row bands"; + return false; + } + const uint16_t* band = reinterpret_cast(raw + (size_t)block * bf16_bytes); + for (size_t i = 0; i < (size_t)Q4NX_TILE_ROWS * (size_t)cols; ++i) ple_block_[i] = bf16_to_f32(band[i]); + } else if ((size_t)td->data_size == plain_bytes) { // PLAIN int8 + per-32-column scale (Gemma4): only the requested band of // 32 rows is needed, so it is dequantized in place rather than the whole // [vocab, NL*ple_dim] table. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 29b1decb6..75e4b7b0b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include #include @@ -503,6 +505,27 @@ std::vector fconst19_header(int heads) { // True when `fc` is a 1.9 Token .fconst of this geometry: exactly `layers` headers, evenly spaced. // Fills the first header's offset and the block stride. False means the 1.8 form (or a shape this // importer does not know, which the caller decides about). +// engine#1503: the run scan on its own. `fconst19_detect` additionally requires ONE stride, which the mobius line +// (gemma-4-e2b: 35 blocks of 21.87-288.06 MiB) does not have -- so a caller that needs to tell "this is not a 1.9 fconst" +// from "this is a 1.9 fconst whose blocks are not uniform" needs the starts before the stride test. +bool fconst19_block_starts(const Fconst& fc, int heads, int layers, std::vector* starts) { + if (heads <= 0 || layers < 2) return false; + const std::vector pat = fconst19_header(heads); + std::vector runs; + for (size_t i = 0; i + pat.size() <= fc.size;) { + if (fc.map[i] == pat[0] && fc.map[i + 2] == 0x7f && fc.map[i + 3] == 0xff && + std::memcmp(fc.map + i, pat.data(), pat.size()) == 0) { + runs.push_back(int64_t(i)); + i += pat.size(); + } else { + ++i; + } + } + if (int(runs.size()) != layers) return false; + if (starts) *starts = std::move(runs); + return true; +} + bool fconst19_detect(const Fconst& fc, int heads, int layers, int64_t* hdr_off, int64_t* stride) { if (heads <= 0 || layers < 2) return false; const std::vector pat = fconst19_header(heads); @@ -537,6 +560,29 @@ std::string canonical_token_fconst(const std::string& path, int hidden, int head Fconst fc = open_fconst(path); int64_t hdr_off = 0, stride9 = 0; if (!fconst19_detect(fc, heads, layers, &hdr_off, &stride9)) { + // Distinguish "not a 1.9 fconst" from "a 1.9 fconst whose blocks are NOT uniform": the mobius line (gemma-4-e2b + // measured: 35 blocks, 8 anchors each, sizes 21.87-288.06 MiB) is the second, and the rule table cannot describe + // it -- one stride per file is the table's model (engine#1503). Refusing with the block table is the difference + // between a reader that looks unmeasured and a model that is wrong for the family. + std::vector starts; + if (fconst19_block_starts(fc, heads, layers, &starts)) { + std::string sizes; + int64_t lo = INT64_MAX, hi = 0; + for (size_t k = 0; k < starts.size(); ++k) { + const int64_t end = (k + 1 < starts.size()) ? starts[k + 1] : int64_t(fc.size); + const int64_t sz = end - starts[k]; + if (sz < lo) lo = sz; + if (sz > hi) hi = sz; + } + unmap_file(fc.map, fc.size); + throw std::runtime_error( + path + ": this IS a Ryzen AI 1.9 Token .fconst (" + std::to_string(layers) + " blocks of " + + std::to_string(heads * 4) + " B headers at " + std::to_string(starts.front()) + " B) but its blocks are " + + "NOT uniform: " + std::to_string(lo / 1024) + " KiB to " + std::to_string(hi / 1024) + + " KiB, i.e. a variable-stride container. The rule table describes one stride per file, so no row can fit " + "it; this family (mobius/MatFormer) needs the per-layer block table instead -- the anchors give the block " + "starts, and each block's size follows that layer's own projection shapes (engine#1503)"); + } unmap_file(fc.map, fc.size); return path; } @@ -947,6 +993,205 @@ DeclaredMap read_declared(const std::string& pack_dir) { // Returns [V, H] f32 (row-major), empty when the pack has no such component. Everything it needs already existed: // walk_graph() reads the graph with its external-data declarations, decode_matmulnbits() is block-parameterised, and // the scale-per-32-column-group layout is the container's own. +// engine#1725: the per-layer half of a multi-component embedding. Same component path resolution as the token table +// (genai_config.json's model.embedding.filename), same external-data rule (resolved beside the graph), and the same +// dtype rule slice 1 established: f32 is read, anything else unquantised is refused by dtype rather than mis-read. +// Kept separate from read_embedding_component() because the shapes differ (V x width and a width-wide norm, not V x H) +// and because the consumer differs: the token table is the head's ORACLE, these are a decoder INPUT. +void implicit_zp_zero(Q4& q, int bits); // defined below, beside decode_matmulnbits +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why) { + EmbedPerLayer out; + std::string gpath = pack_dir + "/embedding.onnx"; + { + std::ifstream gf(pack_dir + "/genai_config.json"); + if (gf) { + json g; + try { g = json::parse(gf); } catch (...) { g = json::object(); } + if (g.is_object() && g.contains("model") && g["model"].is_object() && g["model"].contains("embedding") && + g["model"]["embedding"].is_object()) { + const json& emb = g["model"]["embedding"]; + if (emb.contains("filename") && emb["filename"].is_string()) { + const std::string rel = emb["filename"].get(); + if (!rel.empty() && rel.front() != '/' && rel.find("..") == std::string::npos) { + std::error_code nec; + if (fs::is_regular_file(pack_dir + "/" + rel, nec)) gpath = pack_dir + "/" + rel; + } + } + } + } + } + std::error_code ec; + if (!fs::is_regular_file(gpath, ec)) { if (why) *why = "no embedding component graph at " + gpath; return out; } + OnnxModel em; + try { em = parse_onnx(gpath); } catch (const std::exception& e) { if (why) *why = e.what(); return out; } + const OnnxTensor* table = nullptr; + const OnnxTensor* norm = nullptr; + for (const auto& t : em.tensors) { + if (t.name.find("embed_tokens_per_layer") != std::string::npos) table = &t; + else if (norm == nullptr && t.name.find("per_layer_projection_norm") != std::string::npos) norm = &t; + } + if (table == nullptr) { if (why) *why = "the component carries no embed_tokens_per_layer table"; return out; } + if (table->dims.size() != 2 || table->dims.at(0) <= 0 || table->dims.at(1) <= 0) { + if (why) *why = "embed_tokens_per_layer is not 2-D with positive dims"; return out; + } + if (table->type != kF32) { + if (why) *why = "embed_tokens_per_layer has dtype " + std::to_string(table->type) + ", and only f32 is read here"; + return out; + } + const int64_t V = table->dims.at(0), width = table->dims.at(1); + // Every external-data file this component names is mapped once and held by `out.keep` (the table's pointer lives in + // one of them); the norm is small and copied out. + struct Maps { + std::vector> all; + std::vector inline_table; + ~Maps() { + for (const auto& [m, n] : all) unmap_file(m, n); + } + }; + auto maps = std::make_shared(); + std::map> by_path; + const auto mapped = [&](const OnnxTensor& t, const char* what, size_t* sz) -> const uint8_t* { + const std::string base = fs::path(gpath).parent_path().string() + "/" + t.location; + const auto it = by_path.find(base); + if (it != by_path.end()) { *sz = it->second.second; return it->second.first; } + std::string w2; + const uint8_t* m = map_file(base, *sz, &w2); + if (!m) { if (why) *why = std::string(what) + ": " + w2; return nullptr; } + maps->all.emplace_back(m, *sz); + by_path[base] = {m, *sz}; + return m; + }; + const auto read_f32 = [&](const OnnxTensor& t, size_t count, const char* what, std::vector* dst) { + size_t sz = 0; + const uint8_t* m = mapped(t, what, &sz); + if (!m) return false; + const uint64_t need = uint64_t(count) * 4ull; + if (t.offset + need > sz) { if (why) *why = std::string(what) + " declares more bytes than its data carries"; return false; } + dst->resize(count); + std::memcpy(dst->data(), m + t.offset, size_t(need)); + return true; + }; + const uint8_t* table_f32 = nullptr; + if (!table->location.empty()) { + // The table is POINTED AT, not read: V x width f32 is 9.4 GiB on gemma-4-e2b. Its extent is checked here, once, + // so a band read cannot run off the mapping later. + size_t sz = 0; + const uint8_t* m = mapped(*table, "embed_tokens_per_layer", &sz); + if (!m) return out; + if (table->offset + uint64_t(V) * uint64_t(width) * 4ull > sz) { + if (why) *why = "embed_tokens_per_layer declares more bytes than its data carries"; + return out; + } + table_f32 = m + table->offset; + } else if (table->raw != nullptr && table->raw_len >= size_t(V) * size_t(width) * 4) { + // an inline initializer is small by construction (it sits inside the graph file); copy it so nothing depends on + // the parsed graph outliving this call + maps->inline_table.resize(size_t(V) * size_t(width)); + std::memcpy(maps->inline_table.data(), table->raw, maps->inline_table.size() * sizeof(float)); + table_f32 = reinterpret_cast(maps->inline_table.data()); + } else { + if (why) *why = "embed_tokens_per_layer has no readable data"; return out; + } + // The norm is P wide and SHARED by every layer: gemma-4-e2b states it as (256,) beside a table 8960 = 35 x 256 + // wide. Requiring it to be `width` long (as this did) skipped it on exactly that pack, and the loader then refuses + // the container on `model.per_layer_proj_norm.weight`. Any length that divides the table's width is a per-layer + // width; one that does not is not this norm. + if (norm != nullptr && norm->type == kF32 && norm->dims.size() == 1 && norm->dims.at(0) > 0 && + width % norm->dims.at(0) == 0 && !norm->location.empty()) { + if (!read_f32(*norm, size_t(norm->dims.at(0)), "per_layer_projection_norm", &out.norm)) return out; + } + // The context projection: `per_layer_model_projection.weight_t_Q4` [width][H] at block 32, with `..._t_scales` and + // packed `..._t_zero_point`. Dequantised through the file's own MatMulNBits decoder so the arithmetic is the one the + // rest of the importer uses, then split into one [P][H] matrix per layer for the loader's naming. + { + const OnnxTensor* pw = nullptr; + const OnnxTensor* ps = nullptr; + const OnnxTensor* pz = nullptr; + for (const auto& t : em.tensors) { + if (t.name.find("per_layer_model_projection") == std::string::npos) continue; + if (t.name.find("scale") != std::string::npos) ps = &t; + else if (t.name.find("zero_point") != std::string::npos) pz = &t; + else if (t.name.find("_Q4") != std::string::npos) pw = &t; + } + if (pw && ps && hidden > 0 && width > 0) { + const std::string gdir = fs::path(gpath).parent_path().string(); + const auto blob = [&](const OnnxTensor& t, std::string* w4) -> const uint8_t* { + if (t.location.empty()) return t.raw; + size_t sz = 0; + const uint8_t* m = mapped(t, "per_layer_model_projection", &sz); + if (!m && w4 && why) *w4 = *why; + return m ? m + t.offset : nullptr; + }; + std::string wa, wb, wc; + const uint8_t* qw = blob(*pw, &wa); + const uint8_t* sc = blob(*ps, &wb); + const uint8_t* zp = pz ? blob(*pz, &wc) : nullptr; + if (qw && sc) { + const int64_t H = hidden; + Q4 q = decode_matmulnbits(qw, reinterpret_cast(sc), zp, int(width), int(H), 32, + false, zp == nullptr); + // no zero-point tensor means the implicit 2^(bits-1), not 0 (see implicit_zp_zero: ONNX Runtime's rule, + // measured on this family's own embedding component) + if (zp == nullptr) implicit_zp_zero(q, 4); + if (q.N == int(width) && q.K == int(H)) { + // NL = width / P; the norm is P wide, so P comes from it when the component states one. + const int64_t Pn = !out.norm.empty() ? int64_t(out.norm.size()) : width / (width / 256 ? width / 256 : 1); + out.P = Pn > 0 ? Pn : width; + out.NL = width / out.P; + if (out.NL > 0 && out.NL * out.P == width) { + out.model_proj.assign(size_t(out.NL) * size_t(out.P) * size_t(H), 0.0f); + for (int64_t n = 0; n < width; ++n) + for (int64_t k = 0; k < H; ++k) { + const size_t ci = size_t(n) * size_t(H) + size_t(k); + const size_t gi = size_t(n) * size_t(H / 32) + size_t(k / 32); + out.model_proj[size_t(n) * size_t(H) + size_t(k)] = + float(q.codes[ci]) * q.scale[gi] + q.zero[gi]; + } + out.H = H; + } + } + } + } + } + if (out.norm.empty()) { + // No projection norm: the table is still usable, but say so — a consumer that expects the projection to be + // normalised would otherwise silently use raw rows. + if (why) *why = "no usable per_layer_projection_norm beside the table (rows are NOT projected)"; + } + out.V = V; + out.width = width; + out.table_f32 = table_f32; + out.keep = maps; + if (out.P <= 0) out.P = int64_t(out.norm.size()); // the norm is P wide: it states the per-layer width + return out; +} + +// engine#1725: one band of the per-layer token table, f32 -> BF16. The component states the table as f32 (V x NL*P, +// 9.4 GiB on gemma-4-e2b); `meta` already holds the mapping and has checked the table's extent, so this is a bounds +// check on the band and a conversion loop -- `f32_to_bf16`, round to nearest even, the conversion every other tensor in +// the container gets. (It used to re-parse the graph and re-map the data on every call, 8,192 times on e2b, and round +// "up iff the kept lsb is 1", which moved half of all values by one ulp -- including ones bf16 holds exactly.) +int64_t read_per_layer_table_band(const EmbedPerLayer& meta, int64_t first_row, int64_t rows, uint8_t* out, + std::string* why) { + if (out == nullptr || !meta.ok()) { + if (why) *why = "no per-layer table (call read_embedding_per_layer first)"; + return -1; + } + if (first_row < 0 || rows <= 0 || first_row + rows > meta.V) { + if (why) *why = "band out of range"; + return -1; + } + const uint8_t* src = meta.table_f32 + size_t(first_row) * size_t(meta.width) * 4; + const size_t n = size_t(rows) * size_t(meta.width); + for (size_t i = 0; i < n; ++i) { + float f; + std::memcpy(&f, src + i * 4, 4); + const uint16_t h = f32_to_bf16(f); + std::memcpy(out + i * 2, &h, 2); + } + return rows; +} + std::vector read_embedding_component(const std::string& pack_dir, std::string* why) { // A multi-component pack (`type: compositemodel`) names its embedding component in genai_config.json // (`model.embedding.filename = "embedding/model.onnx"`), exactly as it names its decoder; a single-graph pack keeps @@ -998,19 +1243,11 @@ std::vector read_embedding_component(const std::string& pack_dir, std::st // and a 256-wide projection norm, and no reader here implements that mechanism yet. So: read the token table, refuse // by NAME when the per-layer table is present, instead of reporting the shape mismatch as a missing table. if (table && !scales) { - const bool per_layer = [&] { - for (const auto& t2 : em.tensors) - if (t2.name.find("embed_tokens_per_layer") != std::string::npos) return true; - return false; - }(); - if (per_layer) { - if (why) - *why = "the component carries embedding.embed_tokens_per_layer.weight, i.e. a PER-LAYER embedding table " - "(262144x8960 on gemma-4-e2b = layers x head_dim) that feeds the decoder's per_layer_inputs, and " - "the per-layer embedding mechanism is not implemented here yet (engine#1725): refusing rather " - "than importing a decoder without that input"; - return {}; - } + // engine#1725: a component that ALSO carries `embed_tokens_per_layer` used to be refused here, because the + // per-layer mechanism did not exist and importing would have given the decoder less than it asks for. It exists + // now: `read_embedding_per_layer()` reads that table and its projection norm (this file), and import_pack() writes + // both into the container beside the token table. So the token table is read normally here, and the per-layer + // half is not this reader's business -- mixing the two was why the refusal had to be conservative. if (table->type != kF32) { if (why) *why = "embed_tokens is unquantised with dtype " + std::to_string(table->type) + ", and only f32 is read here (engine#1725)"; @@ -3164,6 +3401,16 @@ struct Writer { offset += bytes.size(); blobs.emplace_back(name, std::move(bytes)); } + // A tensor too large to hold: `produce` writes exactly `bytes` bytes when the container is written, in table order. + // (engine#1725: gemma-4-e2b's per-layer table is 4.7 GiB as BF16 and comes from a 9.4 GiB f32 mapping.) + void add_stream(const std::string& name, const std::string& dtype, const std::vector& shape, uint64_t bytes, + std::function produce) { + table[name] = {{"dtype", dtype}, {"shape", shape}, {"data_offsets", {offset, offset + bytes}}}; + offset += bytes; + streams[blobs.size()] = {bytes, std::move(produce)}; + blobs.emplace_back(name, std::vector()); + } + std::map>> streams; void finish(const json& metadata) { json t = json::object(); t["__metadata__"] = metadata; @@ -3175,8 +3422,20 @@ struct Writer { const uint64_t len = header.size(); f.write(reinterpret_cast(&len), 8); f.write(header.data(), std::streamsize(header.size())); - for (const auto& [name, bytes] : blobs) - f.write(reinterpret_cast(bytes.data()), std::streamsize(bytes.size())); + for (size_t i = 0; i < blobs.size(); ++i) { + const auto st = streams.find(i); + if (st == streams.end()) { + f.write(reinterpret_cast(blobs[i].second.data()), std::streamsize(blobs[i].second.size())); + continue; + } + const std::streampos before = f.tellp(); + st->second.second(f); + // the header already states this tensor's extent, so a producer that wrote anything else has corrupted + // every offset after it + if (!f || uint64_t(f.tellp() - before) != st->second.first) + throw std::runtime_error(blobs[i].first + ": the streamed tensor did not write the " + + std::to_string(st->second.first) + " B the container's header states"); + } if (!f) throw std::runtime_error("short write into " + dir + "/model.q4nx"); } }; @@ -3647,6 +3906,48 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons embed_tbl = EmbedTable{embed_bf16.data(), EmbedTable::Fmt::BF16, int(H)}; w.add("model.embed_tokens.weight", "BF16", {V, H}, embed_bf16); ++r.tensors; + // engine#1725: a multi-component pack's embedding also carries the PER-LAYER table, its projection norm and the + // context projection. Written under the names the FORWARD loads (q4nx_forward.cpp's build_ple_inputs/load_ple_block), + // because the arithmetic there is already this model's: context = rmsnorm((emb @ W_l) * H**-0.5, norm), + // identity = 16 * table[id], per_layer_input = (context + identity)/sqrt(2). Absent for a single-graph pack. + { + std::string pl_why; + const EmbedPerLayer pl = read_embedding_per_layer(pack_dir, H, &pl_why); + if (pl.ok()) { + // The table is STREAMED into the container when it is written: V x (NL*P) f32 is 9.4 GiB on e2b and + // 4.7 GiB as BF16, so neither form is held -- the producer converts 32 rows at a time (573 KB there) + // out of the component's mapping, which the captured `pl` keeps alive until the write. + const uint64_t tbl_bytes = uint64_t(pl.V) * uint64_t(pl.width) * 2ull; + w.add_stream("model.per_layer_token_embd.weight", "BF16", {pl.V, pl.width}, tbl_bytes, + [pl](std::ostream& f) { + std::vector band(size_t(32) * size_t(pl.width) * 2); + std::string bw; + for (int64_t row = 0; row < pl.V; row += 32) { + const int64_t n = (pl.V - row < 32) ? (pl.V - row) : 32; + if (read_per_layer_table_band(pl, row, n, band.data(), &bw) != n) + throw std::runtime_error("model.per_layer_token_embd.weight: " + bw); + f.write(reinterpret_cast(band.data()), + std::streamsize(size_t(n) * size_t(pl.width) * 2)); + } + }); + if (!pl.norm.empty()) + w.add("model.per_layer_proj_norm.weight", "BF16", {pl.P}, + as_bf16(reinterpret_cast(pl.norm.data()), kF32, pl.norm.size())); + r.tensors += 1 + (pl.norm.empty() ? 0 : 1); + if (pl.complete()) { + for (int64_t l = 0; l < pl.NL; ++l) { + w.add("model.per_layer_model_proj.weight_layer" + std::to_string(l), "BF16", {pl.P, pl.H}, + as_bf16(reinterpret_cast(pl.model_proj.data() + + size_t(l) * size_t(pl.P) * size_t(pl.H)), + kF32, size_t(pl.P) * size_t(pl.H))); + ++r.tensors; + } + // what switches the block on in the container's config (written further down, by another writer) + r.per_layer_P = pl.P; + r.per_layer_V = pl.V; + } + } + } } // genre A = the graph carries real weights and there is no .fconst to read (the same test plan() uses) @@ -4556,6 +4857,33 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons c["vocab_size"] = p.dims.vocab; c["model_type"] = p.dims.model_type; c["rope_theta"] = p.dims.rope_theta; + // engine#1725: the PLE block is switched ON by `hidden_size_per_layer_input > 0` in the container's semantics + // (q4nx_semantics.c), so a container that carries the per-layer tensors but not these two keys would import them + // and never use them -- the forward skips the block silently. Taken from the component itself, which is where they + // are measured (gemma-4-e2b: 256 and 262144), rather than from the pack's config, which states them one level down. + // They come from the component, measured when its tensors were written above (gemma-4-e2b: 256 and 262144), and + // are set only when the whole block -- table, norm and context projection -- went into the container. + if (r.per_layer_P > 0 && r.per_layer_V > 0) { + c["hidden_size_per_layer_input"] = r.per_layer_P; + c["vocab_size_per_layer_input"] = r.per_layer_V; + } + // The Gemma-4 layer-shape switches the semantics reader also consults, carried through when the pack states them. + { + std::ifstream mc(pack_dir + "/model_config.json"); + if (mc) { + json m; + try { m = json::parse(mc); } catch (...) { m = json::object(); } + const json* t2 = nullptr; + if (m.is_object() && m.contains("config") && m["config"].is_object() && + m["config"].contains("model_attributes") && m["config"]["model_attributes"].is_object() && + m["config"]["model_attributes"].contains("text_config") && + m["config"]["model_attributes"]["text_config"].is_object()) + t2 = &m["config"]["model_attributes"]["text_config"]; + if (t2 != nullptr) + for (const char* k : {"num_kv_shared_layers", "use_double_wide_mlp", "extra_layer_norms"}) + if (t2->contains(k) && !(*t2)[k].is_null()) c[k] = (*t2)[k]; + } + } // A pack that ships its own precomputed RoPE (cos_cache/sin_cache) is not necessarily built for the theta // its genai_config.json states -- pi-inferzo measured 10000 declared against a model that needs 1e6 on // Qwen2.5 rai_1.7.1_hybrid, and our forward computes RoPE from theta rather than reading a cache. That is a diff --git a/npu/pack_import.h b/npu/pack_import.h index bb50c25ab..91a24a4b7 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -35,6 +35,8 @@ #include +#include +#include #include #include @@ -97,6 +99,54 @@ std::string qk_norm_refusal(bool declared, bool found); // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. std::vector read_embedding_component(const std::string& pack_dir, std::string* why); +// The per-layer half of a multi-component pack's embedding (engine#1725). gemma-4 components carry, beside the token +// table, `embed_tokens_per_layer.weight` (262144 x 8960 on e2b = layers x head_dim) and a 256-wide projection norm; +// the decoder consumes them as `per_layer_inputs`, so a container without them gives the decoder less than it asks for. +// Read separately from the token table because they have different shapes and different consumers: the token table +// becomes the head oracle, this becomes an input. Empty (with `why`) when the component has no per-layer table. +struct EmbedPerLayer { + // [V][layers*P] f32, row-major (embed_tokens_per_layer.weight) -- NOT copied. It is 9.4 GiB on gemma-4-e2b, so the + // reader hands out a pointer into the component's mapped external data and the importer converts it a band at a + // time. `keep` owns whatever that pointer lives in (the mapping, or a copy of an inline initializer) and unmaps it + // when the last copy of this struct goes. + const uint8_t* table_f32 = nullptr; + std::shared_ptr keep; + std::vector norm; // [P] (per_layer_projection_norm.weight, shared by every layer) + // The CONTEXT projection: the component carries it quantised as + // `per_layer_model_projection.weight_t_Q4` [layers*P][H] plus scales and packed zero points. The loader wants it + // dequantised, one [P][H] matrix per layer (`model.per_layer_model_proj.weight_layer{l}`), so it is dequantised here + // -- without it the per-layer input is `16*table/sqrt(2)`, plausible and wrong at every layer (engine#1725). + std::vector model_proj; // [layers][P][H] + int64_t V = 0, width = 0, P = 0, H = 0, NL = 0; + // `ok()` is what the reader guarantees: the table (bounds-checked against its data) and the projection norm. + // `complete()` additionally requires the context projection, which the PLE contract needs -- a component with + // `ok()` but not `complete()` imports the table and will be refused by the loader on + // `per_layer_model_proj.weight_layer0`, which is the honest place to refuse (the importer does not know whether the + // decoder declares per_layer_inputs). + bool ok() const { + return V > 0 && width > 0 && table_f32 != nullptr && (!norm.empty() ? norm.size() == size_t(P) : true); + } + bool complete() const { return ok() && P > 0 && H > 0 && NL > 0 && model_proj.size() == size_t(NL) * size_t(P) * size_t(H); } + // one value of the table, for tests and spot checks + float at(int64_t row, int64_t col) const { + float f; + std::memcpy(&f, table_f32 + (size_t(row) * size_t(width) + size_t(col)) * 4, 4); + return f; + } +}; +// `hidden` is the decoder's hidden size (the context projection's K); it comes from the pack's geometry, not from +// the component, which is why the caller passes it. Parses the component graph and maps its external data ONCE; the +// table's bytes are not read. +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why); + +// One band of the per-layer token table as BF16 (round to nearest even, the same `f32_to_bf16` every other tensor +// gets), straight from the mapping `meta` holds: no graph parse, no file open, nothing allocated. The importer +// streams the whole table through this in 32-row bands (573 KB at a time on gemma-4-e2b) while the container is +// written, so the table is never held -- not as f32, not as BF16 (engine#1725). +// Returns the rows written, or -1 with `why` set. `out` must hold `rows * width * 2` bytes. +int64_t read_per_layer_table_band(const EmbedPerLayer& meta, int64_t first_row, int64_t rows, uint8_t* out, + std::string* why); + // One tensor the pack describes, and where its values would come from. struct Planned { std::string name; // Q4NX tensor name (model.layers.N.<...>.weight) @@ -319,6 +369,8 @@ struct Report { // walk and the pack is tied). Empty means it was read, as usual. Carried on the Report because the config JSON is // built after the .fconst block that discovers this. std::string head_synthesized_note; + // engine#1725: the per-layer embedding block the container carries (0 = none): row width per layer, and rows + int64_t per_layer_P = 0, per_layer_V = 0; bool prefill_tail_seen = false; // a second copy was readable in the prefill .fconst bool final_norm_crosscheck = false; // ... and it is byte-identical to the Token_Token copy diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index b40a08fa1..acd95805f 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -3753,12 +3753,340 @@ void test_embedding_f32_component(const std::string& root) { std::string why2; const std::vector none = pi::read_embedding_component(b.string(), &why2); std::printf("f32 embedding (per-layer): %zu values, why: %s\n", none.size(), why2.c_str()); - check(none.empty() && why2.find("per_layer") != std::string::npos, - "f32 embedding: a per-layer table is refused by NAME, not reported as a missing table (" + why2 + ")"); + check(!none.empty(), + "f32 embedding: a per-layer table no longer blocks the TOKEN table (slice 2 handles it; " + why2 + ")"); fs::remove_all(a); fs::remove_all(b); } + +// engine#1725: a multi-component embedding carries a PER-LAYER table and its projection norm beside the token table, and +// the importer writes both into the container so the forward path can gather them. Measured shapes on +// amd/gemma-4-e2b-it-mobius-int4_rai_1.9.0_npu: embed_tokens.weight (262144,1536) f32, +// embed_tokens_per_layer.weight (262144,8960) f32, per_layer_projection_norm.weight (256,) -- all f32, no scales. +// This drives the writer on a crafted pack of that shape (V=4, width=6) and reads the container header back. +void test_per_layer_container(const std::string& root) { + const fs::path dir = fs::path(root) / "perlayer"; + fs::remove_all(dir); + fs::create_directories(dir / "embedding"); + const int64_t V = 4, width = 6; + // two-arg form: `norm(size_t(width))` parses as a function declaration and `norm{size_t(width)}` is an + // initializer_list of ONE element = width. Both mistakes produce a plausible-looking fixture. + std::vector tok(size_t(V) * 3, 0.0f), per(size_t(V) * size_t(width), 0.0f), norm(size_t(width), 0.0f); + for (size_t i = 0; i < tok.size(); ++i) tok[i] = float(i) * 0.5f; + for (size_t i = 0; i < per.size(); ++i) per[i] = float(int(i) - 8) * 0.25f; + for (size_t i = 0; i < norm.size(); ++i) norm[i] = 1.0f + float(i) * 0.125f; + std::string data; + const size_t tok_bytes = tok.size() * 4, per_bytes = per.size() * 4; + data.append(reinterpret_cast(tok.data()), tok_bytes); + data.append(reinterpret_cast(per.data()), per_bytes); + data.append(reinterpret_cast(norm.data()), norm.size() * 4); + put_file((dir / "embedding" / "model.onnx.data").string(), data); + put_file((dir / "embedding" / "model.onnx").string(), + pb::model({pb::tensor("embedding.embed_tokens.weight", {V, 3}, "model.onnx.data", 0, tok_bytes, 1), + pb::tensor("embedding.embed_tokens_per_layer.weight", {V, width}, "model.onnx.data", tok_bytes, + per_bytes, 1), + pb::tensor("embedding.per_layer_projection_norm.weight", {width}, "model.onnx.data", + tok_bytes + per_bytes, norm.size() * 4, 1)}, + {})); + put_file((dir / "genai_config.json").string(), + R"({"model":{"type":"gemma4","embedding":{"filename":"embedding/model.onnx"},)" + R"("decoder":{"filename":"decoder/model.onnx"}}})"); + put_file((dir / "config.json").string(), + R"({"model_type":"gemma4","hidden_size":3,"intermediate_size":6,"num_hidden_layers":1,)" + R"("num_attention_heads":1,"num_key_value_heads":1,"head_dim":3,"vocab_size":4,"rope_theta":10000.0})"); + std::string why; + const pi::EmbedPerLayer pl = pi::read_embedding_per_layer(dir.string(), 3, &why); + std::printf("per-layer: V %lld width %lld, why: %s (tok %zu per %zu norm %zu bytes)\n", (long long) pl.V, + (long long) pl.width, why.c_str(), tok_bytes, per_bytes, norm.size() * 4); + check(pl.ok(), "per-layer: the table and its projection norm are read from the component genai_config.json names"); + bool same = pl.ok() && pl.V == V && pl.width == width; + for (size_t i = 0; same && i < per.size(); ++i) same = pl.at(int64_t(i) / width, int64_t(i) % width) == per[i]; + for (size_t i = 0; same && i < norm.size(); ++i) same = pl.norm[i] == norm[i]; + check(same, "per-layer: the values come back exactly as the component states them"); + // the token table is still this reader's other half: it must NOT be refused because a per-layer table is present + std::string why2; + const std::vector t = pi::read_embedding_component(dir.string(), &why2); + check(t.size() == size_t(V) * 3, "per-layer: the token table still reads beside it (" + std::to_string(t.size()) + ")"); + + // #1825 review: the band converter had no arm, and it rounded "up iff the kept lsb is 1" -- half of all values off by + // one ulp, including values bf16 holds EXACTLY. per[] here is multiples of 0.25 (exact); the fixture below adds the + // three cases that separate the rules: an exact value with an odd kept lsb (1.0078125 = 0x3F81'0000 must stay + // 0x3F81), a tie on an even lsb (stays), a tie on an odd lsb (goes up to even), and one just above a tie. + { + const auto f32_of = [](uint32_t bits) { + float f; + std::memcpy(&f, &bits, 4); + return f; + }; + const fs::path d2 = fs::path(root) / "perlayer-band"; + fs::remove_all(d2); + fs::create_directories(d2 / "embedding"); + const int64_t V2 = 70, W2 = 5; // 70 rows: two whole 32-row bands and a 6-row one + std::vector per2(size_t(V2) * size_t(W2)); + for (size_t i = 0; i < per2.size(); ++i) per2[i] = 0.001f * float(int(i * 37 % 211) - 100); // inexact in bf16 + per2[0] = f32_of(0x3F810000u); // exact, odd lsb -> 0x3F81 + per2[1] = f32_of(0x3F808000u); // tie, even lsb -> 0x3F80 + per2[2] = f32_of(0x3F818000u); // tie, odd lsb -> 0x3F82 + per2[3] = f32_of(0x3F808001u); // just above a tie -> 0x3F81 + per2[size_t(69) * size_t(W2) + 4] = f32_of(0xBF810000u); // the last value of the short band, exact and negative + std::vector norm2(size_t(W2), 1.0f), tok2(size_t(V2) * 3, 0.5f); + std::string data2; + data2.append(reinterpret_cast(tok2.data()), tok2.size() * 4); + data2.append(reinterpret_cast(per2.data()), per2.size() * 4); + data2.append(reinterpret_cast(norm2.data()), norm2.size() * 4); + put_file((d2 / "embedding" / "model.onnx.data").string(), data2); + put_file((d2 / "embedding" / "model.onnx").string(), + pb::model({pb::tensor("embedding.embed_tokens.weight", {V2, 3}, "model.onnx.data", 0, tok2.size() * 4, 1), + pb::tensor("embedding.embed_tokens_per_layer.weight", {V2, W2}, "model.onnx.data", + tok2.size() * 4, per2.size() * 4, 1), + pb::tensor("embedding.per_layer_projection_norm.weight", {W2}, "model.onnx.data", + tok2.size() * 4 + per2.size() * 4, norm2.size() * 4, 1)}, + {})); + put_file((d2 / "genai_config.json").string(), + R"({"model":{"type":"gemma4","embedding":{"filename":"embedding/model.onnx"},)" + R"("decoder":{"filename":"decoder/model.onnx"}}})"); + std::string why3; + const pi::EmbedPerLayer p2 = pi::read_embedding_per_layer(d2.string(), 3, &why3); + check(p2.ok() && p2.V == V2 && p2.width == W2, "per-layer band: the table is located (" + why3 + ")"); + if (p2.ok()) { + // the reference is as_bf16-equivalent: pi::bf16_bits_of is the file's exported round-to-nearest-even + std::vector want(per2.size()); + for (size_t i = 0; i < per2.size(); ++i) want[i] = pi::bf16_bits_of(per2[i]); + check(want[0] == 0x3F81 && want[1] == 0x3F80 && want[2] == 0x3F82 && want[3] == 0x3F81, + "per-layer band: the reference rounds to nearest even on the four separating values"); + std::vector got(per2.size(), 0xFFFF); + bool bands = true; + for (int64_t row = 0; row < V2; row += 32) { + const int64_t n = std::min(32, V2 - row); + std::string bw; + bands = bands && pi::read_per_layer_table_band( + p2, row, n, reinterpret_cast(got.data() + size_t(row) * size_t(W2)), &bw) == n; + } + check(bands, "per-layer band: 32 + 32 + 6 rows read"); + size_t differ = 0; + for (size_t i = 0; i < got.size(); ++i) differ += got[i] != want[i]; + check(differ == 0, "per-layer band: every banded value is round-to-nearest-even bf16 (" + + std::to_string(differ) + " of " + std::to_string(got.size()) + " differ)"); + check(got[0] == 0x3F81, "per-layer band: an exactly representable value with an odd lsb is not moved"); + std::string bw; + std::vector scratch(size_t(W2) * 2 * 2); + check(pi::read_per_layer_table_band(p2, V2 - 1, 2, scratch.data(), &bw) == -1 && + pi::read_per_layer_table_band(p2, -1, 1, scratch.data(), &bw) == -1, + "per-layer band: a band past the table, or before it, is refused"); + } + // a table that declares more rows than its data carries is refused when it is LOCATED, not at some later band + put_file((d2 / "embedding" / "model.onnx").string(), + pb::model({pb::tensor("embedding.embed_tokens_per_layer.weight", {V2 + 1000, W2}, "model.onnx.data", + tok2.size() * 4, per2.size() * 4, 1)}, + {})); + std::string why4; + const pi::EmbedPerLayer p3 = pi::read_embedding_per_layer(d2.string(), 3, &why4); + check(!p3.ok() && why4.find("more bytes than its data carries") != std::string::npos, + "per-layer band: a table longer than its data is refused by the reader (" + why4 + ")"); + fs::remove_all(d2); + } + fs::remove_all(dir); +} + + + +// engine#1725, #1825 review: the per-layer block THROUGH import_pack(). The reader arms above call the helpers; nothing +// ran the writer, so the streamed table, the offsets of every tensor after it, the per-layer split of the context +// projection and the two config keys that switch the block on were all unpinned. The fixture is the tied block-32 one +// (it reaches the embedding writer under allow-incomplete) plus an embedding component shaped like gemma-4's: a table +// NL x P wide, a norm P wide (shared by the layers -- not `width` wide), and the context projection quantised at +// block 32, stored at a NON-ZERO offset of the external data. +void test_per_layer_import(const std::string& root) { + const fs::path dir = fs::path(root) / "perlayer-import"; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + fs::create_directories(dir / "embedding"); + const int H = 128, L = 8, V = 64; + const int64_t P = 4, NL = 2, W = P * NL; + { + const int64_t stride = 1 << 20, second = 262144; + std::string fc(size_t(int64_t(L) * stride + 64 + size_t(H) * 2), '\0'); + auto put_eps = [&](int64_t off) { + fc[size_t(off)] = char(0x86); + fc[size_t(off) + 1] = char(0x35); + }; + for (int l = 0; l < L; ++l) { + put_eps(int64_t(l) * stride); + put_eps(int64_t(l) * stride + second); + } + put_eps(int64_t(L) * stride); + for (int i = 0; i < H; ++i) { + const uint16_t bits = pi::bf16_bits_of(float(i + 1)); + fc[size_t(int64_t(L) * stride + 64 + 2 * i)] = char(bits & 0xFF); + fc[size_t(int64_t(L) * stride + 64 + 2 * i) + 1] = char(bits >> 8); + } + put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); + } + put_file((dir / "config.json").string(), + R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06})"); + { + std::string data(size_t(V) * size_t(H) * 2, '\0'); + for (int r = 0; r < V; ++r) + for (int c = 0; c < H; ++c) { + const uint16_t bits = pi::bf16_bits_of(0.05f * float((r * 7 + c * 13) % 97) - 2.4f); + data[(size_t(r) * H + c) * 2] = char(bits & 0xFF); + data[(size_t(r) * H + c) * 2 + 1] = char(bits >> 8); + } + put_file((dir / "model.onnx.data").string(), data); + } + put_file((dir / "cache" / "lm_head_meta.json").string(), + R"({"op_list":[{"type":"MladfMatMul","name":"lm_head.MatMulNBits",)" + R"("attrs":{"N":{"value":[64]},"K":{"value":[128]},"block_size":{"value":[32]},"bits":{"value":[4]}}}],)" + R"("tensor_map":{}})"); + put_file((dir / "model.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", 0, + uint64_t(V) * uint64_t(H) * 2, 10), + pb::placeholder("lm_head.weight.MatMulNBits.qweight"), + pb::placeholder("lm_head.weight.MatMulNBits.scales"), + pb::placeholder("lm_head.weight.MatMulNBits.zeros")}, + {})); + // the embedding component: table [V][W] f32, norm [P] f32, projection [W][H] int4 at block 32 + f32 scales + std::vector per(size_t(V) * size_t(W), 0.0f), norm(size_t(P), 0.0f), scales(size_t(W) * size_t(H / 32), 0.5f); + for (size_t i = 0; i < per.size(); ++i) per[i] = 0.003f * float(int(i * 53 % 409) - 200); + { + uint32_t odd = 0x3F810000u; // exact in bf16 with an odd kept lsb: must come back as 0x3F81 + std::memcpy(&per[0], &odd, 4); + } + for (size_t i = 0; i < norm.size(); ++i) norm[i] = 1.0f + 0.25f * float(i); + const std::string qw(size_t(W) * size_t(H) / 2, char(0xBB)); // every code 11 -> (11 - 8) x 0.5 = 1.5 + std::string data; + data.append(reinterpret_cast(per.data()), per.size() * 4); + const uint64_t norm_off = data.size(); + data.append(reinterpret_cast(norm.data()), norm.size() * 4); + const uint64_t qw_off = data.size(); + data += qw; + const uint64_t sc_off = data.size(); + data.append(reinterpret_cast(scales.data()), scales.size() * 4); + put_file((dir / "embedding" / "model.onnx.data").string(), data); + put_file((dir / "embedding" / "model.onnx").string(), + pb::model({pb::tensor("embedding.embed_tokens_per_layer.weight", {V, W}, "model.onnx.data", 0, per.size() * 4, 1), + pb::tensor("embedding.per_layer_projection_norm.weight", {P}, "model.onnx.data", norm_off, + norm.size() * 4, 1), + pb::tensor("embedding.per_layer_model_projection.weight_t_Q4", {W, int64_t(H / 32), 16}, + "model.onnx.data", qw_off, qw.size(), 2), + pb::tensor("embedding.per_layer_model_projection.weight_t_scales", {W * int64_t(H / 32)}, + "model.onnx.data", sc_off, scales.size() * 4, 1)}, + {})); + put_file((dir / "genai_config.json").string(), R"({"model":{"embedding":{"filename":"embedding/model.onnx"}}})"); + + const std::string out = + (fs::temp_directory_path() / ("oga_pack_import_perlayer." + std::to_string(::getpid()))).string(); + fs::remove_all(out); + pi::Options allow; + allow.allow_incomplete = true; + std::string why; + pi::Report r; + try { + r = pi::import_pack(dir.string(), out, allow); + } catch (const std::exception& e) { + why = e.what(); + } + check(why.empty(), "per-layer import: the pack imports (" + why.substr(0, 200) + ")"); + check(r.per_layer_P == P && r.per_layer_V == V, "per-layer import: the report carries P " + std::to_string(r.per_layer_P) + + " and V " + std::to_string(r.per_layer_V)); + if (fs::exists(out + "/model.q4nx")) { + const json tbl = container_table(out + "/model.q4nx"); + const std::vector blob = container_payload(out + "/model.q4nx"); + const auto u16 = [&](const std::string& name, size_t i) { + const auto offs = tbl[name]["data_offsets"].get>(); + return uint16_t(uint16_t(blob[size_t(offs[0]) + 2 * i]) | uint16_t(uint16_t(blob[size_t(offs[0]) + 2 * i + 1]) << 8)); + }; + const bool have = tbl.contains("model.per_layer_token_embd.weight") && tbl.contains("model.per_layer_proj_norm.weight") && + tbl.contains("model.per_layer_model_proj.weight_layer0") && + tbl.contains("model.per_layer_model_proj.weight_layer1") && + !tbl.contains("model.per_layer_model_proj.weight_layer2"); + check(have, "per-layer import: the container carries the table, the norm and one projection per layer (2)"); + if (have) { + const auto shape = tbl["model.per_layer_token_embd.weight"]["shape"].get>(); + const auto offs = tbl["model.per_layer_token_embd.weight"]["data_offsets"].get>(); + check(shape == std::vector({V, W}) && offs[1] - offs[0] == uint64_t(V) * uint64_t(W) * 2 && + tbl["model.per_layer_token_embd.weight"]["dtype"] == "BF16", + "per-layer import: the table is declared BF16 [V, NL*P] with exactly that many bytes"); + size_t differ = 0; + for (size_t i = 0; i < per.size(); ++i) differ += u16("model.per_layer_token_embd.weight", i) != pi::bf16_bits_of(per[i]); + check(differ == 0, "per-layer import: every value of the STREAMED table is round-to-nearest-even bf16 (" + + std::to_string(differ) + " of " + std::to_string(per.size()) + " differ)"); + check(u16("model.per_layer_token_embd.weight", 0) == 0x3F81, + "per-layer import: an exactly representable value with an odd lsb is not moved"); + // the tensors AFTER the streamed one: their offsets are only right if the stream wrote what the header says + bool norm_ok = tbl["model.per_layer_proj_norm.weight"]["shape"].get>() == std::vector({P}); + for (size_t i = 0; norm_ok && i < norm.size(); ++i) + norm_ok = u16("model.per_layer_proj_norm.weight", i) == pi::bf16_bits_of(norm[i]); + check(norm_ok, "per-layer import: the P-wide norm follows the streamed table intact (shared by the layers)"); + bool proj_ok = tbl["model.per_layer_model_proj.weight_layer1"]["shape"].get>() == + std::vector({P, int64_t(H)}); + for (size_t i = 0; proj_ok && i < size_t(P) * size_t(H); ++i) + proj_ok = u16("model.per_layer_model_proj.weight_layer0", i) == pi::bf16_bits_of(1.5f) && + u16("model.per_layer_model_proj.weight_layer1", i) == pi::bf16_bits_of(1.5f); + check(proj_ok, "per-layer import: the context projection is read at ITS offset of the external data and split " + "per layer ((11 - 8) x 0.5 = 1.5 everywhere)"); + } + std::ifstream cf(out + "/config.json"); + check(cf.good(), "per-layer import: the container carries a config.json"); + if (cf.good()) { + const json cfg = json::parse(cf); + check(cfg.value("hidden_size_per_layer_input", int64_t(0)) == P && cfg.value("vocab_size_per_layer_input", int64_t(0)) == V, + "per-layer import: config.json switches the block on (hidden_size_per_layer_input 4, vocab 64)"); + } + } else { + check(false, "per-layer import: no container was written"); + } + fs::remove_all(out); + fs::remove_all(dir); +} + +// engine#1503: a 1.9 fconst whose blocks are NOT uniform (the mobius line). `fconst19_detect` requires one stride for the +// whole file, so it fails -- and the failure used to be indistinguishable from "not a 1.9 fconst at all". The refusal now +// names the structure and says the rule table cannot describe it. Measured on gemma-4-e2b: 35 blocks of 8 anchors, sizes +// 21.87-288.06 MiB. +void test_fconst_variable_stride(const std::string& root) { + const fs::path dir = fs::path(root) / "fconst-varstride"; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + const int H = 128, L = 4, heads = 2; + const int64_t hdr = heads * 4; + // four blocks, each header then a body of a DIFFERENT size: 64 KiB, 96 KiB, 128 KiB, 160 KiB + const int64_t bodies[4] = {64 << 10, 96 << 10, 128 << 10, 160 << 10}; + std::string fc; + for (int l = 0; l < L; ++l) { + for (int h = 0; h < heads; ++h) { // 00 00 7f ff per head + fc.push_back('\0'); fc.push_back('\0'); fc.push_back(char(0x7f)); fc.push_back(char(0xff)); + } + fc.append(size_t(bodies[l]), '\0'); + } + check(fc.size() > size_t(L * hdr), "variable stride: the crafted fconst is larger than its headers"); + put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); + put_file((dir / "config.json").string(), + R"({"model_type":"gemma4","hidden_size":128,"intermediate_size":256,"num_hidden_layers":4,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,"rope_theta":10000.0})"); + put_file((dir / "model.onnx").string(), pb::model({}, {})); + put_file((dir / "model.onnx.data").string(), std::string(4096, '\x11')); + pi::Options allow; + allow.allow_incomplete = true; + const std::string out = (fs::temp_directory_path() / ("varstride_out." + std::to_string(::getpid()))).string(); + fs::remove_all(out); + bool refused = false; + std::string what; + try { + (void) pi::import_pack(dir.string(), out, allow); + } catch (const std::exception& e) { + what = e.what(); + refused = what.find("variable-stride") != std::string::npos || what.find("NOT uniform") != std::string::npos; + } + std::printf("variable stride: %s\n", refused ? "refused by name (variable-stride container)" : what.substr(0, 140).c_str()); + check(refused, "variable stride: a 1.9 fconst whose blocks differ in size is refused BY NAME, not as 'not a 1.9 fconst'"); + fs::remove_all(out); + fs::remove_all(dir); +} + // engine#1740 (W1a): the tail-head placement arithmetic, pinned without a pack, a device or a fixture. The rows, the // tile size and the envelope are the hidden-896 pack's, so the numbers here are the real ones: 1,856 B envelope, // 2,374 tiles of 122,880 B, tail 1,856 + 2,374 x 122,880 = 291,653,952 B. @@ -3907,6 +4235,7 @@ int main(int argc, char** argv) { test_attn_tile(golden_dir); test_mlp_gu(golden_dir); test_fconst_eps(golden_dir.empty() ? "." : golden_dir); + test_fconst_variable_stride(golden_dir.empty() ? "." : golden_dir); test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); @@ -3923,6 +4252,8 @@ int main(int argc, char** argv) { test_im_from_node_attrs(golden_dir.empty() ? "." : golden_dir); test_fconst_in_component(golden_dir.empty() ? "." : golden_dir); test_embedding_in_component(golden_dir.empty() ? "." : golden_dir); + test_per_layer_container(golden_dir.empty() ? "." : golden_dir); + test_per_layer_import(golden_dir.empty() ? "." : golden_dir); test_embedding_f32_component(golden_dir.empty() ? "." : golden_dir); test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir);