diff --git a/docs/npu.md b/docs/npu.md index bfc37430e..ef85b0523 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1840,6 +1840,18 @@ runs unchanged. Rebuilt this way, the Llama file is the 1.8 twin's size. Every t least 96.4 % byte-equal to the twin at its mapped place (a one-unit misplacement scores 8.5 %), and the rest is the two exports' re-quantization. +**The 1.7 `Gemma-3-4b-it-mm-onnx-ryzenai-npu` embedding (engine#2663).** `genai_config.json` names the embedding +graph `gemma-3-embedding.onnx`. It is a `Gather` over **`lm_head.weight` [262208, 2560] f32** (the model is tied; the +decoder's 238 `MatMulNBitsBf` are exactly 34 layers x 7, with no head), followed by a multiply by the 0-D +`model.language_model.embed_tokens.embed_scale`. When no `*embed_tokens*` table is named, the reader takes the +component's only 2-D f32 initializer as the table; with two candidates it guesses neither. A 0-D tensor is never taken +as the 4-bit table's scales (those are 2-D `[V, groups]`). The container stores a gemma embedding pre-scaled: the +forward's lookup applies no gemma scale, and its tied head divides sqrt(H) back out. So `embed_scale` is applied to +the table, and only when it equals sqrt(hidden); any other value is refused by name. A small inline initializer +payload (64 B or less) is now copied when the graph is parsed, because the graph's mapping is gone by the time the +reader looks. Inline tables larger than that still point into the unmapped file (engine#2668); the packs measured so far keep +their tables in external data. + With `config.json` empty, the model's numbers come from the pack itself: - **rms_norm_eps** from the graph's RMSNorm nodes (18 nodes state 1e-5). The default was 1e-6. When the nodes state different epsilons nothing is taken, and the import prints a WARNING naming the conflict and the value it keeps (`graph_eps_decision()`). - **RoPE** from `cos_cache`. The theta fit (500000) looks only at the first 4,096 positions, where llama3 scaling is diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index fccdc184b..3f166f78e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -134,6 +134,10 @@ struct OnnxTensor { uint64_t offset = 0, length = 0; const uint8_t* raw = nullptr; size_t raw_len = 0; + // engine#2663: `raw` points into the graph file's mapping, which parse_onnx() unmaps before it returns, so it is only + // valid inside the parse. A small inline payload (a scalar such as gemma-3's embed_scale) is copied here and stays + // readable for the tensor's lifetime. + std::vector small; }; struct OnnxMatMul { @@ -204,6 +208,7 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m, const std::string& bra if (const auto* d = field1(t, 9)) { x.raw = d->p; x.raw_len = d->n; + if (d->n <= 64) x.small.assign(d->p, d->p + d->n); } for (const auto* d : fields(t, 13)) { const auto kv = pb(d->p, d->n); @@ -1563,13 +1568,27 @@ std::vector read_embedding_component(const std::string& pack_dir, std::st } const OnnxTensor* table = nullptr; const OnnxTensor* scales = nullptr; + const OnnxTensor* embed_scale = nullptr; // engine#2663: a 0-D multiplier the component applies after its Gather for (const auto& t : em.tensors) { const bool emb = t.name.find("embed_tokens") != std::string::npos; if (!emb) continue; - if (t.name.find("scale") != std::string::npos) scales = &t; + // engine#2663: `...embed_tokens.embed_scale` is a 0-D f32 (Gemma-3-4b-it-mm 1.7), not the 4-bit table's scales, + // which are 2-D [V, groups]. Taking it as scales left the reader with "no table + scales". + if (t.dims.empty() && t.name.find("embed_scale") != std::string::npos) { embed_scale = &t; continue; } + if (t.name.find("scale") != std::string::npos) { if (t.dims.size() == 2) scales = &t; } else if (t.name.size() > 3 && t.name.compare(t.name.size() - 3, 3, "_Q4") == 0) table = &t; else if (!table && t.dims.size() == 2) table = &t; // an unquantised [V, H] table } + // engine#2663: Gemma-3-4b-it-mm's component keeps its table as `lm_head.weight` [262208, 2560] f32 (the model is tied, + // and the decoder has no head matmul). When no `embed_tokens` table is named, the component's ONLY 2-D f32 initializer + // is the table; with two candidates nothing is guessed. + if (!table && !scales) { + const OnnxTensor* only = nullptr; + int n2 = 0; + for (const auto& t : em.tensors) + if (t.dims.size() == 2 && t.type == kF32) { only = &t; ++n2; } + if (n2 == 1) table = only; + } // engine#1725: the gemma-4 components are f32 and UNQUANTISED -- an `embed_tokens` table with no `*scales*` beside // it -- while the shape above is the 4-bit packed one another family ships. Reading the f32 table is worth doing // (the container wants BF16 and the conversion is exact); claiming the pack is not, because its decoder also carries @@ -1612,6 +1631,27 @@ std::vector read_embedding_component(const std::string& pack_dir, std::st if (raw == nullptr) { if (why) *why = "an f32 embed_tokens table with no data"; return {}; } std::vector out(size_t(Vf) * size_t(Hf)); std::memcpy(out.data(), raw, out.size() * sizeof(float)); + // engine#2663: the container stores the embedding PRE-SCALED (the forward's lookup applies no gemma scale and its + // tied head divides sqrt(H) back out; q4nx_forward.cpp), so a component that multiplies its raw table by a 0-D + // embed_scale is applied here -- and only when that scale is the sqrt(H) the forward will divide out. + if (embed_scale) { + float v = 0.0f; + if (embed_scale->type != kF32 || embed_scale->small.size() != 4) { + if (why) *why = "embed_scale is not an inline f32 scalar, so the table's scale is unknown (engine#2663)"; + return {}; + } + std::memcpy(&v, embed_scale->small.data(), 4); + const double want = std::sqrt(double(Hf)); + if (!(std::fabs(double(v) - want) <= 1e-3 * want)) { + if (why) *why = "embed_scale is " + std::to_string(v) + ", not sqrt(hidden) = " + std::to_string(want) + + ", which is the only scale the forward divides back out (engine#2663)"; + return {}; + } + for (auto& x : out) x *= v; + std::fprintf(stderr, "note: the embedding component's table is %s, scaled by its embed_scale %.6g (= sqrt(%lld)) " + "into the container's pre-scaled form (engine#2663)\n", + table->name.c_str(), double(v), (long long)Hf); + } return out; } if (!table || !scales) { if (why) *why = "embedding.onnx has no embed_tokens table + scales"; return {}; } diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index cdc41868b..6c247d1c4 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5417,6 +5417,102 @@ void test_embedding_in_component(const std::string& root) { } +// engine#2663: Gemma-3-4b-it-mm 1.7's component -- the table is `lm_head.weight` f32 (tied; no `embed_tokens` name) and a +// 0-D `model.language_model.embed_tokens.embed_scale` multiplies it. The table is read and scaled by embed_scale into the +// container's pre-scaled form; a scale other than sqrt(H) is refused; two unnamed 2-D candidates are not guessed between; +// and the control without embed_scale reads the raw table. +void test_embedding_gemma3mm_component(const std::string& root) { + const int64_t V = 8, H = 256; // sqrt(256) = 16, exact in f32 + std::vector tok(size_t(V) * size_t(H)); + for (size_t i = 0; i < tok.size(); ++i) tok[i] = float(int(i % 97) - 48) * 0.25f; + const auto scalar = [](const std::string& name, float v) { + std::string t, raw(4, '\0'); + std::memcpy(&raw[0], &v, 4); + pb::ints(t, 2, 1); // data_type FLOAT, no dims: 0-D + pb::str(t, 8, name); + pb::str(t, 9, raw); // raw_data + return t; + }; + const auto build = [&](const fs::path& dir, bool with_scale, float scale, bool second_table) { + fs::remove_all(dir); + fs::create_directories(dir); + std::string data(reinterpret_cast(tok.data()), tok.size() * sizeof(float)); + std::vector inits{pb::tensor("lm_head.weight", {V, H}, "gemma-3-embedding.onnx.data", 0, data.size(), 1)}; + if (second_table) { + const size_t off = data.size(); + data.append(reinterpret_cast(tok.data()), tok.size() * sizeof(float)); + inits.push_back(pb::tensor("other.weight", {V, H}, "gemma-3-embedding.onnx.data", off, tok.size() * 4, 1)); + } + if (with_scale) inits.push_back(scalar("model.language_model.embed_tokens.embed_scale", scale)); + put_file((dir / "gemma-3-embedding.onnx.data").string(), data); + put_file((dir / "gemma-3-embedding.onnx").string(), pb::model(inits, {})); + put_file((dir / "genai_config.json").string(), + R"({"model":{"type":"gemma3","embedding":{"filename":"gemma-3-embedding.onnx"},)" + R"("decoder":{"filename":"model.onnx"}}})"); + }; + const fs::path dir = fs::path(root) / ("g3mm-embed." + std::to_string(::getpid())); + { + build(dir, true, 16.0f, false); + std::string why; + const std::vector got = pi::read_embedding_component(dir.string(), &why); + bool scaled = got.size() == tok.size(); + for (size_t i = 0; scaled && i < tok.size(); ++i) scaled = got[i] == tok[i] * 16.0f; + check(scaled, "gemma3-mm embedding: lm_head.weight is the table and embed_scale (16 = sqrt(256)) is applied (" + + std::to_string(got.size()) + " values, " + why + ")"); + } + { + build(dir, false, 0.0f, false); + std::string why; + const std::vector got = pi::read_embedding_component(dir.string(), &why); + check(got == tok, "gemma3-mm embedding: control -- without embed_scale the raw table is read (" + why + ")"); + } + { + build(dir, true, 10.0f, false); + std::string why; + check(pi::read_embedding_component(dir.string(), &why).empty() && why.find("embed_scale is") != std::string::npos, + "gemma3-mm embedding: an embed_scale that is not sqrt(H) is refused by name (" + why + ")"); + } + { + build(dir, true, 16.0f, true); + std::string why; + check(pi::read_embedding_component(dir.string(), &why).empty(), + "gemma3-mm embedding: two unnamed 2-D f32 tables are not guessed between (" + why + ")"); + } + fs::remove_all(dir); + // #2664 review: at H = 256 the scale the arm quotes (16) IS sqrt(256), so a hardcoded 16 would pass. The real pack's + // width, 2560: sqrt(2560) = 50.596... is accepted and applied, and 16 is refused naming the sqrt it wanted. + { + const int64_t V2 = 4, H2 = 2560; + std::vector t2(size_t(V2) * size_t(H2)); + for (size_t i = 0; i < t2.size(); ++i) t2[i] = float(int(i % 31) - 15) * 0.125f; + const float s2 = float(std::sqrt(2560.0)); + const auto build2 = [&](float scale) { + fs::remove_all(dir); + fs::create_directories(dir); + const std::string data(reinterpret_cast(t2.data()), t2.size() * sizeof(float)); + put_file((dir / "gemma-3-embedding.onnx.data").string(), data); + put_file((dir / "gemma-3-embedding.onnx").string(), + pb::model({pb::tensor("lm_head.weight", {V2, H2}, "gemma-3-embedding.onnx.data", 0, data.size(), 1), + scalar("model.language_model.embed_tokens.embed_scale", scale)}, + {})); + put_file((dir / "genai_config.json").string(), + R"({"model":{"type":"gemma3","embedding":{"filename":"gemma-3-embedding.onnx"}}})"); + }; + build2(s2); + std::string why; + const std::vector got = pi::read_embedding_component(dir.string(), &why); + bool ok = got.size() == t2.size(); + for (size_t i = 0; ok && i < t2.size(); ++i) ok = got[i] == t2[i] * s2; + check(ok, "gemma3-mm embedding: at H 2560 the scale sqrt(2560) = 50.596 is accepted and applied (" + why + ")"); + build2(16.0f); + std::string why16; + check(pi::read_embedding_component(dir.string(), &why16).empty() && why16.find("50.59") != std::string::npos, + "gemma3-mm embedding: at H 2560 a scale of 16 is refused, naming sqrt(hidden) = 50.59 (" + why16 + ")"); + fs::remove_all(dir); + } +} + + // engine#1725: the gemma-4 component is f32 and unquantised -- an `embed_tokens` table with no scales beside it -- and its // decoder also needs a PER-LAYER table. Two arms: the f32 token table alone is read, and a per-layer table is refused by // name rather than reported as a missing table. Measured sizes on amd/gemma-4-e2b-it-mobius-int4_rai_1.9.0_npu: @@ -7350,6 +7446,7 @@ int main(int argc, char** argv) { test_per_layer_container(golden_dir.empty() ? "." : golden_dir); test_per_layer_import(golden_dir.empty() ? "." : golden_dir); test_embedding_f32_component(golden_dir.empty() ? "." : golden_dir); + test_embedding_gemma3mm_component(golden_dir.empty() ? "." : golden_dir); test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); test_tied_block32_head(golden_dir.empty() ? "." : golden_dir);