diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index eb8118c4d..d80030732 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -672,6 +672,93 @@ DeclaredMap read_declared(const std::string& pack_dir) { return out; } + +} // namespace: read_embedding_component is declared in pack_import.h so a test can drive it + +// ── the embedding component ─────────────────────────────────────────────────── +// A hybrid pack keeps its embedding table in its OWN graph: `embedding.onnx` (+ `embedding.onnx.data`) holds +// `language_model.embed_tokens.weight_Q4` [V, H/2] UINT8 and `..._weight_scales` [V, groups] FLOAT, gathered by +// `GatherBlockQuantized`. The main graph then has no `model.embed_tokens.weight` at all, which is what the importer +// used to refuse on (measured on amd/Qwen3-VL-4B-Instruct_rai_1.9.0_npu: 151936 x 1280 UINT8 + 151936 x 20 FLOAT, +// i.e. 4-bit, group 128, hidden 2560; engine #1503). +// +// Returns [V, H] f32 (row-major), empty when the pack has no such component. Everything it needs already existed: +// walk_graph() reads the graph with its external-data declarations, decode_matmulnbits() is block-parameterised, and +// the scale-per-32-column-group layout is the container's own. +std::vector read_embedding_component(const std::string& pack_dir, std::string* why) { + const std::string gpath = pack_dir + "/embedding.onnx"; + std::error_code ec; + if (!fs::is_regular_file(gpath, ec)) { if (why) *why = "no embedding.onnx"; return {}; } + // parse_onnx() unwraps the ModelProto's graph field and walks it; walk_graph() alone expects the GRAPH protobuf, + // which is why handing it the whole file yields zero tensors. + OnnxModel em; + try { + em = parse_onnx(gpath); + } catch (const std::exception& e) { + if (why) *why = e.what(); + return {}; + } + const OnnxTensor* table = nullptr; + const OnnxTensor* scales = nullptr; + for (const auto& t : em.tensors) { + const bool emb = t.name.find("embed_tokens") != std::string::npos; + if (!emb) continue; + if (t.name.find("scale") != std::string::npos) scales = &t; + else if (t.name.size() > 3 && t.name.compare(t.name.size() - 3, 3, "_Q4") == 0) table = &t; + else if (!table && t.dims.size() == 2) table = &t; // an unquantised [V, H] table + } + if (!table || !scales) { if (why) *why = "embedding.onnx has no embed_tokens table + scales"; return {}; } + if (table->dims.size() != 2 || scales->dims.size() != 2) { if (why) *why = "embed_tokens dims are not 2-D"; return {}; } + const int64_t V = table->dims.at(0), packed = table->dims.at(1), groups = scales->dims.at(1); + if (V <= 0 || packed <= 0 || groups <= 0) { if (why) *why = "embed_tokens dims are not positive"; return {}; } + const int64_t H = packed * 2; // 4-bit: two codes per byte + if (H % groups || (H / groups) % 32) { if (why) *why = "embed_tokens group " + std::to_string(H / groups) + " is not a multiple of 32"; return {}; } + // the component's own external data, bounds-checked the same way the main graph's is + std::map ext; + std::map ext_size; + for (const OnnxTensor* t : {table, scales}) { + if (t->location.empty() || ext.count(t->location)) continue; + size_t sz = 0; + std::string w2; + const uint8_t* m = map_file(pack_dir + "/" + t->location, sz, &w2); + if (!m) { if (why) *why = w2; return {}; } + ext[t->location] = m; + ext_size[t->location] = sz; + } + auto get = [&](const OnnxTensor& t, uint64_t need) -> const uint8_t* { + const auto it = ext.find(t.location); + if (it == ext.end()) throw std::runtime_error(t.name + ": no data"); + if (!ext_range_ok(ext_size[t.location], t.offset, t.length)) + throw std::runtime_error(t.name + ": external data past the end of " + t.location); + if (need > t.length) + throw std::runtime_error(t.name + ": needs " + std::to_string(need) + " B, declares " + + std::to_string(t.length)); + return it->second + t.offset; + }; + const uint64_t n_scales = uint64_t(V) * uint64_t(groups); + if (scales->type != kF32) { if (why) *why = "embed_tokens scales are not f32"; return {}; } + const uint8_t* sc = get(*scales, n_scales * 4); + const uint8_t* qw = get(*table, uint64_t(V) * uint64_t(packed)); + const int block = int(H / groups); + // Two conventions, both settled against ONNX Runtime running this pack's own embedding.onnx rather than assumed: + // the first element of a pair is the LOW nibble (`swap_nibbles = false`), and the codes are **asymmetric int4 with + // zero point 8**, i.e. value = (code - 8) * scale. Measured on token 1 of + // amd/Qwen3-VL-4B-Instruct_rai_1.9.0_npu: low-first + zp 8 reproduces ORT's row with cosine 1.000000 and max|d| 0, + // while low-first + zp 0 gives cosine 0.317 and either high-first variant gives ~-0.05. + const int zp = 8; // 2^(bits-1) for GatherBlockQuantized's unsigned 4-bit codes + const Q4 q = decode_matmulnbits(qw, reinterpret_cast(sc), nullptr, int(V), int(H), block, false, true); + std::vector out(size_t(V) * size_t(H)); + for (int64_t v = 0; v < V; ++v) + for (int64_t k = 0; k < H; ++k) { + const size_t gi = size_t(v) * size_t(H / 32) + size_t(k) / 32; + out[size_t(v) * size_t(H) + size_t(k)] = + float(int(q.codes[size_t(v) * size_t(H) + size_t(k)]) - zp) * q.scale[gi]; + } + return out; +} + +namespace { + // ── the pack's geometry, from wherever this generation put it ──────────────── // Ryzen AI 1.6 ships `config.json = {}` and 1.7/1.7.1 ship **no config.json at all**; all of them carry // genai_config.json with the decoder block. So the geometry comes from config.json when it is usable and from @@ -1380,11 +1467,26 @@ Plan plan(const std::string& pack_dir) { const OnnxTensor* embed = nullptr; for (const auto& t : onnx.tensors) if (t.name == "model.embed_tokens.weight") embed = &t; - if (!embed) throw std::runtime_error("the pack has no model.embed_tokens.weight"); - if (d.vocab == 0) p.dims.vocab = int(embed->dims.at(0)); - const int64_t H = embed->dims.at(1); - p.tensors.push_back(make("model.embed_tokens.weight", embed->location, "BF16", - {embed->dims.at(0), H})); + if (!embed) { + // A hybrid pack keeps the table in its own graph; if that component is there, this pack HAS an embedding and the + // refusal below would be wrong about it. + std::string emb_why; + const std::vector comp = read_embedding_component(pack_dir, &emb_why); + if (comp.empty()) + throw std::runtime_error("the pack has no model.embed_tokens.weight, and " + emb_why); + const int64_t Vc2 = d.vocab > 0 ? d.vocab : 0, Hc2 = d.hidden > 0 ? d.hidden : 0; + if (d.vocab == 0 || d.hidden == 0) + throw std::runtime_error("the pack's embedding comes from embedding.onnx but the geometry does not give V/H"); + if (comp.size() != size_t(Vc2) * size_t(Hc2)) + throw std::runtime_error("embedding.onnx carries " + std::to_string(comp.size()) + " values, not V*H = " + + std::to_string(int64_t(Vc2) * Hc2)); + p.tensors.push_back(make("model.embed_tokens.weight", "embedding.onnx", "BF16", {Vc2, Hc2})); + } + if (embed) { + if (d.vocab == 0) p.dims.vocab = int(embed->dims.at(0)); + p.tensors.push_back(make("model.embed_tokens.weight", embed->location, "BF16", + {embed->dims.at(0), embed->dims.at(1)})); + } for (const auto& mm : onnx.matmuls) { const auto it = onnx.index.find(mm.weight); @@ -1405,6 +1507,7 @@ Plan plan(const std::string& pack_dir) { // refusal here is empty and import_pack reports what the anchors actually found. const std::string norm_source = "fconst:Llm_Token_Token"; const std::string norm_refusal; + const int64_t H = d.hidden; // the geometry's hidden; the embedding, when the main graph has none, must match it // Genre A: the norms are ordinary F16 initializers named after themselves, with real offsets -- not the // .fconst tail envelope -- so the refusal names the initializer and where it sits. auto graph_norm_refusal = [&](const std::string& name) -> std::string { @@ -1909,19 +2012,38 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons const OnnxTensor* embed = nullptr; for (const auto& t : onnx.tensors) if (t.name == "model.embed_tokens.weight") embed = &t; - if (!embed) throw std::runtime_error("no model.embed_tokens.weight"); - if (embed->dims.size() != 2) throw std::runtime_error("model.embed_tokens.weight is not 2-D"); - const int64_t V = embed->dims.at(0), H = embed->dims.at(1); - const uint64_t Vc = dim_count("model.embed_tokens.weight", V), Hc = dim_count("model.embed_tokens.weight", H); - uint64_t elems = 0; - if (!mul_ok(Vc, Hc, elems)) throw std::runtime_error("model.embed_tokens.weight: V*H overflows"); - const uint64_t src_width = embed->type == kF32 ? 4 : 2; - uint64_t embed_need = 0; - if (!mul_ok(elems, src_width, embed_need)) - throw std::runtime_error("model.embed_tokens.weight: V*H*width overflows"); - const uint8_t* embed_bytes = bytes_of(*embed, embed_need); - w.add("model.embed_tokens.weight", "BF16", {V, H}, as_bf16(embed_bytes, embed->type, size_t(elems))); - ++r.tensors; + if (embed && embed->dims.size() != 2) throw std::runtime_error("model.embed_tokens.weight is not 2-D"); + // V/H are read further down for the tied-head and vocab checks, so they exist whether the table came from the main + // graph or from the embedding component (where the geometry states them instead). + const int64_t V = embed ? embed->dims.at(0) : int64_t(p.dims.vocab); + const int64_t H = embed ? embed->dims.at(1) : int64_t(p.dims.hidden); + // The container takes BF16, and the tied-head self-check reads these same bytes, so convert once into a buffer that + // outlives both. For the component path the source is f32 that the reader already dequantised. + std::vector embed_bf16; + const uint8_t* embed_bytes = nullptr; + { + const uint64_t Vc = dim_count("model.embed_tokens.weight", V), Hc = dim_count("model.embed_tokens.weight", H); + uint64_t elems = 0; + if (!mul_ok(Vc, Hc, elems)) throw std::runtime_error("model.embed_tokens.weight: V*H overflows"); + if (embed) { + const uint64_t src_width = embed->type == kF32 ? 4 : 2; + uint64_t embed_need = 0; + if (!mul_ok(elems, src_width, embed_need)) + throw std::runtime_error("model.embed_tokens.weight: V*H*width overflows"); + embed_bf16 = as_bf16(bytes_of(*embed, embed_need), embed->type, size_t(elems)); + } else { + std::string emb_why; + const std::vector comp = read_embedding_component(pack_dir, &emb_why); + if (comp.empty()) throw std::runtime_error("no model.embed_tokens.weight, and " + emb_why); + if (comp.size() != size_t(elems)) + throw std::runtime_error("embedding.onnx carries " + std::to_string(comp.size()) + " values, not V*H = " + + std::to_string(elems)); + embed_bf16 = as_bf16(reinterpret_cast(comp.data()), kF32, size_t(elems)); + } + embed_bytes = embed_bf16.data(); + w.add("model.embed_tokens.weight", "BF16", {V, H}, embed_bf16); + ++r.tensors; + } // genre A = the graph carries real weights and there is no .fconst to read (the same test plan() uses) auto has_real_graph_weight = [&]() { diff --git a/npu/pack_import.h b/npu/pack_import.h index 87306bdf4..8d70fdb0c 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -69,6 +69,11 @@ struct Dims { std::vector eos; }; +// A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with +// GatherBlockQuantized). Returns [V, H] f32 row-major, empty when the pack has no such component, with `why` naming the +// reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. +std::vector read_embedding_component(const std::string& pack_dir, std::string* why); + // One tensor the pack describes, and where its values would come from. struct Planned { std::string name; // Q4NX tensor name (model.layers.N.<...>.weight) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index c66a66d34..f00751cd1 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -121,10 +121,10 @@ std::string kv(const std::string& k, const std::string& v) { } // a tensor whose values live in an external file, optionally with dims that overstate it std::string tensor(const std::string& name, const std::vector& dims, const std::string& location, - uint64_t offset, uint64_t length) { + uint64_t offset, uint64_t length, int64_t type = 1) { std::string t; for (auto d : dims) ints(t, 1, d); // dims - ints(t, 2, 1); // data_type (FLOAT; the importer only uses it for width) + ints(t, 2, type); // data_type: FLOAT unless the caller says otherwise (a quantised table is UINT8) str(t, 8, name); for (const auto& [k, v] : {std::pair{"location", location}, {"offset", std::to_string(offset)}, {"length", std::to_string(length)}}) { @@ -1490,6 +1490,57 @@ void test_attn_payload_off() { // the Token program's op_list) and declares `o_proj N=2560 K=4096` on the 4B, i.e. o_proj's K is // q_proj's output width, not hidden. The two 256-byte gains match Qwen/Qwen3-1.7B's and -4B's public // q_norm/k_norm weights bit for bit at those offsets (128 of 128 values, and 127-128 of 128 on the swap). + +// The embedding component (engine #1503): a hybrid pack keeps its table in `embedding.onnx` (+ `.data`), gathered by +// GatherBlockQuantized. Shape measured on amd/Qwen3-VL-4B-Instruct_rai_1.9.0_npu: weight_Q4 [V, H/2] UINT8 and +// weight_scales [V, groups] FLOAT with group 128. Driving the reader against known codes and scales pins the +// dequantisation without a 3 GB pack. +void test_embedding_component(const std::string& root) { + const fs::path dir = fs::path(root) / "embedcomp"; + fs::remove_all(dir); + fs::create_directories(dir); + const int64_t V = 8, H = 256, groups = H / 128; + auto want = [&](int64_t v, int64_t k) { return uint8_t((v * 7 + k * 5) % 16); }; + std::vector table(size_t(V) * size_t(H / 2), 0); + std::vector scales(size_t(V) * size_t(groups), 0.0f); + for (int64_t v = 0; v < V; ++v) { + for (int64_t g = 0; g < groups; ++g) scales[size_t(v) * groups + size_t(g)] = 0.5f * float(g + 1); + for (int64_t k = 0; k < H; k += 2) + table[size_t(v) * size_t(H / 2) + size_t(k / 2)] = uint8_t(want(v, k) | (want(v, k + 1) << 4)); + } + std::string data; + data.append(reinterpret_cast(table.data()), table.size()); + const size_t table_bytes = table.size(); + data.append(reinterpret_cast(scales.data()), scales.size() * sizeof(float)); + put_file((dir / "embedding.onnx.data").string(), data); + const auto t = pb::tensor("language_model.embed_tokens.weight_Q4", {V, H / 2}, "embedding.onnx.data", 0, + table_bytes, 2); + const auto sc = pb::tensor("language_model.embed_tokens.weight_scales", {V, groups}, "embedding.onnx.data", + table_bytes, scales.size() * sizeof(float), 1); + put_file((dir / "embedding.onnx").string(), pb::model({t, sc}, {})); + + std::string why; + const std::vector got = pi::read_embedding_component(dir.string(), &why); + check(got.size() == size_t(V) * size_t(H), + "embedding component: reads V*H values (" + std::to_string(got.size()) + ", why: " + why + ")"); + bool ok = got.size() == size_t(V) * size_t(H); + for (int64_t v = 0; v < V && ok; ++v) + for (int64_t k = 0; k < H; ++k) { + // asymmetric int4: the codes are centred on 8, which is what ONNX Runtime's GatherBlockQuantized does + const float exp = (float(want(v, k)) - 8.0f) * 0.5f * float(k / 128 + 1); + if (std::fabs(got[size_t(v) * size_t(H) + size_t(k)] - exp) > 1e-6f) { ok = false; break; } + } + check(ok, "embedding component: (code - 8) times its group's scale, group 128, low nibble first"); + std::printf("embedding component: %zu values, V=%lld H=%lld group 128\n", got.size(), (long long)V, (long long)H); + + const fs::path none = dir / "none"; + fs::create_directories(none); + std::string why2; + check(pi::read_embedding_component(none.string(), &why2).empty() && !why2.empty(), + "embedding component: absent -> empty with a reason (" + why2 + ")"); + if (!getenv("KEEP_EMBEDCOMP")) fs::remove_all(dir); +} + void test_qwen3_attn_sections() { struct Pack { const char* name; int64_t hidden, q_out, kv_out, hd, h0, h1, h2; }; const Pack packs[] = {{"Qwen3-1.7B", 2048, 2048, 1024, 128, 4739136, 4739456, 7107264}, @@ -1769,6 +1820,35 @@ void test_fconst_eps(const std::string& root) { const fs::path dir = fs::path fs::remove_all(dir); } +// Opt-in: read a REAL pack's embedding component (NPU_EMBED_COMPONENT=) and dump one row for an oracle comparison. +// The 4B pack's component is 206 MB, not 3 GB, so this is the cheap way to settle the two facts the reader's comment +// calls pack-settled -- the nibble order and the implicit zero point -- against ONNX Runtime running the pack's own +// embedding.onnx. +static void dump_real_component(const std::string& dir) { + std::string why; + const std::vector v = pi::read_embedding_component(dir, &why); + if (v.empty()) { + std::printf("real component: EMPTY (%s)\n", why.c_str()); + return; + } + const std::string cfg = dir + "/config.json"; + size_t H = 0; + if (FILE* f = std::fopen(cfg.c_str(), "rb")) { fclose(f); } + // H comes from the file the caller names next to the dump; keep this independent of the JSON parser. + H = v.size() / 151936; // the 4B shape; asserted below against the advertised V + std::printf("real component: %zu values, H=%zu\n", v.size(), H); + const int row = 1; // token id 1: a plain text token ORT embeds + if ((row + 1) * H > v.size()) return; + const std::string out = dir + "/eng_row.f32"; + if (FILE* f = std::fopen(out.c_str(), "wb")) { + std::fwrite(&v[size_t(row) * H], sizeof(float), H, f); + std::fclose(f); + double s = 0; + for (size_t k = 0; k < H; ++k) s += std::fabs(v[size_t(row) * H + k]); + std::printf("real component: row %d -> %s (mean|x| %.6f)\n", row, out.c_str(), s / double(H)); + } +} + int main(int argc, char** argv) { const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; @@ -1791,6 +1871,8 @@ int main(int argc, char** argv) { test_mlp_gu(golden_dir); test_fconst_eps(golden_dir.empty() ? "." : golden_dir); test_attn_payload_off(); + test_embedding_component(golden_dir.empty() ? "." : golden_dir); + if (const char* rc = std::getenv("NPU_EMBED_COMPONENT")) dump_real_component(rc); test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir);