diff --git a/docs/npu.md b/docs/npu.md index 38052dae3..bfc37430e 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1820,6 +1820,37 @@ then zeros. On SmolLM2-135M the block is 2,360,292 B (q at 1,216, k at 231,616, **The 1.8 file is the same block in another order (engine#2594).** `SmolLM2-135M-Instruct_rai_1.8.0_npu_4K`'s Token `.fconst` is 90,469,696 B, 1,080 B less than the 1.9 file, which is 30 layers x 4 x 9 heads. It writes `k_proj` before `q_proj` and has no `4 x heads` field: k at 1,216, q at 78,016, v at 308,416, o at 385,216, stride 2,360,256, head at 70,808,896. Rebuilt into the 1.9 order, it equals the 1.9 file byte for byte, except 112 code bytes in layer 3's `o_proj`. `plan()` takes this form only when the file fits it and does not fit the one above. The envelopes move 4 x heads bytes a layer, so on a multi-layer file only one of the two can pass, and `npu-pack info` names the form in `block32_layers.form`. The tied-head self-check cannot see a q/k swap. With the sink field dropped but q still first, the import passed its own checks (head vs embedding corr 0.999999, 60/60 norm anchors), and the device scored 2/64 on the teacher-forced probes. With k first, the device scores 53/64, the same as the 1.9 pack (53/64; RTN int4 of the HF model scores 47/64). It answers "Neil Armstrong … July 20, 1969". It declines the France question, and so do the host f32 and host int8 GEMMs on the same container, so that comes from the pack's 4-bit weights; the HF fp32 model answers Paris. +**The Ryzen AI 1.7 `-onnx-ryzenai-npu` Token file has no norm envelopes (engine#2634).** These packs ship +`dd_metastate_Llm_Token_MatMulNBits_2_0.fconst` and an empty `config.json`. Measured on Llama-3.2-1B-Instruct against +its 1.8 `npu_4K` twin, a layer block holds the same tiles in a different arrangement: + +| 1.7 `-onnx-ryzenai-npu` | 1.8 | +|---|---| +| q, k, v, o tiles, **q first** | header + input gain, then k, q, v, o | +| post_attention gain bf16[H], **no header** | header + post gain | +| MLP gate/up then down, **no 64-byte separator** | the MLP with a separator (`01`, then zeros) | +| the **next** layer's input gain (after the last layer: the final norm) | -- | +| tail: the head's tiles | header + final norm, then the head's tiles | + +Layer 0's input_layernorm is not in the file. It is the graph's fp16 `model.layers.0.input_layernorm.weight` (in +`fusion.onnx`'s `If` branches), and every gain the file does carry equals bf16 of the graph's fp16 exactly. +`materialise_token_fconst()` detects this form by its arithmetic: the file must be exactly layers x stride + head, and +it must not open with an epsilon header. It then rewrites the file into the 1.8 form, so every check of the 1.8 reader +runs unchanged. Rebuilt this way, the Llama file is the 1.8 twin's size. Every tile, gate/up unit and down group is at +least 96.4 % byte-equal to the twin at its mapped place (a one-unit misplacement scores 8.5 %), and the rest is the two +exports' re-quantization. + +With `config.json` empty, the model's numbers come from the pack itself: +- **rms_norm_eps** from the graph's RMSNorm nodes (18 nodes state 1e-5). The default was 1e-6. When the nodes state different epsilons nothing is taken, and the import prints a WARNING naming the conflict and the value it keeps (`graph_eps_decision()`). +- **RoPE** from `cos_cache`. The theta fit (500000) looks only at the first 4,096 positions, where llama3 scaling is + invisible. So the whole cache is also compared, by max error, with plain RoPE and with llama3 variants: + - this pack's cache is **factor 32 with original_max_position_embeddings 131072**, max error 0.009, against 1.995 + for plain RoPE; + - the upstream config states 8192, and that fits no better than plain, so the pack's own cache wins. + +On the NPU, the result is "The capital of France is Paris.", "12 * 7 = 84" and "Neil Armstrong ... July 20, 1969" +(all stop, 3.9-5.0 tok/s). + The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are `gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index fcf6cd275..27ce52a7b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -178,6 +178,12 @@ struct OnnxModel { // other branch is diagnosable rather than silently misread. std::string decode_branch; std::string branch_note; + // engine#2634: the epsilon the graph's RMSNorm nodes state (SimplifiedLayerNormalization, SkipSimplified..., SSMLP), and + // how many nodes stated it. A pack whose config.json is `{}` (the 1.7 -onnx-ryzenai-npu line) states its epsilon only + // here; 0 when no node does, or when two nodes disagree (then nothing is taken from the graph). + float norm_eps = 0.0f; + int norm_eps_nodes = 0; + bool norm_eps_conflict = false; }; // A matmul is decode-path when its branch is the decode branch, or when it is in no branch at all (a pack whose graph @@ -225,6 +231,17 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m, const std::string& bra const auto* nm = field1(at, 1); if (!nm) continue; if (const auto* iv = field1(at, 3)) attrs[pb_str(*nm)] = int64_t(iv->v); + // AttributeProto.f (field 2, fixed32): the RMSNorm epsilon (engine#2634) + if (pb_str(*nm) == "epsilon" && + (op == "SimplifiedLayerNormalization" || op == "SkipSimplifiedLayerNormalization" || op == "SSMLP")) + if (const auto* fv = field1(at, 2); fv && fv->wire == 5) { + const uint32_t bits = uint32_t(fv->v); + float e = 0.0f; + std::memcpy(&e, &bits, 4); + if (m.norm_eps_nodes == 0) m.norm_eps = e; + else if (m.norm_eps != e) m.norm_eps_conflict = true; + ++m.norm_eps_nodes; + } // AttributeProto.graph is field 6 (a GRAPH), graphs is field 11 (a repeated GRAPH) -- so a branch arrives // in field 6 and its name is the attribute's own, `then_branch` / `else_branch`. const std::string sub_branch = nm ? pb_str(*nm) : std::string(); @@ -896,6 +913,127 @@ std::string find_token_fconst(const std::string& dir, std::string* why = nullptr // .fconst?" predicate, which cost a second full rewrite per `npu-pack` command (measured: two O_TRUNC opens of the same // 291 MB file for one `npu-pack info`). Call it ONCE per plan/import and pass the path around. "Is there an .fconst?" is // find_token_fconst(), which touches nothing. +std::vector as_bf16(const uint8_t* src, int type, size_t count); +std::string graph_filename(const std::string& pack_dir); + +// engine#2634: the Ryzen AI 1.7 `-onnx-ryzenai-npu` Token .fconst (`dd_metastate_Llm_Token_MatMulNBits_2_0.fconst`). +// Measured on Llama-3.2-1B-Instruct against its 1.8 npu_4K twin: the same tiles in another arrangement, with no norm +// envelopes. Layer block L (stride S17 below) is +// [q, k, v, o tiles -- q FIRST; 1.8 is k, q, v, o][post_attention_layernorm bf16[H], no header] +// [MLP: gate/up units, then down groups -- no 64-byte separator between them][layer L+1's input_layernorm bf16[H]] +// (after the last layer that last gain is the final norm), then the head's tiles, the same as 1.8's after its envelope. +// Layer 0's input_layernorm is not in the file: it is the graph's fp16 `model.layers.0.input_layernorm.weight`, and +// every gain the file does carry equals bf16(the graph's fp16) exactly, so it is the same number either way. Rebuilt +// with this map the file is the 1.8 twin's size, and every tile / unit / group is >= 96.4 % byte-equal to it at its +// mapped place (a one-unit misplacement scores 8.5 %); the rest is the two exports' re-quantization. +struct Fconst17Geom { + int64_t tile_h = 0, tile_o = 0, n_q = 0, n_kv = 0, n_o = 0, payload = 0, gu = 0, down = 0, stride = 0, head = 0; +}; +// The arithmetic, and whether `size` is exactly layers x stride + head. Empty when it is; otherwise why not. +std::string fconst17_geometry(uint64_t size, const Dims& d, Fconst17Geom* g) { + const int64_t H = d.hidden, IM = d.intermediate, Nq = int64_t(d.heads) * d.head_dim, + Nkv = int64_t(d.kv_heads) * d.head_dim, V = d.vocab; + if (H <= 0 || IM <= 0 || Nq <= 0 || Nkv <= 0 || V <= 0 || d.layers <= 0) return "the geometry is incomplete"; + if (Nq % 64 || Nkv % 64 || H % 64 || IM % 64) return "a width is not whole 64-row tiles"; + g->tile_h = attn_tile_bytes(H); + g->tile_o = attn_tile_bytes(Nq); + if (g->tile_h <= 0 || g->tile_o <= 0) return "hidden or q width is not whole 128-column groups"; + g->n_q = Nq / 64, g->n_kv = Nkv / 64, g->n_o = H / 64; + g->payload = (g->n_q + 2 * g->n_kv) * g->tile_h + g->n_o * g->tile_o; + g->gu = (IM / 64) * group_bytes(H, 128); + g->down = (H / 64) * group_bytes(IM, 64); + if (g->gu <= 0 || g->down <= 0) return "the MLP is not whole groups"; + g->stride = g->payload + 2 * H + g->gu + g->down + 2 * H; + g->head = ((V + 63) / 64) * g->tile_h; + const uint64_t want = uint64_t(d.layers) * uint64_t(g->stride) + uint64_t(g->head); + if (size != want) + return "the file is " + std::to_string(size) + " B, and the 1.7 headerless form of this geometry is " + + std::to_string(d.layers) + " x " + std::to_string(g->stride) + " + a " + std::to_string(g->head) + + " B head = " + std::to_string(want) + " B"; + return ""; +} + +// Write the 1.8 form of a 1.7 headerless Token .fconst to `work`. Returns `work`, or the source path unchanged when the +// file is not in that form (so a 1.8 / 1.9 pack is untouched). Throws when it IS that form and the graph does not give +// layer 0's input norm. +std::string canonical_token_fconst17(const std::string& path, const std::string& pack_dir, const Dims& d, + const std::string& work) { + Fconst fc = open_fconst(path); + Fconst17Geom g; + // A 1.8 file opens with an epsilon header; this form opens with q's first tile. Both checks, so a 1.8 pack whose + // size happened to fit the arithmetic is never rewritten. + const bool opens_with_eps = fc.size >= 2 && fc.map[0] == uint8_t(d.eps_bits & 0xFF) && + fc.map[1] == uint8_t(d.eps_bits >> 8); + if (opens_with_eps || !fconst17_geometry(fc.size, d, &g).empty()) { + unmap_file(fc.map, fc.size); + return path; + } + const int64_t H = d.hidden; + // layer 0's input_layernorm, from the graph (either branch: the two copies are one tensor) + std::vector in0; + { + const OnnxModel m = parse_onnx(pack_dir + "/" + graph_filename(pack_dir)); + const auto it = m.index.find("model.layers.0.input_layernorm.weight"); + if (it == m.index.end()) { + unmap_file(fc.map, fc.size); + throw std::runtime_error(path + ": a Ryzen AI 1.7 headerless Token .fconst (engine#2634), whose layer-0 " + "input_layernorm is in the graph, and the graph has no " + "model.layers.0.input_layernorm.weight"); + } + const OnnxTensor& t = m.tensors[it->second]; + const size_t es = t.type == kF32 ? 4 : 2; + const uint8_t* src = t.raw; + size_t ext_size = 0; + const uint8_t* ext = nullptr; + if (!src) { + std::string why; + ext = map_file(pack_dir + "/" + t.location, ext_size, &why); + if (!ext || t.offset + uint64_t(H) * es > ext_size) { + if (ext) unmap_file(ext, ext_size); + unmap_file(fc.map, fc.size); + throw std::runtime_error(path + ": layer-0 input_layernorm: " + (why.empty() ? "past the end" : why)); + } + src = ext + t.offset; + } + in0 = as_bf16(src, t.type, size_t(H)); + if (ext) unmap_file(ext, ext_size); + } + std::ofstream o(work, std::ios::binary | std::ios::trunc); + if (!o) { + unmap_file(fc.map, fc.size); + throw std::runtime_error("cannot write " + work); + } + char hdr[64] = {0}; + hdr[0] = char(d.eps_bits & 0xFF), hdr[1] = char(d.eps_bits >> 8); + char sep[64] = {0}; + sep[0] = 1; // 1.8's gate/up | down separator: 01 then 63 zeros (measured) + const auto put = [&](const uint8_t* p, int64_t n) { o.write(reinterpret_cast(p), std::streamsize(n)); }; + for (int L = 0; L < d.layers; ++L) { + const uint8_t* b = fc.map + size_t(L) * size_t(g.stride); + o.write(hdr, 64); + if (L == 0) put(in0.data(), 2 * H); + else put(b - 2 * H, 2 * H); // the previous block's trailing gain + put(b + g.n_q * g.tile_h, g.n_kv * g.tile_h); // k + put(b, g.n_q * g.tile_h); // q + put(b + (g.n_q + g.n_kv) * g.tile_h, g.n_kv * g.tile_h); // v + put(b + (g.n_q + 2 * g.n_kv) * g.tile_h, g.n_o * g.tile_o); // o + o.write(hdr, 64); + put(b + g.payload, 2 * H); // post_attention_layernorm + put(b + g.payload + 2 * H, g.gu); + o.write(sep, 64); + put(b + g.payload + 2 * H + g.gu, g.down); + } + o.write(hdr, 64); + put(fc.map + size_t(d.layers) * size_t(g.stride) - size_t(2 * H), 2 * H); // the final norm + put(fc.map + size_t(d.layers) * size_t(g.stride), g.head); + o.close(); + unmap_file(fc.map, fc.size); + if (!o) throw std::runtime_error("writing " + work + " failed"); + std::fprintf(stderr, "1bit npu-pack: note: a Ryzen AI 1.7 headerless Token .fconst (engine#2634) was rewritten into its " + "1.8 form at %s (remove it when done)\n", work.c_str()); + return work; +} + std::string materialise_token_fconst(const std::string& pack_dir, const Dims& dims) { const std::string found = find_token_fconst(pack_dir); if (found.empty()) return found; @@ -983,6 +1121,10 @@ std::string materialise_token_fconst(const std::string& pack_dir, const Dims& di "its 1.8 form before it can be read, and no writable directory is available " "for that copy (tried the temporary directory, /tmp and the current " "directory)"); + // engine#2634: the 1.7 headerless form is rewritten first; a file not in that form comes back unchanged and goes on to + // the 1.9 check exactly as before. + const std::string f17 = canonical_token_fconst17(found, pack_dir, dims, work); + if (f17 != found) return f17; return canonical_token_fconst(found, dims.hidden, dims.heads, dims.layers, work); } @@ -1690,6 +1832,7 @@ Dims read_dims(const json& c, const std::string& source = "config.json") { d.rope_theta = opt_field(c, "rope_theta", 10000.0f); d.model_type = opt_field(c, "model_type", ""); d.rms_norm_eps = opt_field(c, "rms_norm_eps", 1e-6f); + d.eps_stated = c.contains("rms_norm_eps") && c["rms_norm_eps"].is_number(); // engine#1889: the pack's own statement about per-head q/k gains. `opt_field` because the key is absent on // every pack that has no such norm, and absent must mean false rather than a parse error. d.use_qk_norm = opt_field(c, "use_qk_norm", false); @@ -4375,6 +4518,18 @@ Plan plan(const std::string& pack_dir) { throw std::runtime_error("the pack's graph file " + graph + " is not in " + pack_dir + " (named by genai_config.json's model.decoder.filename)"); const OnnxModel onnx = parse_onnx(pack_dir + "/" + graph); + // engine#2634: a pack that states no rms_norm_eps (config.json `{}`) states it in its graph's RMSNorm nodes. Taken + // before the layer blocks are located, because the block headers are found by this same epsilon. + { + std::string msg; + const float e = graph_eps_decision(p.dims.eps_stated, p.dims.rms_norm_eps, onnx.norm_eps_nodes, + onnx.norm_eps_conflict, onnx.norm_eps, &msg); + if (!msg.empty()) std::fprintf(stderr, "%s\n", msg.c_str()); + if (e != p.dims.rms_norm_eps) { + p.dims.rms_norm_eps = e; + p.dims.eps_bits = bf16_bits_of(e); + } + } // Census the graph's own weight carrier (see plan_import.h's note): this is what tells a 1.3/1.6/1.7.1 pack // (real ONNX initializers) from a 1.8/R1-Distill pack (placeholders, weights in model.bin) without parsing // anything outside the pack. @@ -5183,6 +5338,53 @@ std::string kernel_set_refusal(const std::vector& file_names, const // Copies the tokenizer/chat files the container needs beside model.q4nx; defined past kTextFiles' declaration. static int copy_text_artifacts(const std::string& pack_dir, const std::string& out_dir); +// engine#2634: whether a cos_cache is llama3-scaled RoPE (Llama-3.1/3.2: rope_scaling {rope_type llama3, factor, +// low_freq_factor, high_freq_factor, original_max_position_embeddings}). The theta fit above looks at the first 4,096 +// positions, where llama3 scaling -- which only slows the low-frequency columns -- is invisible, so a pack with no +// config.json (the 1.7 -onnx-ryzenai-npu line) passed that fit at the right theta and was served WITHOUT the scaling. +// This compares the cache over its whole length, by max |error|, with plain RoPE and with llama3 variants (factor +// 8/16/32/64, low 1, high 4, original_max_position_embeddings 8192..131072), and returns the best variant only when it +// clearly explains the cache (err < 0.02, the fp16 cache's own floor at long positions, where plain is > 0.5 and at +// least 10x worse). The original window is searched, not assumed: Llama-3.2-1B-Instruct-onnx-ryzenai-npu's cache is +// factor 32 with original_max_position_embeddings 131072 (max error 0.009; plain 1.995; the 8192 the upstream config +// states fits no better than plain), and running the pack as published means its own cache. +RopeLlama3Fit rope_cache_llama3(const std::vector& cache, int64_t rows, int64_t half, double theta) { + RopeLlama3Fit r; + if (rows < 2 || half < 2 || !(theta > 0)) return r; + const int64_t HD2 = 2 * half; + const double pi = 3.14159265358979323846; + const auto max_err = [&](const std::vector& inv) { + double e = 0; + const int64_t step = std::max(1, rows / 2048); + for (int64_t p = 0; p < rows; p += step) + for (int64_t i = 0; i < half; ++i) + e = std::max(e, std::fabs(double(cache[size_t(p) * size_t(half) + size_t(i)]) - std::cos(double(p) * inv[size_t(i)]))); + return e; + }; + std::vector base(static_cast(half), 0.0); + for (int64_t i = 0; i < half; ++i) base[size_t(i)] = 1.0 / std::pow(theta, double(2 * i) / double(HD2)); + r.plain_err = max_err(base); + struct V { double factor, low, high, orig; }; + std::vector cands; + for (double fac : {8.0, 16.0, 32.0, 64.0}) + for (double orig : {8192.0, 16384.0, 32768.0, 65536.0, 131072.0}) cands.push_back(V{fac, 1, 4, orig}); + for (const V& v : cands) { + std::vector inv(base); + const double low_wl = v.orig / v.low, high_wl = v.orig / v.high; + for (auto& f : inv) { // transformers' _compute_llama3_parameters + const double wl = 2 * pi / f; + if (wl < high_wl) continue; + if (wl > low_wl) { f = f / v.factor; continue; } + const double smooth = (v.orig / wl - v.low) / (v.high - v.low); + f = (1 - smooth) * f / v.factor + smooth * f; + } + const double e = max_err(inv); + if (e < r.err) { r.err = e; r.factor = v.factor; r.low = v.low; r.high = v.high; r.orig = v.orig; } + } + r.found = r.err < 0.02 && r.plain_err > 0.5 && r.err * 10 < r.plain_err; + return r; +} + double rope_cache_best_theta(const std::vector& cache, int64_t rows, int64_t half, const std::vector& cands, double* best_theta) { // ONE Pearson r of the cache against cos(p * theta^(-2i/d)) over the whole matrix (the per-column mean is noise @@ -6989,6 +7191,19 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons "attention may be wrong from the first token\n", best_corr, best_theta, (double)p.dims.rope_theta); } + // engine#2634: a stated rope_scaling wins; otherwise the cache over its whole length decides llama3. + if (best_corr > 0.99 && !c.contains("rope_scaling") && c.contains("rope_theta") && + c["rope_theta"].is_number()) { + const RopeLlama3Fit l3 = rope_cache_llama3(cache, rows, half, c["rope_theta"].get()); + if (l3.found) { + c["rope_scaling"] = {{"rope_type", "llama3"}, {"factor", l3.factor}, + {"low_freq_factor", l3.low}, {"high_freq_factor", l3.high}, + {"original_max_position_embeddings", l3.orig}}; + std::fprintf(stderr, "note: rope_scaling llama3 (factor %g) taken from the pack's cos_cache: " + "max error %.2g over its %lld positions, against %.2g for plain RoPE\n", + l3.factor, l3.err, (long long)rows, l3.plain_err); + } + } } } } @@ -7548,4 +7763,44 @@ HeadFit fit_head(const Q4& q, const EmbedTable& embed, int rows, int block) { return f; } +// engine#2634: the rms_norm_eps a pack is served with, given what config.json and the graph say. config.json wins when it +// states one. Otherwise the graph's RMSNorm nodes: taken when they agree, and when they DISAGREE nothing is taken and +// that is said -- a pack that states two epsilons must not be served at one of them, or at the default, in silence +// (#2638 review). `msg` is the line to print ("" when there is nothing to say). +float graph_eps_decision(bool config_stated, float current, int nodes, bool conflict, float graph_eps, std::string* msg) { + char buf[320]; + if (msg) msg->clear(); + if (config_stated || nodes <= 0) return current; + if (conflict) { + std::snprintf(buf, sizeof buf, + "WARNING: rms_norm_eps: the graph's %d RMSNorm nodes state DIFFERENT epsilons and config.json states " + "none; keeping %.6g, which may not be this model's", nodes, double(current)); + if (msg) *msg = buf; + return current; + } + if (!(graph_eps > 0.0f)) return current; + if (graph_eps != current) { + std::snprintf(buf, sizeof buf, "note: rms_norm_eps %.6g taken from the graph (%d RMSNorm nodes state it; " + "config.json does not)", double(graph_eps), nodes); + if (msg) *msg = buf; + } + return graph_eps; +} + +// engine#2634: test hooks for the 1.7 headerless Token .fconst and the graph's own epsilon. +std::string fconst17_geometry_exported(uint64_t size, const Dims& d) { + Fconst17Geom g; + return fconst17_geometry(size, d, &g); +} +std::string canonical_token_fconst17_exported(const std::string& path, const std::string& pack_dir, const Dims& d, + const std::string& work) { + return canonical_token_fconst17(path, pack_dir, d, work); +} +float graph_norm_eps_exported(const std::string& onnx_path, int* nodes, bool* conflict) { + const OnnxModel m = parse_onnx(onnx_path); + if (nodes) *nodes = m.norm_eps_nodes; + if (conflict) *conflict = m.norm_eps_conflict; + return m.norm_eps; +} + } // namespace onebit::npu::pack_import diff --git a/npu/pack_import.h b/npu/pack_import.h index c34f193b2..60a0d6b76 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -71,6 +71,7 @@ struct Dims { // packs declare; Llama-3.2-1B declares 1e-5 (28 37) and so had no anchors at all. float rms_norm_eps = 1e-6f; unsigned short eps_bits = 0x3586; + bool eps_stated = false; // config.json stated rms_norm_eps (else plan() may take it from the graph, engine#2634) std::string model_type; std::vector eos; // config.json's `use_qk_norm` (engine#1889). Hunyuan-7B declares it and puts 32x128 + 8x128 per-head q/k @@ -988,6 +989,21 @@ unsigned short bf16_bits_of(float v); // the default rather than throw. Exported so tests/npu_pack_import_test.cpp can pin it without a // pack; `source` names where the geometry came from, for the refusal messages. Dims read_dims_exported(const nlohmann::json& c, const std::string& source); +// engine#2634: the Ryzen AI 1.7 `-onnx-ryzenai-npu` Token .fconst -- no norm envelopes, q first, the MLP without its +// 64-byte separator, each block ending with the NEXT layer's input_layernorm, layer 0's in the graph. "" when `size` is +// exactly that form's layers x stride + head for `d`, else why not. +std::string fconst17_geometry_exported(uint64_t size, const Dims& d); +// The 1.8 form of such a file written to `work` (returns `work`), or `path` unchanged when it is not in that form. +std::string canonical_token_fconst17_exported(const std::string& path, const std::string& pack_dir, const Dims& d, + const std::string& work); +// The epsilon the graph's RMSNorm nodes state (0 when none), how many state it, and whether two disagree. +float graph_norm_eps_exported(const std::string& onnx_path, int* nodes, bool* conflict); +// The rms_norm_eps to serve with: config.json's when it states one, else the graph's when its nodes agree; when they +// disagree, `current` is kept and `msg` says so (a WARNING, never silence). `msg` is "" when there is nothing to say. +float graph_eps_decision(bool config_stated, float current, int nodes, bool conflict, float graph_eps, std::string* msg); +// engine#2634: whether a cos_cache is llama3-scaled RoPE, searched over factor and original window (see the .cpp). +struct RopeLlama3Fit { bool found = false; double factor = 0, low = 0, high = 0, orig = 0, err = 1e9, plain_err = 1e9; }; +RopeLlama3Fit rope_cache_llama3(const std::vector& cache, int64_t rows, int64_t half, double theta); // engine#1503: what the graph walker RECORDED, without exposing its model type. A three-input MatMulNBits (no qzeros, // weights spelled `.MatMul.weight_Q4`) used to be dropped entirely, which is why Qwen3-VL-4B imported one tensor. // Returns the matmul count; `without_zeros` counts those that carry no qzeros tensor (the implicit-zero-point case), and diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 9a0ac8666..f640b13cf 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -342,6 +342,168 @@ void test_rope_cache_best_theta() { check(th2 != 250000.0, "rope cache: without the right candidate the fit does not invent it"); } +// engine#2634: llama3 RoPE read from a cos_cache over its whole length. Plain RoPE must not be called llama3, and the +// original window is searched: Llama-3.2-1B-Instruct-onnx-ryzenai-npu's cache is factor 32 / original 131072. +void test_rope_cache_llama3() { + const int64_t rows = 131072, half = 32; + const double theta = 500000.0, pi_ = 3.14159265358979323846; + const auto cache_of = [&](double fac, double orig) { + std::vector c(size_t(rows) * size_t(half)); + for (int64_t i = 0; i < half; ++i) { + double f = 1.0 / std::pow(theta, double(2 * i) / double(2 * half)); + if (fac > 0) { + const double wl = 2 * pi_ / f; + if (wl > orig) f /= fac; + else if (wl >= orig / 4) { + const double sm = (orig / wl - 1) / 3; + f = (1 - sm) * f / fac + sm * f; + } + } + for (int64_t p = 0; p < rows; ++p) c[size_t(p) * size_t(half) + size_t(i)] = float(std::cos(double(p) * f)); + } + return c; + }; + const pi::RopeLlama3Fit a = pi::rope_cache_llama3(cache_of(32, 131072), rows, half, theta); + check(a.found && a.factor == 32 && a.orig == 131072 && a.err < 0.02, + "rope llama3: factor 32 / original 131072 is recovered (err " + std::to_string(a.err) + ")"); + const pi::RopeLlama3Fit b = pi::rope_cache_llama3(cache_of(8, 8192), rows, half, theta); + check(b.found && b.factor == 8 && b.orig == 8192, "rope llama3: factor 8 / original 8192 is recovered"); + const pi::RopeLlama3Fit c = pi::rope_cache_llama3(cache_of(0, 0), rows, half, theta); + check(!c.found && c.plain_err < 1e-3, "rope llama3: a plain cache is not called llama3 (plain err " + + std::to_string(c.plain_err) + ")"); +} + +// engine#2634: the 1.7 -onnx-ryzenai-npu Token .fconst -> its 1.8 form, byte for byte, on a synthetic geometry. The 1.8 +// stream is built first with every field its own bytes; the 1.7 stream is derived from it with the measured map (q first, +// no headers, no MLP separator, each block ending with the next layer's input gain, layer 0's input gain only in the +// graph); the rewrite has to give the 1.8 stream back exactly. +void test_fconst17_rewrite() { + pi::Dims d; + d.hidden = 128, d.intermediate = 256, d.layers = 2, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = 64; + d.model_type = "llama"; + d.rms_norm_eps = 1e-5f; + d.eps_bits = pi::bf16_bits_of(1e-5f); + const int64_t H = 128, T = pi::attn_tile_bytes(H), nq = 2, nkv = 1, no = 2; + const int64_t gu = (256 / 64) * pi::group_bytes(H, 128), down = (H / 64) * pi::group_bytes(256, 64); + const int64_t head = 1 * T; // vocab 64 = one 64-row tile + uint32_t seed = 12345; + const auto rnd = [&](int64_t n) { + std::string b(size_t(n), '\0'); + for (auto& ch : b) { seed = seed * 1103515245u + 12345u; ch = char(seed >> 16); } + return b; + }; + std::vector gain0_bf(static_cast(H), 0), gain0_f16(static_cast(H), 0); + for (int64_t i = 0; i < H; ++i) { + const int m = int(i % 128); + gain0_bf[size_t(i)] = pi::bf16_bits_of(1.0f + float(m) / 128.0f); + gain0_f16[size_t(i)] = uint16_t((15 << 10) | (m << 3)); // 1 + m/128 in fp16, exactly + } + const auto bytes_of = [](const std::vector& v) { + return std::string(reinterpret_cast(v.data()), v.size() * 2); + }; + std::string hdr(64, '\0'); + hdr[0] = char(d.eps_bits & 0xFF), hdr[1] = char(d.eps_bits >> 8); + std::string sep(64, '\0'); + sep[0] = 1; + // per-layer fields + std::vector K, Q, Vv, O, POST, GU, DN, IN; + for (int L = 0; L < 2; ++L) { + K.push_back(rnd(nkv * T)); Q.push_back(rnd(nq * T)); Vv.push_back(rnd(nkv * T)); O.push_back(rnd(no * T)); + POST.push_back(rnd(2 * H)); GU.push_back(rnd(gu)); DN.push_back(rnd(down)); + IN.push_back(L == 0 ? bytes_of(gain0_bf) : rnd(2 * H)); + } + const std::string FIN = rnd(2 * H), HEAD = rnd(head); + std::string f18, f17; + for (int L = 0; L < 2; ++L) { + f18 += hdr + IN[L] + K[L] + Q[L] + Vv[L] + O[L] + hdr + POST[L] + GU[L] + sep + DN[L]; + f17 += Q[L] + K[L] + Vv[L] + O[L] + POST[L] + GU[L] + DN[L] + (L + 1 < 2 ? IN[L + 1] : FIN); + } + f18 += hdr + FIN + HEAD; + f17 += HEAD; + check(pi::fconst17_geometry_exported(f17.size(), d).empty(), "1.7 fconst: the synthetic stream fits the arithmetic"); + check(!pi::fconst17_geometry_exported(f17.size() + 64, d).empty(), "1.7 fconst: 64 B more does not"); + const fs::path dir = fs::temp_directory_path() / ("fconst17." + std::to_string(::getpid())); + fs::remove_all(dir); + fs::create_directories(dir); + const std::string tok = (dir / "dd_metastate_Llm_Token_MatMulNBits_2_0.fconst").string(); + put_file(tok, f17); + put_file((dir / "genai_config.json").string(), R"({"model":{"decoder":{"filename":"fusion.onnx"}}})"); + put_file((dir / "fusion.onnx.data").string(), bytes_of(gain0_f16)); + put_file((dir / "fusion.onnx").string(), + pb::model({pb::tensor("model.layers.0.input_layernorm.weight", {H}, "fusion.onnx.data", 0, uint64_t(2 * H), 10)}, + {})); + const std::string work = (dir / "canonical.fconst").string(); + std::string said; + std::string got; + try { + got = pi::canonical_token_fconst17_exported(tok, dir.string(), d, work); + } catch (const std::exception& e) { + said = e.what(); + } + check(said.empty() && got == work, "1.7 fconst: the rewrite runs and writes the copy (" + said + ")"); + std::string out; + { + std::ifstream f(work, std::ios::binary); + out.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + } + check(out == f18, "1.7 fconst: the rewrite is the 1.8 stream byte for byte (" + std::to_string(out.size()) + " vs " + + std::to_string(f18.size()) + " B)"); + // controls: a 1.8 file and a file of the wrong size come back as themselves, untouched + const std::string tok18 = (dir / "as18.fconst").string(); + put_file(tok18, f18); + check(pi::canonical_token_fconst17_exported(tok18, dir.string(), d, work + ".2") == tok18, + "1.7 fconst: a 1.8 file is not rewritten"); + const std::string tokx = (dir / "odd.fconst").string(); + put_file(tokx, f17 + std::string(64, '\0')); + check(pi::canonical_token_fconst17_exported(tokx, dir.string(), d, work + ".3") == tokx, + "1.7 fconst: a file that does not fit the arithmetic is not rewritten"); + // the graph's RMSNorm epsilon (a node with a float attribute), read back + std::string node, attr; + pb::str(node, 4, "SimplifiedLayerNormalization"); + pb::str(attr, 1, "epsilon"); + pb::tag(attr, 2, 5); + const float e = 1e-5f; + char eb[4]; + std::memcpy(eb, &e, 4); + attr.append(eb, 4); + pb::ints(attr, 20, 1); // AttributeProto.type = FLOAT + pb::str(node, 5, attr); + put_file((dir / "eps.onnx").string(), pb::model({}, {node, node})); + int n_eps = 0; + bool conflict = true; + const float got_eps = pi::graph_norm_eps_exported((dir / "eps.onnx").string(), &n_eps, &conflict); + check(got_eps == 1e-5f && n_eps == 2 && !conflict, "1.7 fconst: the graph's RMSNorm epsilon is read (" + + std::to_string(got_eps) + ", " + std::to_string(n_eps) + " nodes)"); + // #2638 review: two nodes that DISAGREE are a conflict, and the decision says so instead of falling back in silence + const auto eps_node = [&](float v) { + std::string nd, at; + pb::str(nd, 4, "SimplifiedLayerNormalization"); + pb::str(at, 1, "epsilon"); + pb::tag(at, 2, 5); + char vb[4]; + std::memcpy(vb, &v, 4); + at.append(vb, 4); + pb::ints(at, 20, 1); + pb::str(nd, 5, at); + return nd; + }; + put_file((dir / "eps2.onnx").string(), pb::model({}, {eps_node(1e-5f), eps_node(1e-6f)})); + int n2 = 0; + bool conflict2 = false; + (void)pi::graph_norm_eps_exported((dir / "eps2.onnx").string(), &n2, &conflict2); + check(conflict2 && n2 == 2, "1.7 fconst: two RMSNorm nodes stating different epsilons are a conflict"); + std::string msg; + const float kept = pi::graph_eps_decision(false, 1e-6f, 2, true, 1e-5f, &msg); + check(kept == 1e-6f && msg.find("WARNING") != std::string::npos && msg.find("DIFFERENT") != std::string::npos, + "1.7 fconst: on a conflict the configured eps is kept and a WARNING says why (" + msg + ")"); + const float taken = pi::graph_eps_decision(false, 1e-6f, 18, false, 1e-5f, &msg); + check(taken == 1e-5f && msg.find("taken from the graph") != std::string::npos, + "1.7 fconst: agreeing nodes give their eps, with a note"); + const float stated = pi::graph_eps_decision(true, 1e-6f, 18, false, 1e-5f, &msg); + check(stated == 1e-6f && msg.empty(), "1.7 fconst: a stated config.json eps wins, silently"); + fs::remove_all(dir); +} + // The conversion half of the writer (engine#1503): the pack's `(code - zp)*scale` and the container's // `code*d + m` are different conventions, and the mapping has to reproduce the pack's own numbers. The obvious // alternative -- dequantise and re-quantise with the container's quantiser -- was measured on a real phi-4 tile and @@ -7092,6 +7254,8 @@ int main(int argc, char** argv) { test_bounds(); test_external_tile_decode(); test_rope_cache_best_theta(); + test_rope_cache_llama3(); + test_fconst17_rewrite(); test_external_to_container(); test_external_roles(); test_external_container_names();