diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index a5ccca7ad..926b8738d 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -1493,18 +1493,13 @@ int run_forward_serve(int argc, char** argv) { // contributes nothing, and the load line below shows what the serve stops on either way. std::string turn_end = chatml_route ? std::string("<|im_end|>") : std::string(); if (can_render) { - static const char* const kProbe = "\xC2\xA4turn-end-probe\xC2\xA4"; - try { - nlohmann::ordered_json exchange = nlohmann::ordered_json::array(); - exchange.push_back({{"role", "user"}, {"content", "u"}}); - exchange.push_back({{"role", "assistant"}, {"content", kProbe}}); - const std::string rendered = npu::render_chat_template(e.tmpl, exchange, /*add_generation_prompt=*/false, - nlohmann::ordered_json::object(), e.bos_text, e.eos_text); - const size_t at = rendered.find(kProbe); - if (at != std::string::npos) turn_end = npu::turn_end_marker(rendered.substr(at + std::strlen(kProbe))); - } catch (const std::exception& ex) { - std::fprintf(stderr, "1bit forward-serve: the template did not render a two-message exchange (%s); no turn-end marker from it\n", ex.what()); - } + // engine#2604: three user-content shapes, first that renders (translategemma's template takes only its own) + const npu::TurnEndProbe probe = npu::turn_end_from_template(e.tmpl, e.bos_text, e.eos_text); + if (probe.form.empty()) + std::fprintf(stderr, "1bit forward-serve: the template did not render a two-message exchange (%s); no turn-end marker from it\n", probe.why.c_str()); + else if (probe.form != "plain") + std::fprintf(stderr, "1bit forward-serve: turn-end marker read with %s user content (the plain probe did not render)\n", probe.form.c_str()); + turn_end = probe.marker; } e.eos = npu::stop_token_ids(npu::read_json_or_null(cfg), npu::read_json_or_null(dir + "/tokenizer_config.json"), turn_end, [&](const std::string& text) { return e.tok->token_id(text); }); diff --git a/app/npu_template.h b/app/npu_template.h index 2a448e035..e5e4373b6 100644 --- a/app/npu_template.h +++ b/app/npu_template.h @@ -48,6 +48,8 @@ // something different from a build that has the renderer. #pragma once +#include +#include "npu_stop_tokens.h" #include "template_pin.h" // container_chat_template_text, the same read the pin and the load line use #include @@ -231,4 +233,48 @@ inline std::string render_chat_template(const ChatTemplate& tmpl, const nlohmann #endif } +// engine#2604: the marker a template closes an assistant turn with, found by rendering a two-message exchange and +// reading what follows the assistant's text (turn_end_marker). The user turn is tried in three shapes, first that +// renders wins: a plain string, OpenAI-style list content, and translategemma's list content with language codes -- +// its template raises on the other two ("User role must provide `content` as an iterable with exactly one item..."), +// so with only the plain probe the serve never learned and every answer ran on as +// "Le chat dort sur la table...." to the token budget (AMD's pack names only eos 1). +struct TurnEndProbe { + std::string marker; // "" when no shape rendered, or the text after the turn is not a <...> token + std::string form; // which user-content shape rendered: "plain", "list" or "translation" + std::string why; // the first shape's error, when none rendered +}; +inline TurnEndProbe turn_end_from_template(const ChatTemplate& tmpl, const std::string& bos_token, + const std::string& eos_token) { + static const char* const kProbe = "\xC2\xA4turn-end-probe\xC2\xA4"; + const std::pair shapes[] = { + {"plain", "u"}, + {"list", nlohmann::ordered_json::parse(R"([{"type":"text","text":"u"}])")}, + {"translation", + nlohmann::ordered_json::parse(R"([{"type":"text","source_lang_code":"en","target_lang_code":"fr","text":"u"}])")}, + }; + TurnEndProbe r; + for (const auto& [name, user] : shapes) { + try { + nlohmann::ordered_json exchange = nlohmann::ordered_json::array(); + exchange.push_back({{"role", "user"}, {"content", user}}); + exchange.push_back({{"role", "assistant"}, {"content", kProbe}}); + const std::string rendered = render_chat_template(tmpl, exchange, /*add_generation_prompt=*/false, + nlohmann::ordered_json::object(), bos_token, eos_token); + const size_t at = rendered.find(kProbe); + if (at == std::string::npos) { + if (r.why.empty()) r.why = std::string("the ") + name + " probe rendered without the assistant's text"; + continue; + } + r.marker = turn_end_marker(rendered.substr(at + std::strlen(kProbe))); + r.form = name; + r.why.clear(); + return r; + } catch (const std::exception& ex) { + if (r.why.empty()) r.why = ex.what(); + } + } + return r; +} + } // namespace onebit::npu diff --git a/docs/npu.md b/docs/npu.md index 0fa479493..38052dae3 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -85,8 +85,13 @@ exchange at load. The two files need not agree: FastFlowLM's `Qwen3.5-2B-NPU2` container names `<|endoftext|>` in the first and `<|im_end|>` in the second, and the model ends a chat turn with the second. And neither file has to name the turn end at all: FastFlowLM's `Phi4-mini-Instruct-NPU2` names `<|endoftext|>` in both, its template closes every turn with `<|end|>`, and before -the template was consulted its answer ran on past `Paris<|end|><|assistant|>` to the token budget. The load prints -the list (`forward-serve: stops on …`). +the template was consulted its answer ran on past `Paris<|end|><|assistant|>` to the token budget. The exchange's user +turn is tried in three shapes, and the first one that renders is used: a plain string, list content +(`[{"type":"text",...}]`), and translategemma's list content with `source_lang_code` / `target_lang_code`. +translategemma's template raises on a plain string (engine#2604), and its pack names only `eos_token_id: 1`. With +only the plain probe, "Le chat dort sur la table." ran on as `…` to the budget; it now +stops after 7 tokens. The load prints the list (`forward-serve: stops on …`), and names the shape when it was not +the plain one. `1bit serve -m ` serves such a directory on the NPU behind the OpenAI-compatible API ([serve.md](serve.md)); inside Lemonade, that is what its `onebit` backend runs (`--onebit npu`). @@ -1670,6 +1675,42 @@ their hidden sizes differ). `plan()` accepts a block of 2 or 4 headers, checks e that arithmetic to the byte, and refuses any other count by name. FACTS §7.24 has the tables and the checkpoint evidence. +**gemma3 carries six headers, and two block sizes** (engine#2604). The 1.8 gemma3 packs +(`translategemma-4b-it_rai_1.8.0_npu_4K`, and `Gemma-3-4b_rai_1.8.0_npu_16K`, whose Token `.fconst` +is the same size, 2,189,399,392 B) are the Qwen3 block plus Gemma-3's two feed-forward norms: + +| header | section after it | +|---|---| +| h0 `input_layernorm` | k, q, v tiles at K = hidden (k 0-15, q 16-47, v 48-63 at hidden 2560) | +| h1 `k_norm` | bf16 `[head_dim]` | +| h2 `q_norm` | bf16 `[head_dim]`, then (sliding-window layers only) `4 x kv_heads` records of 8 B, then o_proj | +| h3 `post_attention_layernorm` | bf16 `[hidden]` | +| h4 `pre_feedforward_layernorm` | bf16 `[hidden]`, then the MLP (the Qwen3 format) | +| h5 `post_feedforward_layernorm` | bf16 `[hidden]` | + +The global-attention layers (every sixth: 5, 11, 17, 23, 29 of 34) do not carry the 8-byte records, +each of which is bf16 `0xFF7F` (about -3.4e38) followed by zeros. Those layers are 32 B shorter, so the +blocks are not one stride: `fconst_six_header_blocks()` takes each layer's start from its own header +(all 6 x layers + 1 of them, read in the strict 64-byte form), stores the global form, and gives every +layer a shift of 0 or the field's size, applied from o_proj on. The import checks the field byte for +byte before reading past it. The order and contents were checked against an independent reference, +`amd/Gemma-3-4b-it-mm-onnx-ryzenai-npu`, whose norms are f32 initializers and whose projections are +`.packed` blobs. All six norms equal it exactly in layers 0, 5 and 33. o_proj is byte-identical to the +reference's `.packed` at +32 on a local layer and at +0 on a global one, and k and v are tile-for-tile +identical at tiles 0-15 and 48-63. + +**The embedding is pre-scaled.** A gemma pack stores its embedding multiplied by sqrt(hidden), and so +does the forward (`embed_scale`); on a tied model the LM head divides that factor back out. The pack's +4-bit head, though, is the raw table: best-fit scale 50.33 against sqrt(2560) = 50.60. So +`gemma_prescale_head()` multiplies the head's scales and zeros by sqrt(hidden) before it is tiled and +self-checked, which is exact for an affine code. Every head gate then runs unchanged: on +translategemma the fit is scale 0.995, rms ratio 1.0014, rel 1.12x its 4-bit floor. Device (chat route, translategemma's own content shape): +English to French gives "Le chat dort sur la table." (stop, 7 tokens), and English to German gives "Wo ist der +Bahnhof?" (stop, 5 tokens). In the Gemma turn format, "What is the capital of France?" gives "The capital of France +is... Paris." +`Gemma-3-4b_rai_1.8.0_npu_16K` still refuses, because its `config.json` is empty (`{}`) and nothing in +the pack states `intermediate_size`. + **A 2-header block can carry the gains too, in the other order** (engine#1889, engine#1934). The Hunyuan packs declare `use_qk_norm: true` and keep one header per block: the gains are the 640 B between the last q/k/v tile and o_proj, which the pack's metadata names as one tensor diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 9f548eb52..fcf6cd275 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1797,13 +1797,24 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden) // vote as the fallback for packs with no 1.9 header -- the 1.8 / R1-Distill line, where it works today. FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& pack_dir, const Dims& dims) { FconstGeom g = locate_fconst(tok, dims.eps_bits); + // engine#2604: the 1.8 gemma3 6-header form -- per-layer starts from every header, not one stride. Only on a file with + // no 1.9 header (below, a 6-header 1.9 block is refused by name). + auto six_18 = [&]() -> FconstGeom { + if (g.stride != 0 && g.six_headers()) + if (const std::string why = fconst_six_header_blocks(tok, dims.eps_bits, dims.layers, dims.kv_heads, g); + !why.empty()) { + g.why = why; + g.stride = 0; + } + return g; + }; const std::string src = find_token_fconst(pack_dir); - if (src.empty()) return g; + if (src.empty()) return six_18(); Fconst fc = open_fconst(src); int64_t hdr_off = 0, stride9 = 0; const bool is19 = fconst19_detect(fc, dims.heads, dims.layers, &hdr_off, &stride9); unmap_file(fc.map, fc.size); - if (!is19) return g; // no 1.9 header: the eps vote is the only witness, keep it + if (!is19) return six_18(); // no 1.9 header: the eps vote is the only witness, keep it const int64_t stride8 = stride9 - int64_t(dims.heads) * 4; if (stride8 <= 0) return g; if (g.stride != 0 && g.stride != stride8) { @@ -1814,6 +1825,12 @@ FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& g.stride = 0; return g; } + if (g.six_headers()) { // engine#2604 review: the 6-header form is measured on 1.8 only + g.why = "a 6-header layer block in a file with a 1.9 header; 6-header blocks are read only in the measured 1.8 " + "gemma3 form (engine#2604), so refusing"; + g.stride = 0; + return g; + } g.stride = stride8; g.why.clear(); // The second header offset (`h1`) is what tells the payload reader whether a block holds 2 eps headers or 4, so the eps @@ -1844,9 +1861,19 @@ FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& // only about half the blocks, so neither `pos.size()/4` nor a fixed fraction of the window separates the two cases // (review of engine#2021, gaps 1 and 2). const std::vector inner = fconst_block_headers_observed(pos, stride8); + // engine#2604 review: a 6-header block is read only in its measured 1.8 form, whose per-layer starts come from every + // header (fconst_six_header_blocks). On this 1.9 path the stride is the header's and there are no per-layer starts, + // so a 6-header block here refuses by name instead of being read at one stride. + if (inner.size() == 5) { + g.why = "the 1.9 header gives stride " + std::to_string(stride8) + " and the anchors show a 6-header block; " + "6-header blocks are read only in the measured 1.8 gemma3 form (engine#2604), so refusing"; + g.stride = 0; + return g; + } if (!fconst_set_block_headers(g, inner)) { g.why = "the 1.9 header gives stride " + std::to_string(stride8) + ", but the epsilon anchors repeat at " + - std::to_string(inner.size()) + " in-block offset(s), and a layer block has 1 (2-header) or 3 (4-header); " + std::to_string(inner.size()) + " in-block offset(s), and a layer block has 1 (2-header), 3 (4-header) or, " + "on the 1.8 gemma3 form only, 5 (6-header); " "refusing rather than guessing the layout"; g.stride = 0; } @@ -1900,15 +1927,77 @@ std::vector fconst_block_headers(const std::vector& pos, int64 // engine#1831: the fields `locate_fconst()` sets from the same list -- `hdr` (whose SIZE is the block layout), `h1` (its // first entry) and `period` (headers per block, counting the block's own). One inner header is a 2-header block -// (Qwen2.5/Llama), three is a 4-header block (Qwen3); anything else is not a layout this reader knows, so nothing is set. +// (Qwen2.5/Llama), three is a 4-header block (Qwen3), five is the 6-header gemma3 1.8 block (engine#2604; usable only +// once fconst_six_header_blocks() has given it per-layer starts); anything else is not a layout this reader knows, so +// nothing is set. bool fconst_set_block_headers(FconstGeom& g, const std::vector& inner) { - if (inner.size() != 1 && inner.size() != 3) return false; + if (inner.size() != 1 && inner.size() != 3 && inner.size() != 5) return false; // 5: gemma3 1.8 (engine#2604) g.hdr = inner; g.h1 = inner.front(); g.period = int(inner.size()) + 1; return true; } +std::string fconst_six_header_blocks(const std::string& path, unsigned short eps_bits, int layers, int kv_heads, + FconstGeom& g) { + // Every header in the file, from a mapping (no read-chunk seams) and in the strict 64-byte form -- epsilon, then 62 + // zero bytes -- because the 12-byte prefix eps_header_positions() votes with is not rare inside 4-bit weights, and + // one coincidence here would shift a block start. + std::vector pos; + { + Fconst fc = open_fconst(path); + const uint8_t lo8 = uint8_t(eps_bits & 0xFF), hi8 = uint8_t(eps_bits >> 8); + for (size_t i = 0; i + 64 <= fc.size;) { + const void* hit = std::memchr(fc.map + i, lo8, fc.size - 64 - i + 1); + if (!hit) break; + const size_t j = size_t(static_cast(hit) - fc.map); + bool ok = fc.map[j + 1] == hi8; + for (int k = 2; ok && k < 64; ++k) ok = fc.map[j + size_t(k)] == 0; + if (ok) { + pos.push_back(int64_t(j)); + i = j + 64; + } else { + i = j + 1; + } + } + unmap_file(fc.map, fc.size); + } + const size_t want = size_t(layers) * 6 + 1; + if (layers <= 0 || pos.size() != want || pos[0] != 0) + return "a 6-header layer block needs 6 x " + std::to_string(layers) + " + 1 = " + std::to_string(want) + + " epsilon headers from offset 0, and the file has " + std::to_string(pos.size()); + const int64_t field = int64_t(kv_heads) * 8; + std::vector rel_q(static_cast(layers), 0); // h3 - h2: the only section whose size varies + int64_t lo = INT64_MAX, hi = 0; + for (int L = 0; L < layers; ++L) { + const int64_t* b = &pos[size_t(L) * 6]; + if (b[1] - b[0] != pos[1] - pos[0] || b[2] - b[1] != pos[2] - pos[1] || b[4] - b[3] != pos[4] - pos[3] || + b[5] - b[4] != pos[5] - pos[4] || b[6] - b[5] != pos[6] - pos[5]) + return "layer " + std::to_string(L) + "'s headers are not at the first block's offsets outside the " + "q_norm/o_proj section"; + rel_q[size_t(L)] = b[3] - b[2]; + lo = std::min(lo, rel_q[size_t(L)]); + hi = std::max(hi, rel_q[size_t(L)]); + } + if (hi != lo && hi - lo != field) + return "the q_norm/o_proj section comes in sizes " + std::to_string(lo) + " and " + std::to_string(hi) + + " B; the sliding-window field is 4 x kv_heads x 8 = " + std::to_string(field) + " B"; + g.bases.assign(size_t(layers) + 1, 0); + g.shifts.assign(size_t(layers), 0); + for (int L = 0; L < layers; ++L) { + g.bases[size_t(L)] = pos[size_t(L) * 6]; + g.shifts[size_t(L)] = rel_q[size_t(L)] - lo; + } + g.bases[size_t(layers)] = pos[size_t(layers) * 6]; + // the global form: block 0's offsets less its own shift from h3 on + const int64_t s0 = g.shifts[0]; + g.hdr = {pos[1], pos[2], pos[3] - s0, pos[4] - s0, pos[5] - s0}; + g.h1 = pos[1]; + g.stride = pos[6] - s0; + g.period = 6; + return ""; +} + std::string find_token_fconst_exported(const std::string& pack_dir) { return find_token_fconst(pack_dir); } // engine#1786 (item F1): read a FastFlowLM NPU2 model source's container header. Safetensors layout: an 8-byte @@ -2458,6 +2547,20 @@ double embed_at(const EmbedTable& t, int row, int k) { return t.fmt == EmbedTable::Fmt::BF16 ? bf16_to_f32(v) : f16_to_f32(v); } +// engine#2604: a gemma pack stores its embedding PRE-SCALED by sqrt(hidden), and so does the forward +// (q4nx_semantics: embed_scale = sqrt(hidden); on a tied model the LM head divides its output by it), while the pack's +// 4-bit head is the raw table -- measured on translategemma-4b-it 1.8: best-fit scale 50.33 against sqrt(2560) = +// 50.60. Scaling the head's scale and zero by sqrt(hidden) is exact for an affine code and gives the forward the head +// it divides back out. Applied before the head is tiled and fitted, so every self-check gate then runs unchanged and a +// head off by anything other than that factor is still refused. No-op for any other model type. +bool gemma_prescale_head(Q4& q, const std::string& model_type, int64_t hidden) { + if (model_type.rfind("gemma", 0) != 0 || hidden <= 0) return false; + const float s = float(std::sqrt(double(hidden))); + for (auto& v : q.scale) v *= s; + for (auto& v : q.zero) v *= s; + return true; +} + double corr_bf16(const Q4& q, const uint8_t* embed_bf16, int H, int rows) { return corr(q, EmbedTable{embed_bf16, EmbedTable::Fmt::BF16, H}, rows); } @@ -4627,7 +4730,10 @@ Plan plan(const std::string& pack_dir) { p.o_k = int(H); } - } else if (g.hdr.size() == 3) { + } else if (g.hdr.size() == 3 || g.six_headers()) { + // engine#2604: the gemma3 6-header block opens the same way (its hdr[0..2] are k_norm, q_norm and + // post_attention_layernorm, in the global form); the per-layer sliding-window field is applied + // as a shift by the import. // engine#979: the Qwen3 families split the attention section in four. Measured on // both shipped packs: [k,q,v tiles][header + per-head k_norm][header + per-head q_norm // + o_proj tiles], with o_proj's tiles at K = q_proj's output width. @@ -4657,12 +4763,14 @@ Plan plan(const std::string& pack_dir) { } } else { refuse("the layer block has " + std::to_string(g.hdr.size() + 1) + - " epsilon headers, and this importer reads blocks of 2 (q/k/v/o in one payload) " - "or 4 (q/k/v, two per-head QK norms, o_proj) -- recorded 2026-11-04 on Qwen3"); + " epsilon headers, and this importer reads blocks of 2 (q/k/v/o in one payload), " + "4 (q/k/v, two per-head QK norms, o_proj) or 6 (the 4-header block plus gemma3's " + "pre/post-feedforward norms, engine#2604); any other count is refused"); } - // the MLP section starts right after the block's post-attention norm - const int64_t mlp_off = g.post_attn_hdr() + kNormOff + H * 2; - const MlpGeom mg = mlp_geom(H, d.intermediate, mlp_off, g.stride); + // the MLP section starts right after the block's post-attention norm -- or, on the gemma3 6-header + // block, after pre_feedforward_layernorm, and ends at post_feedforward_layernorm's header (engine#2604) + const int64_t mlp_off = (g.six_headers() ? g.hdr[3] : g.post_attn_hdr()) + kNormOff + H * 2; + const MlpGeom mg = mlp_geom(H, d.intermediate, mlp_off, g.six_headers() ? g.hdr[4] : g.stride); p.mlp_note = mg.why; if (mg.ok()) { p.mlp_gu_units = mg.gu_units; @@ -5901,6 +6009,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) if (!have_zp) implicit_zp_zero(q, zp_bits_for_spelling(mm.weight, mm.bits)); + if (q.N == int(V) && q.K == int(H) && gemma_prescale_head(q, p.dims.model_type, H)) + r.head_prescaled = true; std::vector tiled; long long pinexact = 0; tile_q4_1_f32(q, tiled, &pinexact); @@ -6173,7 +6283,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons bool head32_read = false; std::string head32_why; if (p.head_block == 32) { - const int64_t tail_off = int64_t(p.dims.layers) * g.stride; + const int64_t tail_off = g.base_of(p.dims.layers); const int64_t env = attn_payload_off(H); if (tail_off + env > int64_t(fc.size)) { head32_why = "the .fconst ends before the tail's envelope"; @@ -6258,7 +6368,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons (long long)p.head_block); } } else if (onnx.matmuls.empty()) { - const int64_t tail_off = int64_t(p.dims.layers) * g.stride; + const int64_t tail_off = g.base_of(p.dims.layers); const int64_t tile = attn_tile_bytes(H); // ceil, not floor: a vocab that is not a multiple of 64 is padded up to whole tiles in the pack // (Hunyuan-0.5B: 120,818 rows = 1,887.8 tiles, so the tail holds 1,888 and the last one is partly @@ -6311,6 +6421,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons "note: the head is stored as %d rows for a vocabulary of %lld (the pack pads " "to whole 64-row tiles); vocab_size stays %lld in the container\n", q.N, (long long)V, (long long)V); + if (q.K == int(H) && gemma_prescale_head(q, p.dims.model_type, H)) r.head_prescaled = true; std::vector tiled; long long pinexact = 0; tile_q4_1_f32(q, tiled, &pinexact); @@ -6366,18 +6477,25 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons ++r.tensors; }; for (int L = 0; L < p.dims.layers; ++L) { - const int64_t base = int64_t(L) * g.stride; + const int64_t base = g.base_of(L), sh = g.shift_of(L); + const std::string ln = "model.layers." + std::to_string(L) + "."; r.norm_anchors_total += 2; r.norm_anchors_ok += int(eps_at(fc, base, p.dims.eps_bits)) + - int(eps_at(fc, base + g.post_attn_hdr(), p.dims.eps_bits)); - add_norm(base + kNormOff, "model.layers." + std::to_string(L) + ".input_layernorm.weight"); - add_norm(base + g.post_attn_hdr() + kNormOff, - "model.layers." + std::to_string(L) + ".post_attention_layernorm.weight"); + int(eps_at(fc, base + g.post_attn_hdr() + sh, p.dims.eps_bits)); + add_norm(base + kNormOff, ln + "input_layernorm.weight"); + add_norm(base + g.post_attn_hdr() + sh + kNormOff, ln + "post_attention_layernorm.weight"); + if (g.six_headers()) { // engine#2604: gemma3's two feed-forward norms, the block's 5th and 6th headers + r.norm_anchors_total += 2; + r.norm_anchors_ok += int(eps_at(fc, base + g.hdr[3] + sh, p.dims.eps_bits)) + + int(eps_at(fc, base + g.hdr[4] + sh, p.dims.eps_bits)); + add_norm(base + g.hdr[3] + sh + kNormOff, ln + "pre_feedforward_layernorm.weight"); + add_norm(base + g.hdr[4] + sh + kNormOff, ln + "post_feedforward_layernorm.weight"); + } } // the final norm: the tail envelope opens at `layers x stride`, the norm sits at +64, and the // prefill file (when the pack ships one) holds a second copy that has to match byte for byte { - const int64_t tail = int64_t(p.dims.layers) * g.stride; + const int64_t tail = g.base_of(p.dims.layers); if (!eps_at(fc, tail, p.dims.eps_bits)) { late.push_back("model.norm.weight: no RMSNorm epsilon at " + std::to_string(tail) + " B, where the .fconst tail envelope opens, so its position is not " @@ -6459,15 +6577,29 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons {"v_proj", kv_out, attn_off, int(H), ft.v}, {"o_proj", int(H), p.o_off, p.o_k, 0}}; for (int L = 0; L < p.dims.layers; ++L) { - const int64_t block = int64_t(L) * g.stride; + const int64_t block = g.base_of(L); + // engine#2604: a gemma3 sliding-window layer carries 4 x kv_heads records of 8 B (bf16 0xFF7F, + // then zeros) between q_norm's gain and o_proj. Checked byte for byte before o_proj is read past it. + const int64_t sh = g.shift_of(L); + if (sh) { + const uint8_t* fld = at(fc, block + p.o_off, sh); + bool fld_ok = sh == int64_t(p.dims.kv_heads) * 8; + for (int64_t i = 0; fld_ok && i < sh; ++i) + fld_ok = fld[i] == ((i % 8) == 0 ? 0x7f : (i % 8) == 1 ? 0xff : 0x00); + if (!fld_ok) + late.push_back("model.layers." + std::to_string(L) + ".self_attn.o_proj.weight: the " + + std::to_string(sh) + " B before o_proj are not 4 x kv_heads records of " + "bf16 0xFF7F and zeros, so o_proj's start is not known"); + } for (const auto& sl : slots) { + const int64_t sl_base = sl.base + (std::strcmp(sl.proj, "o_proj") == 0 ? sh : 0); const int nt = sl.rows / kTileRows; const int tile_bytes = int(attn_tile_bytes(sl.k)); std::vector tiles; tiles.reserve(size_t(nt)); bool exact = true; for (int i = 0; i < nt; ++i) { - const uint8_t* src = at(fc, block + sl.base + int64_t(sl.first + i) * tile_bytes, + const uint8_t* src = at(fc, block + sl_base + int64_t(sl.first + i) * tile_bytes, tile_bytes); tiles.push_back(decode_tile(src, sl.k)); ++r.attn_tiles_total; @@ -6543,7 +6675,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // not survive that is refused by name. The geometry is re-derived from this block rather // than taken from plan(), so the stride identity is checked against the file being read. if (p.mlp_gu_units) { - const MlpGeom mg = mlp_geom(H, p.dims.intermediate, p.mlp_off, g.stride); + const MlpGeom mg = mlp_geom(H, p.dims.intermediate, p.mlp_off, g.six_headers() ? g.hdr[4] : g.stride); if (!mg.ok()) { late.push_back("the MLP section: " + mg.why); } else { @@ -6551,7 +6683,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons const int64_t gu_unit = group_bytes(H, kMlpRows); const int64_t down_group = group_bytes(inter, kTileRows); for (int L = 0; L < p.dims.layers; ++L) { - const int64_t base_gu = int64_t(L) * g.stride + mg.off; + const int64_t base_gu = g.base_of(L) + g.shift_of(L) + mg.off; const int64_t base_down = base_gu + mg.gu + mg.gap; std::vector gate, up, down; bool gate_ok = true, up_ok = true, down_ok = true; diff --git a/npu/pack_import.h b/npu/pack_import.h index ae82a0314..c34f193b2 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -667,6 +667,7 @@ struct Report { bool selfcheck_scaled = false; double lmhead_corr = 0, lmhead_rel = 0, ctrl_corr_nibble = 0, ctrl_corr_nozp = 0; HeadFit head_fit; // engine#1503: scale, rms ratio and the 4-bit noise floor beside lmhead_rel + bool head_prescaled = false; // engine#2604: a gemma head scaled by sqrt(hidden) to the forward's convention std::string model_type; // the pack's, so the CLI can refuse a known-bad architecture by name // The tail (.fconst) path has no MatMulNBits bytes to mis-decode, so its control is an ALIGNMENT one: the same // tiles, the same decoder, rows matched against the neighbouring embedding row. A correct decode correlates ~1 @@ -957,9 +958,29 @@ struct FconstGeom { // post_attention_layernorm weight and the first byte of the MLP section are read from here, so this is // the one rule the two readers share -- reading `h1` unconditionally imported the k_norm section's // padding as post_attention_layernorm on every Qwen3 pack (engine#1003). - int64_t post_attn_hdr() const { return hdr.empty() ? h1 : hdr.back(); } + int64_t post_attn_hdr() const { return hdr.size() == 5 ? hdr[2] : hdr.empty() ? h1 : hdr.back(); } + // engine#2604: the gemma3 1.8 block has SIX headers -- input_layernorm, k_norm, q_norm, post_attention_layernorm, + // pre_feedforward_layernorm, post_feedforward_layernorm -- and two sizes: a sliding-window layer carries a 4*kv_heads*8 + // byte field (each 8 B record bf16 0xFF7F then zeros; not a weight) between q_norm's gain and o_proj, a global layer + // does not. So a 6-header geometry is stored in its GLOBAL form (`hdr`, `stride` = the shorter block) and every layer + // has its own start (`base_of`, from a full-file header scan) and its own shift (`shift_of`: 0, or the field's size), + // applied to every offset from o_proj on. Empty `bases` (every other layout) means `L x stride` and shift 0. + std::vector bases; // layers + 1 entries when set: each block's start, then the tail's + std::vector shifts; // layers entries when set + int64_t base_of(int L) const { return bases.empty() ? int64_t(L) * stride : bases[size_t(L)]; } + int64_t shift_of(int L) const { return shifts.empty() ? 0 : shifts[size_t(L)]; } + bool six_headers() const { return hdr.size() == 5; } std::string why; // non-empty when the geometry could not be established }; +// engine#2604: a gemma head's scale and zero multiplied by sqrt(hidden), to the forward's pre-scaled-embedding +// convention; false (and q untouched) for any other model type. +bool gemma_prescale_head(Q4& q, const std::string& model_type, int64_t hidden); +// engine#2604: fill `g.bases` / `g.shifts` for a 6-header (gemma3 1.8) geometry from EVERY epsilon header in `path`, and +// rewrite `g.hdr` / `g.stride` to the global (shorter) form. Empty on success; otherwise why the blocks do not fit (the +// header count is not 6 x layers + 1, a block's k/q-norm offsets differ from the first's, or the blocks come in more than +// two sizes, or the size difference is not 4 x kv_heads x 8). +std::string fconst_six_header_blocks(const std::string& path, unsigned short eps_bits, int layers, int kv_heads, + FconstGeom& g); unsigned short bf16_bits_of(float v); // engine#1650: config.json may declare an OPTIONAL field as JSON null (AMD's 1.9 diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 2846056cf..9a0ac8666 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4449,6 +4449,64 @@ void test_fconst_block_headers() { g.post_attn_hdr() == 340, "block headers: and is set as a 4-header block whose post-attention header is the last one, not k_norm's"); } + // engine#2604: the gemma3 1.8 6-header block, two sizes. A synthetic Token .fconst: 4 layers, kv_heads 2, so the + // sliding-window field is 2 x 8 = 16 B; layer 2 is the global one (no field). Headers are the strict 64-byte form. + { + const unsigned short eps = 0x3586; + const int64_t glob[6] = {0, 1000, 1200, 2000, 2200, 3000}, gsize = 3200, field = 16; + const bool local[4] = {true, true, false, true}; + const auto file_of = [&](int drop, int64_t extra_on_l1) { + std::vector pos; + int64_t at = 0; + for (int L = 0; L < 4; ++L) { + const int64_t sh = (local[L] ? field : 0) + (L == 1 ? extra_on_l1 : 0); + for (int j = 0; j < 6; ++j) pos.push_back(at + glob[j] + (j >= 3 ? sh : 0)); + at += gsize + sh; + } + pos.push_back(at); // the tail's header (final norm) + std::string b(size_t(at + 4096), '\0'); + for (size_t i = 0; i < pos.size(); ++i) { + if (int(i) == drop) continue; + b[size_t(pos[i])] = char(eps & 0xFF); + b[size_t(pos[i]) + 1] = char(eps >> 8); + } + return b; + }; + const std::string path = (fs::temp_directory_path() / ("six_hdr." + std::to_string(::getpid()))).string(); + put_file(path, file_of(-1, 0)); + pi::FconstGeom g; + g.hdr = {1000, 1200, 2016, 2216, 3016}; // what the eps vote records from block 0 (a local layer) + g.stride = gsize + field; + const std::string why = pi::fconst_six_header_blocks(path, eps, 4, 2, g); + check(why.empty() && g.six_headers() && g.stride == gsize && g.hdr == std::vector{1000, 1200, 2000, 2200, 3000} && + g.post_attn_hdr() == 2000, + "six-header blocks: the global form -- stride 3,200, post_attention at h3, not the last header (" + why + ")"); + check(g.base_of(0) == 0 && g.base_of(1) == 3216 && g.base_of(2) == 6432 && g.base_of(3) == 9632 && + g.base_of(4) == 12848, + "six-header blocks: every layer's start from its own header, and the tail's after the last layer"); + check(g.shift_of(0) == field && g.shift_of(1) == field && g.shift_of(2) == 0 && g.shift_of(3) == field, + "six-header blocks: the sliding-window layers shift by the 16-byte field, the global one by 0"); + pi::FconstGeom g2; + put_file(path, file_of(9, 0)); + check(pi::fconst_six_header_blocks(path, eps, 4, 2, g2).find("epsilon headers") != std::string::npos, + "six-header blocks: a missing header refuses by count, it does not shift the blocks after it"); + pi::FconstGeom g3; + put_file(path, file_of(-1, 8)); + check(pi::fconst_six_header_blocks(path, eps, 4, 2, g3).find("sizes") != std::string::npos, + "six-header blocks: a third block size refuses by name"); + fs::remove(path); + } + { // engine#2604: the gemma head pre-scale, to the forward's convention (embedding pre-scaled by sqrt(hidden)) + pi::Q4 q; + q.scale = {1.0f, 2.0f}; + q.zero = {-1.0f, 0.5f}; + pi::Q4 r = q; + check(pi::gemma_prescale_head(q, "gemma3_text", 2560) && std::fabs(q.scale[1] - 2.0f * std::sqrt(2560.0f)) < 1e-3f && + std::fabs(q.zero[0] + std::sqrt(2560.0f)) < 1e-3f, + "gemma head: scale and zero multiplied by sqrt(hidden)"); + check(!pi::gemma_prescale_head(r, "qwen2", 2560) && r.scale[1] == 2.0f && r.zero[0] == -1.0f, + "gemma head: any other model type is untouched"); + } // engine#1985: the cut has to come from the observed distribution, not from a constant or from a fraction of the window. // 16 blocks of a uniform 4-header pack: the three real offsets appear 16 times each, a coincidence appears 3 times. // The constant keeps the coincidence -- four offsets, which `fconst_set_block_headers` refuses -- and the observed floor diff --git a/tests/npu_template_test.cpp b/tests/npu_template_test.cpp index e8fae2d53..32cea4ca5 100644 --- a/tests/npu_template_test.cpp +++ b/tests/npu_template_test.cpp @@ -121,6 +121,34 @@ int main(int argc, char** argv) { run_fixture(a); } + // engine#2604: the turn-end probe takes the first user-content shape a template renders. A template that refuses a + // string (translategemma's) still yields its marker, and an ordinary one is read with the plain shape. + { + onebit::npu::ChatTemplate strict; + strict.text = "{% for m in messages %}{% if m.role == 'user' %}{% if m.content is string %}" + "{{ raise_exception('User role must provide content as an iterable') }}{% endif %}" + "{% if m.content[0].source_lang_code is not defined %}{{ raise_exception('no language codes') }}" + "{% endif %}" + "user\n{{ m.content[0].source_lang_code }}:{{ m.content[0].text }}\n" + "{% else %}model\n{{ m.content }}\n{% endif %}{% endfor %}"; + strict.source = "test"; + const onebit::npu::TurnEndProbe p = onebit::npu::turn_end_from_template(strict, "", ""); + check(p.marker == "" && p.form == "translation", + "turn-end probe: a template that refuses string content still yields (form " + p.form + + ", " + p.why + ")"); + onebit::npu::ChatTemplate plain; + plain.text = "{% for m in messages %}<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n{% endfor %}"; + plain.source = "test"; + const onebit::npu::TurnEndProbe q = onebit::npu::turn_end_from_template(plain, "", ""); + check(q.marker == "<|im_end|>" && q.form == "plain", "turn-end probe: an ordinary template is read with the plain shape"); + onebit::npu::ChatTemplate broken; + broken.text = "{{ raise_exception('never renders') }}"; + broken.source = "test"; + const onebit::npu::TurnEndProbe b = onebit::npu::turn_end_from_template(broken, "", ""); + check(b.marker.empty() && b.form.empty() && b.why.find("never renders") != std::string::npos, + "turn-end probe: a template that renders no shape yields no marker and says why"); + } + // The chat route's own variables (engine#1788): a request's `tools` reach the template, in the // caller's key order, and an empty list is "none offered". The route used to pass only // chat_template_kwargs and enable_thinking, so a template's tools block never rendered.