From d6316a845ed899aa28214c7d7dae14eaf98851d3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 22:14:02 -0400 Subject: [PATCH 1/6] npu-pack: locate the 1.9 head in the .fconst tail instead of requiring it to end the file (engine#1740 W1a) The tail is [envelope][rows x tile][padding] and the importer required `have == env + rows*tile` exactly. That is true for a pack that ends with the head and false for one that ends with padding -- 110,208,000 B of it on the hidden-1536 family -- so 12+ of the 42 1.9 packs had their output layer refused for a reason about PLACEMENT rather than about the fields. The refusal was named and correct; it was the placement assumption that was wrong. The placement is now measured: * the trailing all-zero run at EOF is counted, whatever its alignment; * `plan_tail_head()` (exposed in the header) turns (bytes, tail_off, envelope, rows, tile, zeros) into the earliest and latest tile-aligned candidate, or a refusal carrying both lengths. It is arithmetic only, so it is pinned by a host test with no pack, no device and no fixture; * `locate_tail_head()` walks those candidates, requires decode -> encode to be exact, and then -- only then -- correlating against the pack's own embedding decides. That order matters: an all-zero tile decodes to zeros AND re-encodes to zeros, so the round-trip proves the FIELD ORDER and nothing about placement. Above `kTieCorr` is the head; two candidates above it, or none, is an ambiguity refusal naming both correlations and both offsets. Tied and untied are reported rather than assumed. A tied pack gets the correlation as its self-check; an untied pack has no reference in the file, so it is taken only when the tail is exact, and the report says the placement is unverified instead of printing a number that looks like a check. `tie_word_embeddings` is still read from the pack, never inferred. Gate 1, `Llama-3.1-8B_rai_1.9.0_npu` (untied, hidden 4096, 916,096 tiles): imports, 291 tensors, no gaps; attention 5120/5120 and MLP 9216/9216 .fconst tiles survive decode->encode byte for byte; fp32 scale/zero exact for 100.00% of groups; self-check states the untied case rather than claiming a correlation. Regression, `Qwen2.5-0.5B-Instruct 1.9` (tied, hidden 896): unchanged -- 291 tensors / 67,088 tiles, corr 0.999999, rel_err 0.000700, row-shifted control 0.0716. Block-32 heads (the 1.5B trio) still refuse by name: `attn_tile_bytes(H)` is not their tile, so no amount of skipping zeros fixes them, and W1b's placement is the thing that will. One dead branch removed on the way: with `have >= want` and a tile-aligned `want`, the "fewer than `rows` whole tiles" case cannot arise, so it is gone rather than tested. --- npu/pack_import.cpp | 169 +++++++++++++++++++++++++++++++-- npu/pack_import.h | 16 ++++ tests/npu_pack_import_test.cpp | 63 ++++++++++++ 3 files changed, 241 insertions(+), 7 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 19c171787..e25ddd5bf 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1809,6 +1809,161 @@ std::vector encode_tile(const TileW& t) { return out; } + +constexpr int kLocateSlack = 8; // tiles of search before the zero-run position + +// The placement arithmetic on its own: no file, no decode, no correlation. Split out because it is the half that can be +// wrong without anything throwing -- an off-by-one tile here reads a plausible region and only the correlation notices +// -- so it is pinned by a host test that needs no pack, no device and no fixture. +// TailHeadPlacement is declared in the header: the placement arithmetic is pinned by a host test. +// bytes = the whole .fconst's length; zeros = trailing all-zero run measured by the caller. +TailHeadPlacement plan_tail_head(int64_t bytes, int64_t tail_off, int64_t env, int64_t rows, int64_t tile, + int64_t zeros) { + TailHeadPlacement p; + p.rows = rows; + p.zeros = zeros; + p.have = bytes - tail_off; + p.want = env + rows * tile; + if (p.have < p.want) { + p.why = "the .fconst tail is " + std::to_string(p.have) + " B, shorter than the " + std::to_string(p.want) + + " B its envelope and head tiles would take"; + return p; + } + p.exact = (p.have == p.want); + const int64_t base = tail_off + env; + p.last = base + ((bytes - zeros - base) / tile - rows) * tile; + p.first = p.last - kLocateSlack * tile; + if (p.first < base) p.first = base; + if (p.last < base) p.last = base; + return p; +} + +// engine#1740: where the 1.9 Token .fconst keeps the lm head. +// +// The tail is [envelope][rows x tile][padding], and only the middle term is the head. The importer used to require +// `have == env + rows*tile` exactly, which is true for a pack that ends with the head and false for one that ends with +// padding -- and on the hidden-1536 family the padding is 110,208,000 B, 11 % of the file. Refusing there is a named +// refusal, but it refuses the head for a reason that is about placement, not about the fields, and 12+ of the 42 1.9 +// packs keep their head only here. +// +// So the placement is measured rather than assumed, in this order: +// 1. the run of all-zero bytes at EOF (`zeros`), whatever its alignment -- a pack can also end in junk that is not a +// whole number of tiles (Hunyuan-0.5B's tail is 28 B off), and that has to be visible rather than rounded away; +// 2. `rows` consecutive tile-aligned candidates, each starting at most kSlack tiles before the position the zero run +// implies, because "the data ends where the zeros begin" is arithmetic and the head may not sit there; +// 3. the round-trip (decode -> encode) proves the FIELD ORDER and nothing else: an all-zero tile decodes to zeros and +// re-encodes to zeros exactly, so padding passes it. Placement therefore has to be decided by the correlation +// against the pack's own embedding when the pack is tied, and refused when it is not. +// +// The outcome is a value, not a std::string, so the caller cannot accidentally proceed on a refusal. +struct TailHeadLocate { + int64_t off = -1; // byte offset of head tile 0 + int64_t rows = 0; // tiles of the head + int64_t zeros = 0; // trailing all-zero bytes at EOF + int64_t slack = 0; // tiles the accepted placement sits before the zero-run position + double corr = 0.0; // correlation of the accepted placement against the embedding (tied only) + double runner = 0.0; // best rejected placement's correlation, for the ambiguity message + bool verified = false;// corr above kTieCorr: this is the head, not merely a decodable region + bool exact = false; // the tail length matched with no padding at all (the pre-#1740 path) + std::string why; // set on refusal: what was measured, in numbers +}; + +static int64_t trailing_zero_run(const Fconst& fc, int64_t from) { + const uint8_t* m = fc.map; + int64_t i = fc.size; + while (i > from && m[i - 1] == 0) --i; + return int64_t(fc.size) - i; +} + +// `want` is env + rows*tile at this geometry; `tail_off` is where the tail begins (layers x stride). +static TailHeadLocate locate_tail_head(const Fconst& fc, int64_t tail_off, int64_t env, int64_t rows, int64_t tile, + int64_t k, const EmbedTable& embed, bool tied) { + TailHeadLocate r; + r.rows = rows; + r.zeros = trailing_zero_run(fc, tail_off); + const TailHeadPlacement plan = plan_tail_head(int64_t(fc.size), tail_off, env, rows, tile, r.zeros); + r.exact = plan.exact; + if (!plan.why.empty()) { + r.why = plan.why; + return r; + } + // Candidate 0 is where the zero run says the head ends; earlier candidates cover a head that does not end at the + // last non-zero byte. Every candidate must be tile-aligned to the same phase as the envelope, so a jumbled offset + // cannot be mistaken for a placement. + const int64_t base = tail_off + env; + // The planner gives the earliest and latest tile-aligned candidate; the zero run decides the latest. + const int64_t k0 = (plan.last - base) / tile; + int64_t best_slack = -1, second_slack = -1; + double best = -2.0, second = -2.0; + int64_t tried = 0; + for (int64_t cand = (plan.first - base) / tile; cand <= k0; ++cand) { + const int64_t s = base + cand * tile; + if (s < base || s + rows * tile > int64_t(fc.size)) continue; + std::vector tiles; + tiles.reserve(size_t(rows)); + bool exact_rt = true; + for (int64_t i = 0; i < rows; ++i) { + const uint8_t* src = fc.map + s + i * tile; + tiles.push_back(decode_tile(src, int(k))); + const std::vector back = encode_tile(tiles.back()); + if (std::memcmp(back.data(), src, size_t(tile)) != 0) exact_rt = false; + } + ++tried; + if (!exact_rt) continue; + double c = 0.0; + if (tied) { + const Q4 q = q_from_tiles(tiles); + if (q.N < int(embed.H) || q.K != int(embed.H)) continue; + c = corr(q, embed, int(embed.H)); + } + if (c > best) { + second = best; second_slack = best_slack; + best = c; best_slack = cand; + } else if (c > second) { + second = c; second_slack = cand; + } + } + if (tried == 0) { + r.why = "no tile-aligned placement in the last " + std::to_string(rows + kLocateSlack) + " tiles decodes and " + "re-encodes exactly, so the head's fields are not understood at this geometry"; + return r; + } + if (best_slack < 0) { + r.why = "the tail holds " + std::to_string(tried) + " tile-aligned placement(s) and none of them survives " + "decode->encode, so the head's offsets are not established"; + return r; + } + r.off = base + best_slack * tile; + r.slack = best_slack; + r.corr = best; + r.runner = second; + if (!tied) { + // No reference inside the pack: only the exact placement is taken, and it is reported as unverified. + if (!r.exact || best_slack != 0) { + r.why = "this pack is untied, so the head cannot be verified against its embedding, and its tail carries " + + std::to_string(r.zeros) + " B of trailing zeros -- the placement would be a guess"; + return r; + } + r.off = base; + r.why.clear(); + return r; + } + if (best < kTieCorr) { + r.why = "the best placement correlates " + std::to_string(best) + " with the pack's own embedding (the tie " + "threshold is " + std::to_string(kTieCorr) + "), so no placement in the tail is the head"; + return r; + } + if (second > kTieCorr) { + r.why = "two placements both correlate with the embedding (" + std::to_string(best) + " at offset " + + std::to_string(base + best_slack * tile) + ", " + std::to_string(second) + " at offset " + + std::to_string(base + (second_slack < 0 ? 0 : second_slack) * tile) + + "), so the head's offset is ambiguous"; + return r; + } + r.verified = true; + return r; +} + Q4 q_from_tiles(const std::vector& tiles) { if (tiles.empty()) throw std::runtime_error("no tiles"); const int k = tiles.at(0).k, kbs = k / kKBlock, rows = tiles.at(0).rows; @@ -3505,18 +3660,18 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // byte, which the constant would have missed by 2,304. const int64_t env = attn_payload_off(H); const int64_t want = env + rows * tile; - const int64_t have = int64_t(fc.size) - tail_off; - if (want != have) { - late.push_back("lm_head.weight: the .fconst tail is " + std::to_string(have) + " B, not the " + - std::to_string(want) + " B its envelope and head tiles would take (" + - std::to_string(env) + " + " + std::to_string(rows) + " x " + - std::to_string(tile) + " B), so the head's offsets are not established"); + // engine#1740: the head is LOCATED, not assumed to end the file. `want == have` was an equality + // check on the tail length, so a pack whose .fconst ends in padding (110,208,000 B on the + // hidden-1536 family) was refused for a reason about placement rather than about its fields. + const TailHeadLocate loc = locate_tail_head(fc, tail_off, env, rows, tile, H, embed_tbl, tie_declared && tied); + if (loc.off < 0) { + late.push_back("lm_head.weight: " + loc.why); } else { std::vector tiles; tiles.reserve(size_t(rows)); bool exact = true; for (int64_t i = 0; i < rows; ++i) { - const uint8_t* src = at(fc, tail_off + env + i * tile, tile); + const uint8_t* src = at(fc, loc.off + i * tile, tile); tiles.push_back(decode_tile(src, int(H))); const std::vector back = encode_tile(tiles.back()); if (std::memcmp(back.data(), src, size_t(tile)) != 0) exact = false; diff --git a/npu/pack_import.h b/npu/pack_import.h index 1a4f96da4..10f2420b7 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -188,6 +188,22 @@ struct TextArtifacts { }; TextArtifacts scan_text_artifacts(const std::string& pack_dir); +// engine#1740 (W1a): where the 1.9 Token .fconst keeps its output layer. The arithmetic is exposed because it is the +// half that can be wrong silently -- a one-tile error reads a plausible region and only a correlation against the +// embedding notices -- so it is pinned by a host test with no pack, no device and no fixture. +struct TailHeadPlacement { + int64_t first = -1; // earliest tile-aligned candidate (== tail_off + env) + int64_t last = -1; // candidate the trailing zero run implies + int64_t rows = 0; + int64_t zeros = 0; + int64_t have = 0; + int64_t want = 0; + bool exact = false; // have == want: the head ends the file + std::string why; // non-empty when no candidate exists +}; +TailHeadPlacement plan_tail_head(int64_t bytes, int64_t tail_off, int64_t env, int64_t rows, int64_t tile, + int64_t zeros); + struct Options { bool allow_incomplete = false; // write what is representable and name the rest // The device weight layout the container will declare (`q4nx_weight_layout`). Empty means the diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a639cc241..2ff7a4f2c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2907,7 +2907,70 @@ void test_embedding_f32_component(const std::string& root) { fs::remove_all(b); } +// engine#1740 (W1a): the tail-head placement arithmetic, pinned without a pack, a device or a fixture. The rows, the +// tile size and the envelope are the hidden-896 pack's, so the numbers here are the real ones: 1,856 B envelope, +// 2,374 tiles of 122,880 B, tail 1,856 + 2,374 x 122,880 = 291,653,952 B. +void test_tail_head_placement() { + const int64_t env = 1856, rows = 2374, tile = 122880; + const int64_t head = rows * tile; // 291,741,120 B... one tile more than the real tail + const int64_t exact_bytes = 896 * 0 + env + head; // the file ends with the head + // (a) the head ends the file: exactly one candidate, no slack + { + const pi::TailHeadPlacement p = pi::plan_tail_head(exact_bytes, 0, env, rows, tile, 0); + check(p.why.empty() && p.exact, "tail placement: a tail that ends with the head is exact"); + check(p.first == env && p.last == env, "tail placement: and the only candidate is the envelope's end"); + } + // (b) n whole zero tiles after the head: the candidate walks back by exactly n tiles, never past the envelope + for (int64_t n : {1, 3, 9}) { + const int64_t bytes = exact_bytes + n * tile; + const pi::TailHeadPlacement p = pi::plan_tail_head(bytes, 0, env, rows, tile, n * tile); + check(p.why.empty() && !p.exact, "tail placement: trailing padding is not the exact case"); + check(p.last == env, + "tail placement: " + std::to_string(n) + " zero tile(s) at EOF still land the head at the envelope's end"); + check(p.first == env, "tail placement: and the search never walks to before the envelope"); + } + // (c) a clean zero run of ANY length puts the head at the envelope's end -- the padding's size is not a displacement + { + const int64_t bytes = exact_bytes + 20 * tile; + const pi::TailHeadPlacement p = pi::plan_tail_head(bytes, 0, env, rows, tile, 20 * tile); + check(p.why.empty() && p.first == env && p.last == env, + "tail placement: trailing padding of 20 tiles still ends the head at the envelope's end"); + } + // (c2) but when the file ends in NON-zero bytes after the head, the zero run is short and the candidates span the + // slack -- this is the shape the caller's correlation has to decide (the hidden-1536 case) + { + const int64_t bytes = exact_bytes + 20 * tile; + const pi::TailHeadPlacement p = pi::plan_tail_head(bytes, 0, env, rows, tile, 3 * tile); + check(p.why.empty() && p.first > env && p.last > p.first, + "tail placement: a short zero run before non-zero junk leaves a span of candidates, not one guess"); + } + // (d) refinements: a tail shorter than envelope + head is refused by name, with both numbers in it + { + const pi::TailHeadPlacement p = pi::plan_tail_head(env + head - 1, 0, env, rows, tile, 0); + check(!p.why.empty(), "tail placement: a short tail is refused, not read"); + check(p.why.find(std::to_string(env + head)) != std::string::npos, + "tail placement: and the refusal carries the length it wanted (" + p.why + ")"); + } + // (e) one byte short is refused for the length, and one byte more is a candidate rather than a refusal: the + // boundary is exact, which is what keeps a part-tile payload from being rounded up into a head + { + const pi::TailHeadPlacement short_by_one = pi::plan_tail_head(env + head - 1, 0, env, rows, tile, 0); + const pi::TailHeadPlacement one_more = pi::plan_tail_head(env + head + 1, 0, env, rows, tile, 0); + check(!short_by_one.why.empty() && one_more.why.empty(), + "tail placement: the length boundary is exact (short refused, one byte more is a candidate)"); + check(one_more.last == env, "tail placement: and the extra byte does not move the head"); + } + // (f) a tail_off that moves the window is honoured: the same arithmetic one layer block later + { + const int64_t tail_off = 5 * 8708288; + const pi::TailHeadPlacement p = pi::plan_tail_head(tail_off + exact_bytes, tail_off, env, rows, tile, 0); + check(p.why.empty() && p.last == tail_off + env, "tail placement: the window is relative to tail_off"); + } +} + + int main(int argc, char** argv) { + test_tail_head_placement(); const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; // The pack leg is the only one that can regress a real pack, and ctest runs this binary with the From 449a2ab047c7a0c14271a0e9abfd5cc608300931 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 22:17:10 -0400 Subject: [PATCH 2/6] npu-pack: report WHICH tail placement the head was taken from "imported" cannot distinguish a tail that ends with the head from one that ends in padding, and that difference is the whole point of the search. The line names tile 0's offset, the tile count, the envelope it follows, the trailing zero run, the slack, whether the tail ends with the head, and whether the placement is verified (tied) or merely taken (untied). Measured through it: Qwen2.5-0.5B (tied) tile 0 at 209,000,768 B, 2,374 tiles after a 1,856 B envelope, 0 B zeros, 0 slack, exact, verified by correlation; Mistral-7B-Instruct-v0.3 (untied) tile 0 at 3,771,611,200 B, 512 tiles after an 8,256 B envelope, 0 B zeros, 0 slack, exact, taken not verified. --- npu/pack_import.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index e25ddd5bf..30930cd3e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3667,6 +3667,17 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (loc.off < 0) { late.push_back("lm_head.weight: " + loc.why); } else { + // engine#1740: say WHICH placement was taken. "imported" alone cannot distinguish a tail that ends + // with the head from one that ends in padding, and that difference is the whole point of the + // search -- so the number a reviewer needs is here rather than inferred from the file size. + std::fprintf(stderr, + " lm head: tile 0 at %lld B, %lld tile(s) after a %lld B envelope; %lld B of trailing " + "zeros, %lld tile(s) of slack%s%s\n", + (long long)loc.off, (long long)rows, (long long)env, (long long)loc.zeros, + (long long)loc.slack, + loc.exact ? ", the tail ends with the head" : "", + loc.verified ? " (tied: located by correlation with the pack's embedding)" + : " (untied: placement taken, not verified)"); std::vector tiles; tiles.reserve(size_t(rows)); bool exact = true; From 40c70c96112884692f0864733c2b3edd4d501aa3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 22:44:06 -0400 Subject: [PATCH 3/6] npu: apply the dynamic-NTK rope base rule instead of refusing it (engine#1740 W2, Hunyuan-0.5B) The 1.9 Hunyuan packs declare `rope_scaling: {type: "dynamic", factor: 1.0}` with a 262,144 window, and the forward refused them by name -- correctly, under the #1087 rule that a declared scaling which is not applied must not be served as plain rope. This is the rule, so the refusal can go. Dynamic NTK is a BASE rule, not a per-dim one, so it is a pure function of (theta, factor, window, position, dim): below the window: base = theta -- bit for bit the same as a model with no scaling past it: base = theta * (factor * pos / window - (factor - 1)) ^ (dim / (dim - 2)) which is HF's `_compute_dynamic_ntk_parameters` and what ggml applies through rope_yarn. Because the base moves with the position it is computed once per position rather than once per dim pair, and because the rule is a pure function it is pinned by the existing rope test with no device, no pack and no fixture: * below the window the base IS theta (position 0, and one position short of the window), and Hunyuan's factor 1.0 at the window is a no-op bit for bit -- which is what makes accepting that specific block safe rather than a guess; * factor 4 at twice the window reproduces `theta * 5^(dim/(dim-2))` to 1e-6; * an unusable factor, an unusable window, and `on == 0` all leave the base alone -- the same rule the llama3 helper follows, so a malformed block cannot re-frequency a model in a way no reference reproduces; * the support predicate accepts "dynamic"/"dynamic_ntk" ONLY when the numbers were read, and still refuses yarn, linear and mrope. `all llama3 rope checks passed`, including the new arms. What this does NOT yet claim: that Hunyuan-0.5B serves. That needs an npu-hold run of the real pack (import is already clean at 219 tensors), and it is the next step rather than something this commit can assert. --- npu/forward/include/q4nx_forward.h | 1 + npu/forward/include/q4nx_rope.h | 37 ++++++++++++++++++++++++++++-- npu/forward/src/q4nx_forward.cpp | 34 ++++++++++++++++++++++++--- tests/npu_rope_llama3_test.c | 36 +++++++++++++++++++++++++---- 4 files changed, 99 insertions(+), 9 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index b37b33176..770d31197 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -427,6 +427,7 @@ class Q4nxNpuForward { std::vector> q_bias_, k_bias_, v_bias_; // [NL][NH*HD]/[NL][NKV*HD] attention biases std::vector rope_scale_; // LongRoPE short_factor (per-half-dim), empty when none Q4nxRopeLlama3 rope_llama3_; // rope_type "llama3" (Llama-3.1/3.2/3.3); on == 0 for everything else + Q4nxRopeDynamic rope_dynamic_; // rope_type "dynamic" (NTK base rule, engine#1740 W2); on == 0 elsewhere // MiniCPM4 scales (config "embedding_scale" / "residual_scale"); 1.0 for // every other architecture. MiniCPM4 scales the embedding by 12 and each // residual branch by scale_depth/sqrt(NL) (0.2475 for 32 layers); skipping diff --git a/npu/forward/include/q4nx_rope.h b/npu/forward/include/q4nx_rope.h index 36d666b7c..946e41670 100644 --- a/npu/forward/include/q4nx_rope.h +++ b/npu/forward/include/q4nx_rope.h @@ -37,6 +37,7 @@ */ #pragma once +#include #include #ifdef __cplusplus @@ -51,6 +52,34 @@ typedef struct { double original_max_position; /* 8192 — the band is in THIS length's units, not the served ctx */ } Q4nxRopeLlama3; +/* Dynamic NTK ("dynamic"/"dynamic_ntk") — the one rope_scaling type the 1.9 Hunyuan packs declare, and the reason + * their serve refuses today. This is a BASE rule rather than a per-dim rule: below the model's original window the + * base is the plain theta (so a short prompt is bit-identical to a model that never declared scaling), and above it + * + * base = theta * ((factor * pos / original) - (factor - 1)) ^ (dim / (dim - 2)) + * + * which is the formula HF applies in _compute_dynamic_ntk_parameters and ggml applies through rope_yarn. The base + * therefore has to be recomputed when a position crosses the window -- unlike llama3 and longrope, which are functions + * of the dimension alone -- so this is a pure function of (theta, factor, original, pos, dim) with no state, and a + * caller that computes it once per position pays one pow per position, not one per dim pair. + * + * `on == 0` (or unusable numbers) returns `theta` unchanged, so a malformed block cannot silently re-frequency a model + * in a way no reference reproduces -- the same rule the llama3 helper follows. */ +typedef struct { + int on; /* config declared rope_type "dynamic" with usable numbers */ + double factor; /* 1.0 on Hunyuan-0.5B -- a factor of 1 is a no-op until pos > original */ + double original_max_position; /* 262144 on Hunyuan-0.5B */ +} Q4nxRopeDynamic; + +static inline double q4nx_rope_dynamic_base(double theta, const Q4nxRopeDynamic* d, double pos, int dim) { + if (!d || !d->on) return theta; + if (!(d->factor > 0.0) || !(d->original_max_position > 0.0) || dim <= 2) return theta; + if (!(pos >= d->original_max_position)) return theta; /* inside the window: plain rope, bit for bit */ + const double scaled = d->factor * pos / d->original_max_position - (d->factor - 1.0); + if (!(scaled > 0.0)) return theta; /* a factor below 1 can drive this negative */ + return theta * pow(scaled, (double)dim / (double)(dim - 2)); +} + /* One dim pair's inverse frequency, after the llama3 rule. `inv` is the plain theta^(-i/half). * Returns `inv` unchanged when `s` is null or off, or when the block is not usable (a factor of 0, * a band with high <= low), because a malformed scaling block must not silently re-frequency a model @@ -133,12 +162,16 @@ static inline void q4nx_rope_declared_type(const char* json, char* out, int cap) * usable numbers (a declared-but-unusable block is not the same thing as no block) and `longrope` * needs the short_factor the forward reads. */ static inline int q4nx_rope_type_supported(const char* type, const Q4nxRopeLlama3* llama3, - int has_short_factor) { + int has_short_factor, int has_dynamic_numbers) { if (!type || !type[0]) return 1; /* no block declared: plain rope */ if (!strcmp(type, "default") || !strcmp(type, "none")) return 1; if (!strcmp(type, "llama3")) return llama3 && llama3->on; if (!strcmp(type, "longrope")) return has_short_factor ? 1 : 0; - return 0; /* yarn, linear, dynamic, mrope, ... */ + /* "dynamic"/"dynamic_ntk" is accepted only with usable numbers: the base rule needs the factor and the window, + * and a declared-but-unusable block must be refused rather than served as plain rope (engine#1087). The + * predicate cannot see the numbers, so the caller passes `has_short_factor` as "the numbers were read". */ + if (!strcmp(type, "dynamic") || !strcmp(type, "dynamic_ntk")) return has_dynamic_numbers ? 1 : 0; + return 0; /* yarn, linear, mrope, ... */ } #ifdef __cplusplus diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 444356ee1..c0ca7d998 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -193,6 +193,29 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa // transformers' _compute_llama3_parameters applies. `on == 0` unless the config names the type, so // every other architecture builds exactly the table it built before. Keys are searched with their // quotes, which is what keeps "factor" from matching inside "high_freq_factor". +// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`. Hunyuan's +// 1.9 packs declare factor 1.0 with a 262,144 window, i.e. a no-op until a prompt passes that window -- which is why +// accepting it is safe rather than a guess: below the window the base IS theta, so the served bytes are plain rope. +static Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { + Q4nxRopeDynamic d{}; + std::ifstream f(cfg_path); + if (!f) return d; + std::string t((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + char type[64]; + q4nx_rope_declared_type(t.c_str(), type, (int)sizeof type); + if (std::strcmp(type, "dynamic") && std::strcmp(type, "dynamic_ntk")) return d; + auto number = [&](const char* key) { + const std::string k = std::string("\"") + key + "\":"; + const size_t at = t.find(k); + if (at == std::string::npos) return 0.0; + return std::strtod(t.c_str() + at + k.size(), nullptr); + }; + d.factor = number("factor"); + d.original_max_position = number("max_position_embeddings"); + d.on = (d.factor > 0.0 && d.original_max_position > 0.0) ? 1 : 0; + return d; +} + static Q4nxRopeLlama3 read_rope_llama3(const std::string& cfg_path) { Q4nxRopeLlama3 s{}; std::ifstream f(cfg_path); @@ -579,6 +602,7 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, } rope_scale_ = read_rope_short_factor(cfg_path); // LongRoPE (MiniCPM4) rope_llama3_ = read_rope_llama3(cfg_path); // rope_type "llama3" (engine#1048) + rope_dynamic_ = read_rope_dynamic(cfg_path); // rope_type "dynamic" (NTK by base, engine#1740 W2) { // A declared scaling this forward does not apply is refused BY NAME (engine#1087): serving it // would run plain rope, stay fluent and be wrong past the original window, with nothing in the @@ -587,7 +611,7 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, std::string t((std::istreambuf_iterator(f)), std::istreambuf_iterator()); char declared[64]; q4nx_rope_declared_type(t.c_str(), declared, (int)sizeof declared); - if (!q4nx_rope_type_supported(declared, &rope_llama3_, rope_scale_.empty() ? 0 : 1)) { + if (!q4nx_rope_type_supported(declared, &rope_llama3_, rope_scale_.empty() ? 0 : 1, rope_dynamic_.on)) { if (!std::strcmp(declared, "llama3")) err_ = "config.json declares rope_type \"llama3\" but its numbers are not usable (factor > 0, " "original_max_position_embeddings > 0, high_freq_factor > low_freq_factor), so the " @@ -4187,13 +4211,17 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int { const int half = proportional ? HD / 2 : rot / 2; std::vector cos_(half), sin_(half); + // Dynamic NTK: the base moves with the position, so it is one pow per position rather than one per dim pair. + // Below the model's original window this returns theta unchanged, which is what makes a short prompt + // bit-identical to the same model served without the scaling. + const double rope_theta = q4nx_rope_dynamic_base((double)theta, &rope_dynamic_, (double)pos, (int)HD); for (int i = 0; i < half; i++) { double inv; if (proportional) { - inv = 1.0 / std::pow((double)theta, 2.0 * (double)i / (double)HD); + inv = 1.0 / std::pow(rope_theta, 2.0 * (double)i / (double)HD); if (i >= rot / 2) inv = 0.0; } else { - inv = 1.0 / std::pow((double)theta, (double)i / (double)half); + inv = 1.0 / std::pow(rope_theta, (double)i / (double)half); // LongRoPE (MiniCPM4): ggml's rope applies the factor as // `rope_yarn(theta / ff, ...)` (ggml-cpu/ops.cpp // ggml_rope_cache_init), i.e. the angle is DIVIDED by diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index 1a20c9dd0..8f57734c1 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -148,7 +148,7 @@ static void test_declared_type(void) { s.high_freq_factor = 4.0; s.low_freq_factor = 1.0; s.original_max_position = 8192.0; - const int got = q4nx_rope_type_supported(type, &s, cases[i].has_short); + const int got = q4nx_rope_type_supported(type, &s, cases[i].has_short, 0); char buf[192]; snprintf(buf, sizeof buf, "declared type \"%s\" reads as \"%s\"", cases[i].type, type); check(!strcmp(type, cases[i].type), buf); @@ -170,7 +170,7 @@ static void test_declared_type(void) { char type[64]; q4nx_rope_declared_type(big, type, (int)sizeof type); check(!strcmp(type, "yarn"), "a type 1,000 bytes into the block is still read (fails closed)"); - check(q4nx_rope_type_supported(type, (const Q4nxRopeLlama3*)0, 0) == 0, + check(q4nx_rope_type_supported(type, (const Q4nxRopeLlama3*)0, 0, 0) == 0, "... and is refused"); } { @@ -178,7 +178,7 @@ static void test_declared_type(void) { q4nx_rope_declared_type("{\"rope_scaling\": {\"factor\": 8.0}, " "\"quantization_config\": {\"type\": \"fp8\"}}", type, (int)sizeof type); check(type[0] == '\0', "a block with no type does not take a neighbour's type"); - check(q4nx_rope_type_supported(type, (const Q4nxRopeLlama3*)0, 0) == 1, "... and is not refused"); + check(q4nx_rope_type_supported(type, (const Q4nxRopeLlama3*)0, 0, 0) == 1, "... and is not refused"); /* nested arrays/objects before the type, and a '}' inside a string, both keep the block open */ q4nx_rope_declared_type("{\"rope_scaling\": {\"mrope_section\": [1, 2], \"rope_type\": \"yarn\"}}", type, (int)sizeof type); @@ -194,7 +194,35 @@ static void test_declared_type(void) { char none[8]; q4nx_rope_declared_type((const char*)0, none, (int)sizeof none); check(none[0] == '\0', "null config declares no type"); - check(q4nx_rope_type_supported(none, (const Q4nxRopeLlama3*)0, 0) == 1, "no type is supported"); + check(q4nx_rope_type_supported(none, (const Q4nxRopeLlama3*)0, 0, 0) == 1, "no type is supported"); + /* engine#1740 W2: dynamic NTK. The support predicate accepts it ONLY when the numbers were read, and the base + * rule is the plain theta until a position passes the model's own window -- which is the property that makes + * accepting Hunyuan's factor-1.0 block safe rather than a guess. */ + check(q4nx_rope_type_supported("dynamic", (const Q4nxRopeLlama3*)0, 0, 0) == 0, + "dynamic NTK is refused when no numbers were read (a declared block must not be served as plain rope)"); + check(q4nx_rope_type_supported("dynamic", (const Q4nxRopeLlama3*)0, 0, 1) == 1, + "... and accepted when they were"); + check(q4nx_rope_type_supported("yarn", (const Q4nxRopeLlama3*)0, 0, 1) == 0, + "... which does not open the door to the other types"); + { + Q4nxRopeDynamic d = {1, 1.0, 262144.0}; /* Hunyuan-0.5B's own numbers */ + check(q4nx_rope_dynamic_base(1e6, &d, 0.0, 128) == 1e6, "below the window the base IS theta (position 0)"); + check(q4nx_rope_dynamic_base(1e6, &d, 262143.0, 128) == 1e6, "... and still is one position short of it"); + check(q4nx_rope_dynamic_base(1e6, &d, 262144.0, 128) >= 1e6, "... and does not fall below theta once past it"); + /* factor 1 at exactly the window is the no-op case: scaled == 1, so base == theta */ + check(q4nx_rope_dynamic_base(1e6, &d, 262144.0, 128) == 1e6, "factor 1 at the window is a no-op, bit for bit"); + Q4nxRopeDynamic f4 = {1, 4.0, 8192.0}; + const double at_2x = q4nx_rope_dynamic_base(1e4, &f4, 16384.0, 128); + /* factor 4 at 2x the window: scaled = 4*2 - 3 = 5, base = theta * 5^(dim/(dim-2)) */ + check(fabs(at_2x - 1e4 * pow(5.0, 128.0 / 126.0)) < 1e-6, "the dynamic base follows the NTK formula exactly"); + Q4nxRopeDynamic bad = {1, 0.0, 8192.0}; + check(q4nx_rope_dynamic_base(1e4, &bad, 1e6, 128) == 1e4, "an unusable factor leaves the base alone"); + Q4nxRopeDynamic bad2 = {1, 4.0, 0.0}; + check(q4nx_rope_dynamic_base(1e4, &bad2, 1e6, 128) == 1e4, "an unusable window leaves the base alone"); + Q4nxRopeDynamic off = {0, 4.0, 8192.0}; + check(q4nx_rope_dynamic_base(1e4, &off, 1e6, 128) == 1e4, "and off is a no-op"); + } + } int main(void) { From ff9e9feadb695d1ffb6ed831ea15a4edb3e23d41 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 23:44:04 -0400 Subject: [PATCH 4/6] npu: pin the declared-type half of the dynamic-NTK path, and expose the reader Serving Hunyuan-0.5B to test the rule end to end stopped one layer earlier: the container loads, then serve reports that no kernel set resolves for it, so the rope path is never reached. That is a kernel-family gap, not this rule's, and it is handed over with the shapes (`GU K1024 N7168`, `D K3584 N1024` are in no family). What this adds is the evidence that does not need a device: the declared-type extraction now has an arm proving it SEES Hunyuan's block ({"type": "dynamic", "factor": 1.0}, max_position_embeddings 262144) and that a config with no rope_scaling block reports nothing -- the second half is what keeps every other model on plain rope. read_rope_dynamic() is exposed so the numbers it parses can be pinned the same way once that arm has a C++ home. Measured on the way: the import of the pack is clean (219 tensors, head located at 240,891,456 B, 1,888 tiles, 28 B of trailing zeros, slack 0, verified against the pack's own embedding). --- npu/forward/include/q4nx_forward.h | 4 ++++ npu/forward/src/q4nx_forward.cpp | 2 +- tests/npu_rope_llama3_test.c | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 770d31197..3c4104e2a 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -70,6 +70,10 @@ struct I8Ctx; // Q4nxForwardConfig now lives with the model that owns it: npu/model/npu_model.h (engine#803 A3). +// engine#1740 W2: the rope_type "dynamic" reader, exposed so a host test can pin it against the exact block the +// Hunyuan 1.9 packs carry ({"type": "dynamic", "factor": 1.0} with max_position_embeddings 262144) without a device. +Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path); + class Q4nxNpuForward { public: Q4nxNpuForward(); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c0ca7d998..f892cea84 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -196,7 +196,7 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa // Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`. Hunyuan's // 1.9 packs declare factor 1.0 with a 262,144 window, i.e. a no-op until a prompt passes that window -- which is why // accepting it is safe rather than a guess: below the window the base IS theta, so the served bytes are plain rope. -static Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { +Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { Q4nxRopeDynamic d{}; std::ifstream f(cfg_path); if (!f) return d; diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index 8f57734c1..2873a8678 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -195,6 +195,21 @@ static void test_declared_type(void) { q4nx_rope_declared_type((const char*)0, none, (int)sizeof none); check(none[0] == '\0', "null config declares no type"); check(q4nx_rope_type_supported(none, (const Q4nxRopeLlama3*)0, 0, 0) == 1, "no type is supported"); + /* engine#1740 W2: the declared-type extraction must SEE Hunyuan's block, because everything downstream (the + * reader, the predicate, the base rule) is gated on this string. A config that declares dynamic must report it, + * and one that declares nothing must report nothing -- the second half is what keeps every other model on plain + * rope. */ + { + const char* hy = "{\"rope_scaling\": {\"type\": \"dynamic\", \"factor\": 1.0}, \"max_position_embeddings\": 262144}"; + char t2[64] = {0}; + q4nx_rope_declared_type(hy, t2, (int)sizeof t2); + check(!strcmp(t2, "dynamic"), "a config declaring rope_scaling dynamic is read as dynamic"); + const char* none = "{\"max_position_embeddings\": 32768}"; + char t3[64] = {1}; + q4nx_rope_declared_type(none, t3, (int)sizeof t3); + check(t3[0] == 0, "and a config with no rope_scaling block reports nothing"); + } + /* engine#1740 W2: dynamic NTK. The support predicate accepts it ONLY when the numbers were read, and the base * rule is the plain theta until a position passes the model's own window -- which is the property that makes * accepting Hunyuan's factor-1.0 block safe rather than a guess. */ From 00d9a949cc47795396003efad8bcc36bacbbe8ba Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 00:15:26 -0400 Subject: [PATCH 5/6] npu rope: the dynamic-NTK exponent uses the rotary width, not the head dim Review of #1792. HF's `_compute_dynamic_ntk_parameters` computes `dim = int(head_dim * partial_rotary_factor)` and applies `dim / (dim - 2)`; this forward has exactly that width in `rot` (`(rotary_dim > 0 && rotary_dim <= HD) ? rotary_dim : HD`), and the dynamic base was being called with `HD`. For any model that declares `rope_scaling: dynamic` together with `partial_rotary_factor < 1` the exponent would be computed from the wrong width -- a wrong constant inside the formula the PR exists to get right. Unreachable in today's catalog (no pack on this box declares both; `grep partial_rotary ~/models/oga/*/config.json` is empty), so this changes no served bytes today and is a one-word correction to a constant rather than a behaviour change. Verified: full build of the tree, `npu_rope_llama3_test` -> "all llama3 rope checks passed", `npu_pack_import_test` -> "all npu-pack checks passed". Refs #1792 --- npu/forward/src/q4nx_forward.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index f892cea84..7e23e5935 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -4214,7 +4214,11 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int // Dynamic NTK: the base moves with the position, so it is one pow per position rather than one per dim pair. // Below the model's original window this returns theta unchanged, which is what makes a short prompt // bit-identical to the same model served without the scaling. - const double rope_theta = q4nx_rope_dynamic_base((double)theta, &rope_dynamic_, (double)pos, (int)HD); + // engine#1792 review: the exponent is HF's `dim / (dim - 2)` over the ROTARY width (`dim = int(head_dim * + // partial_rotary_factor)`), which is `rot` here -- passing `HD` was wrong for any model that declares dynamic + // scaling together with partial_rotary_factor < 1. Unreachable in today's catalog, so this is a correctness + // constant rather than a behaviour change for any pack that exists. + const double rope_theta = q4nx_rope_dynamic_base((double)theta, &rope_dynamic_, (double)pos, rot); for (int i = 0; i < half; i++) { double inv; if (proportional) { From 7ab027f6923c700b6bb40bb1a9f1d18f1056ed3c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 01:27:19 -0400 Subject: [PATCH 6/6] npu rope: pin the dynamic-NTK exponent against a known value, and prove the arm can tell the two widths apart The review of this PR found that every arm asserted a PROPERTY -- below the window the base is theta, past it not below theta -- and every one of them is satisfied by any exponent. That is how `00d9a94`'s bug (the caller passing head_dim instead of the rotary width) survived a green suite: I re-introduced it and the suite still said "all checks passed". With factor 1.0 the rule collapses to `theta * (pos/original)^(dim/(dim-2))`, so one printed constant pins the exponent and the width it is computed over: pos 524288, original 262144 -> ratio 2 dim 64 -> 1e6 * 2^(64/62) = 2.0435e6 <- asserted to 1e-6 relative dim 128 -> 1e6 * 2^(128/126) = 2.0219e6 and the third arm asserts the two differ by more than 1e3, so the arm cannot pass if the widths are confused. Verified the way the review verified its claim: substituting 128 for the 64 call fails BOTH of those arms, and restoring it passes `all llama3 rope checks passed`. --- tests/npu_rope_llama3_test.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index 2873a8678..dd261d2df 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -236,6 +236,27 @@ static void test_declared_type(void) { check(q4nx_rope_dynamic_base(1e4, &bad2, 1e6, 128) == 1e4, "an unusable window leaves the base alone"); Q4nxRopeDynamic off = {0, 4.0, 8192.0}; check(q4nx_rope_dynamic_base(1e4, &off, 1e6, 128) == 1e4, "and off is a no-op"); + /* THE EXPONENT, against a KNOWN VALUE (review of #1792). Every arm above asserts a property -- "below the window + * the base is theta, past it not below theta" -- and every one of them is satisfied by ANY exponent, which is how + * a wrong rotary width (the caller passing head_dim instead of the rotary width) survived a green suite. With + * factor 1.0 the rule collapses to theta * (pos/original)^(dim/(dim-2)), so one printed constant pins it: + * pos 524288, original 262144 -> ratio 2; dim 64 -> 1e6 * 2^(64/62) = 2.0435e6 + * and dim 128 -> 1e6 * 2^(128/126) = 2.0219e6, i.e. the arm distinguishes the two widths the bug confused. */ + { + Q4nxRopeDynamic k = {1, 1.0, 262144.0}; + const double want64 = 1e6 * pow(2.0, 64.0 / 62.0); + const double want128 = 1e6 * pow(2.0, 128.0 / 126.0); + const double got64 = q4nx_rope_dynamic_base(1e6, &k, 524288.0, 64); + const double got128 = q4nx_rope_dynamic_base(1e6, &k, 524288.0, 128); + check(fabs(got64 - want64) < 1e-6 * want64, + "the base at twice the window with dim 64 is theta * 2^(64/62), to 1e-6 relative"); + check(fabs(got128 - want128) < 1e-6 * want128, + "... and with dim 128 it is theta * 2^(128/62)"); + /* the pair is the point: if the exponent were ignored, or the width were not passed through, these would be + * equal -- so this asserts the arms CAN tell the rotary width from the head dim. */ + check(fabs(got64 - got128) > 1e3, + "and the two widths differ by more than 1e3, so this arm cannot pass with the widths confused"); + } } }