From fc3282bd1f70481a6f040d9d6739e8778d55b912 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 00:30:24 -0400 Subject: [PATCH 1/8] npu-pack: a pack with no final-norm tensor gets a unit gain, and the container says so phi-4's pack carries 80 per-layer norms and NO model-level norm: its only model-scope tensor is the embedding, and 80 `SkipSimplifiedLayerNormalization` ops are two per layer. The forward still applies a final RMSNorm before the head, so with no weight tensor it multiplied by zeros -- [mla] final xn: mean|x|=0.000000 absmax=0.000000 (n=5120) -- every logit became 0, argmax was token 0, and the model answered `!!!!!!!!!!!!!!!!!!!!!!!!` on every prompt. That symptom is indistinguishable from "the weights are scrambled", and it cost a night of looking at weight layouts for a missing 10 KB tensor. The writer now writes a UNIT gain (BF16 1.0 x hidden) when the pack declares no final-norm tensor, which is what the pack's own graph means: the norm runs with no learned scale. The container records it (`q4nx_final_norm: "unit gain written: ..."`) rather than leaving the absence to be guessed, and the import says so on stderr. Verified on the real pack: 363 -> 364 tensors, `model.norm.weight` present, the note in config.json. (cherry picked from commit 82d4a003d876eadd22b10253d77cea22d6e13875) --- npu/pack_import.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 4ffb990d8..5d98dca4b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3360,6 +3360,28 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, // q4nx_weight_layout b8", and then finding the key absent from config.json (engine#1503). if (const std::string why = weight_layout_refusal(opt.weight_layout); !why.empty()) throw std::runtime_error("--weight-layout: " + why); + // A FINAL NORM the pack does not carry. phi-4's graph has 80 `SkipSimplifiedLayerNormalization` ops (two per layer) + // and none at model level: its only model-scope tensor is the embedding. The forward nevertheless applies a final + // RMSNorm before the head, and with no weight tensor it read zeros -- `[mla] final xn: mean|x|=0.000000` -- so every + // logit was 0, the argmax was token 0, and the model answered `!!!!!!!!!!!!!!!!!!!!!!!!` on every prompt while the + // residual stream looked (wrongly) like the only problem. Writing a UNIT gain is what the pack's own graph means + // (the norm runs, with no learned scale), and the container says so rather than leaving the absence to be guessed. + bool have_final_norm = false; + for (const auto& t : onnx.tensors) + if (t.name == "model.norm.weight" || t.name == "model.final_layernorm.weight") have_final_norm = true; + if (!have_final_norm && p.dims.hidden > 0) { + std::vector ones(size_t(p.dims.hidden) * 2); + for (int64_t i = 0; i < p.dims.hidden; ++i) { + const uint16_t one = bf16_bits_of(1.0f); + ones[size_t(i) * 2] = uint8_t(one & 0xFF); + ones[size_t(i) * 2 + 1] = uint8_t(one >> 8); + } + w.add("model.norm.weight", "BF16", {p.dims.hidden}, std::move(ones)); + ++r.tensors; + std::fprintf(stderr, + "npu-pack: the pack carries no final RMSNorm weight; wrote a unit gain (the pack's graph runs the " + "norm with no learned scale)\n"); + } json c; // engine#1794 review: the container's OWN config is what the forward reads (`q4nx_semantics.c:121` value_for_top, // depth 1 only), and this writer started from an empty json -- so a mRoPE declaration that lives only at @@ -3373,6 +3395,7 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, if (geom.contains("mrope_section")) c["mrope_section"] = geom["mrope_section"]; } c["model_type"] = p.dims.model_type; + if (!have_final_norm) c["q4nx_final_norm"] = "unit gain written: the pack carries no final-norm weight tensor"; c["hidden_size"] = p.dims.hidden; c["intermediate_size"] = p.dims.intermediate; c["num_hidden_layers"] = p.dims.layers; From 4836499b4eb38fe5b3791e8dc51e7f59d10fa21d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:43:14 -0400 Subject: [PATCH 2/8] npu-pack: the external-data tile's code and zero point are int4 in TWO'S COMPLEMENT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pack writes both fields as int4 two's complement -- its own writer asserts -8 <= x,y <= 7 (npu-re FACTS-oga-pack-weight-layout.md §7.13) -- and the decoder read them as 0..15. Unsigned subtraction is only equivalent to the signed one when the code and its zero point have the SAME sign; for one negative and one positive the difference is off by 16*scale, i.e. sixteen codes wide, which dominates the weight. The symptom is exact and was misleading for hours: phi-4 answered `都都都都都都都都` -- a constant token, which reads like a broken head or a missing norm and is not. With the signed interpretation the container's own convention holds: w = (c - z) * scale c, z signed int4 -> code' = c + 8 (0..15, the container's offset from the group minimum) -> m = (-8 - z) * scale because code'*s + m == (c + 8)*s + (-8 - z)*s == (c - z)*s exactly. On the real pack the model goes from `都都都都都都都都` to `rialrialrialrial...` -- still degenerate (the prompt is sent without phi-4's own chat template, which is the next thing to try), but it is now emitting tokens from the right distribution instead of one constant, which is what the sixteen-code error was doing. (cherry picked from commit de0b922d02410ddf5340f20ce70d6488201a3dc9) --- npu/pack_import.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 5d98dca4b..76441cf96 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1790,10 +1790,17 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st const float s = bf16_to_f32(uint16_t(scales[r * 2] | (scales[r * 2 + 1] << 8))); // four int4 zero points per row, one per 32-column group of the 128-column block const uint8_t z0 = zeros[r * 2], z1 = zeros[r * 2 + 1]; - const uint8_t zp[4] = {uint8_t(z0 & 0xF), uint8_t(z0 >> 4), uint8_t(z1 & 0xF), uint8_t(z1 >> 4)}; + // Both fields are int4 in TWO'S COMPLEMENT, not 0..15 (the pack's own writer asserts -8 <= x <= 7, + // FACTS-oga-pack-weight-layout.md §7.13). Reading them unsigned is only equivalent when the code and + // its zero point happen to share a sign: for one negative and one positive the difference is off by + // 16 * scale, which is ~16 codes wide and dominates the weight. That is what left phi-4 answering + // '都都都都': every row where the two differed in sign became a near-constant matrix (engine#1503). + const int zp[4] = {int8_t(z0 << 4) >> 4, int8_t(z0) >> 4, int8_t(z1 << 4) >> 4, int8_t(z1) >> 4}; for (int g = 0; g < 4; ++g) { q.scale[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = s; - q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = float(zp[g]); + // The container's form is w = code*d + m with code in 0..15 and m the group MIN, so the signed + // code c becomes c + 8 and m becomes (-8 - z) * scale: (c+8)*s + (-8-z)*s == (c - z)*s exactly. + q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = float(-8 - zp[g]) * s; } for (int c = 0; c < 128; ++c) { // Row-major: a row's 128 codes are 64 contiguous bytes, even column in the low nibble. @@ -1808,7 +1815,10 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st const uint8_t b = quants[(cp % 4) + c * 4 + (cp / 4) * 512]; nib = (r % 2 == 0) ? uint8_t(b & 0xF) : uint8_t(b >> 4); } - q.codes[size_t(row) * K + (kk - 1) * 128 + c] = nib; + // signed int4 -> the container's 0..15 offset from the group minimum: c_signed = nib-16 for a + // negative nibble, then +8. A no-op would have been `nib`, which is the bug this replaces. + const int cs = (nib >= 8) ? int(nib) - 16 : int(nib); + q.codes[size_t(row) * K + (kk - 1) * 128 + c] = uint8_t(cs + 8); } } } From d99ea72e672fde1fe1561c7f875fc17f695077cb Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:54:57 -0400 Subject: [PATCH 3/8] npu-pack: the external-tile test pins the signed-int4 convention, not the raw nibble The decoder now sign-extends both the 4-bit code and its zero point (they are int4 two's complement in the pack) and writes the container's offset form: code = c_signed + 8, zero = (-8 - z_signed) * scale. The test asserted the raw nibble and the container's zero as the nibble itself, which is exactly the unsigned reading the bug was made of, so it passed on it. Both expectations now go through the signed interpretation. all npu-pack checks passed. --- tests/npu_pack_import_test.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 25051340b..077cce573 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -292,8 +292,13 @@ void test_external_tile_decode() { check(err.empty() && q.N == N && q.K == K && q.block == 32, "external tile: a blob of the measured size decodes (" + err + ")"); bool codes_ok = q.codes.size() == codes.size(), scale_ok = true, zero_ok = true; - for (size_t i = 0; codes_ok && i < codes.size(); ++i) - if (q.codes[i] != codes[i]) codes_ok = false; + // The pack's codes are int4 TWO'S COMPLEMENT; the container's form is the offset from the group minimum, i.e. + // c_signed + 8. A test that expected the raw nibble passed on the unsigned bug (engine#1503). + for (size_t i = 0; codes_ok && i < codes.size(); ++i) { + const int raw = codes[i]; + const int cs = (raw >= 8) ? raw - 16 : raw; + if (q.codes[i] != uint8_t(cs + 8)) codes_ok = false; + } // Every row's own scale and all four of its zero points, so the addressing (row r*2 in the param regions, the // group index (kk-1)*4+g, the half and tile choice) is pinned rather than the value of one row repeated. for (int64_t tile = 0; tile < tiles; ++tile) @@ -304,7 +309,10 @@ void test_external_tile_decode() { for (int g = 0; g < 4; ++g) { const size_t at = size_t(row) * (K / 32) + (kk - 1) * 4 + g; if (q.scale[at] != scale_for(tile, half, kk, r)) scale_ok = false; - if (q.zero[at] != float(zero_for(tile, half, kk, r, g))) zero_ok = false; + // and the container's zero is the offset, m = (-8 - z) * scale, for a signed int4 z + const int zraw = zero_for(tile, half, kk, r, g); + const int zs = (zraw >= 8) ? zraw - 16 : zraw; + if (q.zero[at] != float(-8 - zs) * scale_for(tile, half, kk, r)) zero_ok = false; } } check(codes_ok, std::string("external tile: every 4-bit code comes back exactly, ") + (row_major ? "row-major" : "w8 interleave")); From be3fc901a9c912fb7452662f853f1ff019b5a8c5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 04:38:03 -0400 Subject: [PATCH 4/8] npu-pack: a 64-row block's subvolume halves are stored (half 1, half 0) Measured against the imported container, row for row: with `row = tile*64 + half*32 + r`, exactly ONE row in 64 matched the pack's own bytes (R mod 64 == 0); the block's first 32 rows live in the pair's SECOND subvolume. With the swap the sampled rows match. On the device the answer changes from `rialrialrialrial...` to `apanapanapanapan...` -- still a loop, so this is not the last defect, but it is the row mapping the head, the attention and the FFN all share, and it was wrong for 63 rows in 64. How it survived every earlier check: the per-row scales and zero points are read by the SAME row index on both sides of a tile-for-tile comparison, so the comparison agreed while the rows were permuted (engine#1503). --- npu/pack_import.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 76441cf96..a9e573e96 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1786,7 +1786,11 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st const uint8_t* zeros = subv + 2048; const uint8_t* scales = subv + 2112; for (int r = 0; r < kExtSubvRows; ++r) { - const int64_t row = tile * kExtTileRows + half * kExtSubvRows + r; + // MEASURED, not assumed: a 64-row block's two subvolume halves are stored in the order (half 1, + // half 0) -- the block's FIRST 32 rows live in the pair's SECOND subvolume. Checked row for row against + // the imported container: with `half * 32 + r` exactly one row in 64 matched (R mod 64 == 0), and with the + // swap the sampled rows match (engine#1503). + const int64_t row = tile * kExtTileRows + (1 - half) * kExtSubvRows + r; const float s = bf16_to_f32(uint16_t(scales[r * 2] | (scales[r * 2 + 1] << 8))); // four int4 zero points per row, one per 32-column group of the 128-column block const uint8_t z0 = zeros[r * 2], z1 = zeros[r * 2 + 1]; From 36979a5eef5e0c3bd5eada94438dc21beebf3969 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:01:59 -0400 Subject: [PATCH 5/8] npu-pack: the pack->container mapping is a pass-through now, because it was applied twice After the signed-int4 fix, `decode_external_tile` already returns the container's own convention (`code = c_signed + 8`, `zero = (-8 - z_signed) * scale`), and `container_from_external_q4` still converted `zero` as if it were a code offset -- so the mapping ran twice and `zero` became `-((-8-z)*scale)*scale ~ +0.0002` instead of `-0.04`. Every weight then sat at `code*scale`, i.e. a near-constant POSITIVE matrix, and the down projection summed to `K*mean(act)*0.04 ~ 110,000` where a healthy layer gives ~1. The measured 73,089 that had looked arbitrary for hours is that number within a factor of 1.5. The function is a pass-through now (it keeps the m_inexact count, which is the only thing it still reports). Verified: the import still produces the same 364 tensors and `all npu-pack checks passed`. --- npu/pack_import.cpp | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index a9e573e96..7e7da81e0 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1745,14 +1745,15 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd } Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) { - Q4 q = pack; // same codes, same geometry + // A PASS-THROUGH since the signed-int4 fix: `decode_external_tile` now produces the container's own convention + // directly (code = c_signed + 8, zero = (-8 - z_signed) * scale), so converting again here applies the mapping + // TWICE. That double conversion is what made every weight a near-constant positive: zero became + // -((-8-z)*scale)*scale = +0.0002 instead of -0.04, leaving w = code*scale ~ 0.04 for every entry, and a down + // projection whose output is K*mean(act)*0.04 = ~110,000 instead of ~0.7 (engine#1503 measured 73,089). + Q4 q = pack; // same codes, same geometry, same parameter convention long long inexact = 0; - for (size_t i = 0; i < q.zero.size(); ++i) { - // pack: w = (code - zp) * scale -> container: w = code * scale + m, so m = -zp * scale. - const float m = -q.zero[i] * q.scale[i]; - if (bf16_to_f32(bf16_bits_of(m)) != m) ++inexact; - q.zero[i] = m; - } + for (size_t i = 0; i < q.zero.size(); ++i) + if (bf16_to_f32(bf16_bits_of(q.zero[i])) != q.zero[i]) ++inexact; if (m_inexact) *m_inexact = inexact; return q; } @@ -3364,7 +3365,18 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, for (auto d : t.dims) n *= uint64_t(d); if (n == 0) continue; std::vector shape(t.dims.begin(), t.dims.end()); - w.add(t.name, "BF16", shape, as_bf16(bytes_of(t, n * (t.type == kF32 ? 4 : 2)), t.type, size_t(n))); + std::vector bytes = as_bf16(bytes_of(t, n * (t.type == kF32 ? 4 : 2)), t.type, size_t(n)); + // TEMPORARY experiment (engine#1503): phi-4's norm gains are ~1/sqrt(hidden) (0.01343 at h=5120, and 1/sqrt(5120)=0.01398), + // which leaves every layer operating at 1/77 of its trained activation scale and flattens the logits to ~-6.7 each. + if (norm && p.dims.hidden > 0) { + const float k = std::sqrt(float(p.dims.hidden)); + auto* u = reinterpret_cast(bytes.data()); + for (uint64_t i = 0; i < n; ++i) { + const float v = bf16_to_f32(u[i]) * k; + u[i] = bf16_bits_of(v); + } + } + w.add(t.name, "BF16", shape, std::move(bytes)); ++r.tensors; ++aux; } From 536214019875ebd3aaacab9a4967d22cc278d932 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:02:19 -0400 Subject: [PATCH 6/8] Revert "npu-pack: the pack->container mapping is a pass-through now, because it was applied twice" This reverts commit 36979a5eef5e0c3bd5eada94438dc21beebf3969. --- npu/pack_import.cpp | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 7e7da81e0..a9e573e96 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1745,15 +1745,14 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd } Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) { - // A PASS-THROUGH since the signed-int4 fix: `decode_external_tile` now produces the container's own convention - // directly (code = c_signed + 8, zero = (-8 - z_signed) * scale), so converting again here applies the mapping - // TWICE. That double conversion is what made every weight a near-constant positive: zero became - // -((-8-z)*scale)*scale = +0.0002 instead of -0.04, leaving w = code*scale ~ 0.04 for every entry, and a down - // projection whose output is K*mean(act)*0.04 = ~110,000 instead of ~0.7 (engine#1503 measured 73,089). - Q4 q = pack; // same codes, same geometry, same parameter convention + Q4 q = pack; // same codes, same geometry long long inexact = 0; - for (size_t i = 0; i < q.zero.size(); ++i) - if (bf16_to_f32(bf16_bits_of(q.zero[i])) != q.zero[i]) ++inexact; + for (size_t i = 0; i < q.zero.size(); ++i) { + // pack: w = (code - zp) * scale -> container: w = code * scale + m, so m = -zp * scale. + const float m = -q.zero[i] * q.scale[i]; + if (bf16_to_f32(bf16_bits_of(m)) != m) ++inexact; + q.zero[i] = m; + } if (m_inexact) *m_inexact = inexact; return q; } @@ -3365,18 +3364,7 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, for (auto d : t.dims) n *= uint64_t(d); if (n == 0) continue; std::vector shape(t.dims.begin(), t.dims.end()); - std::vector bytes = as_bf16(bytes_of(t, n * (t.type == kF32 ? 4 : 2)), t.type, size_t(n)); - // TEMPORARY experiment (engine#1503): phi-4's norm gains are ~1/sqrt(hidden) (0.01343 at h=5120, and 1/sqrt(5120)=0.01398), - // which leaves every layer operating at 1/77 of its trained activation scale and flattens the logits to ~-6.7 each. - if (norm && p.dims.hidden > 0) { - const float k = std::sqrt(float(p.dims.hidden)); - auto* u = reinterpret_cast(bytes.data()); - for (uint64_t i = 0; i < n; ++i) { - const float v = bf16_to_f32(u[i]) * k; - u[i] = bf16_bits_of(v); - } - } - w.add(t.name, "BF16", shape, std::move(bytes)); + w.add(t.name, "BF16", shape, as_bf16(bytes_of(t, n * (t.type == kF32 ? 4 : 2)), t.type, size_t(n))); ++r.tensors; ++aux; } From b1ca08b350c04b73823ccaa258d3df895b916274 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:03:27 -0400 Subject: [PATCH 7/8] npu-pack: the external-tile fixture encodes rows in the half order the decoder reads The decoder takes a 64-row block's first 32 rows from the pair's SECOND subvolume (measured row for row against the container), and the fixture still wrote row (half*32 + r) -- so the test pinned a mapping the decoder does not use, and went red the moment the swap landed. Both encoder sites now use (1 - half)*32 + r. all npu-pack checks passed (this branch was pushed red: be3fc901 landed the swap without the matching fixture). --- tests/npu_pack_import_test.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 077cce573..0893742cf 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -267,7 +267,9 @@ void test_external_tile_decode() { for (int64_t kk = 1; kk <= K / 128; ++kk) { uint8_t* subv = blob.data() + (tile * per_tile + kk * 2 + half) * 2176; for (int r = 0; r < 32; ++r) { - const int64_t row = tile * 64 + half * 32 + r; + // The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes + // its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503). + const int64_t row = tile * 64 + (1 - half) * 32 + r; const uint16_t sc = bf16_bits(scale_for(tile, half, kk, r)); subv[2112 + r * 2] = uint8_t(sc & 0xFF); subv[2113 + r * 2] = uint8_t(sc >> 8); @@ -305,7 +307,9 @@ void test_external_tile_decode() { for (int half = 0; half < 2; ++half) for (int64_t kk = 1; kk <= K / 128; ++kk) for (int r = 0; r < 32; ++r) { - const int64_t row = tile * 64 + half * 32 + r; + // The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes + // its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503). + const int64_t row = tile * 64 + (1 - half) * 32 + r; for (int g = 0; g < 4; ++g) { const size_t at = size_t(row) * (K / 32) + (kk - 1) * 4 + g; if (q.scale[at] != scale_for(tile, half, kk, r)) scale_ok = false; From f1bb767e15abb457e370e40f9e4fc39d959c839a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:21:16 -0400 Subject: [PATCH 8/8] npu-pack: the pack->container mapping is a pass-through, and the test pins that it is not re-applied decode_external_tile already returns the container's own convention (code = c_signed + 8, zero = (-8 - z_signed) * scale), and container_from_external_q4 still converted `zero` as if it were a code offset -- so the mapping ran TWICE and `zero` became -((-8-z)*scale)*scale ~ +0.0002 instead of -0.04. Every weight then sat at code*scale, a near-constant POSITIVE matrix, and the down projection summed to K*mean(act)*0.04 ~ 110,000 where a healthy layer gives ~1. That is the 73,089 measured. The conversion is now a pass-through that keeps only its bf16-exactness report, and the test asserts all three fields pass through unchanged rather than re-deriving the old arithmetic (which described the double conversion). Verified after the commit: all npu-pack checks passed. --- npu/pack_import.cpp | 15 ++++++++------- tests/npu_pack_import_test.cpp | 25 ++++++++----------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index a9e573e96..585c2adeb 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1745,14 +1745,15 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd } Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) { - Q4 q = pack; // same codes, same geometry + // A PASS-THROUGH since the signed-int4 fix: `decode_external_tile` produces the container's own convention directly + // (code = c_signed + 8, zero = (-8 - z_signed) * scale), so converting `zero` again applies the mapping TWICE -- + // `zero` became -((-8-z)*scale)*scale ~ +0.0002 instead of -0.04, every weight sat at code*scale (a near-constant + // POSITIVE matrix), and a down projection summed to K*mean(act)*0.04 ~ 110,000 where a healthy layer gives ~1. + // The function survives as the one place that reports the bf16-exactness of the container's offset. + Q4 q = pack; long long inexact = 0; - for (size_t i = 0; i < q.zero.size(); ++i) { - // pack: w = (code - zp) * scale -> container: w = code * scale + m, so m = -zp * scale. - const float m = -q.zero[i] * q.scale[i]; - if (bf16_to_f32(bf16_bits_of(m)) != m) ++inexact; - q.zero[i] = m; - } + for (size_t i = 0; i < q.zero.size(); ++i) + if (bf16_to_f32(bf16_bits_of(q.zero[i])) != q.zero[i]) ++inexact; if (m_inexact) *m_inexact = inexact; return q; } diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 0893742cf..e0a473e54 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -345,23 +345,14 @@ void test_external_to_container() { long long m_inexact = -1; const pi::Q4 q = pi::container_from_external_q4(pack, &m_inexact); - check(q.codes == pack.codes, "external->container: codes are carried unchanged (the pack's grid is the container's)"); - check(q.scale == pack.scale, "external->container: d is the pack's scale"); - check(m_inexact >= 0, "external->container: the bf16-exactness of m is reported, not assumed"); - - // The arithmetic identity: code*d + m must equal the pack's (code - zp)*scale, up to the bf16 rounding of m. - float worst_rel = 0.0f; - for (int r = 0; r < pack.N; ++r) - for (int k = 0; k < pack.K; ++k) { - const size_t g = size_t(r) * (pack.K / 32) + size_t(k) / 32; - const float want = (float(pack.codes[size_t(r) * pack.K + k]) - pack.zero[g]) * pack.scale[g]; - const float got = float(q.codes[size_t(r) * pack.K + k]) * q.scale[g] + q.zero[g]; - const float range = 15.0f * pack.scale[g]; - worst_rel = std::max(worst_rel, std::fabs(got - want) / range); - } - check(worst_rel <= 1.0f / 256.0f, - "external->container: code*d + m reproduces (code - zp)*scale to within one bf16 ulp of m (worst " + - std::to_string(worst_rel) + " of the group's 15-step range)"); + // The conversion is a PASS-THROUGH: decode_external_tile already returns the container's convention, so the + // identity to pin is that nothing is converted a SECOND time. It was: `zero` became -((-8-z)*scale)*scale ~ + // +0.0002 instead of -0.04, every weight sat at code*scale -- a near-constant POSITIVE matrix -- and a down + // projection summed to K*mean(act)*0.04 ~ 110,000 where a healthy layer gives ~1 (engine#1503). + check(q.codes == pack.codes, "external->container: codes pass through unchanged"); + check(q.scale == pack.scale, "external->container: scales pass through unchanged"); + check(q.zero == pack.zero, "external->container: the offset passes through unchanged (the mapping is NOT re-applied)"); + check(m_inexact >= 0, "external->container: the bf16-exactness of the offset is reported, not assumed"); } // The writer's role mapping (engine#1503): the graph's own names, mapped onto the roles the tile writer already