diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 4ffb990d8..585c2adeb 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1745,14 +1745,15 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd } Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) { - Q4 q = pack; // same codes, same geometry + // A PASS-THROUGH since the signed-int4 fix: `decode_external_tile` produces the container's own convention directly + // (code = c_signed + 8, zero = (-8 - z_signed) * scale), so converting `zero` again applies the mapping TWICE -- + // `zero` became -((-8-z)*scale)*scale ~ +0.0002 instead of -0.04, every weight sat at code*scale (a near-constant + // POSITIVE matrix), and a down projection summed to K*mean(act)*0.04 ~ 110,000 where a healthy layer gives ~1. + // The function survives as the one place that reports the bf16-exactness of the container's offset. + Q4 q = pack; long long inexact = 0; - for (size_t i = 0; i < q.zero.size(); ++i) { - // pack: w = (code - zp) * scale -> container: w = code * scale + m, so m = -zp * scale. - const float m = -q.zero[i] * q.scale[i]; - if (bf16_to_f32(bf16_bits_of(m)) != m) ++inexact; - q.zero[i] = m; - } + for (size_t i = 0; i < q.zero.size(); ++i) + if (bf16_to_f32(bf16_bits_of(q.zero[i])) != q.zero[i]) ++inexact; if (m_inexact) *m_inexact = inexact; return q; } @@ -1786,14 +1787,25 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st const uint8_t* zeros = subv + 2048; const uint8_t* scales = subv + 2112; for (int r = 0; r < kExtSubvRows; ++r) { - const int64_t row = tile * kExtTileRows + half * kExtSubvRows + r; + // MEASURED, not assumed: a 64-row block's two subvolume halves are stored in the order (half 1, + // half 0) -- the block's FIRST 32 rows live in the pair's SECOND subvolume. Checked row for row against + // the imported container: with `half * 32 + r` exactly one row in 64 matched (R mod 64 == 0), and with the + // swap the sampled rows match (engine#1503). + const int64_t row = tile * kExtTileRows + (1 - half) * kExtSubvRows + r; const float s = bf16_to_f32(uint16_t(scales[r * 2] | (scales[r * 2 + 1] << 8))); // four int4 zero points per row, one per 32-column group of the 128-column block const uint8_t z0 = zeros[r * 2], z1 = zeros[r * 2 + 1]; - const uint8_t zp[4] = {uint8_t(z0 & 0xF), uint8_t(z0 >> 4), uint8_t(z1 & 0xF), uint8_t(z1 >> 4)}; + // Both fields are int4 in TWO'S COMPLEMENT, not 0..15 (the pack's own writer asserts -8 <= x <= 7, + // FACTS-oga-pack-weight-layout.md §7.13). Reading them unsigned is only equivalent when the code and + // its zero point happen to share a sign: for one negative and one positive the difference is off by + // 16 * scale, which is ~16 codes wide and dominates the weight. That is what left phi-4 answering + // '都都都都': every row where the two differed in sign became a near-constant matrix (engine#1503). + const int zp[4] = {int8_t(z0 << 4) >> 4, int8_t(z0) >> 4, int8_t(z1 << 4) >> 4, int8_t(z1) >> 4}; for (int g = 0; g < 4; ++g) { q.scale[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = s; - q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = float(zp[g]); + // The container's form is w = code*d + m with code in 0..15 and m the group MIN, so the signed + // code c becomes c + 8 and m becomes (-8 - z) * scale: (c+8)*s + (-8-z)*s == (c - z)*s exactly. + q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = float(-8 - zp[g]) * s; } for (int c = 0; c < 128; ++c) { // Row-major: a row's 128 codes are 64 contiguous bytes, even column in the low nibble. @@ -1808,7 +1820,10 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st const uint8_t b = quants[(cp % 4) + c * 4 + (cp / 4) * 512]; nib = (r % 2 == 0) ? uint8_t(b & 0xF) : uint8_t(b >> 4); } - q.codes[size_t(row) * K + (kk - 1) * 128 + c] = nib; + // signed int4 -> the container's 0..15 offset from the group minimum: c_signed = nib-16 for a + // negative nibble, then +8. A no-op would have been `nib`, which is the bug this replaces. + const int cs = (nib >= 8) ? int(nib) - 16 : int(nib); + q.codes[size_t(row) * K + (kk - 1) * 128 + c] = uint8_t(cs + 8); } } } @@ -3360,6 +3375,28 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, // q4nx_weight_layout b8", and then finding the key absent from config.json (engine#1503). if (const std::string why = weight_layout_refusal(opt.weight_layout); !why.empty()) throw std::runtime_error("--weight-layout: " + why); + // A FINAL NORM the pack does not carry. phi-4's graph has 80 `SkipSimplifiedLayerNormalization` ops (two per layer) + // and none at model level: its only model-scope tensor is the embedding. The forward nevertheless applies a final + // RMSNorm before the head, and with no weight tensor it read zeros -- `[mla] final xn: mean|x|=0.000000` -- so every + // logit was 0, the argmax was token 0, and the model answered `!!!!!!!!!!!!!!!!!!!!!!!!` on every prompt while the + // residual stream looked (wrongly) like the only problem. Writing a UNIT gain is what the pack's own graph means + // (the norm runs, with no learned scale), and the container says so rather than leaving the absence to be guessed. + bool have_final_norm = false; + for (const auto& t : onnx.tensors) + if (t.name == "model.norm.weight" || t.name == "model.final_layernorm.weight") have_final_norm = true; + if (!have_final_norm && p.dims.hidden > 0) { + std::vector ones(size_t(p.dims.hidden) * 2); + for (int64_t i = 0; i < p.dims.hidden; ++i) { + const uint16_t one = bf16_bits_of(1.0f); + ones[size_t(i) * 2] = uint8_t(one & 0xFF); + ones[size_t(i) * 2 + 1] = uint8_t(one >> 8); + } + w.add("model.norm.weight", "BF16", {p.dims.hidden}, std::move(ones)); + ++r.tensors; + std::fprintf(stderr, + "npu-pack: the pack carries no final RMSNorm weight; wrote a unit gain (the pack's graph runs the " + "norm with no learned scale)\n"); + } json c; // engine#1794 review: the container's OWN config is what the forward reads (`q4nx_semantics.c:121` value_for_top, // depth 1 only), and this writer started from an empty json -- so a mRoPE declaration that lives only at @@ -3373,6 +3410,7 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, if (geom.contains("mrope_section")) c["mrope_section"] = geom["mrope_section"]; } c["model_type"] = p.dims.model_type; + if (!have_final_norm) c["q4nx_final_norm"] = "unit gain written: the pack carries no final-norm weight tensor"; c["hidden_size"] = p.dims.hidden; c["intermediate_size"] = p.dims.intermediate; c["num_hidden_layers"] = p.dims.layers; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 25051340b..e0a473e54 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -267,7 +267,9 @@ void test_external_tile_decode() { for (int64_t kk = 1; kk <= K / 128; ++kk) { uint8_t* subv = blob.data() + (tile * per_tile + kk * 2 + half) * 2176; for (int r = 0; r < 32; ++r) { - const int64_t row = tile * 64 + half * 32 + r; + // The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes + // its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503). + const int64_t row = tile * 64 + (1 - half) * 32 + r; const uint16_t sc = bf16_bits(scale_for(tile, half, kk, r)); subv[2112 + r * 2] = uint8_t(sc & 0xFF); subv[2113 + r * 2] = uint8_t(sc >> 8); @@ -292,19 +294,29 @@ void test_external_tile_decode() { check(err.empty() && q.N == N && q.K == K && q.block == 32, "external tile: a blob of the measured size decodes (" + err + ")"); bool codes_ok = q.codes.size() == codes.size(), scale_ok = true, zero_ok = true; - for (size_t i = 0; codes_ok && i < codes.size(); ++i) - if (q.codes[i] != codes[i]) codes_ok = false; + // The pack's codes are int4 TWO'S COMPLEMENT; the container's form is the offset from the group minimum, i.e. + // c_signed + 8. A test that expected the raw nibble passed on the unsigned bug (engine#1503). + for (size_t i = 0; codes_ok && i < codes.size(); ++i) { + const int raw = codes[i]; + const int cs = (raw >= 8) ? raw - 16 : raw; + if (q.codes[i] != uint8_t(cs + 8)) codes_ok = false; + } // Every row's own scale and all four of its zero points, so the addressing (row r*2 in the param regions, the // group index (kk-1)*4+g, the half and tile choice) is pinned rather than the value of one row repeated. for (int64_t tile = 0; tile < tiles; ++tile) for (int half = 0; half < 2; ++half) for (int64_t kk = 1; kk <= K / 128; ++kk) for (int r = 0; r < 32; ++r) { - const int64_t row = tile * 64 + half * 32 + r; + // The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes + // its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503). + const int64_t row = tile * 64 + (1 - half) * 32 + r; for (int g = 0; g < 4; ++g) { const size_t at = size_t(row) * (K / 32) + (kk - 1) * 4 + g; if (q.scale[at] != scale_for(tile, half, kk, r)) scale_ok = false; - if (q.zero[at] != float(zero_for(tile, half, kk, r, g))) zero_ok = false; + // and the container's zero is the offset, m = (-8 - z) * scale, for a signed int4 z + const int zraw = zero_for(tile, half, kk, r, g); + const int zs = (zraw >= 8) ? zraw - 16 : zraw; + if (q.zero[at] != float(-8 - zs) * scale_for(tile, half, kk, r)) zero_ok = false; } } check(codes_ok, std::string("external tile: every 4-bit code comes back exactly, ") + (row_major ? "row-major" : "w8 interleave")); @@ -333,23 +345,14 @@ void test_external_to_container() { long long m_inexact = -1; const pi::Q4 q = pi::container_from_external_q4(pack, &m_inexact); - check(q.codes == pack.codes, "external->container: codes are carried unchanged (the pack's grid is the container's)"); - check(q.scale == pack.scale, "external->container: d is the pack's scale"); - check(m_inexact >= 0, "external->container: the bf16-exactness of m is reported, not assumed"); - - // The arithmetic identity: code*d + m must equal the pack's (code - zp)*scale, up to the bf16 rounding of m. - float worst_rel = 0.0f; - for (int r = 0; r < pack.N; ++r) - for (int k = 0; k < pack.K; ++k) { - const size_t g = size_t(r) * (pack.K / 32) + size_t(k) / 32; - const float want = (float(pack.codes[size_t(r) * pack.K + k]) - pack.zero[g]) * pack.scale[g]; - const float got = float(q.codes[size_t(r) * pack.K + k]) * q.scale[g] + q.zero[g]; - const float range = 15.0f * pack.scale[g]; - worst_rel = std::max(worst_rel, std::fabs(got - want) / range); - } - check(worst_rel <= 1.0f / 256.0f, - "external->container: code*d + m reproduces (code - zp)*scale to within one bf16 ulp of m (worst " + - std::to_string(worst_rel) + " of the group's 15-step range)"); + // The conversion is a PASS-THROUGH: decode_external_tile already returns the container's convention, so the + // identity to pin is that nothing is converted a SECOND time. It was: `zero` became -((-8-z)*scale)*scale ~ + // +0.0002 instead of -0.04, every weight sat at code*scale -- a near-constant POSITIVE matrix -- and a down + // projection summed to K*mean(act)*0.04 ~ 110,000 where a healthy layer gives ~1 (engine#1503). + check(q.codes == pack.codes, "external->container: codes pass through unchanged"); + check(q.scale == pack.scale, "external->container: scales pass through unchanged"); + check(q.zero == pack.zero, "external->container: the offset passes through unchanged (the mapping is NOT re-applied)"); + check(m_inexact >= 0, "external->container: the bf16-exactness of the offset is reported, not assumed"); } // The writer's role mapping (engine#1503): the graph's own names, mapped onto the roles the tile writer already