Skip to content
Merged
60 changes: 49 additions & 11 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1745,14 +1745,15 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd
}

Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) {
Q4 q = pack; // same codes, same geometry
// A PASS-THROUGH since the signed-int4 fix: `decode_external_tile` produces the container's own convention directly
// (code = c_signed + 8, zero = (-8 - z_signed) * scale), so converting `zero` again applies the mapping TWICE --
// `zero` became -((-8-z)*scale)*scale ~ +0.0002 instead of -0.04, every weight sat at code*scale (a near-constant
// POSITIVE matrix), and a down projection summed to K*mean(act)*0.04 ~ 110,000 where a healthy layer gives ~1.
// The function survives as the one place that reports the bf16-exactness of the container's offset.
Q4 q = pack;
long long inexact = 0;
for (size_t i = 0; i < q.zero.size(); ++i) {
// pack: w = (code - zp) * scale -> container: w = code * scale + m, so m = -zp * scale.
const float m = -q.zero[i] * q.scale[i];
if (bf16_to_f32(bf16_bits_of(m)) != m) ++inexact;
q.zero[i] = m;
}
for (size_t i = 0; i < q.zero.size(); ++i)
if (bf16_to_f32(bf16_bits_of(q.zero[i])) != q.zero[i]) ++inexact;
if (m_inexact) *m_inexact = inexact;
return q;
}
Expand Down Expand Up @@ -1786,14 +1787,25 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st
const uint8_t* zeros = subv + 2048;
const uint8_t* scales = subv + 2112;
for (int r = 0; r < kExtSubvRows; ++r) {
const int64_t row = tile * kExtTileRows + half * kExtSubvRows + r;
// MEASURED, not assumed: a 64-row block's two subvolume halves are stored in the order (half 1,
// half 0) -- the block's FIRST 32 rows live in the pair's SECOND subvolume. Checked row for row against
// the imported container: with `half * 32 + r` exactly one row in 64 matched (R mod 64 == 0), and with the
// swap the sampled rows match (engine#1503).
const int64_t row = tile * kExtTileRows + (1 - half) * kExtSubvRows + r;
const float s = bf16_to_f32(uint16_t(scales[r * 2] | (scales[r * 2 + 1] << 8)));
// four int4 zero points per row, one per 32-column group of the 128-column block
const uint8_t z0 = zeros[r * 2], z1 = zeros[r * 2 + 1];
const uint8_t zp[4] = {uint8_t(z0 & 0xF), uint8_t(z0 >> 4), uint8_t(z1 & 0xF), uint8_t(z1 >> 4)};
// Both fields are int4 in TWO'S COMPLEMENT, not 0..15 (the pack's own writer asserts -8 <= x <= 7,
// FACTS-oga-pack-weight-layout.md §7.13). Reading them unsigned is only equivalent when the code and
// its zero point happen to share a sign: for one negative and one positive the difference is off by
// 16 * scale, which is ~16 codes wide and dominates the weight. That is what left phi-4 answering
// '都都都都': every row where the two differed in sign became a near-constant matrix (engine#1503).
const int zp[4] = {int8_t(z0 << 4) >> 4, int8_t(z0) >> 4, int8_t(z1 << 4) >> 4, int8_t(z1) >> 4};
for (int g = 0; g < 4; ++g) {
q.scale[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = s;
q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = float(zp[g]);
// The container's form is w = code*d + m with code in 0..15 and m the group MIN, so the signed
// code c becomes c + 8 and m becomes (-8 - z) * scale: (c+8)*s + (-8-z)*s == (c - z)*s exactly.
q.zero[size_t(row) * (K / 32) + (kk - 1) * 4 + g] = float(-8 - zp[g]) * s;
}
for (int c = 0; c < 128; ++c) {
// Row-major: a row's 128 codes are 64 contiguous bytes, even column in the low nibble.
Expand All @@ -1808,7 +1820,10 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st
const uint8_t b = quants[(cp % 4) + c * 4 + (cp / 4) * 512];
nib = (r % 2 == 0) ? uint8_t(b & 0xF) : uint8_t(b >> 4);
}
q.codes[size_t(row) * K + (kk - 1) * 128 + c] = nib;
// signed int4 -> the container's 0..15 offset from the group minimum: c_signed = nib-16 for a
// negative nibble, then +8. A no-op would have been `nib`, which is the bug this replaces.
const int cs = (nib >= 8) ? int(nib) - 16 : int(nib);
q.codes[size_t(row) * K + (kk - 1) * 128 + c] = uint8_t(cs + 8);
}
}
}
Expand Down Expand Up @@ -3360,6 +3375,28 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir,
// q4nx_weight_layout b8", and then finding the key absent from config.json (engine#1503).
if (const std::string why = weight_layout_refusal(opt.weight_layout); !why.empty())
throw std::runtime_error("--weight-layout: " + why);
// A FINAL NORM the pack does not carry. phi-4's graph has 80 `SkipSimplifiedLayerNormalization` ops (two per layer)
// and none at model level: its only model-scope tensor is the embedding. The forward nevertheless applies a final
// RMSNorm before the head, and with no weight tensor it read zeros -- `[mla] final xn: mean|x|=0.000000` -- so every
// logit was 0, the argmax was token 0, and the model answered `!!!!!!!!!!!!!!!!!!!!!!!!` on every prompt while the
// residual stream looked (wrongly) like the only problem. Writing a UNIT gain is what the pack's own graph means
// (the norm runs, with no learned scale), and the container says so rather than leaving the absence to be guessed.
bool have_final_norm = false;
for (const auto& t : onnx.tensors)
if (t.name == "model.norm.weight" || t.name == "model.final_layernorm.weight") have_final_norm = true;
if (!have_final_norm && p.dims.hidden > 0) {
std::vector<uint8_t> ones(size_t(p.dims.hidden) * 2);
for (int64_t i = 0; i < p.dims.hidden; ++i) {
const uint16_t one = bf16_bits_of(1.0f);
ones[size_t(i) * 2] = uint8_t(one & 0xFF);
ones[size_t(i) * 2 + 1] = uint8_t(one >> 8);
}
w.add("model.norm.weight", "BF16", {p.dims.hidden}, std::move(ones));
++r.tensors;
std::fprintf(stderr,
"npu-pack: the pack carries no final RMSNorm weight; wrote a unit gain (the pack's graph runs the "
"norm with no learned scale)\n");
}
json c;
// engine#1794 review: the container's OWN config is what the forward reads (`q4nx_semantics.c:121` value_for_top,
// depth 1 only), and this writer started from an empty json -- so a mRoPE declaration that lives only at
Expand All @@ -3373,6 +3410,7 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir,
if (geom.contains("mrope_section")) c["mrope_section"] = geom["mrope_section"];
}
c["model_type"] = p.dims.model_type;
if (!have_final_norm) c["q4nx_final_norm"] = "unit gain written: the pack carries no final-norm weight tensor";
c["hidden_size"] = p.dims.hidden;
c["intermediate_size"] = p.dims.intermediate;
c["num_hidden_layers"] = p.dims.layers;
Expand Down
47 changes: 25 additions & 22 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,9 @@ void test_external_tile_decode() {
for (int64_t kk = 1; kk <= K / 128; ++kk) {
uint8_t* subv = blob.data() + (tile * per_tile + kk * 2 + half) * 2176;
for (int r = 0; r < 32; ++r) {
const int64_t row = tile * 64 + half * 32 + r;
// The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes
// its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503).
const int64_t row = tile * 64 + (1 - half) * 32 + r;
const uint16_t sc = bf16_bits(scale_for(tile, half, kk, r));
subv[2112 + r * 2] = uint8_t(sc & 0xFF);
subv[2113 + r * 2] = uint8_t(sc >> 8);
Expand All @@ -292,19 +294,29 @@ void test_external_tile_decode() {
check(err.empty() && q.N == N && q.K == K && q.block == 32,
"external tile: a blob of the measured size decodes (" + err + ")");
bool codes_ok = q.codes.size() == codes.size(), scale_ok = true, zero_ok = true;
for (size_t i = 0; codes_ok && i < codes.size(); ++i)
if (q.codes[i] != codes[i]) codes_ok = false;
// The pack's codes are int4 TWO'S COMPLEMENT; the container's form is the offset from the group minimum, i.e.
// c_signed + 8. A test that expected the raw nibble passed on the unsigned bug (engine#1503).
for (size_t i = 0; codes_ok && i < codes.size(); ++i) {
const int raw = codes[i];
const int cs = (raw >= 8) ? raw - 16 : raw;
if (q.codes[i] != uint8_t(cs + 8)) codes_ok = false;
}
// Every row's own scale and all four of its zero points, so the addressing (row r*2 in the param regions, the
// group index (kk-1)*4+g, the half and tile choice) is pinned rather than the value of one row repeated.
for (int64_t tile = 0; tile < tiles; ++tile)
for (int half = 0; half < 2; ++half)
for (int64_t kk = 1; kk <= K / 128; ++kk)
for (int r = 0; r < 32; ++r) {
const int64_t row = tile * 64 + half * 32 + r;
// The decoder reads the block's FIRST 32 rows from the pair's SECOND subvolume, so the fixture encodes
// its rows in that order; without this the test pins a mapping the decoder does not use (engine#1503).
const int64_t row = tile * 64 + (1 - half) * 32 + r;
for (int g = 0; g < 4; ++g) {
const size_t at = size_t(row) * (K / 32) + (kk - 1) * 4 + g;
if (q.scale[at] != scale_for(tile, half, kk, r)) scale_ok = false;
if (q.zero[at] != float(zero_for(tile, half, kk, r, g))) zero_ok = false;
// and the container's zero is the offset, m = (-8 - z) * scale, for a signed int4 z
const int zraw = zero_for(tile, half, kk, r, g);
const int zs = (zraw >= 8) ? zraw - 16 : zraw;
if (q.zero[at] != float(-8 - zs) * scale_for(tile, half, kk, r)) zero_ok = false;
}
}
check(codes_ok, std::string("external tile: every 4-bit code comes back exactly, ") + (row_major ? "row-major" : "w8 interleave"));
Expand Down Expand Up @@ -333,23 +345,14 @@ void test_external_to_container() {

long long m_inexact = -1;
const pi::Q4 q = pi::container_from_external_q4(pack, &m_inexact);
check(q.codes == pack.codes, "external->container: codes are carried unchanged (the pack's grid is the container's)");
check(q.scale == pack.scale, "external->container: d is the pack's scale");
check(m_inexact >= 0, "external->container: the bf16-exactness of m is reported, not assumed");

// The arithmetic identity: code*d + m must equal the pack's (code - zp)*scale, up to the bf16 rounding of m.
float worst_rel = 0.0f;
for (int r = 0; r < pack.N; ++r)
for (int k = 0; k < pack.K; ++k) {
const size_t g = size_t(r) * (pack.K / 32) + size_t(k) / 32;
const float want = (float(pack.codes[size_t(r) * pack.K + k]) - pack.zero[g]) * pack.scale[g];
const float got = float(q.codes[size_t(r) * pack.K + k]) * q.scale[g] + q.zero[g];
const float range = 15.0f * pack.scale[g];
worst_rel = std::max(worst_rel, std::fabs(got - want) / range);
}
check(worst_rel <= 1.0f / 256.0f,
"external->container: code*d + m reproduces (code - zp)*scale to within one bf16 ulp of m (worst " +
std::to_string(worst_rel) + " of the group's 15-step range)");
// The conversion is a PASS-THROUGH: decode_external_tile already returns the container's convention, so the
// identity to pin is that nothing is converted a SECOND time. It was: `zero` became -((-8-z)*scale)*scale ~
// +0.0002 instead of -0.04, every weight sat at code*scale -- a near-constant POSITIVE matrix -- and a down
// projection summed to K*mean(act)*0.04 ~ 110,000 where a healthy layer gives ~1 (engine#1503).
check(q.codes == pack.codes, "external->container: codes pass through unchanged");
check(q.scale == pack.scale, "external->container: scales pass through unchanged");
check(q.zero == pack.zero, "external->container: the offset passes through unchanged (the mapping is NOT re-applied)");
check(m_inexact >= 0, "external->container: the bf16-exactness of the offset is reported, not assumed");
}

// The writer's role mapping (engine#1503): the graph's own names, mapped onto the roles the tile writer already
Expand Down
Loading