Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 58 additions & 5 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <optional>
#include "pack_import.h"

#include "onnx_graph.h"
Expand Down Expand Up @@ -4525,6 +4526,7 @@ Plan plan(const std::string& pack_dir) {
const int64_t attn_off = attn_payload_off(H);
const int64_t per_head = int64_t(d.head_dim) * 2; // one QK norm gain, bf16
const int64_t tile_o = attn_tile_bytes(q_out);
std::optional<int64_t> mlp_off_6h; // engine#2604: set only by the 6-header form
const int64_t n_kv = kv_out / 64, n_q = q_out / 64;
auto refuse = [&](const std::string& s) {
p.attn_note = s;
Expand Down Expand Up @@ -4655,14 +4657,65 @@ Plan plan(const std::string& pack_dir) {
p.o_off = g.hdr[1] + kNormOff + per_head;
p.o_k = q_out;
}
} else if (g.hdr.size() == 5) {
// engine#2604: the gemma3 1.8 Token block -- Qwen3's four headers plus Gemma-3's pre- and
// post-feedforward norms, i.e. six per layer. Measured on amd/translategemma-4b-it_rai_1.8.0_npu_4K
// (hidden 2560, 34 layers, 8 heads, head_dim 256): headers are `eps` + 62 zero bytes, 205 of them
// (34 x 6 + the tail), and the block spans are
// h0->h1 q/k/v tiles | h1->h2 first per-head gain | h2->h3 second per-head gain [+32 B on the 29
// SLIDING-window layers] then o_proj | h3->h4 post_attention | h4->h5 pre/post-feedforward | MLP.
// Both spans reproduce exactly: 64 + 512 + 578*5120 = 2,959,936 on the 5 full-attention layers, and
// + 32 = 2,959,968 on the other 29. The 32 B is 16 bf16 = 2 x num_attention_heads: a per-head scalar
// pair the reader has to SKIP, not interpret. Its presence is cross-checked against the span itself,
// so nothing here is inferred from a version or a config.
const int64_t kv_tiles = n_kv + n_q + n_kv;
const int64_t s0 = g.hdr[0] - attn_off;
const int64_t s1 = g.hdr[1] - g.hdr[0];
const int64_t s2 = g.hdr[2] - g.hdr[1];
const int64_t after_gains = s2 - (kNormOff + per_head); // o_proj tiles, plus that 32 B
const int64_t extra = after_gains >= 0 ? after_gains % tile_o : -1;
const int64_t n_otiles = after_gains >= 0 ? after_gains / tile_o : 0;
if (s0 != kv_tiles * tile_bytes)
refuse("the attention payload before the first per-head gain is " + std::to_string(s0) +
" B; k/q/v at 64 rows each are " + std::to_string(kv_tiles * tile_bytes) + " B");
else if (s1 != kNormOff + per_head)
refuse("the section between the block's first two gains is " + std::to_string(s1) + " B; a " +
std::to_string(kNormOff) + "-byte header plus a bf16 [head_dim " +
std::to_string(d.head_dim) + "] gain is " + std::to_string(kNormOff + per_head));
else if (n_otiles != H / kTileRows || (extra != 0 && extra != 32))
refuse("the o_proj section is " + std::to_string(after_gains) + " B after the gains; hidden " +
std::to_string(H) + " rows at K = q_proj's " + std::to_string(q_out) + " are " +
std::to_string(H / kTileRows) + " tiles of " + std::to_string(tile_o) +
" B, optionally preceded by 32 B of per-head scalars on a sliding-window layer");
else {
p.attn_tiles = kv_tiles;
p.knorm_off = g.hdr[0] + kNormOff;
p.qnorm_off = g.hdr[1] + kNormOff;
p.o_off = g.hdr[1] + kNormOff + per_head + extra; // skip the per-head scalar pair
p.o_k = q_out;
// engine#2604: the MLP sits BETWEEN the pre- and post-feedforward norms, so `hdr.back()`
// (the post-feedforward header) is PAST it -- measured on translategemma-4b: h3 8,814,816,
// h4 8,820,000, MLP 42,921,088, h5 51,741,088. The MLP therefore starts after h4.
mlp_off_6h = g.hdr[3] + kNormOff + H * 2;
}
} else {
refuse("the layer block has " + std::to_string(g.hdr.size() + 1) +
" epsilon headers, and this importer reads blocks of 2 (q/k/v/o in one payload) "
"or 4 (q/k/v, two per-head QK norms, o_proj) -- recorded 2026-11-04 on Qwen3");
" epsilon headers, and this importer reads blocks of 2 (q/k/v/o in one payload), "
"4 (q/k/v, two per-head QK norms, o_proj) or 6 (the same plus pre/post-feedforward "
"norms, engine#2604) -- recorded 2026-11-04 on Qwen3");
}
// the MLP section starts right after the block's post-attention norm
const int64_t mlp_off = g.post_attn_hdr() + kNormOff + H * 2;
const MlpGeom mg = mlp_geom(H, d.intermediate, mlp_off, g.stride);
// the MLP section starts right after the block's post-attention norm -- except on the 6-header
// form, where the pre/post-feedforward norms straddle it and the branch above moves mlp_off.
int64_t mlp_end_trim = 0;
const int64_t mlp_off = mlp_off_6h ? *mlp_off_6h
: mlp_start_in_block(g.hdr, kNormOff, H, &mlp_end_trim);
// engine#2604: on the 6-header form the MLP is followed by the post-feedforward norm, which is an
// envelope + bf16[H] (5,184 B at hidden 2560) INSIDE the block -- so the section ends that much
// before the block does. Passing the full stride made the decoder expect 5,184 B more than the
// file has (measured: "ends at 51,741,088 in a 51,746,272-byte block", the two differing by
// exactly h5's own envelope+weight).
const int64_t mlp_end = g.stride - mlp_end_trim;
const MlpGeom mg = mlp_geom(H, d.intermediate, mlp_off, mlp_end);
p.mlp_note = mg.why;
if (mg.ok()) {
p.mlp_gu_units = mg.gu_units;
Expand Down
18 changes: 18 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -960,6 +960,24 @@ struct FconstGeom {
int64_t post_attn_hdr() const { return hdr.empty() ? h1 : hdr.back(); }
std::string why; // non-empty when the geometry could not be established
};
// engine#2604: where the MLP section starts inside a layer block, and how many bytes of the block's tail are NOT
// part of it. Two forms, and the difference is a whole norm:
// * 2- and 4-header blocks (Qwen2.5/Llama/Qwen3): the MLP follows the block's LAST header (post_attention), and
// nothing trails it -- `end_trim` is 0.
// * the 6-header gemma3 form: the pre-feedforward norm opens the MLP and the POST-feedforward norm closes it, so
// the MLP starts after the SECOND-TO-LAST header and the block ends `kNormOff + 2H` bytes after the MLP does.
// Measured on translategemma-4b (hidden 2560): h3 8,814,816, h4 8,820,000, MLP 42,921,088, h5 51,741,088.
// Reading the last header on the 6-header form put the MLP one envelope past its start, which is the refusal
// "the MLP section ends at 51,741,088 B in a 51,746,272-byte layer block".
inline int64_t mlp_start_in_block(const std::vector<int64_t>& hdr, int64_t kNormOff, int64_t H, int64_t* end_trim) {
const int64_t last = hdr.empty() ? 0 : hdr.back();
if (hdr.size() == 5) { // gemma3: five in-block headers, the last one AFTER the MLP
*end_trim = kNormOff + H * 2;
return hdr[3] + kNormOff + H * 2;
}
*end_trim = 0;
return last + kNormOff + H * 2;
}
unsigned short bf16_bits_of(float v);

// engine#1650: config.json may declare an OPTIONAL field as JSON null (AMD's 1.9
Expand Down
29 changes: 29 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3904,6 +3904,34 @@ void test_import_fp32_params(const std::string& root) {
// payload came out as "whole tiles plus a stray run" and the attention section was refused. The two
// numbers below are the packs' OWN measured second-header offsets, so this is a known-answer test: with
// the derived offset each section is exactly the tiles q/k/v/o need, and with the old constant it is not.
// engine#2604: the MLP offset rule, with the real pack's own header offsets. The 6-header gemma3 block puts the
// pre-feedforward norm BEFORE the MLP and the post-feedforward norm AFTER it, so the block's last header is past
// the section at BOTH ends -- reading it as the MLP start is the refusal "the MLP section ends at 51,741,088 B in a
// 51,746,272-byte layer block", and reading the full stride as the MLP's length is 5,184 B too generous.
void test_mlp_start_in_block() {
const int64_t kNormOff = 64, H = 2560;
// translategemma-4b layer 0, measured: q/k/v | h1 5,854,272 | h2 5,854,848 | h3 8,814,816 | h4 8,820,000 |
// MLP 42,921,088 | h5 51,741,088 | next block
const std::vector<int64_t> six{5854272, 5854848, 8814816, 8820000, 51741088};
int64_t trim = -1;
const int64_t start = pi::mlp_start_in_block(six, kNormOff, H, &trim);
check(start == 8820000 + 64 + H * 2, "mlp start: the 6-header form opens the MLP after h4, not h5 (" +
std::to_string(start) + " vs " + std::to_string(8820000 + 64 + H * 2) + ")");
check(start != six.back() + 64 + H * 2, "mlp start: h5 is past the MLP, so it is not the start");
check(trim == 64 + H * 2, "mlp start: the block ends one envelope+weight after the MLP (" + std::to_string(trim) + ")");
check(51746272 - start - trim == 42915904, "mlp start: the section is 42,915,904 B, as the file has");
// a 4-header block is unchanged by the rule: last header is the post-attention norm, nothing trails it
const std::vector<int64_t> four{1000, 2000, 3000};
int64_t trim4 = -1;
const int64_t s4 = pi::mlp_start_in_block(four, kNormOff, H, &trim4);
check(s4 == 3000 + 64 + H * 2 && trim4 == 0, "mlp start: a 4-header block keeps the last header and trims nothing");
// a 2-header block likewise
const std::vector<int64_t> two{5000};
int64_t trim2 = -1;
check(pi::mlp_start_in_block(two, kNormOff, H, &trim2) == 5000 + 64 + H * 2 && trim2 == 0,
"mlp start: a 2-header block keeps its only in-block header");
}

void test_attn_payload_off() {
check(pi::attn_payload_off(896) == 1856,
"attn off: hidden 896 is the 1,856 the importer shipped with (64 + 2 x 896)");
Expand Down Expand Up @@ -7055,6 +7083,7 @@ int main(int argc, char** argv) {
test_mlp_gu(golden_dir);
test_fconst_eps(golden_dir.empty() ? "." : golden_dir);
test_fconst_variable_stride(golden_dir.empty() ? "." : golden_dir);
test_mlp_start_in_block();
test_attn_payload_off();
test_text_config_geometry(golden_dir.empty() ? "." : golden_dir);
test_use_qk_norm_declaration(golden_dir.empty() ? "." : golden_dir);
Expand Down
Loading