From 50beacbb11ac1746f8e16be034b9a0fc00a57f2d Mon Sep 17 00:00:00 2001 From: agent Date: Sat, 14 Nov 2026 14:31:39 -0400 Subject: [PATCH 1/3] qwen4exp, blackmamba: undo two #327 merge slips; test-llama-archs runs every architecture (engine#2444) qwen4exp (Qwen3.8-Flash-Next) could not load at all on this line since #327: - build_attn_qsa kept the pre-merge index scatter (ggml_set_rows of a top-k index list into the kq mask) next to AMD's version, where `sel` is already the mask and goes to build_attn_mha as is. Its result was never used, but building it asserts: set_rows needs I32/I64 indices and `sel` is the F16/F32 mask. Removed; the function is now identical to 9bf5871's. - build_hc_combine kept the fork's sigmoid(inject / hc) where AMD's tree has sigmoid(inject). With the old block removed the model loads and produces "\n\nThe\n\n\n\n##"; with AMD's line it produces the same text as the pre-merge fork and as 9bf5871. blackmamba read expert_feed_forward_length with get_key into element 0, so the model saver's per-layer array ([n_ff, 0]) did not load back. Read it with get_key_or_arr like every other MoE architecture; a real BlackMamba-2.8B GGUF generates byte-identical text before and after. test-llama-archs: BlackMamba is MoE-only and alternates Mamba and MoE layers, so the fixture writes no dense variant and a per-layer n_ff of [0, n_ff]. zamba2 and zaya need hparams the generic fixture does not write; they are skipped with a FIXME instead of aborting every later architecture. Co-Authored-By: Claude Opus 5.5 --- src/models/blackmamba.cpp | 2 +- src/models/qwen4exp.cpp | 30 +----------------------------- tests/test-llama-archs.cpp | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/src/models/blackmamba.cpp b/src/models/blackmamba.cpp index 671ebe12c9e2..46f50c782bb2 100644 --- a/src/models/blackmamba.cpp +++ b/src/models/blackmamba.cpp @@ -28,7 +28,7 @@ void llama_model_blackmamba::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr[0]); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); switch (hparams.n_embd) { case 1152: type = LLM_TYPE_1_5B; break; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 7dd764cdebd0..599760e6f45b 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -428,7 +428,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( const int64_t nt = residual->ne[2]; // 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add - ggml_tensor * w = ggml_sigmoid(ctx0, ggml_scale(ctx0, inject, 1.0f / (float) hc)); + ggml_tensor * w = ggml_sigmoid(ctx0, inject); w = ggml_scale(ctx0, w, 2.0f); ggml_tensor * cur = nullptr; @@ -1014,34 +1014,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa( ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il)); } - ggml_tensor * kq_mask = inp->get_kq_mask(); - - // prepare new kq mask - starts filled with -INFINITY - ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY); - - // reshape KQ mask into tensor with rows of size 1: - // [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream] - kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0); - - // reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1] - ggml_tensor * sel_3d = ggml_view_4d(ctx0, sel, sel->ne[0], sel->ne[1], sel->ne[3], 1, sel->nb[1], sel->nb[2], sel->ne[3]*sel->nb[3], 0); - - // prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream] - // this will be our source of zero values for unmasking top k mask elements - ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, sel_3d->ne[0], sel_3d->ne[1], sel_3d->ne[2]); - zeros = ggml_fill(ctx0, zeros, 0.0f); - - // modify KQ mask by unmasking elements that are in top_k indices - // ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1]) - ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, sel_3d); - - // reshape to restore the original shape of KQ mask: - // [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream] - kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0); - - // combine with the original kq mask - kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask); - ggml_tensor * q = q_cur; ggml_tensor * k = mctx_cur->get_k(ctx0, il); ggml_tensor * v = mctx_cur->get_v(ctx0, il); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 43197926be07..01998d074ea4 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -327,6 +327,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_ff_per_layer.push_back(il <= 1 ? 0 : n_ff); } ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer); + } else if (arch == LLM_ARCH_BLACKMAMBA) { + // layers alternate a Mamba mixer (n_ff == 0) and a Switch MoE; with no 0 the graph has no mixer and + // never consumes the recurrent-state input, which then has no buffer at set_input + std::vector n_ff_per_layer; + n_ff_per_layer.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + n_ff_per_layer.push_back(il % 2 == 0 ? 0 : n_ff); + } + ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer); } else { ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff); } @@ -939,6 +948,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MAPLE: case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_KOLIBRI1: + case LLM_ARCH_BLACKMAMBA: // every layer is a Mamba mixer + routed MoE: blackmamba.cpp reads expert_feed_forward_length unconditionally return true; default: return false; @@ -998,6 +1008,12 @@ static bool arch_supported(const llm_arch arch) { if (arch == LLM_ARCH_PLM) { return false; // TODO tensor shapes } + if (arch == LLM_ARCH_ZAMBA2 || arch == LLM_ARCH_ZAYA) { + // FIXME engine#2444: the generic fixture does not write these hybrids' hparams (zaya: 2-tap CCA conv and its + // state widths; zamba2: the shared attention block's width), so one aborts at load and the other in build_qkv + // and either took every later architecture down with it + return false; + } if (arch == LLM_ARCH_CLEF || arch == LLM_ARCH_VLLM_DECISION2) { return false; // TODO decision head tensors } From 4e1deb2ca68e74424e8ed1eb9d94191e81cc3eec Mon Sep 17 00:00:00 2001 From: agent Date: Sat, 14 Nov 2026 14:41:59 -0400 Subject: [PATCH 2/3] test-llama-archs: zaya and zamba2 run instead of being skipped (engine#2444) - zaya puts the CCA attention and the MoE in every layer, so it is MoE-only like BlackMamba (with no expert_feed_forward_length its n_ff_exp is 0 and a 0-wide tensor fails the load), and its CCA graph is 2-tap: the fixture writes ssm.conv_kernel = 2 for it. - zamba2's shared attention runs on [hidden, embedding] concatenated, so q/k/v are 2*n_embd wide: the fixture writes key/value lengths of 2*n_embd_head, where the default n_embd/n_head did not divide them. All 174 architectures in llama-arch.cpp now run in test-llama-archs on the CPU with no failure; test-generate-models and test-llama-context pass. Co-Authored-By: Claude Opus 5.5 --- tests/test-llama-archs.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 01998d074ea4..0241d8fe93dd 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -369,7 +369,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f); - if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) { + if (arch == LLM_ARCH_ZAMBA2) { + // the shared attention block runs on [hidden, embedding] concatenated: q/k/v are 2*n_embd wide + ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, 2*n_embd_head); + ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, 2*n_embd_head); + } else if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) { ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head); ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head); ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2); @@ -676,7 +680,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_XIELU_BETA, 1.0f); ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f); ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP ? 256 : 2*n_embd); - ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4)); + ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(arch == LLM_ARCH_ZAYA ? 2 : 4)); // zaya: the CCA graph is 2-tap only ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128)); ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head); ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2)); @@ -948,6 +952,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MAPLE: case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_KOLIBRI1: + case LLM_ARCH_ZAYA: // every layer has CCA attention and the MoE case LLM_ARCH_BLACKMAMBA: // every layer is a Mamba mixer + routed MoE: blackmamba.cpp reads expert_feed_forward_length unconditionally return true; default: @@ -1008,12 +1013,6 @@ static bool arch_supported(const llm_arch arch) { if (arch == LLM_ARCH_PLM) { return false; // TODO tensor shapes } - if (arch == LLM_ARCH_ZAMBA2 || arch == LLM_ARCH_ZAYA) { - // FIXME engine#2444: the generic fixture does not write these hybrids' hparams (zaya: 2-tap CCA conv and its - // state widths; zamba2: the shared attention block's width), so one aborts at load and the other in build_qkv - // and either took every later architecture down with it - return false; - } if (arch == LLM_ARCH_CLEF || arch == LLM_ARCH_VLLM_DECISION2) { return false; // TODO decision head tensors } From cf0e7fb7daceb1d1ed302fa40cb24232315acd6b Mon Sep 17 00:00:00 2001 From: agent Date: Sat, 14 Nov 2026 14:51:07 -0400 Subject: [PATCH 3/3] qwen4exp: say at both sites why the pre-#327 code is absent (review on #381) Both removals are deletions of pre-#327 fork code, not restores of it: AMD's design (9bf5871) replaced each one, and #327 kept the old next to the new. A comment at each site says so, so the next merge does not re-add them as an omission. Co-Authored-By: Claude Opus 5.5 --- src/models/qwen4exp.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 599760e6f45b..98bee2599cd0 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -428,6 +428,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( const int64_t nt = residual->ne[2]; // 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add + // inject arrives scaled by 1/hc (both branches above); the pre-#327 sigmoid(inject / hc) here scaled it twice (engine#2444) ggml_tensor * w = ggml_sigmoid(ctx0, inject); w = ggml_scale(ctx0, w, 2.0f); @@ -1018,6 +1019,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa( ggml_tensor * k = mctx_cur->get_k(ctx0, il); ggml_tensor * v = mctx_cur->get_v(ctx0, il); + // no top-k scatter into kq_mask here: build_qsa_sel already returns the mask. The pre-#327 index scatter + // (kq_mask_top_k) that #327 kept fed this mask to ggml_set_rows as indices and could not build (engine#2444) // the selection replaces the kq_mask: split it into the streams of the K view const int64_t n_stream = k->ne[3]; GGML_ASSERT(sel->ne[0] == k->ne[2] && sel->ne[1] % n_stream == 0);