diff --git a/src/models/blackmamba.cpp b/src/models/blackmamba.cpp index 671ebe12c9e2..46f50c782bb2 100644 --- a/src/models/blackmamba.cpp +++ b/src/models/blackmamba.cpp @@ -28,7 +28,7 @@ void llama_model_blackmamba::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr[0]); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); switch (hparams.n_embd) { case 1152: type = LLM_TYPE_1_5B; break; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 7dd764cdebd0..98bee2599cd0 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -428,7 +428,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( const int64_t nt = residual->ne[2]; // 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add - ggml_tensor * w = ggml_sigmoid(ctx0, ggml_scale(ctx0, inject, 1.0f / (float) hc)); + // inject arrives scaled by 1/hc (both branches above); the pre-#327 sigmoid(inject / hc) here scaled it twice (engine#2444) + ggml_tensor * w = ggml_sigmoid(ctx0, inject); w = ggml_scale(ctx0, w, 2.0f); ggml_tensor * cur = nullptr; @@ -1014,38 +1015,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa( ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il)); } - ggml_tensor * kq_mask = inp->get_kq_mask(); - - // prepare new kq mask - starts filled with -INFINITY - ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY); - - // reshape KQ mask into tensor with rows of size 1: - // [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream] - kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0); - - // reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1] - ggml_tensor * sel_3d = ggml_view_4d(ctx0, sel, sel->ne[0], sel->ne[1], sel->ne[3], 1, sel->nb[1], sel->nb[2], sel->ne[3]*sel->nb[3], 0); - - // prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream] - // this will be our source of zero values for unmasking top k mask elements - ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, sel_3d->ne[0], sel_3d->ne[1], sel_3d->ne[2]); - zeros = ggml_fill(ctx0, zeros, 0.0f); - - // modify KQ mask by unmasking elements that are in top_k indices - // ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1]) - ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, sel_3d); - - // reshape to restore the original shape of KQ mask: - // [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream] - kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0); - - // combine with the original kq mask - kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask); - ggml_tensor * q = q_cur; ggml_tensor * k = mctx_cur->get_k(ctx0, il); ggml_tensor * v = mctx_cur->get_v(ctx0, il); + // no top-k scatter into kq_mask here: build_qsa_sel already returns the mask. The pre-#327 index scatter + // (kq_mask_top_k) that #327 kept fed this mask to ggml_set_rows as indices and could not build (engine#2444) // the selection replaces the kq_mask: split it into the streams of the K view const int64_t n_stream = k->ne[3]; GGML_ASSERT(sel->ne[0] == k->ne[2] && sel->ne[1] % n_stream == 0); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 43197926be07..0241d8fe93dd 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -327,6 +327,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_ff_per_layer.push_back(il <= 1 ? 0 : n_ff); } ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer); + } else if (arch == LLM_ARCH_BLACKMAMBA) { + // layers alternate a Mamba mixer (n_ff == 0) and a Switch MoE; with no 0 the graph has no mixer and + // never consumes the recurrent-state input, which then has no buffer at set_input + std::vector n_ff_per_layer; + n_ff_per_layer.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + n_ff_per_layer.push_back(il % 2 == 0 ? 0 : n_ff); + } + ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer); } else { ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff); } @@ -360,7 +369,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f); - if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) { + if (arch == LLM_ARCH_ZAMBA2) { + // the shared attention block runs on [hidden, embedding] concatenated: q/k/v are 2*n_embd wide + ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, 2*n_embd_head); + ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, 2*n_embd_head); + } else if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) { ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head); ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head); ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2); @@ -667,7 +680,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_XIELU_BETA, 1.0f); ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f); ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP ? 256 : 2*n_embd); - ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4)); + ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(arch == LLM_ARCH_ZAYA ? 2 : 4)); // zaya: the CCA graph is 2-tap only ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128)); ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head); ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2)); @@ -939,6 +952,8 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MAPLE: case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_KOLIBRI1: + case LLM_ARCH_ZAYA: // every layer has CCA attention and the MoE + case LLM_ARCH_BLACKMAMBA: // every layer is a Mamba mixer + routed MoE: blackmamba.cpp reads expert_feed_forward_length unconditionally return true; default: return false;