Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/models/blackmamba.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ void llama_model_blackmamba::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr[0]);
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);

switch (hparams.n_embd) {
case 1152: type = LLM_TYPE_1_5B; break;
Expand Down
33 changes: 4 additions & 29 deletions src/models/qwen4exp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine(
const int64_t nt = residual->ne[2];

// 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add
ggml_tensor * w = ggml_sigmoid(ctx0, ggml_scale(ctx0, inject, 1.0f / (float) hc));
// inject arrives scaled by 1/hc (both branches above); the pre-#327 sigmoid(inject / hc) here scaled it twice (engine#2444)
ggml_tensor * w = ggml_sigmoid(ctx0, inject);
w = ggml_scale(ctx0, w, 2.0f);

ggml_tensor * cur = nullptr;
Expand Down Expand Up @@ -1014,38 +1015,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
}

ggml_tensor * kq_mask = inp->get_kq_mask();

// prepare new kq mask - starts filled with -INFINITY
ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);

// reshape KQ mask into tensor with rows of size 1:
// [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream]
kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0);

// reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1]
ggml_tensor * sel_3d = ggml_view_4d(ctx0, sel, sel->ne[0], sel->ne[1], sel->ne[3], 1, sel->nb[1], sel->nb[2], sel->ne[3]*sel->nb[3], 0);

// prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream]
// this will be our source of zero values for unmasking top k mask elements
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, sel_3d->ne[0], sel_3d->ne[1], sel_3d->ne[2]);
zeros = ggml_fill(ctx0, zeros, 0.0f);

// modify KQ mask by unmasking elements that are in top_k indices
// ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1])
ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, sel_3d);

// reshape to restore the original shape of KQ mask:
// [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream]
kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0);

// combine with the original kq mask
kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask);

ggml_tensor * q = q_cur;
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
ggml_tensor * v = mctx_cur->get_v(ctx0, il);

// no top-k scatter into kq_mask here: build_qsa_sel already returns the mask. The pre-#327 index scatter
// (kq_mask_top_k) that #327 kept fed this mask to ggml_set_rows as indices and could not build (engine#2444)
// the selection replaces the kq_mask: split it into the streams of the K view
const int64_t n_stream = k->ne[3];
GGML_ASSERT(sel->ne[0] == k->ne[2] && sel->ne[1] % n_stream == 0);
Expand Down
19 changes: 17 additions & 2 deletions tests/test-llama-archs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_ff_per_layer.push_back(il <= 1 ? 0 : n_ff);
}
ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer);
} else if (arch == LLM_ARCH_BLACKMAMBA) {
// layers alternate a Mamba mixer (n_ff == 0) and a Switch MoE; with no 0 the graph has no mixer and
// never consumes the recurrent-state input, which then has no buffer at set_input
std::vector<uint32_t> n_ff_per_layer;
n_ff_per_layer.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
n_ff_per_layer.push_back(il % 2 == 0 ? 0 : n_ff);
}
ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer);
} else {
ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff);
}
Expand Down Expand Up @@ -360,7 +369,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
}

ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) {
if (arch == LLM_ARCH_ZAMBA2) {
// the shared attention block runs on [hidden, embedding] concatenated: q/k/v are 2*n_embd wide
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, 2*n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, 2*n_embd_head);
} else if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2);
Expand Down Expand Up @@ -667,7 +680,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_XIELU_BETA, 1.0f);
ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f);
ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP ? 256 : 2*n_embd);
ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(arch == LLM_ARCH_ZAYA ? 2 : 4)); // zaya: the CCA graph is 2-tap only
ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128));
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
Expand Down Expand Up @@ -939,6 +952,8 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_MAPLE:
case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_KOLIBRI1:
case LLM_ARCH_ZAYA: // every layer has CCA attention and the MoE
case LLM_ARCH_BLACKMAMBA: // every layer is a Mamba mixer + routed MoE: blackmamba.cpp reads expert_feed_forward_length unconditionally
return true;
default:
return false;
Expand Down