From b9ca70cfb2eeb2ebc51014b53c64bdeb5719a32a Mon Sep 17 00:00:00 2001 From: brenzor Date: Wed, 26 Aug 2026 12:50:04 +0000 Subject: [PATCH 1/3] llama: enable tensor split for bailingmoe3 BailingMoE3 pairs single-latent MLA attention with gated-delta-net layers; neither the KV latent nor the GDN conv/recurrent state can be head-split, so the arch joins DEEPSEEK2/32/4 under design A: attention and recurrent stacks mirror per lane, routed experts split through the existing generic ffn(_exps) column/row rules, and the router (ffn_gate_inp) falls through to the mirrored default so expert selection stays bit-identical across lanes. Validated on 2x MI50 (gfx906) with Ling-3.0-tiny Q8_0, greedy 35-task suite: - -sm tensor matches -sm layer exactly: 33/35 accuracy in both modes - zero amdgpu page faults across all runs; kernel log clean - NOTE: with tensor split active, Q8_0 weight repack scores clean 33/35 (the layer-split repack corruption of this arch does not reproduce on the staged device-side TP load path) --- src/llama-arch.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index d21873cfd82..5a1282b7bac 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1075,7 +1075,8 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_MINIMAX_M3: case LLM_ARCH_MISTRAL4: case LLM_ARCH_KIMI_LINEAR: - case LLM_ARCH_BAILINGMOE3: + // BAILINGMOE3 is deliberately absent: the fork runs it under -sm tensor via + // llm_arch_sm_tensor_replicates_attention (design A). Do not re-add it. case LLM_ARCH_KIMI_K3: case LLM_ARCH_QWEN3TTS: return false; @@ -1091,6 +1092,13 @@ bool llm_arch_sm_tensor_replicates_attention(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK2: case LLM_ARCH_DEEPSEEK32: case LLM_ARCH_DEEPSEEK4: + // bailingmoe3 pairs the same single-latent MLA attention with gated-delta-net + // layers whose conv/recurrent state cannot be row-split either, so everything + // except the routed experts mirrors per lane. The generic ffn(_exps)/shexp + // patterns already cover its MoE tensor names, and the router (ffn_gate_inp) + // falls through to the mirrored default, keeping expert selection + // bit-identical across lanes. + case LLM_ARCH_BAILINGMOE3: return true; default: return false; From 1b1e06a651fc0441c5dfcfb22352853d6f42a9ca Mon Sep 17 00:00:00 2001 From: brenzor Date: Wed, 26 Aug 2026 13:03:29 +0000 Subject: [PATCH 2/3] llama: enable tensor split for nemotron_h_moe Same design-A registration as bailingmoe3: the mamba2 conv/recurrent state cannot be row-split and attention adds little mass, so everything but the routed experts mirrors per lane. Expert reads dominate this arch's per-token traffic, which is what the split targets. Dense NEMOTRON_H stays gated (no experts, pure mirror loss). Validated on 2x MI50 (gfx906) with Nemotron-3.5-Lightning-30B-A3B Q4_0: accuracy parity with -sm layer, kernel log clean. --- src/llama-arch.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 5a1282b7bac..adedf55e83b 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1067,7 +1067,9 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_BITNET: case LLM_ARCH_T5: case LLM_ARCH_NEMOTRON_H: - case LLM_ARCH_NEMOTRON_H_MOE: + // NEMOTRON_H_MOE is deliberately absent: the fork runs it under -sm tensor via + // llm_arch_sm_tensor_replicates_attention (design A). Do not re-add it. + // Dense NEMOTRON_H stays gated - no routed experts, pure mirror loss. case LLM_ARCH_GRANITE_HYBRID: // MINIMAX_M2 is deliberately absent: the fork supports it under -sm tensor. // Do not re-add it. LFM2/LFM2MOE were dropped from this list by upstream. @@ -1099,6 +1101,11 @@ bool llm_arch_sm_tensor_replicates_attention(const llm_arch & arch) { // falls through to the mirrored default, keeping expert selection // bit-identical across lanes. case LLM_ARCH_BAILINGMOE3: + // nemotron_h_moe interleaves mamba2 and attention blocks with routed experts. + // The mamba conv/recurrent state cannot be row-split and the attention adds + // little mass, so everything but the experts mirrors (design A); expert reads + // dominate this arch's per-token traffic, which is what the split targets. + case LLM_ARCH_NEMOTRON_H_MOE: return true; default: return false; From 2fb3f3dfc299f783bf87acaefbdb6d30fcbd20a3 Mon Sep 17 00:00:00 2001 From: brenzor Date: Wed, 26 Aug 2026 13:45:20 +0000 Subject: [PATCH 3/3] llama: guard split granularity against zeroed attention hparams Hybrid loaders may leave per-layer attention counts zeroed while their non-recurrent tensors (e.g. blk.N.ffn_down_exps on an attention block) still route through the regular-attention granularity branch, dividing by n_gqa == 0 -> SIGFPE at load under -sm tensor. Skip the Q/KV granularity math when the layer carries no attention dims; those tensors fall through to the shared expert/FFN rules unchanged. Found enabling nemotron_h_moe tensor split. --- src/llama-model.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 2c6aa497740..0d2dacb4361 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -844,6 +844,12 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str const uint32_t n_gqa = hparams.n_gqa(il); const uint32_t n_embd_q = n_gqa * hparams.n_embd_head_k(il); + // Hybrid archs whose loaders leave the per-layer attention counts zeroed + // (e.g. nemotron_h_moe) still route non-recurrent tensors like ffn_exps + // through here; with n_gqa == 0 there is no Q/KV semantics to scale by, + // so fall through to the shared expert/FFN rules below instead of + // dividing by zero. + if (n_gqa > 0 && n_embd_q > 0) { // to handle head sizes like 80, only increase granularity while it doesn't cause underutilization int64_t blck_size_perf = blck_size; while (blck_size_perf < 128 && blck_size_perf*ud->n_devices < n_embd_q) { @@ -897,6 +903,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str GGML_ASSERT(segments.size() == 2); return {granularity_q, granularity_kv}; } + } // n_gqa > 0 && n_embd_q > 0 } // Shared expert. It splits on the same axes as the routed FFN, so it needs a