llama: enable tensor split for bailingmoe3 + nemotron_h_moe

src/llama-arch.cpp

@@ -1067,15 +1067,18 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_BITNET:
         case LLM_ARCH_T5:
         case LLM_ARCH_NEMOTRON_H:
-        case LLM_ARCH_NEMOTRON_H_MOE:
+        // NEMOTRON_H_MOE is deliberately absent: the fork runs it under -sm tensor via
+        // llm_arch_sm_tensor_replicates_attention (design A). Do not re-add it.
+        // Dense NEMOTRON_H stays gated - no routed experts, pure mirror loss.
         case LLM_ARCH_GRANITE_HYBRID:
         // MINIMAX_M2 is deliberately absent: the fork supports it under -sm tensor.
         // Do not re-add it. LFM2/LFM2MOE were dropped from this list by upstream.
         case LLM_ARCH_MINIMAX_01:
         case LLM_ARCH_MINIMAX_M3:
         case LLM_ARCH_MISTRAL4:
         case LLM_ARCH_KIMI_LINEAR:
-        case LLM_ARCH_BAILINGMOE3:
+        // BAILINGMOE3 is deliberately absent: the fork runs it under -sm tensor via
+        // llm_arch_sm_tensor_replicates_attention (design A). Do not re-add it.
         case LLM_ARCH_KIMI_K3:
         case LLM_ARCH_QWEN3TTS:
             return false;
@@ -1091,6 +1094,18 @@ bool llm_arch_sm_tensor_replicates_attention(const llm_arch & arch) {
         case LLM_ARCH_DEEPSEEK2:
         case LLM_ARCH_DEEPSEEK32:
         case LLM_ARCH_DEEPSEEK4:
+        // bailingmoe3 pairs the same single-latent MLA attention with gated-delta-net
+        // layers whose conv/recurrent state cannot be row-split either, so everything
+        // except the routed experts mirrors per lane. The generic ffn(_exps)/shexp
+        // patterns already cover its MoE tensor names, and the router (ffn_gate_inp)
+        // falls through to the mirrored default, keeping expert selection
+        // bit-identical across lanes.
+        case LLM_ARCH_BAILINGMOE3:
+        // nemotron_h_moe interleaves mamba2 and attention blocks with routed experts.
+        // The mamba conv/recurrent state cannot be row-split and the attention adds
+        // little mass, so everything but the experts mirrors (design A); expert reads
+        // dominate this arch's per-token traffic, which is what the split targets.
+        case LLM_ARCH_NEMOTRON_H_MOE:
             return true;
         default:
             return false;

src/llama-model.cpp

@@ -844,6 +844,12 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
             const uint32_t n_gqa    = hparams.n_gqa(il);
             const uint32_t n_embd_q = n_gqa * hparams.n_embd_head_k(il);
 
+            // Hybrid archs whose loaders leave the per-layer attention counts zeroed
+            // (e.g. nemotron_h_moe) still route non-recurrent tensors like ffn_exps
+            // through here; with n_gqa == 0 there is no Q/KV semantics to scale by,
+            // so fall through to the shared expert/FFN rules below instead of
+            // dividing by zero.
+            if (n_gqa > 0 && n_embd_q > 0) {
             // to handle head sizes like 80, only increase granularity while it doesn't cause underutilization
             int64_t blck_size_perf = blck_size;
             while (blck_size_perf < 128 && blck_size_perf*ud->n_devices < n_embd_q) {
@@ -897,6 +903,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
                 GGML_ASSERT(segments.size() == 2);
                 return {granularity_q, granularity_kv};
             }
+            } // n_gqa > 0 && n_embd_q > 0
         }
 
         // Shared expert. It splits on the same axes as the routed FFN, so it needs a