[model] qwen3.8-flash-next

src/mcore_bridge/config/model_config.py

@@ -196,6 +196,26 @@ class ModelConfig(TransformerConfig):
     attention_output_gate: bool = False
     linear_decoupled_in_proj: bool = False
 
+    # qwen3.8-flash-next (HC + PLE + QSA)
+    hc_count: Optional[int] = None
+    hc_lowrank: Optional[int] = None
+    ple_layer_ids: Optional[List[int]] = None
+    ple_embed_dim: Optional[int] = None
+    ple_conv_kernel_size: Optional[int] = None
+    ngram_size: Optional[int] = None
+    heads_per_ngram: Optional[int] = None
+    ngram_vocab_size_base: Optional[int] = None
+    make_ngram_vocab_size_divisible_by: Optional[int] = None
+    split_ngram_parts: Optional[int] = None
+    ple_seed: Optional[int] = None
+    eos_token_id: Optional[int] = None
+    indexer_n_heads: Optional[int] = None
+    indexer_kv_heads: Optional[int] = None
+    indexer_head_dim: Optional[int] = None
+    indexer_budget: Optional[int] = None
+    indexer_compress_ratio: Optional[int] = None
+    output_gate_type: Optional[str] = None
+
     # nemotron_h (hybrid mamba2 + attention + moe)
     hybrid_layer_pattern: Optional[str] = None
 

src/mcore_bridge/config/parser.py

@@ -52,6 +52,23 @@
     'linear_key_head_dim': ['linear_key_head_dim'],
     'linear_value_head_dim': ['linear_value_head_dim'],
     'linear_conv_kernel_dim': ['linear_conv_kernel_dim'],
+    # qwen4_exp
+    'hc_count': ['hc_count'],
+    'hc_lowrank': ['hc_lowrank'],
+    'ple_layer_ids': ['ple_layer_ids'],
+    'ple_embed_dim': ['ple_embed_dim'],
+    'ple_conv_kernel_size': ['ple_conv_kernel_size'],
+    'ngram_size': ['ngram_size'],
+    'heads_per_ngram': ['heads_per_ngram'],
+    'ngram_vocab_size_base': ['ngram_vocab_size_base'],
+    'make_ngram_vocab_size_divisible_by': ['make_ngram_vocab_size_divisible_by'],
+    'split_ngram_parts': ['split_ngram_parts'],
+    'indexer_n_heads': ['indexer_n_heads'],
+    'indexer_kv_heads': ['indexer_kv_heads'],
+    'indexer_head_dim': ['indexer_head_dim'],
+    'indexer_budget': ['indexer_budget'],
+    'indexer_compress_ratio': ['indexer_compress_ratio'],
+    'output_gate_type': ['output_gate_type'],
     # dsa
     'dsa_indexer_n_heads': ['index_n_heads'],
     'dsa_indexer_head_dim': ['index_head_dim'],
@@ -244,6 +261,39 @@ def hf_to_mcore_config(hf_config: PretrainedConfig) -> Dict[str, Any]:
         if use_mcore_gdn:
             res['experimental_attention_variant'] = 'gated_delta_net'
         res.setdefault('linear_attention_freq', 4)
+    elif hf_model_type == 'qwen4_exp':
+        use_mcore_gdn = get_env_args('USE_MCORE_GDN', bool, True)
+        res['layernorm_zero_centered_gamma'] = True
+        res['attention_output_gate'] = True
+        res['qk_layernorm'] = True
+        res['linear_decoupled_in_proj'] = True
+        res['moe_shared_expert_gate'] = True
+        if use_mcore_gdn:
+            res['experimental_attention_variant'] = 'gated_delta_net'
+        text_config = getattr(hf_config, 'text_config', hf_config)
+        num_layers = res['num_layers']
+        linear_pattern = ['1' if t == 'linear_attention' else '0' for t in layer_types]
+        res['linear_attention_freq'] = f"[{','.join(linear_pattern)}]"
+        if res.get('num_moe_experts'):
+            res['moe_layer_freq'] = f"[{','.join(['1'] * num_layers)}]"
+        # seed is hardcoded in transformers, not in config
+        res['ple_seed'] = int(getattr(text_config, 'seed', 1234))
+        eos_token_id = getattr(text_config, 'eos_token_id', None)
+        if eos_token_id is not None:
+            res['eos_token_id'] = eos_token_id
+        # These fields must come from the model config: ModelConfig carries no
+        # defaults for them, and a silently substituted value would corrupt the
+        # n-gram hash-table sharding/math at checkpoint conversion.
+        _required = [
+            'hc_count', 'hc_lowrank', 'ple_layer_ids', 'ple_embed_dim', 'ple_conv_kernel_size', 'ngram_size',
+            'heads_per_ngram', 'ngram_vocab_size_base', 'make_ngram_vocab_size_divisible_by', 'split_ngram_parts',
+            'eos_token_id', 'indexer_n_heads', 'indexer_kv_heads', 'indexer_head_dim', 'indexer_budget',
+            'indexer_compress_ratio'
+        ]
+        _missing = [k for k in _required if res.get(k) is None]
+        if _missing:
+            raise ValueError(f'qwen4_exp config is missing required fields: {_missing}. '
+                             'They must be provided by the model config.json.')
     elif llm_model_type == 'minimax_m2':
         res['add_qkv_bias'] = False
     elif llm_model_type == 'olmoe':

src/mcore_bridge/model/constant.py

@@ -25,6 +25,7 @@ class MLLMModelType:
     qwen3_omni = 'qwen3_omni'
     qwen3_asr = 'qwen3_asr'
     qwen3_5 = 'qwen3_5'
+    qwen4_exp = 'qwen4_exp'
     ovis2_5 = 'ovis2_5'
 
     internvl_chat = 'internvl_chat'

src/mcore_bridge/model/gpt_model.py

@@ -309,7 +309,7 @@ def forward(
                 padding_mask = torch.chunk(padding_mask, tp_size, dim=1)[mpu.get_tensor_model_parallel_rank()]
             extra_block_kwargs['padding_mask'] = padding_mask.contiguous()
 
-        if self.config.moe_n_hash_layers > 0:
+        if self.config.moe_n_hash_layers > 0 or getattr(self.config, 'ple_layer_ids', None):
             extra_block_kwargs['input_ids'] = input_ids
 
         # Run decoder.

src/mcore_bridge/model/gpts/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
 from . import (bailing_hybrid, bailing_moe, deepseek_v4, glm4, glm_moe_dsa, hunyuan, llm, minimax_m2, nemotron_h, olmoe,
-               qwen3_emb, qwen3_next)
+               qwen3_emb, qwen3_next, qwen4_exp)

src/mcore_bridge/model/gpts/qwen3_next.py

@@ -79,8 +79,7 @@ def __init__(self, config: ModelConfig, hidden_size: int, eps: float = 1e-5):
         super().__init__()
         self.config = config
         self.eps = eps
-        # Initialize weight to zeros (Zero-Centered), matching HuggingFace Qwen3NextRMSNorm
-        self.weight = torch.nn.Parameter(torch.zeros(hidden_size))
+        self.weight = torch.nn.Parameter(torch.zeros(hidden_size, dtype=config.params_dtype))
         # Mark weight for SP gradient AllReduce across TP domain (consistent with TENorm/MCoreRMSNorm)
         setattr(self.weight, 'sequence_parallel', config.sequence_parallel)