diff --git a/src/mobius/_registry.py b/src/mobius/_registry.py index a9597d76..a0a10410 100644 --- a/src/mobius/_registry.py +++ b/src/mobius/_registry.py @@ -604,6 +604,13 @@ def _detect_fallback_registration(hf_config) -> ModelRegistration | None: "qwen2_vl_text": ModelRegistration(Qwen25VLTextModel), "qwen3_5": ModelRegistration(Qwen35VL3ModelCausalLMModel, task="hybrid-qwen-vl"), "qwen3_5_moe_vl": ModelRegistration(Qwen35MoEVL3ModelCausalLMModel, task="hybrid-qwen-vl"), + # Text-only sibling of ``qwen3_5_moe_vl`` (Qwen3.6-35B-A3B). The MoE + # backbone ``Qwen35MoECausalLMModel`` already strips ``language_model.`` + # and drops ``visual.``/MTP keys, so it consumes the VL checkpoint's text + # weights directly; ``build(..., text_only=True)`` routes here via + # ``_TEXT_ONLY_MODEL_TYPE``. It also matches the VL ``text_config``'s own + # ``model_type=qwen3_5_moe_text`` so that config resolves cleanly. + "qwen3_5_moe_text": ModelRegistration(Qwen35MoECausalLMModel), "qwen3_5_vl": ModelRegistration(Qwen35VL3ModelCausalLMModel, task="hybrid-qwen-vl"), "qwen3_5_vl_text": ModelRegistration(Qwen35VLTextModel), "qwen3_vl": ModelRegistration(Qwen3VL3ModelCausalLMModel, task="qwen-vl"), @@ -791,6 +798,12 @@ def _create_default_registry() -> ModelRegistry: _TEXT_ONLY_MODEL_TYPE: dict[str, str] = { "gemma4_unified": "gemma4_unified_text", "gemma4_unified_text": "gemma4_unified_text", + # Qwen3.5-MoE-VL (Qwen3.6-35B-A3B): export just the hybrid MoE text + # backbone as a standalone decoder-only LLM. The builder overrides + # ``qwen3_5_moe`` -> ``qwen3_5_moe_vl`` when a ``vision_config`` is present, + # so the text-only override keys off the VL type here. + "qwen3_5_moe_vl": "qwen3_5_moe_text", + "qwen3_5_moe_text": "qwen3_5_moe_text", } @@ -894,6 +907,7 @@ def _create_default_registry() -> ModelRegistry: "qwen2_moe": "Qwen/Qwen1.5-MoE-A2.7B-Chat", "qwen3_moe": "Qwen/Qwen3-30B-A3B", "qwen3_5_moe": "Qwen/Qwen3.5-MoE-A3B-128K", + "qwen3_5_moe_text": "Qwen/Qwen3.6-35B-A3B", "qwen3_next": "Qwen/Qwen3-235B-A22B", "granitemoe": "ibm-granite/granite-3.0-1b-a400m-instruct", "olmoe": "allenai/OLMoE-1B-7B-0924", @@ -1141,6 +1155,7 @@ def _create_default_registry() -> ModelRegistry: "qwen3_moe": "qwen", "qwen3_5_text": "qwen", "qwen3_5_moe": "qwen", + "qwen3_5_moe_text": "qwen", "qwen3_next": "qwen", "qwen2_vl": "qwen", "qwen2_vl_text": "qwen", diff --git a/tests/_test_configs.py b/tests/_test_configs.py index c7ebb020..7bfcea0c 100644 --- a/tests/_test_configs.py +++ b/tests/_test_configs.py @@ -519,6 +519,29 @@ def _base_config(config_cls=None, **overrides) -> ArchitectureConfig: }, True, ), + # Text-only sibling of the Qwen3.5-MoE-VL (Qwen3.6-35B-A3B) checkpoint, + # exported via ``build(..., text_only=True)``. Same hybrid MoE backbone as + # ``qwen3_5_moe`` above; registered separately so the VL ``text_config``'s + # ``model_type=qwen3_5_moe_text`` and the text-only override both resolve. + ( + "qwen3_5_moe_text", + { + "hidden_act": "silu", + "layer_types": ["linear_attention", "full_attention"], + "partial_rotary_factor": 0.25, + "mrope_interleaved": True, + "num_local_experts": 4, + "num_experts_per_tok": 2, + "moe_intermediate_size": 32, + "shared_expert_intermediate_size": 32, + "linear_num_value_heads": 4, + "linear_num_key_heads": 2, + "linear_key_head_dim": 16, + "linear_value_head_dim": 16, + "linear_conv_kernel_dim": 4, + }, + True, + ), # === Falcon and Bloom === # dual_ln=True: Falcon with new_decoder_architecture uses separate ln_attn + ln_mlp. # hidden_act="gelu": real Falcon uses GELU (HF FalconConfig.activation default); diff --git a/tests/build_graph_test.py b/tests/build_graph_test.py index a96caec2..c63ce356 100644 --- a/tests/build_graph_test.py +++ b/tests/build_graph_test.py @@ -80,6 +80,7 @@ "minimax", "qwen3_5_text", "qwen3_5_moe", + "qwen3_5_moe_text", "qwen3_next", # Models using LinearAttention / CausalConvWithState custom ops # prevent full shape/type propagation through com.microsoft domain.