From b6d4db0fea25c65931793ff2cfbd73b45c5a25ab Mon Sep 17 00:00:00 2001 From: justinchuby Date: Sat, 1 Aug 2026 04:34:26 +0000 Subject: [PATCH] Register qwen3_5_moe_text for text-only Qwen3.6 export Qwen/Qwen3.6-35B-A3B ships model_type=qwen3_5_moe with a vision_config, so the builder overrides it to qwen3_5_moe_vl and unwraps text_config (whose own model_type is qwen3_5_moe_text). Unlike every other VL family (gemma3_text, qwen3_vl_text, qwen3_5_vl_text), the MoE VL had no registered text-only sibling, so: - registry.get("qwen3_5_moe_text") raised KeyError, and - build(..., text_only=True) on the VL checkpoint failed (no _TEXT_ONLY_MODEL_TYPE entry for qwen3_5_moe_vl). Qwen35MoECausalLMModel already strips the language_model. prefix, drops visual./MTP keys, and unpacks fused experts, so it consumes the VL text weights directly. Register qwen3_5_moe_text -> Qwen35MoECausalLMModel, wire the text-only override (qwen3_5_moe_vl -> qwen3_5_moe_text, idempotent self-map), and add default-id/family map entries. Add an L1 build-graph config (reusing the qwen3_5_moe hybrid MoE tiny config) so registry-completeness passes and the new key builds. Verified against the real Qwen3.6 config that both text_only routing and text_config.model_type now resolve to Qwen35MoECausalLMModel. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: justinchuby --- src/mobius/_registry.py | 15 +++++++++++++++ tests/_test_configs.py | 23 +++++++++++++++++++++++ tests/build_graph_test.py | 1 + 3 files changed, 39 insertions(+) diff --git a/src/mobius/_registry.py b/src/mobius/_registry.py index a9597d76..a0a10410 100644 --- a/src/mobius/_registry.py +++ b/src/mobius/_registry.py @@ -604,6 +604,13 @@ def _detect_fallback_registration(hf_config) -> ModelRegistration | None: "qwen2_vl_text": ModelRegistration(Qwen25VLTextModel), "qwen3_5": ModelRegistration(Qwen35VL3ModelCausalLMModel, task="hybrid-qwen-vl"), "qwen3_5_moe_vl": ModelRegistration(Qwen35MoEVL3ModelCausalLMModel, task="hybrid-qwen-vl"), + # Text-only sibling of ``qwen3_5_moe_vl`` (Qwen3.6-35B-A3B). The MoE + # backbone ``Qwen35MoECausalLMModel`` already strips ``language_model.`` + # and drops ``visual.``/MTP keys, so it consumes the VL checkpoint's text + # weights directly; ``build(..., text_only=True)`` routes here via + # ``_TEXT_ONLY_MODEL_TYPE``. It also matches the VL ``text_config``'s own + # ``model_type=qwen3_5_moe_text`` so that config resolves cleanly. + "qwen3_5_moe_text": ModelRegistration(Qwen35MoECausalLMModel), "qwen3_5_vl": ModelRegistration(Qwen35VL3ModelCausalLMModel, task="hybrid-qwen-vl"), "qwen3_5_vl_text": ModelRegistration(Qwen35VLTextModel), "qwen3_vl": ModelRegistration(Qwen3VL3ModelCausalLMModel, task="qwen-vl"), @@ -791,6 +798,12 @@ def _create_default_registry() -> ModelRegistry: _TEXT_ONLY_MODEL_TYPE: dict[str, str] = { "gemma4_unified": "gemma4_unified_text", "gemma4_unified_text": "gemma4_unified_text", + # Qwen3.5-MoE-VL (Qwen3.6-35B-A3B): export just the hybrid MoE text + # backbone as a standalone decoder-only LLM. The builder overrides + # ``qwen3_5_moe`` -> ``qwen3_5_moe_vl`` when a ``vision_config`` is present, + # so the text-only override keys off the VL type here. + "qwen3_5_moe_vl": "qwen3_5_moe_text", + "qwen3_5_moe_text": "qwen3_5_moe_text", } @@ -894,6 +907,7 @@ def _create_default_registry() -> ModelRegistry: "qwen2_moe": "Qwen/Qwen1.5-MoE-A2.7B-Chat", "qwen3_moe": "Qwen/Qwen3-30B-A3B", "qwen3_5_moe": "Qwen/Qwen3.5-MoE-A3B-128K", + "qwen3_5_moe_text": "Qwen/Qwen3.6-35B-A3B", "qwen3_next": "Qwen/Qwen3-235B-A22B", "granitemoe": "ibm-granite/granite-3.0-1b-a400m-instruct", "olmoe": "allenai/OLMoE-1B-7B-0924", @@ -1141,6 +1155,7 @@ def _create_default_registry() -> ModelRegistry: "qwen3_moe": "qwen", "qwen3_5_text": "qwen", "qwen3_5_moe": "qwen", + "qwen3_5_moe_text": "qwen", "qwen3_next": "qwen", "qwen2_vl": "qwen", "qwen2_vl_text": "qwen", diff --git a/tests/_test_configs.py b/tests/_test_configs.py index c7ebb020..7bfcea0c 100644 --- a/tests/_test_configs.py +++ b/tests/_test_configs.py @@ -519,6 +519,29 @@ def _base_config(config_cls=None, **overrides) -> ArchitectureConfig: }, True, ), + # Text-only sibling of the Qwen3.5-MoE-VL (Qwen3.6-35B-A3B) checkpoint, + # exported via ``build(..., text_only=True)``. Same hybrid MoE backbone as + # ``qwen3_5_moe`` above; registered separately so the VL ``text_config``'s + # ``model_type=qwen3_5_moe_text`` and the text-only override both resolve. + ( + "qwen3_5_moe_text", + { + "hidden_act": "silu", + "layer_types": ["linear_attention", "full_attention"], + "partial_rotary_factor": 0.25, + "mrope_interleaved": True, + "num_local_experts": 4, + "num_experts_per_tok": 2, + "moe_intermediate_size": 32, + "shared_expert_intermediate_size": 32, + "linear_num_value_heads": 4, + "linear_num_key_heads": 2, + "linear_key_head_dim": 16, + "linear_value_head_dim": 16, + "linear_conv_kernel_dim": 4, + }, + True, + ), # === Falcon and Bloom === # dual_ln=True: Falcon with new_decoder_architecture uses separate ln_attn + ln_mlp. # hidden_act="gelu": real Falcon uses GELU (HF FalconConfig.activation default); diff --git a/tests/build_graph_test.py b/tests/build_graph_test.py index a96caec2..c63ce356 100644 --- a/tests/build_graph_test.py +++ b/tests/build_graph_test.py @@ -80,6 +80,7 @@ "minimax", "qwen3_5_text", "qwen3_5_moe", + "qwen3_5_moe_text", "qwen3_next", # Models using LinearAttention / CausalConvWithState custom ops # prevent full shape/type propagation through com.microsoft domain.