From c8062cabc8080d7e0157ea49f0772aa842265b5d Mon Sep 17 00:00:00 2001 From: Xiaoyu Date: Thu, 25 Jun 2026 04:49:34 +0000 Subject: [PATCH 1/3] Fix ORT GenAI export metadata Copy tokenizer files directly for local model directories and emit Qwen3-VL-specific GenAI metadata, including model type and vision/video fields. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Xiaoyu --- .../integrations/ort_genai/auto_export.py | 38 +++++++-- .../ort_genai/auto_export_test.py | 83 ++++++++++++++++++- .../integrations/ort_genai/genai_config.py | 12 +++ 3 files changed, 123 insertions(+), 10 deletions(-) diff --git a/src/mobius/integrations/ort_genai/auto_export.py b/src/mobius/integrations/ort_genai/auto_export.py index 5b6f4932..3657a7a5 100644 --- a/src/mobius/integrations/ort_genai/auto_export.py +++ b/src/mobius/integrations/ort_genai/auto_export.py @@ -86,10 +86,10 @@ # HunYuan-V1 dense / Hy-MT1.5 — generic decoder LLM type accepted by # ORT GenAI (see onnxruntime-genai/src/models/model_type.h LLM list). "hunyuan_v1_dense": "decoder", - # Qwen VL models all use the same GenAI pipeline as qwen2_5_vl + # Qwen VL model families have separate ORT GenAI model types. "qwen2_vl": "qwen2_5_vl", - "qwen3_vl": "qwen2_5_vl", - "qwen3_vl_text": "qwen2_5_vl", + "qwen3_vl": "qwen3_vl", + "qwen3_vl_text": "qwen3_vl", "qwen3_5": "qwen2_5_vl", "qwen3_5_vl": "qwen2_5_vl", } @@ -760,6 +760,16 @@ def _write_genai_config( if sms is not None: vision_kwargs["spatial_merge_size"] = sms vision_kwargs["config_filename"] = "processor_config.json" + if model_type in {"qwen3_vl", "qwen3_vl_text"}: + patch_size = getattr(vision_cfg, "patch_size", None) + window_size = getattr(vision_cfg, "window_size", None) + if patch_size is not None: + vision_kwargs["patch_size"] = patch_size + if window_size is not None: + vision_kwargs["window_size"] = window_size + vision_kwargs["tokens_per_second"] = float( + getattr(config, "tokens_per_second", 2.0) + ) if vision_input_mapping is not None: vision_kwargs["input_names"] = vision_input_mapping @@ -769,6 +779,12 @@ def _write_genai_config( embedding_output_mapping = _introspect_outputs(pkg, "embedding") if embedding_output_mapping is not None: vision_kwargs["embedding_output_names"] = embedding_output_mapping + vision_start_token_id = getattr(config, "vision_start_token_id", None) + video_token_id = getattr(config, "video_token_id", None) + if vision_start_token_id is not None: + vision_kwargs["vision_start_token_id"] = vision_start_token_id + if video_token_id is not None: + vision_kwargs["video_token_id"] = video_token_id generator.with_vision(image_token_id=image_token_id, **vision_kwargs) @@ -826,8 +842,8 @@ def write_ort_genai_config( pkg: Already-built :class:`~mobius._model_package.ModelPackage` with weights applied and ``config`` set. directory: Output directory (created if needed). - hf_model_id: HuggingFace model ID. When provided, used to fetch token - IDs (``bos``/``eos``/``pad``) and download tokenizer files. + hf_model_id: HuggingFace model ID or local model directory. When provided, + used to fetch token IDs (``bos``/``eos``/``pad``) and copy tokenizer files. When ``None``, token IDs are read from ``pkg.config`` fields (``bos_token_id``, ``eos_token_id``, ``pad_token_id``) populated by :meth:`~mobius._configs.ArchitectureConfig.from_transformers`, @@ -951,11 +967,15 @@ def write_ort_genai_config( result: dict[str, str] = {"genai_config": genai_path} - # Copy tokenizer files — HF Hub takes precedence; local dir is the fallback - # for --config mode where no HF model ID is available. + # Copy tokenizer files. A local hf_model_id is a local model directory, not a + # Hub repo id; copy directly instead of calling hf_hub_download. if hf_model_id is not None: - logger.info("Copying tokenizer files from %s", hf_model_id) - tokenizer_files = _copy_tokenizer_files(hf_model_id, directory) + if os.path.isdir(hf_model_id): + logger.info("Copying tokenizer files from local model directory %s", hf_model_id) + tokenizer_files = _copy_tokenizer_files_from_local(hf_model_id, directory) + else: + logger.info("Copying tokenizer files from %s", hf_model_id) + tokenizer_files = _copy_tokenizer_files(hf_model_id, directory) for tf in tokenizer_files: result[tf] = os.path.join(directory, tf) elif local_config_dir is not None: diff --git a/src/mobius/integrations/ort_genai/auto_export_test.py b/src/mobius/integrations/ort_genai/auto_export_test.py index 34500335..6eed5a96 100644 --- a/src/mobius/integrations/ort_genai/auto_export_test.py +++ b/src/mobius/integrations/ort_genai/auto_export_test.py @@ -489,6 +489,36 @@ def test_hf_model_id_takes_precedence_over_local_dir(self, tmp_path): mock_hf.assert_called_once() mock_local.assert_not_called() + def test_local_hf_model_id_uses_local_tokenizer_copy(self, tmp_path): + """A local hf_model_id should copy tokenizer files locally, not call the Hub.""" + src = tmp_path / "local_model" + src.mkdir() + (src / "config.json").write_text( + '{"model_type": "llama", "bos_token_id": 1, "eos_token_id": 2}' + ) + (src / "tokenizer.json").write_text('{"local": true}') + + out = tmp_path / "output" + out.mkdir() + pkg = self._make_pkg() + + with ( + mock.patch( + "mobius.integrations.ort_genai.auto_export._copy_tokenizer_files", + return_value=[], + ) as mock_hub_copy, + mock.patch( + "mobius.integrations.ort_genai.auto_export._copy_tokenizer_files_from_local", + wraps=_copy_tokenizer_files_from_local, + ) as mock_local_copy, + ): + result = write_ort_genai_config(pkg, str(out), hf_model_id=str(src)) + + mock_hub_copy.assert_not_called() + mock_local_copy.assert_called_once_with(str(src), str(out)) + assert "tokenizer.json" in result + assert (out / "tokenizer.json").read_text() == '{"local": true}' + class TestExportForOrtGenai: """Unit tests for write_ort_genai_config().""" @@ -543,7 +573,7 @@ class FakeConfig: }, config=FakeConfig(), ) - result = write_ort_genai_config(pkg, str(tmp_path)) + result = write_ort_genai_config(pkg, str(tmp_path), ep="cuda") assert "processor_config" in result assert os.path.isfile(result["processor_config"]) @@ -557,6 +587,57 @@ class FakeConfig: resize = transforms[2]["operation"]["attrs"] assert resize["patch_size"] == 14 + def test_qwen3_vl_writes_qwen3_vl_model_type_and_vision_fields(self, tmp_path): + import dataclasses + + from mobius._model_package import ModelPackage + from mobius.integrations.ort_genai.auto_export import write_ort_genai_config + + @dataclasses.dataclass + class FakeVision: + image_size: int = 448 + patch_size: int = 16 + spatial_merge_size: int = 2 + window_size: int = 64 + model_type: str | None = None + + @dataclasses.dataclass + class FakeConfig: + model_type: str = "qwen3_vl" + vocab_size: int = 151936 + hidden_size: int = 2048 + num_hidden_layers: int = 1 + num_attention_heads: int = 16 + num_key_value_heads: int = 8 + head_dim: int = 128 + image_token_id: int = 151655 + vision_start_token_id: int = 151652 + video_token_id: int = 151656 + tokens_per_second: float = 2.0 + temporal_patch_size: int = 2 + vision: FakeVision = dataclasses.field(default_factory=FakeVision) + + pkg = ModelPackage( + { + "decoder": mock.MagicMock(), + "vision_encoder": mock.MagicMock(), + "embedding": mock.MagicMock(), + }, + config=FakeConfig(), + ) + + result = write_ort_genai_config(pkg, str(tmp_path), ep="cuda") + + with open(result["genai_config"]) as f: + data = json.load(f) + model = data["model"] + assert model["type"] == "qwen3_vl" + assert model["vision_start_token_id"] == 151652 + assert model["video_token_id"] == 151656 + assert model["vision"]["tokens_per_second"] == pytest.approx(2.0) + assert model["vision"]["patch_size"] == 16 + assert model["vision"]["window_size"] == 64 + def test_processor_config_not_written_without_vision(self, tmp_path): """image_processor.json is NOT written when pkg.config has no vision attr.""" from mobius.integrations.ort_genai.auto_export import write_ort_genai_config diff --git a/src/mobius/integrations/ort_genai/genai_config.py b/src/mobius/integrations/ort_genai/genai_config.py index 9983a932..36c6761e 100644 --- a/src/mobius/integrations/ort_genai/genai_config.py +++ b/src/mobius/integrations/ort_genai/genai_config.py @@ -269,6 +269,9 @@ def with_vision( embedding_output_names: dict[str, str] | None = None, vision_start_token_id: int | None = None, video_token_id: int | None = None, + tokens_per_second: float | None = None, + patch_size: int | None = None, + window_size: int | None = None, ) -> GenaiConfigGenerator: """Add VLM vision + embedding sections. @@ -293,6 +296,9 @@ def with_vision( mapping. Defaults to inputs_embeds. vision_start_token_id: Token ID for ``<|vision_start|>``. video_token_id: Token ID for video placeholders. + tokens_per_second: Video/image timestamp rate for Qwen3-VL. + patch_size: Vision patch size. + window_size: Vision window size. Returns self for chaining. """ @@ -320,6 +326,12 @@ def with_vision( } if spatial_merge_size is not None: self._vision["spatial_merge_size"] = spatial_merge_size + if tokens_per_second is not None: + self._vision["tokens_per_second"] = tokens_per_second + if patch_size is not None: + self._vision["patch_size"] = patch_size + if window_size is not None: + self._vision["window_size"] = window_size self._embedding = { "filename": embedding_filename, From b6e6ab0e608f061d8cc8be9a985ee7f7546adc9d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:50:02 +0000 Subject: [PATCH 2/3] Fix test_qwen35_deltanet_single_layer_parity for transformers 5.14+ API change In transformers 5.14, DynamicCache.layers[N].recurrent_states changed from returning a tensor directly to returning a dict keyed by layer index. Handle both forms so the test works with both old and new transformers versions. --- tests/integration_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index 843594ac..369e5893 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -3703,7 +3703,10 @@ def test_qwen35_deltanet_single_layer_parity(): hidden_states=torch.from_numpy(hidden_np).float(), cache_params=cache, ).numpy() - hf_rec = cache.layers[0].recurrent_states.numpy() + # transformers >=5.14 changed recurrent_states from a tensor to a dict + # keyed by layer index; extract the tensor for either version. + _rec_states = cache.layers[0].recurrent_states + hf_rec = (_rec_states[0] if isinstance(_rec_states, dict) else _rec_states).numpy() # ONNX forward sess = _make_session(onnx_model) From 3e1b2fe44b1a847a856c8bb64a5af18aa0d0f3cb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:04:19 +0000 Subject: [PATCH 3/3] Resolve merge conflicts with origin/main - auto_export.py: add deepseek_v4 mapping from main; keep qwen3_vl->qwen3_vl; merge MTP config block from main alongside local tokenizer copy logic - integration_test.py: keep transformers >=5.14 recurrent_states comment from our branch --- src/mobius/integrations/ort_genai/auto_export.py | 15 +++------------ tests/integration_test.py | 7 ------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/src/mobius/integrations/ort_genai/auto_export.py b/src/mobius/integrations/ort_genai/auto_export.py index 5f2ff7b2..3feabde7 100644 --- a/src/mobius/integrations/ort_genai/auto_export.py +++ b/src/mobius/integrations/ort_genai/auto_export.py @@ -86,12 +86,8 @@ # HunYuan-V1 dense / Hy-MT1.5 — generic decoder LLM type accepted by # ORT GenAI (see onnxruntime-genai/src/models/model_type.h LLM list). "hunyuan_v1_dense": "decoder", -<<<<<<< HEAD - # Qwen VL model families have separate ORT GenAI model types. -======= "deepseek_v4": "decoder", - # Qwen VL models all use the same GenAI pipeline as qwen2_5_vl ->>>>>>> origin/main + # Qwen VL model families have separate ORT GenAI model types. "qwen2_vl": "qwen2_5_vl", "qwen3_vl": "qwen3_vl", "qwen3_vl_text": "qwen3_vl", @@ -1104,10 +1100,6 @@ def write_ort_genai_config( result: dict[str, str] = {"genai_config": genai_path} -<<<<<<< HEAD - # Copy tokenizer files. A local hf_model_id is a local model directory, not a - # Hub repo id; copy directly instead of calling hf_hub_download. -======= if "mtp" in pkg: mtp_model = pkg["mtp"] mtp_path = os.path.join(directory, "mtp_config.json") @@ -1136,9 +1128,8 @@ def write_ort_genai_config( f.write("\n") result["mtp_config"] = mtp_path - # Copy tokenizer files — HF Hub takes precedence; local dir is the fallback - # for --config mode where no HF model ID is available. ->>>>>>> origin/main + # Copy tokenizer files. A local hf_model_id is a local model directory, not a + # Hub repo id; copy directly instead of calling hf_hub_download. if hf_model_id is not None: if os.path.isdir(hf_model_id): logger.info("Copying tokenizer files from local model directory %s", hf_model_id) diff --git a/tests/integration_test.py b/tests/integration_test.py index 21ab612d..99218321 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -3782,17 +3782,10 @@ def test_qwen35_deltanet_single_layer_parity(): hidden_states=torch.from_numpy(hidden_np).float(), cache_params=cache, ).numpy() -<<<<<<< HEAD # transformers >=5.14 changed recurrent_states from a tensor to a dict # keyed by layer index; extract the tensor for either version. _rec_states = cache.layers[0].recurrent_states hf_rec = (_rec_states[0] if isinstance(_rec_states, dict) else _rec_states).numpy() -======= - hf_rec = cache.layers[0].recurrent_states - if isinstance(hf_rec, dict): - hf_rec = hf_rec[0] - hf_rec = hf_rec.numpy() ->>>>>>> origin/main # ONNX forward sess = _make_session(onnx_model)