From 69847a89be6f7085d003ba08b003eb182cc5f88a Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Sun, 16 Aug 2026 05:22:54 +0800 Subject: [PATCH] feat(modelkit): reconstruct Swin2SR CPU support --- .../cpu/cpu/image-to-image_fp16_config.json | 67 +++++++++++++++++ .../cpu/cpu/image-to-image_fp32_config.json | 45 +++++++++++ src/winml/modelkit/eval/evaluate.py | 2 +- tests/unit/eval/test_eval.py | 74 +++++++++++++++++++ tests/unit/models/test_swin2sr_support.py | 27 +++++++ 5 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp16_config.json create mode 100644 examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp32_config.json create mode 100644 tests/unit/models/test_swin2sr_support.py diff --git a/examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp16_config.json b/examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp16_config.json new file mode 100644 index 000000000..8bd3e274f --- /dev/null +++ b/examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp16_config.json @@ -0,0 +1,67 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 64, + 64 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "reconstruction" + } + ], + "compatibility": { + "transformers_attention": "eager" + } + }, + "optim": {}, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "task": "image-to-image", + "model_id": "caidas/swin2SR-classical-sr-x2-64", + "model_type": "swin2sr", + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "image-to-image", + "model_class": "AutoModelForImageToImage", + "model_type": "swin2sr" + } +} \ No newline at end of file diff --git a/examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp32_config.json b/examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp32_config.json new file mode 100644 index 000000000..a23c2e5df --- /dev/null +++ b/examples/recipes/caidas_swin2SR-classical-sr-x2-64/cpu/cpu/image-to-image_fp32_config.json @@ -0,0 +1,45 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 64, + 64 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "reconstruction" + } + ], + "compatibility": { + "transformers_attention": "eager" + } + }, + "optim": {}, + "quant": null, + "compile": null, + "loader": { + "task": "image-to-image", + "model_class": "AutoModelForImageToImage", + "model_type": "swin2sr" + } +} \ No newline at end of file diff --git a/src/winml/modelkit/eval/evaluate.py b/src/winml/modelkit/eval/evaluate.py index 07eefd6ee..9ef8d51b5 100644 --- a/src/winml/modelkit/eval/evaluate.py +++ b/src/winml/modelkit/eval/evaluate.py @@ -537,7 +537,7 @@ def _resolve_task( console.print(f"[dim]Use[/dim] {task} [dim]to evaluate[/dim]") - if task not in _EVALUATOR_REGISTRY: + if config.mode != "compare" and task not in _EVALUATOR_REGISTRY: supported = ", ".join(sorted(_EVALUATOR_REGISTRY)) raise ValueError(f"Task '{task}' is not supported. Supported tasks: {supported}.") return task diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index e41ad6e8e..b9348e11c 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -195,6 +195,26 @@ def test_explicit_task(self): config = WinMLEvaluationConfig(task="image-classification") assert _resolve_task(config) == "image-classification" + @pytest.mark.parametrize("mode", ["onnx", "compare"]) + def test_supported_task_preserved_in_all_modes(self, mode): + from winml.modelkit.eval.evaluate import _resolve_task + + config = WinMLEvaluationConfig(task="image-classification", mode=mode) + assert _resolve_task(config) == "image-classification" + + def test_compare_preserves_task_without_metric_evaluator(self): + from winml.modelkit.eval.evaluate import _resolve_task + + config = WinMLEvaluationConfig(task="image-to-image", mode="compare") + assert _resolve_task(config) == "image-to-image" + + def test_onnx_rejects_task_without_metric_evaluator(self): + from winml.modelkit.eval.evaluate import _resolve_task + + config = WinMLEvaluationConfig(task="image-to-image", mode="onnx") + with pytest.raises(ValueError, match="Task 'image-to-image' is not supported"): + _resolve_task(config) + def test_no_model_id_raises(self): from winml.modelkit.eval.evaluate import _resolve_task @@ -310,6 +330,16 @@ def test_unsupported_task_raises_value_error(self): with pytest.raises(ValueError, match="not supported by `winml eval`"): get_evaluator_class(WinMLEvaluationConfig(task="made-up-task")) + def test_compare_dispatches_unsupported_metric_task_to_tensor_evaluator(self): + from winml.modelkit.eval import WinMLEvaluationConfig, get_evaluator_class + + config = WinMLEvaluationConfig(task="image-to-image", mode="compare") + evaluator_class = get_evaluator_class(config) + + assert evaluator_class.__module__ == "winml.modelkit.eval.tensor_similarity_evaluator" + assert evaluator_class.__name__ == "TensorSimilarityEvaluator" + assert config.task == "image-to-image" + def test_evaluator_registry_matches_schema_tasks(self): from winml.modelkit.eval.evaluate import _EVALUATOR_REGISTRY from winml.modelkit.utils.eval_utils import TASK_SCHEMAS @@ -395,6 +425,50 @@ def test_onnx_compare_skips_task_resolution_and_dataset(self): assert result.metrics == {"cosine_mean": {"logits": 1.0}} load_model.assert_called_once() + def test_compare_reaches_tensor_dispatch_for_task_without_metric_evaluator(self): + """HF-reference compare keeps the model task and uses tensor metrics.""" + import importlib + import sys + + eval_mod = sys.modules.get( + "winml.modelkit.eval.evaluate", + ) or importlib.import_module("winml.modelkit.eval.evaluate") + + config = WinMLEvaluationConfig( + model_id="test/image-to-image-model", + model_path="cand.onnx", + task="image-to-image", + mode="compare", + ) + captured = {} + + class TensorEvaluatorProbe: + def __init__(self, resolved_config, model): + captured["config"] = resolved_config + captured["model"] = model + + def compute(self): + return {"cosine_mean": {"reconstruction": 1.0}} + + def select_evaluator(resolved_config): + captured["evaluator_key"] = ( + "compare-tensor" if resolved_config.mode == "compare" else resolved_config.task + ) + return TensorEvaluatorProbe + + model = MagicMock() + with ( + patch.object(eval_mod, "_load_model", return_value=model), + patch.object(eval_mod, "get_evaluator_class", side_effect=select_evaluator), + ): + result = eval_mod.evaluate(config) + + assert result.config.task == "image-to-image" + assert captured["config"].task == "image-to-image" + assert captured["evaluator_key"] == "compare-tensor" + assert captured["model"] is model + assert result.metrics == {"cosine_mean": {"reconstruction": 1.0}} + def test_load_model_returns_none_for_onnx_compare(self): """_load_model short-circuits (no model_id needed) for two-ONNX compare.""" from winml.modelkit.eval.evaluate import _load_model diff --git a/tests/unit/models/test_swin2sr_support.py b/tests/unit/models/test_swin2sr_support.py new file mode 100644 index 000000000..52be11fb1 --- /dev/null +++ b/tests/unit/models/test_swin2sr_support.py @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Offline tests for vendor-provided Swin2SR support.""" + +from transformers import Swin2SRConfig + +from winml.modelkit.export.io import _get_onnx_config, ensure_hf_models_registered +from winml.modelkit.loader import get_supported_tasks, resolve_task + + +def test_swin2sr_vendor_registration_supports_image_to_image() -> None: + ensure_hf_models_registered() + assert "image-to-image" in get_supported_tasks("swin2sr") + + onnx_config = _get_onnx_config("swin2sr", "image-to-image", Swin2SRConfig()) + assert type(onnx_config).__name__ == "Swin2srOnnxConfig" + + +def test_swin2sr_default_task_resolves_to_image_to_image() -> None: + config = Swin2SRConfig(architectures=["Swin2SRForImageSuperResolution"]) + + resolution = resolve_task(config) + + assert resolution.task == "image-to-image" + assert resolution.model_class.__name__ == "AutoModelForImageToImage"