feat: Add automatic perf runtime selection

src/winml/modelkit/commands/perf.py

@@ -70,12 +70,24 @@
 
 # Inference runtimes selectable via ``--runtime`` (closed set; mirrors the
 # ``--compiler`` / ``COMPILER_NAMES`` convention in utils.constants):
-#   "winml"       -> single-shot ONNX inference (default)
+#   "auto"        -> select from the local model folder contents (default)
+#   "winml"       -> single-shot ONNX inference
 #   "winml-genai" -> onnxruntime-genai decoder-pipeline generation
-RuntimeName = Literal["winml", "winml-genai"]
+RuntimeName = Literal["auto", "winml", "winml-genai"]
 RUNTIME_NAMES: tuple[RuntimeName, ...] = get_args(RuntimeName)
 
 
+def _resolve_runtime(runtime: RuntimeName, model: str) -> RuntimeName:
+    """Resolve ``auto`` from a local model folder, preserving explicit choices."""
+    if runtime != "auto":
+        return runtime
+
+    model_path = Path(model)
+    if model_path.is_dir() and (model_path / "genai_config.json").is_file():
+        return "winml-genai"
+    return "winml"
+
+
 def _detail_fallback_guidance(reason: TraceFallbackReason | None) -> str:
     """Return actionable guidance for a structured detail-trace fallback."""
     from ..session.monitor.op_metrics import TraceFallbackReason
@@ -2276,7 +2288,9 @@ def _autobuild_genai_bundle(
     return bundle_dir, True
 
 
-def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) -> None:
+def _run_genai_runtime(
+    ctx: click.Context, *, model: str, console: Console, json_mode: bool
+) -> None:
     """Validate folder input and dispatch to the winml-genai benchmark path.
 
     The genai imports are function-local so ``winml perf --help`` does not pay
@@ -2292,8 +2306,6 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool)
     )
 
     p = ctx.params
-    model: str = p["model"]
-
     # --module walks a live nn.Module graph; meaningless for a prebuilt bundle.
     if p.get("module_class"):
         raise click.UsageError("--module is not supported with --runtime winml-genai.")
@@ -2442,9 +2454,10 @@ def _validate_duration(
 @click.option(
     "--runtime",
     type=click.Choice(list(RUNTIME_NAMES)),
-    default="winml",
+    default="auto",
     show_default=True,
-    help="Inference runtime. 'winml' benchmarks single-shot ONNX inference; "
+    help="'auto' selects winml-genai for folders containing genai_config.json, "
+    "otherwise winml. 'winml' benchmarks single-shot ONNX inference; "
     "'winml-genai' benchmarks an onnxruntime-genai bundle folder "
     "(LLM generation: TTFT + decode tokens/sec).",
 )
@@ -2752,6 +2765,7 @@ def perf(
     except Exception as e:
         raise click.ClickException(f"Failed to resolve Hub-hosted ONNX path {model!r}: {e}") from e
     model = hf_model
+    runtime = _resolve_runtime(runtime, model)
     # AC 11 (mockup spec): --top-k requires --op-tracing. Outside the
     # op-tracing section the flag is meaningless, so reject it explicitly
     # rather than silently ignoring a user's intent.
@@ -2811,7 +2825,7 @@ def perf(
                 "--input-data is not supported with --runtime winml-genai; "
                 "genai benchmarking is driven by --prompt."
             )
-        _run_genai_runtime(ctx, console=console, json_mode=json_mode)
+        _run_genai_runtime(ctx, model=model, console=console, json_mode=json_mode)
         return
 
     # --duration replaces the fixed iteration count with a wall-clock budget.

tests/unit/commands/test_perf_genai.py

@@ -21,6 +21,7 @@
 from rich.console import Console
 
 from winml.modelkit.commands import _perf_genai as perf_genai
+from winml.modelkit.commands import perf as perf_module
 from winml.modelkit.commands._perf_genai import (
     GenaiBenchmarkResult,
     GenaiPerfBenchmark,
@@ -31,7 +32,7 @@
     run_genai_perf,
     write_genai_report,
 )
-from winml.modelkit.commands.perf import perf
+from winml.modelkit.commands.perf import _resolve_runtime, perf
 from winml.modelkit.session import (
     GenaiNotInstalledError,
     GenaiSessionError,
@@ -802,18 +803,33 @@ def test_explicit_ep_without_device(
         assert cfg.ep == "dml"
         assert cfg.device == "config"
 
-    def test_default_device_is_config(
+    def test_auto_runtime_default_dispatches_genai_bundle(
         self, runner: CliRunner, tmp_path: Path, capture_run: dict
     ) -> None:
-        # Omitting --device is genai's "respect the bundle" default: no EP
-        # override, device recorded as "config".
+        # Omitting --runtime detects the bundle marker and routes to genai.
+        # Omitting --device then respects the bundle's own routing.
         bundle = _make_bundle(tmp_path)
-        result = runner.invoke(perf, ["-m", str(bundle), "--runtime", "winml-genai"])
+        result = runner.invoke(perf, ["-m", str(bundle)])
         assert result.exit_code == 0, result.output
         cfg = capture_run["config"]
         assert cfg.device == "config"
         assert cfg.ep is None
 
+    def test_auto_runtime_dispatches_normalized_bundle_path(
+        self, runner: CliRunner, tmp_path: Path, capture_run: dict, monkeypatch
+    ) -> None:
+        bundle = _make_bundle(tmp_path)
+        monkeypatch.setattr(
+            perf_module.cli_utils,
+            "normalize_model_arg",
+            lambda _model: str(bundle),
+        )
+
+        result = runner.invoke(perf, ["-m", "~/bundle"])
+
+        assert result.exit_code == 0, result.output
+        assert capture_run["config"].bundle_dir == bundle
+
     def test_explicit_device_config_respects_bundle(
         self, runner: CliRunner, tmp_path: Path, capture_run: dict
     ) -> None:
@@ -829,8 +845,8 @@ def test_explicit_device_config_respects_bundle(
         assert cfg.ep is None
 
     def test_onnx_runtime_rejects_device_config(self, runner: CliRunner, tmp_path: Path) -> None:
-        # "config" is a winml-genai-only sentinel; the default (winml) runtime
-        # rejects it with a clear message rather than a generic device error.
+        # "config" is a winml-genai-only sentinel; auto resolves an ONNX path to
+        # winml and rejects it with a clear message rather than a generic error.
         result = runner.invoke(perf, ["-m", str(tmp_path / "model.onnx"), "--device", "config"])
         assert result.exit_code != 0
         assert "winml-genai" in result.output
@@ -1308,10 +1324,34 @@ def test_autobuild_without_recipe_rejected(
         assert "recipe" in result.output.lower()
         assert "config" not in capture_run
 
-    def test_winml_runtime_unaffected(
-        self, runner: CliRunner, tmp_path: Path, capture_run: dict
-    ) -> None:
-        # Default runtime must not route through the genai path.
+    def test_runtime_help_shows_auto_default(self, runner: CliRunner, capture_run: dict) -> None:
         result = runner.invoke(perf, ["--help"])
         assert result.exit_code == 0
+        assert "[auto|winml|winml-genai]" in result.output
+        assert "default: auto" in result.output
         assert "config" not in capture_run
+
+
+class TestAutoRuntime:
+    def test_selects_genai_for_bundle_folder(self, tmp_path: Path) -> None:
+        bundle = _make_bundle(tmp_path)
+        assert _resolve_runtime("auto", str(bundle)) == "winml-genai"
+
+    @pytest.mark.parametrize("model_kind", ["plain-folder", "onnx-file", "model-id"])
+    def test_selects_winml_without_bundle_marker(self, tmp_path: Path, model_kind: str) -> None:
+        if model_kind == "plain-folder":
+            model = tmp_path / "model"
+            model.mkdir()
+            value = str(model)
+        elif model_kind == "onnx-file":
+            model = tmp_path / "model.onnx"
+            model.write_bytes(b"onnx")
+            value = str(model)
+        else:
+            value = "organization/model"
+
+        assert _resolve_runtime("auto", value) == "winml"
+
+    @pytest.mark.parametrize("runtime", ["winml", "winml-genai"])
+    def test_preserves_explicit_runtime(self, runtime: str) -> None:
+        assert _resolve_runtime(runtime, "organization/model") == runtime