From 3c00d7e8f84863ccb39d6df0415b2456a9b2e919 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Thu, 13 Aug 2026 14:42:10 +0800 Subject: [PATCH 1/2] Add automatic perf runtime selection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/winml/modelkit/commands/perf.py | 22 +++++++++--- tests/unit/commands/test_perf_genai.py | 46 ++++++++++++++++++++------ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 0c5b52129..3a5e19331 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -70,12 +70,24 @@ # Inference runtimes selectable via ``--runtime`` (closed set; mirrors the # ``--compiler`` / ``COMPILER_NAMES`` convention in utils.constants): -# "winml" -> single-shot ONNX inference (default) +# "auto" -> select from the local model folder contents (default) +# "winml" -> single-shot ONNX inference # "winml-genai" -> onnxruntime-genai decoder-pipeline generation -RuntimeName = Literal["winml", "winml-genai"] +RuntimeName = Literal["auto", "winml", "winml-genai"] RUNTIME_NAMES: tuple[RuntimeName, ...] = get_args(RuntimeName) +def _resolve_runtime(runtime: RuntimeName, model: str) -> RuntimeName: + """Resolve ``auto`` from a local model folder, preserving explicit choices.""" + if runtime != "auto": + return runtime + + model_path = Path(model) + if model_path.is_dir() and (model_path / "genai_config.json").is_file(): + return "winml-genai" + return "winml" + + def _detail_fallback_guidance(reason: TraceFallbackReason | None) -> str: """Return actionable guidance for a structured detail-trace fallback.""" from ..session.monitor.op_metrics import TraceFallbackReason @@ -2442,9 +2454,10 @@ def _validate_duration( @click.option( "--runtime", type=click.Choice(list(RUNTIME_NAMES)), - default="winml", + default="auto", show_default=True, - help="Inference runtime. 'winml' benchmarks single-shot ONNX inference; " + help="'auto' selects winml-genai for folders containing genai_config.json, " + "otherwise winml. 'winml' benchmarks single-shot ONNX inference; " "'winml-genai' benchmarks an onnxruntime-genai bundle folder " "(LLM generation: TTFT + decode tokens/sec).", ) @@ -2752,6 +2765,7 @@ def perf( except Exception as e: raise click.ClickException(f"Failed to resolve Hub-hosted ONNX path {model!r}: {e}") from e model = hf_model + runtime = _resolve_runtime(runtime, model) # AC 11 (mockup spec): --top-k requires --op-tracing. Outside the # op-tracing section the flag is meaningless, so reject it explicitly # rather than silently ignoring a user's intent. diff --git a/tests/unit/commands/test_perf_genai.py b/tests/unit/commands/test_perf_genai.py index 08616777f..f9f22d619 100644 --- a/tests/unit/commands/test_perf_genai.py +++ b/tests/unit/commands/test_perf_genai.py @@ -31,7 +31,7 @@ run_genai_perf, write_genai_report, ) -from winml.modelkit.commands.perf import perf +from winml.modelkit.commands.perf import _resolve_runtime, perf from winml.modelkit.session import ( GenaiNotInstalledError, GenaiSessionError, @@ -802,13 +802,13 @@ def test_explicit_ep_without_device( assert cfg.ep == "dml" assert cfg.device == "config" - def test_default_device_is_config( + def test_auto_runtime_default_dispatches_genai_bundle( self, runner: CliRunner, tmp_path: Path, capture_run: dict ) -> None: - # Omitting --device is genai's "respect the bundle" default: no EP - # override, device recorded as "config". + # Omitting --runtime detects the bundle marker and routes to genai. + # Omitting --device then respects the bundle's own routing. bundle = _make_bundle(tmp_path) - result = runner.invoke(perf, ["-m", str(bundle), "--runtime", "winml-genai"]) + result = runner.invoke(perf, ["-m", str(bundle)]) assert result.exit_code == 0, result.output cfg = capture_run["config"] assert cfg.device == "config" @@ -829,8 +829,8 @@ def test_explicit_device_config_respects_bundle( assert cfg.ep is None def test_onnx_runtime_rejects_device_config(self, runner: CliRunner, tmp_path: Path) -> None: - # "config" is a winml-genai-only sentinel; the default (winml) runtime - # rejects it with a clear message rather than a generic device error. + # "config" is a winml-genai-only sentinel; auto resolves an ONNX path to + # winml and rejects it with a clear message rather than a generic error. result = runner.invoke(perf, ["-m", str(tmp_path / "model.onnx"), "--device", "config"]) assert result.exit_code != 0 assert "winml-genai" in result.output @@ -1308,10 +1308,34 @@ def test_autobuild_without_recipe_rejected( assert "recipe" in result.output.lower() assert "config" not in capture_run - def test_winml_runtime_unaffected( - self, runner: CliRunner, tmp_path: Path, capture_run: dict - ) -> None: - # Default runtime must not route through the genai path. + def test_runtime_help_shows_auto_default(self, runner: CliRunner, capture_run: dict) -> None: result = runner.invoke(perf, ["--help"]) assert result.exit_code == 0 + assert "[auto|winml|winml-genai]" in result.output + assert "default: auto" in result.output assert "config" not in capture_run + + +class TestAutoRuntime: + def test_selects_genai_for_bundle_folder(self, tmp_path: Path) -> None: + bundle = _make_bundle(tmp_path) + assert _resolve_runtime("auto", str(bundle)) == "winml-genai" + + @pytest.mark.parametrize("model_kind", ["plain-folder", "onnx-file", "model-id"]) + def test_selects_winml_without_bundle_marker(self, tmp_path: Path, model_kind: str) -> None: + if model_kind == "plain-folder": + model = tmp_path / "model" + model.mkdir() + value = str(model) + elif model_kind == "onnx-file": + model = tmp_path / "model.onnx" + model.write_bytes(b"onnx") + value = str(model) + else: + value = "organization/model" + + assert _resolve_runtime("auto", value) == "winml" + + @pytest.mark.parametrize("runtime", ["winml", "winml-genai"]) + def test_preserves_explicit_runtime(self, runtime: str) -> None: + assert _resolve_runtime(runtime, "organization/model") == runtime From b8e16db13b3a8459ee656f24926c929e4e574499 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Fri, 14 Aug 2026 09:44:50 +0800 Subject: [PATCH 2/2] Preserve normalized perf model path Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/winml/modelkit/commands/perf.py | 8 ++++---- tests/unit/commands/test_perf_genai.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 3a5e19331..ed2d38c00 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -2288,7 +2288,9 @@ def _autobuild_genai_bundle( return bundle_dir, True -def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) -> None: +def _run_genai_runtime( + ctx: click.Context, *, model: str, console: Console, json_mode: bool +) -> None: """Validate folder input and dispatch to the winml-genai benchmark path. The genai imports are function-local so ``winml perf --help`` does not pay @@ -2304,8 +2306,6 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) ) p = ctx.params - model: str = p["model"] - # --module walks a live nn.Module graph; meaningless for a prebuilt bundle. if p.get("module_class"): raise click.UsageError("--module is not supported with --runtime winml-genai.") @@ -2825,7 +2825,7 @@ def perf( "--input-data is not supported with --runtime winml-genai; " "genai benchmarking is driven by --prompt." ) - _run_genai_runtime(ctx, console=console, json_mode=json_mode) + _run_genai_runtime(ctx, model=model, console=console, json_mode=json_mode) return # --duration replaces the fixed iteration count with a wall-clock budget. diff --git a/tests/unit/commands/test_perf_genai.py b/tests/unit/commands/test_perf_genai.py index f9f22d619..bc21d372d 100644 --- a/tests/unit/commands/test_perf_genai.py +++ b/tests/unit/commands/test_perf_genai.py @@ -21,6 +21,7 @@ from rich.console import Console from winml.modelkit.commands import _perf_genai as perf_genai +from winml.modelkit.commands import perf as perf_module from winml.modelkit.commands._perf_genai import ( GenaiBenchmarkResult, GenaiPerfBenchmark, @@ -814,6 +815,21 @@ def test_auto_runtime_default_dispatches_genai_bundle( assert cfg.device == "config" assert cfg.ep is None + def test_auto_runtime_dispatches_normalized_bundle_path( + self, runner: CliRunner, tmp_path: Path, capture_run: dict, monkeypatch + ) -> None: + bundle = _make_bundle(tmp_path) + monkeypatch.setattr( + perf_module.cli_utils, + "normalize_model_arg", + lambda _model: str(bundle), + ) + + result = runner.invoke(perf, ["-m", "~/bundle"]) + + assert result.exit_code == 0, result.output + assert capture_run["config"].bundle_dir == bundle + def test_explicit_device_config_respects_bundle( self, runner: CliRunner, tmp_path: Path, capture_run: dict ) -> None: