From d34e4802870d30a89d7d4c438993230423df68dc Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 31 Jul 2026 00:56:29 +0800 Subject: [PATCH 1/5] test(recipes): pin pre-split recipe resolution in a golden fixture Captures what all 168 (recipe, hardware, precision, framework) triples resolved to before the recipe params are split into hardware/capability layers, so the split can be proven output-preserving for instruct models. Params are stored as ordered key/value pairs rather than a mapping: infer_plans.yaml is dumped with sort_keys=False and compared byte-for-byte under --resume, so parameter order is part of the contract and a fixture that sorted keys would hide a reordering. The generator reads the recipes from a git ref instead of the working tree, because the pre-split schema stops existing in the next commit. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/infer/recipes/_capture_golden.py | 76 + .../infer/recipes/golden_recipe_profiles.json | 4538 +++++++++++++++++ 2 files changed, 4614 insertions(+) create mode 100644 tests/unit/infer/recipes/_capture_golden.py create mode 100644 tests/unit/infer/recipes/golden_recipe_profiles.json diff --git a/tests/unit/infer/recipes/_capture_golden.py b/tests/unit/infer/recipes/_capture_golden.py new file mode 100644 index 00000000..821b1da1 --- /dev/null +++ b/tests/unit/infer/recipes/_capture_golden.py @@ -0,0 +1,76 @@ +"""Regenerate the recipe-resolution golden fixture from the pre-split recipes. + +The fixture pins what every ``(recipe, hardware, precision, framework)`` triple +resolved to **before** ``profiles`` was split into ``hardware`` + +``capabilities``, with parameter **order** preserved — ``infer_plans.yaml`` is +compared byte-for-byte under ``--resume``, so ordering is part of the contract. + +It is captured from a git ref rather than from the working tree, because the +pre-split schema no longer exists in the code: pre-split ``resolve_profile`` +returned the YAML leaf dict verbatim, so the leaf order *is* the resolved order. + +Usage: pdm run python tests/unit/infer/recipes/_capture_golden.py [git-ref] + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +import subprocess +import sys +from pathlib import Path + +import yaml + +_OUT = Path(__file__).parent / "golden_recipe_profiles.json" +_RECIPES = ("gpt_oss.yaml", "qwen2_5.yaml", "qwen3.yaml") + + +def _load_at_ref(ref: str, path: str) -> dict: + proc = subprocess.run( + ["git", "show", f"{ref}:{path}"], + capture_output=True, + text=True, + check=True, + ) + return yaml.safe_load(proc.stdout) or {} + + +def capture(ref: str) -> list[dict]: + """Record every pre-split profile leaf, preserving parameter order.""" + records: list[dict] = [] + for name in _RECIPES: + data = _load_at_ref(ref, f"sieval/infer/recipes/{name}") + for entry, raw in data.items(): + if entry.startswith("_") or not isinstance(raw, dict): + continue + for hw_key, prec_map in (raw.get("profiles") or {}).items(): + for precision, fw_map in prec_map.items(): + for framework, params in fw_map.items(): + records.append( + { + "recipe": entry, + "hardware_key": hw_key, + "precision": precision, + "framework": framework, + # List of pairs, not a dict: order is asserted. + "params": [[k, v] for k, v in params.items()], + } + ) + records.sort( + key=lambda r: ( + r["recipe"], + r["hardware_key"], + r["precision"], + r["framework"], + ) + ) + return records + + +if __name__ == "__main__": + ref = sys.argv[1] if len(sys.argv) > 1 else "origin/main" + records = capture(ref) + if not records: + raise SystemExit(f"No pre-split recipes found at {ref!r}") + _OUT.write_text(json.dumps(records, indent=2) + "\n") + print(f"Wrote {len(records)} records from {ref} to {_OUT}") diff --git a/tests/unit/infer/recipes/golden_recipe_profiles.json b/tests/unit/infer/recipes/golden_recipe_profiles.json new file mode 100644 index 00000000..f324c374 --- /dev/null +++ b/tests/unit/infer/recipes/golden_recipe_profiles.json @@ -0,0 +1,4538 @@ +[ + { + "recipe": "gpt-oss-120b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.82 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-120b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.85 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "gpt-oss-120b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-120b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 8192 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 8192 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + } +] From 1d86f8c384fddc4e8e94744b777e32ce849c0524 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 31 Jul 2026 00:56:47 +0800 Subject: [PATCH 2/5] feat(recipes)!: split recipe params into hardware and capability layers The flat profiles[hardware][precision][framework] bag mixed two kinds of param with different owners, and was matched on family + size only. A base checkpoint therefore resolved to the same profile as the instruct one and inherited its parsers and tool-choice flags. That is now a live path, not a hypothetical: Qwen2.5-72B-Base is the comparison target for the ARC and HellaSwag ppl/clp tasks and shares the qwen2.5-72b bucket with the Instruct checkpoint, whose every leaf carries enable_auto_tool_choice and tool_call_parser: hermes. Recipes now declare two layers: hardware[gpu][precision][framework] perf, memory, parallelism, context capabilities[instruct|base][fw] parsers, tool choice resolve_profile is replaced by resolve_hardware_profile and resolve_capability_profile; callers merge them with capabilities last. Collapsing the duplicated capability params takes them from 352 lines to 78. The model type comes from the eval config, mapped chat -> instruct and gen -> base. It is *derived*, not read off the explicit `type:` field: no config in the repo sets `type:`, leaving it to inference from the tasks pointing at the model, so reading the explicit field alone would have resolved every base checkpoint to instruct capabilities and made this change a no-op on real configs. The derivation that the eval session already performed is promoted to a shared derive_model_type, since both sites must reach the same answer for one model. BREAKING CHANGE: the `profiles` key is rejected with a migration error rather than read for compatibility; recipe files ship in-repo, so there is no legacy on-disk state to support, and silently accepting it would keep handing base checkpoints the params this split exists to withhold. User customization via `overrides` is unchanged. Instruct resolution is unchanged in content for all 168 profile triples. Key *order* changes for the two gpt-oss recipes alone, whose capability params sat between hardware params pre-split; the golden test pins that set so no further reordering can slip in. Because infer_plans.yaml is compared byte-for-byte, this must ship as a minor bump, where the resume version gate's (major, minor) break axis rejects a 0.7.x resume with a version message instead of a plan diff. Refs: scitix/sieval#27 Co-Authored-By: Claude Opus 5 (1M context) --- sieval/cli/infer/recipe.py | 52 ++- sieval/cli/leaderboard/session.py | 170 +++++----- sieval/cli/run.py | 18 +- sieval/infer/recipes/__init__.py | 10 +- sieval/infer/recipes/gpt_oss.yaml | 57 ++-- sieval/infer/recipes/qwen2_5.yaml | 204 +++++------- sieval/infer/recipes/qwen3.yaml | 362 +++++++-------------- sieval/infer/recipes/registry.py | 158 ++++++--- sieval/infer/topology/resolver.py | 22 +- tests/unit/cli/infer/test_resolve.py | 118 ++++++- tests/unit/cli/leaderboard/test_session.py | 46 +++ tests/unit/infer/recipes/test_golden.py | 112 +++++++ tests/unit/infer/recipes/test_gpt_oss.py | 76 ++++- tests/unit/infer/test_recipes.py | 228 ++++++++++--- 14 files changed, 1063 insertions(+), 570 deletions(-) create mode 100644 tests/unit/infer/recipes/test_golden.py diff --git a/sieval/cli/infer/recipe.py b/sieval/cli/infer/recipe.py index 68935130..f237d4b1 100644 --- a/sieval/cli/infer/recipe.py +++ b/sieval/cli/infer/recipe.py @@ -15,6 +15,7 @@ import yaml from loguru import logger +from sieval.cli.leaderboard.session import derive_model_type from sieval.infer.config import ParamValue from sieval.infer.introspect import ( GPUInfo, @@ -26,11 +27,13 @@ from sieval.infer.params import merge_params from sieval.infer.recipes import ( Recipe, + capability_model_type, list_recipes, load_family_recipes, load_recipe, match_recipe, - resolve_profile, + resolve_capability_profile, + resolve_hardware_profile, ) from sieval.infer.topology.models import ( CP_KEYS, @@ -108,6 +111,13 @@ async def resolve_infer_config( raw_env = infer_dict.get("env") or {} user_env: dict[str, str] = {k: str(v) for k, v in raw_env.items()} + # Which capability layer to serve. Derived from the same config the eval + # session reads, so a base checkpoint gets base capabilities even when the + # config leaves `type` to task inference (the normal case). + model_type = capability_model_type( + derive_model_type(model_name, mcfg.get("type"), cfg.get("tasks") or {}) + ) + # Recipe resolution recipe_params: dict[str, ParamValue] | None = None recipe_name = infer_dict.get("recipe") @@ -120,6 +130,7 @@ async def resolve_infer_config( checkpoint, backend_name, overrides, + model_type, ) elif checkpoint: # Case 2: no recipe, but checkpoint available → try auto-resolve @@ -128,6 +139,7 @@ async def resolve_infer_config( backend_name=backend_name, overrides=overrides, model_name=model_name, + model_type=model_type, ) elif overrides: # Case 3: no recipe, no checkpoint, but overrides → use as-is @@ -232,13 +244,20 @@ async def _resolve_recipe_params( recipe: Recipe, backend_name: str, overrides: dict[str, ParamValue], + model_type: str, ) -> dict[str, ParamValue]: """Resolve engine params for a recipe. - Pipeline: formula TP/DP → profile → overrides → safety check. + Pipeline: formula TP/DP → hardware profile → capability profile → + overrides → safety check. Extracted from _resolve_with_recipe / _try_auto_resolve_recipe to eliminate duplication. + + Args: + model_type: Recipe capability key (``"instruct"`` / ``"base"``), which + selects the capability layer. A base checkpoint resolves to no + parser or tool-choice params. """ # Normalize overrides once up front so the dtype check below and the # final merge operate on a single canonical key form. @@ -259,16 +278,19 @@ async def _resolve_recipe_params( if dp > 1: params[dp_key] = dp - # Profile overrides formula + # Recipe layers override the formula: hardware first, then the capability + # layer for this model type (a base checkpoint contributes nothing). prec_key = precision_key(identity) gpu_model = gpu.model if gpu else None - profile = resolve_profile(recipe, gpu_model, prec_key, backend_name) + profile = resolve_hardware_profile(recipe, gpu_model, prec_key, backend_name) + capabilities = resolve_capability_profile(recipe, model_type, backend_name) if profile is None and identity.dtype and "dtype" not in overrides: - # No profile → fallback to model's intrinsic dtype (unless user overrode). + # No hardware profile → fall back to the model's intrinsic dtype + # (unless the user overrode it). params["dtype"] = identity.dtype - params = merge_params(params, profile or {}, overrides) + params = merge_params(params, profile or {}, capabilities, overrides) # Safety check if gpu: @@ -282,6 +304,7 @@ async def _resolve_with_recipe( checkpoint: str, backend_name: str, overrides: dict[str, ParamValue], + model_type: str, ) -> dict[str, ParamValue] | None: """Merge params for an already-loaded recipe via shared merge logic. @@ -302,7 +325,13 @@ async def _resolve_with_recipe( ) if identity is not None: - return await _resolve_recipe_params(identity, recipe, backend_name, overrides) + return await _resolve_recipe_params( + identity, + recipe, + backend_name, + overrides, + model_type, + ) else: # No identity — can only use overrides return dict(overrides) if overrides else None @@ -343,6 +372,7 @@ async def _try_auto_resolve_recipe( backend_name: str, overrides: dict[str, ParamValue], model_name: str, + model_type: str, ) -> dict[str, ParamValue] | None: """Attempt to auto-resolve a recipe from checkpoint introspection. @@ -379,7 +409,13 @@ async def _try_auto_resolve_recipe( if recipe is not None: # Matched — use shared merge logic - params = await _resolve_recipe_params(identity, recipe, backend_name, overrides) + params = await _resolve_recipe_params( + identity, + recipe, + backend_name, + overrides, + model_type, + ) family_recipes = load_family_recipes(identity.family) family_names = [r.name for r in family_recipes] diff --git a/sieval/cli/leaderboard/session.py b/sieval/cli/leaderboard/session.py index 72f736ca..11c1484f 100644 --- a/sieval/cli/leaderboard/session.py +++ b/sieval/cli/leaderboard/session.py @@ -633,6 +633,94 @@ def resolve_task_class(class_spec: str) -> type: ) +def derive_model_type( + model_name: str, + explicit_type: str | None, + tasks_cfg: Mapping[str, Mapping[str, Any]], +) -> str: + """Decide whether a config's model is a ``"chat"`` or ``"gen"`` model. + + Priority: + 1. The config's explicit ``type``. + 2. The ``model_type`` declared by the tasks pointing at this model. + 3. Default to ``"chat"``. + + Both the eval session (which model class to construct) and infer recipe + resolution (which capability layer to serve) must reach the same answer for + the same model, so the derivation lives here rather than being read off + ``type`` twice. Explicit-only would silently mean "instruct" for every + config in the wild, since ``type`` is normally left to this inference. + + Args: + model_name: Name of the model in config. + explicit_type: Explicitly specified type from config, if any. + tasks_cfg: The config's ``tasks`` mapping, used for inference. + + Returns: + Model type: ``"chat"`` or ``"gen"``. + + Raises: + ValueError: If tasks pointing at this model require conflicting types. + """ + # 1. User explicitly specified + if explicit_type is not None: + return explicit_type + + # 2. Infer from tasks + required_types: set[tuple[str, str]] = set() + + for task_name, task_cfg in tasks_cfg.items(): + if task_cfg.get("model") != model_name: + continue + + # Resolve task class to check its model_type attribute + task_class_spec = task_cfg.get("class") + if not task_class_spec: + continue + + try: + task_class = resolve_task_class(task_class_spec) + task_model_type = getattr(task_class, "model_type", None) + + if task_model_type is not None: + required_types.add((task_name, task_model_type)) + except (ImportError, AttributeError): + # If we can't resolve the task class yet, skip it + # Validation will catch issues later + continue + + # Check for conflicts + unique_types = {t for _, t in required_types} + + if len(unique_types) > 1: + # Conflicting requirements + conflict_info = "\n".join( + f" - {task_name} requires '{model_type}'" + for task_name, model_type in sorted(required_types) + ) + raise ValueError( + f"Model '{model_name}' is used by tasks requiring different types:\n" + f"{conflict_info}\n" + f"Please either:\n" + f" 1. Explicitly specify 'type: chat' or 'type: gen' in model config\n" + f" 2. Use separate models for different types" + ) + + if len(unique_types) == 1: + # All tasks agree on the same type + inferred_type = unique_types.pop() + logger.info( + "Inferred model '{}' type as '{}' from task requirements", + model_name, + inferred_type, + ) + return inferred_type + + # 3. Default to "chat" + logger.info("Using default type 'chat' for model '{}'", model_name) + return "chat" + + def _guess_submodule_names(class_name: str) -> list[str]: """ Guess possible submodule names from a class name. @@ -871,82 +959,12 @@ def _normalize_list(value: Any, field_name: str) -> list[Any]: return value def _infer_model_type(self, model_name: str, explicit_type: str | None) -> str: - """ - Infer the model type based on task requirements. - - Priority: - 1. User explicitly specifies type in config - 2. Infer from tasks that use this model - 3. Default to "chat" - - Args: - model_name: Name of the model in config - explicit_type: Explicitly specified type from config (if any) - - Returns: - Model type: "chat" or "gen" - - Raises: - ValueError: If tasks require conflicting model types - """ - # 1. User explicitly specified - if explicit_type is not None: - return explicit_type - - # 2. Infer from tasks - tasks_cfg = self._get_named_config_map("tasks") - required_types: set[tuple[str, str]] = set() - - for task_name, task_cfg in tasks_cfg.items(): - if task_cfg.get("model") != model_name: - continue - - # Resolve task class to check its model_type attribute - task_class_spec = task_cfg.get("class") - if not task_class_spec: - continue - - try: - task_class = resolve_task_class(task_class_spec) - task_model_type = getattr(task_class, "model_type", None) - - if task_model_type is not None: - required_types.add((task_name, task_model_type)) - except (ImportError, AttributeError): - # If we can't resolve the task class yet, skip it - # Validation will catch issues later - continue - - # Check for conflicts - unique_types = {t for _, t in required_types} - - if len(unique_types) > 1: - # Conflicting requirements - conflict_info = "\n".join( - f" - {task_name} requires '{model_type}'" - for task_name, model_type in sorted(required_types) - ) - raise ValueError( - f"Model '{model_name}' is used by tasks requiring different types:\n" - f"{conflict_info}\n" - f"Please either:\n" - f" 1. Explicitly specify 'type: chat' or 'type: gen' in model config\n" - f" 2. Use separate models for different types" - ) - - if len(unique_types) == 1: - # All tasks agree on the same type - inferred_type = unique_types.pop() - logger.info( - "Inferred model '{}' type as '{}' from task requirements", - model_name, - inferred_type, - ) - return inferred_type - - # 3. Default to "chat" - logger.info("Using default type 'chat' for model '{}'", model_name) - return "chat" + """Infer this session's model type — see :func:`derive_model_type`.""" + return derive_model_type( + model_name, + explicit_type, + self._get_named_config_map("tasks"), + ) def _setup_models(self) -> None: """Initialize all models from config.""" diff --git a/sieval/cli/run.py b/sieval/cli/run.py index 14ce4a44..e967093a 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -17,13 +17,18 @@ from loguru import logger from sieval.cli.infer import cleanup_model, launch_model, resolve_infer_config -from sieval.cli.leaderboard.session import resolve_deterministic, unwrap_proxies +from sieval.cli.leaderboard.session import ( + derive_model_type, + resolve_deterministic, + unwrap_proxies, +) from sieval.cli.output import CommandResult, OutputFormat, cli_command, render from sieval.core.utils.logging import configure_logging, log_user from sieval.infer.backends import get_translator from sieval.infer.backends.translator import inject_user_env from sieval.infer.config import InferHandle from sieval.infer.deployer import LocalDeployer +from sieval.infer.recipes import capability_model_type from sieval.infer.topology.resolver import auto_resolve_plan @@ -92,10 +97,19 @@ async def _run_all( model_name, ) else: - # Path-only mode: auto-resolve from checkpoint + # Path-only mode: auto-resolve from checkpoint. The capability + # layer follows the same model type the eval session will use, + # which is usually inferred from the tasks rather than declared. checkpoint = model_config["path"] result = await auto_resolve_plan( checkpoint=checkpoint, + model_type=capability_model_type( + derive_model_type( + model_name, + model_config.get("type"), + config.get("tasks") or {}, + ) + ), ) plan = result.plan user_env = {} # path-only mode has no YAML env section diff --git a/sieval/infer/recipes/__init__.py b/sieval/infer/recipes/__init__.py index 6f1578c6..3af2311c 100644 --- a/sieval/infer/recipes/__init__.py +++ b/sieval/infer/recipes/__init__.py @@ -1,19 +1,25 @@ from sieval.infer.recipes.registry import ( + CAPABILITY_MODEL_TYPES, Recipe, + capability_model_type, check_tested_versions, list_recipes, load_family_recipes, load_recipe, match_recipe, - resolve_profile, + resolve_capability_profile, + resolve_hardware_profile, ) __all__ = [ + "CAPABILITY_MODEL_TYPES", "Recipe", + "capability_model_type", "check_tested_versions", "list_recipes", "load_family_recipes", "load_recipe", "match_recipe", - "resolve_profile", + "resolve_capability_profile", + "resolve_hardware_profile", ] diff --git a/sieval/infer/recipes/gpt_oss.yaml b/sieval/infer/recipes/gpt_oss.yaml index da3c3de3..6d76882d 100644 --- a/sieval/infer/recipes/gpt_oss.yaml +++ b/sieval/infer/recipes/gpt_oss.yaml @@ -13,9 +13,18 @@ # gpt-oss-120b H100-80G profile assumes TP>=2. Single-card H100-80G OOMs on # defaults; override gpu_memory_utilization=0.95 and max_num_batched_tokens=1024. # +# Structure: +# hardware[gpu][precision][framework] → perf/memory/parallelism/context +# capabilities[model_type][framework] → behavior (parsers, tool choice) +# # Perf flags (max_num_batched_tokens, max_cudagraph_capture_size, # stream_interval, no_enable_prefix_caching) are copied verbatim from # vllm-project/recipes OpenAI/GPT-OSS_Hopper.yaml. +# +# `no_enable_prefix_caching` is a model-intrinsic recommendation rather than a +# hardware property, so it does not really belong in the hardware layer. It +# stays there for now: prefix caching is being modelled as an engine-level +# constraint (scitix/sieval#47), and moving the flag first would migrate it twice. _family: gpt-oss @@ -26,15 +35,12 @@ gpt-oss-20b: sglang: [">=0.5.2"] known_issues: - "Deterministic inference: use vLLM (gpt-oss batch-invariance fixed in vllm#35404). SGLang's --enable-deterministic-inference is unreliable on gpt-oss (sglang#19431 tracks 120b; no direct 20b repro)." - profiles: + hardware: H100-80G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.90 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -42,16 +48,11 @@ gpt-oss-20b: sglang: context_length: 131072 mem_fraction_static: 0.85 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss H200-141G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.90 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -59,8 +60,18 @@ gpt-oss-20b: sglang: context_length: 131072 mem_fraction_static: 0.85 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss + capabilities: + instruct: + vllm: + reasoning_parser: openai + tool_call_parser: openai + enable_auto_tool_choice: true + sglang: + reasoning_parser: gpt-oss + tool_call_parser: gpt-oss + base: + vllm: {} + sglang: {} gpt-oss-120b: size_range: [100, 150] @@ -69,15 +80,12 @@ gpt-oss-120b: sglang: [">=0.5.2"] known_issues: - "Deterministic inference: use vLLM (gpt-oss batch-invariance fixed in vllm#35404). SGLang's --enable-deterministic-inference diverges at large batch sizes on gpt-oss-120b (sglang#19431, open)." - profiles: + hardware: H100-80G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.85 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -85,16 +93,11 @@ gpt-oss-120b: sglang: context_length: 131072 mem_fraction_static: 0.82 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss H200-141G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.90 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -102,5 +105,15 @@ gpt-oss-120b: sglang: context_length: 131072 mem_fraction_static: 0.85 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss + capabilities: + instruct: + vllm: + reasoning_parser: openai + tool_call_parser: openai + enable_auto_tool_choice: true + sglang: + reasoning_parser: gpt-oss + tool_call_parser: gpt-oss + base: + vllm: {} + sglang: {} diff --git a/sieval/infer/recipes/qwen2_5.yaml b/sieval/infer/recipes/qwen2_5.yaml index 52936f00..6b878aee 100644 --- a/sieval/infer/recipes/qwen2_5.yaml +++ b/sieval/infer/recipes/qwen2_5.yaml @@ -1,7 +1,16 @@ -# Qwen2.5 family recipes — profile-based structure +# Qwen2.5 family recipes — two-layer structure # -# Each profile: profiles[hardware][precision][framework] → complete params -# No inheritance or merge — what you see is what you get. +# hardware[gpu][precision][framework] → perf/memory/parallelism/context +# capabilities[model_type][framework] → behavior (parsers, tool choice) +# +# One entry serves both the instruct and the base checkpoint of a family/size; +# resolution picks the capability layer from the model's `type` (chat→instruct, +# gen→base) and merges it over the hardware layer. Within a layer there is no +# inheritance or merge — what you see is what you get. +# +# The base layer matters here: Qwen2.5-72B-Base is the comparison target for +# the ARC and HellaSwag ppl/clp tasks, and it resolves to the same size bucket +# as Qwen2.5-72B-Instruct. # # Hardware notes: # A100-40G: 40GB HBM2e, Ampere, no native FP8 (bf16 only) @@ -44,459 +53,424 @@ qwen2.5-0.5b: tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-1.5b: size_range: [1, 2] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-3b: size_range: [2, 5] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-7b: size_range: [5, 10] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-14b: size_range: [10, 20] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-32b: size_range: [20, 40] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-72b: size_range: [60, 80] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} diff --git a/sieval/infer/recipes/qwen3.yaml b/sieval/infer/recipes/qwen3.yaml index 8422519c..80be0e61 100644 --- a/sieval/infer/recipes/qwen3.yaml +++ b/sieval/infer/recipes/qwen3.yaml @@ -1,7 +1,12 @@ -# Qwen3 family recipes — profile-based structure +# Qwen3 family recipes — two-layer structure # -# Each profile: profiles[hardware][precision][framework] → complete params -# No inheritance or merge — what you see is what you get. +# hardware[gpu][precision][framework] → perf/memory/parallelism/context +# capabilities[model_type][framework] → behavior (parsers, tool choice) +# +# One entry serves both the instruct and the base checkpoint of a family/size; +# resolution picks the capability layer from the model's `type` (chat→instruct, +# gen→base) and merges it over the hardware layer. Within a layer there is no +# inheritance or merge — what you see is what you get. # # Hardware notes: # A100-40G: 40GB HBM2e, Ampere, no native FP8 (bf16 only) @@ -40,681 +45,564 @@ qwen3-0.6b: tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-1.7b: size_range: [1, 3] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-4b: size_range: [3, 6] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-8b: size_range: [6, 12] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-14b: size_range: [12, 20] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-30b-a3b: size_range: [20, 32] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-32b: size_range: [32, 40] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-72b: size_range: [60, 80] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-235b-a22b: size_range: [200, 250] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 8192 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 8192 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} diff --git a/sieval/infer/recipes/registry.py b/sieval/infer/recipes/registry.py index 61b6d13c..070b0f50 100644 --- a/sieval/infer/recipes/registry.py +++ b/sieval/infer/recipes/registry.py @@ -5,8 +5,14 @@ 1. Family match: model architecture → recipe YAML file (e.g. qwen3 → qwen.yaml) 2. Size match: parameter count → size bucket within that file (e.g. ~4B → qwen3-4b) -Profile resolution uses fuzzy GPU matching to look up a complete, self-contained -parameter set from ``profiles[hardware][precision][framework]``. +A matched recipe carries two independent param layers, resolved separately and +merged by the caller (capabilities last): + +* ``hardware[gpu][precision][framework]`` — perf/memory/parallelism/context, + looked up by fuzzy GPU matching. +* ``capabilities[model_type][framework]`` — behavior (parsers, tool choice), + looked up by model type, so one entry serves a family/size's instruct **and** + base checkpoint without the base one inheriting instruct-only params. AI-Generated Code - Claude Opus 4.6 (Anthropic) """ @@ -38,34 +44,80 @@ def _coerce_param(value: object) -> ParamValue: return str(value) +CAPABILITY_MODEL_TYPES: tuple[str, ...] = ("instruct", "base") +"""Model-type keys a recipe's ``capabilities`` layer may declare.""" + + +def capability_model_type(model_type: str | None) -> str: + """Map an eval config's model ``type`` onto a recipe capability key. + + ``"chat"`` → ``"instruct"``, ``"gen"`` → ``"base"``. Anything else + (including ``None``) falls back to ``"instruct"``, matching the eval + config's own default for an undeclared ``type``. + """ + return "base" if model_type == "gen" else "instruct" + + @dataclass class Recipe: """Typed representation of a model infer recipe. + A recipe entry serves both the instruct and the base checkpoint of a + family/size, so serving params are split by who owns them: + Fields: name: Recipe name (e.g. "qwen3-8b") size_range: [min_b, max_b) parameter count range in billions - profiles: Per-hardware, per-precision, per-framework params, e.g. - {"H100-80G": {"bf16": {"vllm": {"dtype": "bfloat16", ...}}}} + hardware: Per-hardware, per-precision, per-framework perf/memory + params, e.g. {"H100-80G": {"bf16": {"vllm": {"dtype": ...}}}} + capabilities: Per-model-type, per-framework behavior params (parsers, + tool choice), e.g. {"instruct": {"vllm": {"tool_call_parser": ...}}} known_issues: Human-readable issue descriptions tested_versions: Per-framework version specifiers """ name: str = "" size_range: tuple[float, float] = (0.0, float("inf")) - profiles: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = field( + hardware: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = field( + default_factory=dict, + ) + capabilities: dict[str, dict[str, dict[str, ParamValue]]] = field( default_factory=dict, ) known_issues: list[str] = field(default_factory=list) tested_versions: dict[str, list[str]] = field(default_factory=dict) +def _coerce_params(raw: object) -> dict[str, ParamValue]: + """Coerce a raw YAML mapping into a flat framework param dict.""" + if not isinstance(raw, dict): + return {} + return {str(k): _coerce_param(v) for k, v in raw.items()} + + +def _coerce_framework_map(raw: object) -> dict[str, dict[str, ParamValue]]: + """Coerce a raw YAML mapping of ``framework -> params``.""" + if not isinstance(raw, dict): + return {} + return {str(fw): _coerce_params(params) for fw, params in raw.items()} + + def _parse_recipe(name: str, raw: dict[str, object]) -> Recipe: """Parse a raw YAML dict into a typed Recipe. Old fields (``frameworks``, ``hardware_overrides``, ``precision_overrides``) - are silently ignored if present — only ``profiles`` is parsed. + are silently ignored if present. The pre-split ``profiles`` key is rejected + outright — recipe files ship in-repo, so there is no legacy on-disk state to + support, and silently reading it would hand base checkpoints the parser + params the split exists to withhold. """ + if "profiles" in raw: + raise ValueError( + f"Recipe {name!r} uses the removed 'profiles' key. Split it into " + "'hardware' (hw -> precision -> framework: dtype, memory, " + "parallelism, context) and 'capabilities' (instruct|base -> " + "framework: parsers, tool choice)." + ) # size_range (optional) raw_range = raw.get("size_range") if isinstance(raw_range, list) and len(raw_range) == 2: @@ -77,24 +129,30 @@ def _parse_recipe(name: str, raw: dict[str, object]) -> Recipe: else: size_range = (0.0, float("inf")) - # profiles (optional) — hw_key → prec_key → framework → params - raw_profiles = raw.get("profiles", {}) - profiles: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = {} - if isinstance(raw_profiles, dict): - for hw_key, prec_map in raw_profiles.items(): - if isinstance(prec_map, dict): - prec_dict: dict[str, dict[str, dict[str, ParamValue]]] = {} - for prec_key, fw_map in prec_map.items(): - if isinstance(fw_map, dict): - fw_dict: dict[str, dict[str, ParamValue]] = {} - for fw_name, fw_params in fw_map.items(): - if isinstance(fw_params, dict): - typed: dict[str, ParamValue] = {} - for k, v in fw_params.items(): - typed[str(k)] = _coerce_param(v) - fw_dict[str(fw_name)] = typed - prec_dict[str(prec_key)] = fw_dict - profiles[str(hw_key)] = prec_dict + # hardware (optional) — hw_key → prec_key → framework → params + raw_hardware = raw.get("hardware", {}) + hardware: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = {} + if isinstance(raw_hardware, dict): + for hw_key, prec_map in raw_hardware.items(): + if not isinstance(prec_map, dict): + continue + hardware[str(hw_key)] = { + str(prec_key): _coerce_framework_map(fw_map) + for prec_key, fw_map in prec_map.items() + } + + # capabilities (optional) — model_type → framework → params + raw_caps = raw.get("capabilities", {}) + capabilities: dict[str, dict[str, dict[str, ParamValue]]] = {} + if isinstance(raw_caps, dict): + for model_type, fw_map in raw_caps.items(): + if str(model_type) not in CAPABILITY_MODEL_TYPES: + raise ValueError( + f"Recipe {name!r} declares unknown capability model type " + f"{model_type!r}; expected one of " + f"{', '.join(CAPABILITY_MODEL_TYPES)}." + ) + capabilities[str(model_type)] = _coerce_framework_map(fw_map) # known_issues (optional) raw_issues = raw.get("known_issues", []) @@ -115,7 +173,8 @@ def _parse_recipe(name: str, raw: dict[str, object]) -> Recipe: return Recipe( name=name, size_range=size_range, - profiles=profiles, + hardware=hardware, + capabilities=capabilities, known_issues=known_issues, tested_versions=tested_versions, ) @@ -254,21 +313,21 @@ def match_recipe(family: str, param_billions: float) -> Recipe | None: return None -def resolve_profile( +def resolve_hardware_profile( recipe: Recipe, gpu_model: str | None, precision: str | None, framework: str, ) -> dict[str, ParamValue] | None: - """Look up a complete parameter set from the recipe's profiles. + """Look up the perf/memory parameter set from the recipe's hardware layer. - Uses fuzzy GPU matching: splits the profile hardware key on ``[-_\\s]+`` - and checks that every resulting token appears in ``gpu_model`` (case- - insensitive). For example, key ``"H100-80G"`` matches GPU string + Uses fuzzy GPU matching: splits the hardware key on ``[-_\\s]+`` and checks + that every resulting token appears in ``gpu_model`` (case-insensitive). + For example, key ``"H100-80G"`` matches GPU string ``"NVIDIA H100-SXM5-80GB"``. Args: - recipe: Typed Recipe with a ``profiles`` field. + recipe: Typed Recipe with a ``hardware`` field. gpu_model: Detected GPU name (e.g. ``"NVIDIA A100-SXM4-80GB"``). ``None`` → return ``None`` immediately. precision: Precision key (e.g. ``"bf16"``, ``"fp8"``). @@ -285,17 +344,17 @@ def resolve_profile( if precision is None: precision = "bf16" - if not recipe.profiles: + if not recipe.hardware: return None gpu_lower = gpu_model.lower() - for hw_key, prec_map in recipe.profiles.items(): + for hw_key, prec_map in recipe.hardware.items(): key_tokens = re.split(r"[-_\s]+", hw_key.lower()) if all(token in gpu_lower for token in key_tokens): - logger.info("GPU {!r} matched profile hardware key {!r}", gpu_model, hw_key) + logger.info("GPU {!r} matched hardware key {!r}", gpu_model, hw_key) if precision not in prec_map: logger.warning( - "Precision {!r} not in profile for hw {!r} (available: {})", + "Precision {!r} not in hardware {!r} (available: {})", precision, hw_key, ", ".join(prec_map), @@ -304,7 +363,7 @@ def resolve_profile( fw_map = prec_map[precision] if framework not in fw_map: logger.info( - "Framework {!r} not in profile {}[{}] (available: {})", + "Framework {!r} not in hardware {}[{}] (available: {})", framework, hw_key, precision, @@ -314,15 +373,36 @@ def resolve_profile( return dict(fw_map[framework]) logger.info( - "GPU {!r} did not match any profile hardware key " - "in recipe {!r} (available: {})", + "GPU {!r} did not match any hardware key in recipe {!r} (available: {})", gpu_model, recipe.name, - ", ".join(recipe.profiles), + ", ".join(recipe.hardware), ) return None +def resolve_capability_profile( + recipe: Recipe, + model_type: str, + framework: str, +) -> dict[str, ParamValue]: + """Look up behavior params for a model type + framework. + + Args: + recipe: Typed Recipe with a ``capabilities`` field. + model_type: A recipe capability key — ``"instruct"`` or ``"base"``. + Use :func:`capability_model_type` to map an eval config's + ``type`` onto it. + framework: Framework name (e.g. ``"vllm"``, ``"sglang"``). + + Returns: + A shallow copy of the matched params, or ``{}`` when the + ``model_type``/``framework`` entry is absent — the expected case for a + base checkpoint, which declares no capability params. + """ + return dict(recipe.capabilities.get(model_type, {}).get(framework, {})) + + def load_recipe(name: str) -> Recipe: """Load a recipe by exact name from the registry. diff --git a/sieval/infer/topology/resolver.py b/sieval/infer/topology/resolver.py index ffdc3118..dc3d8268 100644 --- a/sieval/infer/topology/resolver.py +++ b/sieval/infer/topology/resolver.py @@ -20,7 +20,11 @@ introspect_checkpoint_with_config, ) from sieval.infer.params import merge_params -from sieval.infer.recipes import match_recipe, resolve_profile +from sieval.infer.recipes import ( + match_recipe, + resolve_capability_profile, + resolve_hardware_profile, +) from sieval.infer.topology.models import ( TOPO_KEYS, DeploymentPlan, @@ -371,11 +375,17 @@ async def auto_resolve_plan( *, backend: str = "sglang", overrides: dict[str, ParamValue] | None = None, + model_type: str = "instruct", ) -> ResolveResult: """Auto-resolve a DeploymentPlan from a checkpoint path. Replaces resolve.auto_resolve(). Same 5-step flow but returns ResolveResult instead of InferConfig. + + Args: + model_type: Recipe capability key (``"instruct"`` / ``"base"``). Base + checkpoints resolve without parser or tool-choice params. Defaults + to ``"instruct"`` for standalone callers with no task context. """ # Normalize keys at the entry: the TOPO_KEYS filter (step 4) and # _overrides_to_hints both look up specific underscore-form keys, so a @@ -407,14 +417,16 @@ async def auto_resolve_plan( gpu_memory_mib=gpu.memory_mib, ) - # 3. Match recipe → resolve profile → recipe_params + # 3. Match recipe → resolve hardware + capability layers → recipe_params recipe = match_recipe(identity.family, identity.param_billions) recipe_params: dict[str, ParamValue] | None = None if recipe is not None: prec_key = precision_key(identity) - profile = resolve_profile(recipe, gpu.model, prec_key, backend) - if profile is not None: - recipe_params = profile + profile = resolve_hardware_profile(recipe, gpu.model, prec_key, backend) + capabilities = resolve_capability_profile(recipe, model_type, backend) + merged = {**(profile or {}), **capabilities} + if merged: + recipe_params = merged # 4. Build user hints from overrides; non-topology keys become engine params hints: UserHints | None = None diff --git a/tests/unit/cli/infer/test_resolve.py b/tests/unit/cli/infer/test_resolve.py index df976822..58fbb96a 100644 --- a/tests/unit/cli/infer/test_resolve.py +++ b/tests/unit/cli/infer/test_resolve.py @@ -684,7 +684,7 @@ async def test_user_override_dash_form_wins_over_profile_underscore( new=AsyncMock(return_value=None), ), patch( - "sieval.cli.infer.recipe.resolve_profile", + "sieval.cli.infer.recipe.resolve_hardware_profile", return_value={"foo_bar": 0}, ), ): @@ -693,6 +693,7 @@ async def test_user_override_dash_form_wins_over_profile_underscore( recipe, backend_name="vllm", overrides={"foo-bar": 42}, + model_type="instruct", ) assert result == {"foo_bar": 42} @@ -716,7 +717,7 @@ async def test_dash_form_override_is_normalized_when_fallback_engages( new=AsyncMock(return_value=None), ), patch( - "sieval.cli.infer.recipe.resolve_profile", + "sieval.cli.infer.recipe.resolve_hardware_profile", return_value=None, # profile-less → dtype fallback engaged ), ): @@ -725,7 +726,120 @@ async def test_dash_form_override_is_normalized_when_fallback_engages( recipe, backend_name="vllm", overrides={"max-model-len": 4096}, + model_type="instruct", ) # Fallback injects identity.dtype; dash-form override normalizes to underscore. assert result == {"dtype": "bfloat16", "max_model_len": 4096} + + +# --------------------------------------------------------------------------- +# Capability layer: which params a model type resolves to +# --------------------------------------------------------------------------- + + +class TestCapabilityLayerResolution: + """`resolve_infer_config` must pick the capability layer from the model type + the eval session will use — which is normally inferred from the tasks, not + declared as `type:` in the config.""" + + _CAPABILITY_KEYS = ("reasoning_parser", "tool_call_parser") + + def _write_cfg( + self, + tmp_path: Path, + model_dir: Path, + model_cfg_extra: dict, + tasks_cfg: dict | None = None, + ) -> Path: + cfg = { + "models": { + "mymodel": { + "infer": { + "backend": "sglang", + "recipe": "qwen3-4b", + "checkpoint": str(model_dir), + }, + **model_cfg_extra, + }, + }, + } + if tasks_cfg is not None: + cfg["tasks"] = tasks_cfg + yaml_path = tmp_path / "config.yaml" + yaml_path.write_text(yaml.dump(cfg)) + return yaml_path + + @pytest.mark.anyio + async def test_instruct_model_gets_parsers(self, tmp_path: Path) -> None: + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg(tmp_path, model_dir, {"type": "chat"}) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + assert params["reasoning_parser"] == "qwen3" + assert params["tool_call_parser"] == "qwen" + + @pytest.mark.anyio + async def test_explicit_gen_type_omits_parsers(self, tmp_path: Path) -> None: + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg(tmp_path, model_dir, {"type": "gen"}) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + for key in self._CAPABILITY_KEYS: + assert key not in params + # The hardware half must survive. + assert params["context_length"] == 32768 + + @pytest.mark.anyio + async def test_gen_inferred_from_task_omits_parsers(self, tmp_path: Path) -> None: + """No `type:` in the config — the base path must still be chosen. + + This is the realistic shape: no config in the repo declares `type:`, so + reading the explicit field alone would resolve every base checkpoint to + instruct capabilities. + """ + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg( + tmp_path, + model_dir, + {}, + tasks_cfg={ + "arc_ppl": {"model": "mymodel", "class": "ARCEasyFewShotPplTask"}, + }, + ) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + for key in self._CAPABILITY_KEYS: + assert key not in params, f"{key} leaked into an inferred-gen model" + + @pytest.mark.anyio + async def test_chat_task_still_gets_parsers(self, tmp_path: Path) -> None: + """The inference must not flip instruct models to base.""" + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg( + tmp_path, + model_dir, + {}, + tasks_cfg={ + "aime": {"model": "mymodel", "class": "AIME2024ZeroShotGenTask"}, + }, + ) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + assert params["tool_call_parser"] == "qwen" diff --git a/tests/unit/cli/leaderboard/test_session.py b/tests/unit/cli/leaderboard/test_session.py index 1292ee87..c7a15622 100644 --- a/tests/unit/cli/leaderboard/test_session.py +++ b/tests/unit/cli/leaderboard/test_session.py @@ -34,6 +34,7 @@ _strip_header, _strip_noncomparable_fields, arun_session, + derive_model_type, load_class_from_name, load_class_from_path, resolve_class, @@ -612,6 +613,51 @@ def mock_resolve(_spec): runner._infer_model_type("m", None) +class TestDeriveModelType: + """`derive_model_type` is shared by the eval session and recipe resolution, + so both reach the same answer for one model. Tested directly because + `sieval run` calls the function, not the session method.""" + + def test_explicit_type_wins(self): + assert derive_model_type("m", "gen", {}) == "gen" + + def test_defaults_to_chat_with_no_tasks(self): + assert derive_model_type("m", None, {}) == "chat" + + def test_infers_gen_from_task_without_explicit_type(self): + """The case explicit-only reading would miss: no `type:` in config.""" + + class FakeTask: + model_type = "gen" + + tasks_cfg = {"t1": {"model": "m", "class": "fake.FakeTask"}} + with patch( + "sieval.cli.leaderboard.session.resolve_task_class", + return_value=FakeTask, + ): + assert derive_model_type("m", None, tasks_cfg) == "gen" + + def test_ignores_tasks_pointing_at_other_models(self): + class FakeTask: + model_type = "gen" + + tasks_cfg = {"t1": {"model": "other", "class": "fake.FakeTask"}} + with patch( + "sieval.cli.leaderboard.session.resolve_task_class", + return_value=FakeTask, + ): + assert derive_model_type("m", None, tasks_cfg) == "chat" + + def test_unresolvable_task_class_is_skipped(self): + """Validation reports import errors; derivation must not raise here.""" + tasks_cfg = {"t1": {"model": "m", "class": "missing.Task"}} + with patch( + "sieval.cli.leaderboard.session.resolve_task_class", + side_effect=ImportError("nope"), + ): + assert derive_model_type("m", None, tasks_cfg) == "chat" + + class TestSetupModelsEngine: """`engine` field dispatches a gen model to GenModel vs SglangGenModel.""" diff --git a/tests/unit/infer/recipes/test_golden.py b/tests/unit/infer/recipes/test_golden.py new file mode 100644 index 00000000..b5ab8f41 --- /dev/null +++ b/tests/unit/infer/recipes/test_golden.py @@ -0,0 +1,112 @@ +"""Characterization tests: the capability split must not move instruct output. + +The fixture (``golden_recipe_profiles.json``, regenerated by +``_capture_golden.py``) records what every profile leaf resolved to before +``profiles`` was split into ``hardware`` + ``capabilities``. An instruct model +must still resolve to exactly the same parameters, because a resolved plan +reaches ``infer_plans.yaml`` and ``--resume`` compares that file byte-for-byte. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +from pathlib import Path + +from sieval.infer.params import merge_params +from sieval.infer.recipes import ( + load_recipe, + resolve_capability_profile, + resolve_hardware_profile, +) + +_GOLDEN = Path(__file__).parent / "golden_recipe_profiles.json" + +# Recipes whose capability params sat *between* hardware params in the pre-split +# leaves, so hoisting them into their own layer necessarily moves them to the +# end. Content is unchanged; only key order differs. This is reachable only by +# resuming a pre-split run, which the resume version gate already rejects across +# the minor boundary this ships in — but the exception is pinned here so a +# future reorder cannot slip in unnoticed. +_KNOWN_REORDERED: frozenset[str] = frozenset({"gpt-oss-20b", "gpt-oss-120b"}) + + +def _golden_records() -> list[dict]: + return json.loads(_GOLDEN.read_text()) + + +def _resolve_instruct(record: dict) -> dict: + """Resolve a golden record's triple the way the instruct path now does.""" + recipe = load_recipe(record["recipe"]) + hardware = resolve_hardware_profile( + recipe, + record["hardware_key"], + record["precision"], + record["framework"], + ) + capabilities = resolve_capability_profile(recipe, "instruct", record["framework"]) + return merge_params(hardware or {}, capabilities) + + +def test_golden_fixture_is_populated(): + records = _golden_records() + assert len(records) == 168, "fixture drifted; regenerate with _capture_golden.py" + assert all(r["params"] for r in records) + + +def test_instruct_resolution_preserves_pre_split_content(): + """Every triple resolves to the same parameters as before the split.""" + for record in _golden_records(): + expected = dict(record["params"]) + actual = _resolve_instruct(record) + assert actual == expected, ( + f"{record['recipe']} {record['hardware_key']} " + f"{record['precision']} {record['framework']}" + ) + + +def test_instruct_resolution_preserves_pre_split_order(): + """Key order is preserved except for the recipes documented above.""" + reordered: set[str] = set() + for record in _golden_records(): + expected = [k for k, _ in record["params"]] + actual = list(_resolve_instruct(record)) + if actual != expected: + assert sorted(actual) == sorted(expected), ( + f"{record['recipe']}: order test saw a content diff" + ) + reordered.add(record["recipe"]) + + assert reordered == _KNOWN_REORDERED, ( + "set of reordered recipes changed; a resolved plan's key order moved, " + "which breaks --resume byte matching" + ) + + +def test_base_resolution_omits_capability_params(): + """A base checkpoint inherits hardware params but no instruct behavior.""" + capability_keys = { + "tool_call_parser", + "enable_auto_tool_choice", + "reasoning_parser", + } + checked = 0 + for record in _golden_records(): + recipe = load_recipe(record["recipe"]) + hardware = resolve_hardware_profile( + recipe, + record["hardware_key"], + record["precision"], + record["framework"], + ) + base = merge_params( + hardware or {}, + resolve_capability_profile(recipe, "base", record["framework"]), + ) + assert not (capability_keys & set(base)), ( + f"{record['recipe']} leaked instruct capabilities into the base path" + ) + # The hardware half must still be there — the split withholds behavior, + # not memory/context sizing. + assert base == (hardware or {}) + checked += 1 + assert checked == 168 diff --git a/tests/unit/infer/recipes/test_gpt_oss.py b/tests/unit/infer/recipes/test_gpt_oss.py index 75d967c0..6ff4f23d 100644 --- a/tests/unit/infer/recipes/test_gpt_oss.py +++ b/tests/unit/infer/recipes/test_gpt_oss.py @@ -3,10 +3,29 @@ AI-Generated Code - Claude Opus 4.7 (Anthropic) """ -from sieval.infer.recipes import Recipe, load_recipe, resolve_profile +from sieval.infer.config import ParamValue +from sieval.infer.recipes import ( + Recipe, + load_recipe, + resolve_capability_profile, + resolve_hardware_profile, +) from sieval.infer.recipes.registry import load_family_recipes +def _instruct_params( + recipe: Recipe, + gpu_model: str, + precision: str, + framework: str, +) -> dict[str, ParamValue] | None: + """Compose the two layers the way an instruct model resolves them.""" + hardware = resolve_hardware_profile(recipe, gpu_model, precision, framework) + if hardware is None: + return None + return {**hardware, **resolve_capability_profile(recipe, "instruct", framework)} + + class TestGptOssRecipeShape: def test_family_loads_two_buckets(self) -> None: names = {r.name for r in load_family_recipes("gpt-oss")} @@ -22,22 +41,22 @@ def test_120b_size_range(self) -> None: def test_20b_has_both_hardware_tiers(self) -> None: recipe = load_recipe("gpt-oss-20b") - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware def test_120b_has_both_hardware_tiers(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware def test_only_mxfp4_precision(self) -> None: - h100 = load_recipe("gpt-oss-120b").profiles["H100-80G"] + h100 = load_recipe("gpt-oss-120b").hardware["H100-80G"] assert "mxfp4" in h100 assert "bf16" not in h100 assert "fp8" not in h100 def test_both_frameworks_per_profile(self) -> None: - h100_mxfp4 = load_recipe("gpt-oss-120b").profiles["H100-80G"]["mxfp4"] + h100_mxfp4 = load_recipe("gpt-oss-120b").hardware["H100-80G"]["mxfp4"] assert "vllm" in h100_mxfp4 assert "sglang" in h100_mxfp4 @@ -61,7 +80,7 @@ def test_20b_known_issues(self) -> None: class TestGptOss120bProfiles: def test_h100_vllm_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") assert params is not None assert params["max_model_len"] == 131072 assert params["gpu_memory_utilization"] == 0.85 @@ -75,7 +94,7 @@ def test_h100_vllm_profile(self) -> None: def test_h100_sglang_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") assert params is not None assert params["context_length"] == 131072 assert params["mem_fraction_static"] == 0.82 @@ -84,7 +103,7 @@ def test_h100_sglang_profile(self) -> None: def test_h200_vllm_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H200-141GB", "mxfp4", "vllm") + params = _instruct_params(recipe, "NVIDIA H200-141GB", "mxfp4", "vllm") assert params is not None assert params["gpu_memory_utilization"] == 0.90 assert params["max_num_batched_tokens"] == 8192 @@ -93,7 +112,7 @@ def test_h200_vllm_profile(self) -> None: def test_h200_sglang_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H200-141GB", "mxfp4", "sglang") + params = _instruct_params(recipe, "NVIDIA H200-141GB", "mxfp4", "sglang") assert params is not None assert params["mem_fraction_static"] == 0.85 @@ -101,7 +120,7 @@ def test_h200_sglang_profile(self) -> None: class TestGptOss20bProfiles: def test_h100_vllm_profile(self) -> None: recipe = load_recipe("gpt-oss-20b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") assert params is not None assert params["max_model_len"] == 131072 assert params["gpu_memory_utilization"] == 0.90 @@ -109,20 +128,45 @@ def test_h100_vllm_profile(self) -> None: def test_h100_sglang_profile(self) -> None: recipe = load_recipe("gpt-oss-20b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") assert params is not None assert params["mem_fraction_static"] == 0.85 +class TestGptOssBasePath: + """A gpt-oss base checkpoint keeps the perf flags but loses the parsers.""" + + def test_base_omits_parsers_but_keeps_prefix_cache_flag(self) -> None: + recipe = load_recipe("gpt-oss-120b") + hardware = resolve_hardware_profile( + recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm" + ) + assert hardware is not None + base = {**hardware, **resolve_capability_profile(recipe, "base", "vllm")} + assert "reasoning_parser" not in base + assert "tool_call_parser" not in base + assert "enable_auto_tool_choice" not in base + # Model-intrinsic, currently declared on the hardware layer, so it must + # survive the base path too. + assert base["no_enable_prefix_caching"] is True + assert base["max_model_len"] == 131072 + + class TestGptOssNegativeLookups: def test_bf16_not_defined(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") is None + assert ( + resolve_hardware_profile(recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") + is None + ) def test_a100_not_in_scope(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert resolve_profile(recipe, "NVIDIA A100-SXM4-80GB", "mxfp4", "vllm") is None + assert ( + resolve_hardware_profile(recipe, "NVIDIA A100-SXM4-80GB", "mxfp4", "vllm") + is None + ) def test_b200_not_in_scope(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert resolve_profile(recipe, "NVIDIA B200", "mxfp4", "vllm") is None + assert resolve_hardware_profile(recipe, "NVIDIA B200", "mxfp4", "vllm") is None diff --git a/tests/unit/infer/test_recipes.py b/tests/unit/infer/test_recipes.py index b177f51e..177e5250 100644 --- a/tests/unit/infer/test_recipes.py +++ b/tests/unit/infer/test_recipes.py @@ -1,4 +1,4 @@ -"""Tests for sieval.infer.recipes.registry — recipe loading and profile resolution. +"""Tests for sieval.infer.recipes.registry — recipe loading and layer resolution. AI-Generated Code - Claude Opus 4.6 (Anthropic) """ @@ -10,10 +10,12 @@ from sieval.infer.recipes import ( Recipe, + capability_model_type, check_tested_versions, list_recipes, load_recipe, - resolve_profile, + resolve_capability_profile, + resolve_hardware_profile, ) from sieval.infer.recipes.registry import _parse_recipe @@ -28,17 +30,17 @@ def test_list_recipes(self) -> None: class TestLoadRecipe: def test_load_recipe(self) -> None: - """Verify loading qwen3-8b returns expected profile-based structure.""" + """Verify loading qwen3-8b returns the expected two-layer structure.""" recipe = load_recipe("qwen3-8b") assert isinstance(recipe, Recipe) assert recipe.known_issues == [] # Hardware tiers present - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware # H100-80G / bf16 / vllm has correct params - h100_bf16_vllm = recipe.profiles["H100-80G"]["bf16"]["vllm"] + h100_bf16_vllm = recipe.hardware["H100-80G"]["bf16"]["vllm"] assert h100_bf16_vllm["dtype"] == "bfloat16" assert h100_bf16_vllm["gpu_memory_utilization"] == 0.95 assert h100_bf16_vllm["max_model_len"] == 32768 @@ -48,8 +50,8 @@ def test_load_recipe_qwen3_06b(self) -> None: recipe = load_recipe("qwen3-0.6b") assert isinstance(recipe, Recipe) assert recipe.size_range == (0.3, 1.0) - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware def test_load_recipe_not_found(self) -> None: """Verify LookupError for nonexistent recipe.""" @@ -79,7 +81,7 @@ def test_load_recipe_underscore_prefix_rejected(self) -> None: def test_load_recipe_qwen3_235b_a22b_has_fp8_profile(self) -> None: """Verify qwen3-235b-a22b H100 has fp8 with correct max_model_len.""" recipe = load_recipe("qwen3-235b-a22b") - fp8_vllm = recipe.profiles["H100-80G"]["fp8"]["vllm"] + fp8_vllm = recipe.hardware["H100-80G"]["fp8"]["vllm"] assert fp8_vllm["max_model_len"] == 32768 def test_hardware_overrides_removed(self) -> None: @@ -90,13 +92,13 @@ def test_hardware_overrides_removed(self) -> None: assert not hasattr(recipe, "precision_overrides") -class TestParseRecipeProfiles: - """Tests for _parse_recipe with the new profiles structure.""" +class TestParseRecipeLayers: + """Tests for _parse_recipe with the hardware + capabilities structure.""" - def test_parse_profiles_structure(self) -> None: + def test_parse_hardware_structure(self) -> None: """Verify 4-level nested dict parsed correctly.""" raw = { - "profiles": { + "hardware": { "H100-80G": { "bf16": { "vllm": { @@ -126,27 +128,27 @@ def test_parse_profiles_structure(self) -> None: }, } recipe = _parse_recipe("test-model", raw) - assert "H100-80G" in recipe.profiles - assert "A100-80G" in recipe.profiles - assert "bf16" in recipe.profiles["H100-80G"] - assert "vllm" in recipe.profiles["H100-80G"]["bf16"] - assert recipe.profiles["H100-80G"]["bf16"]["vllm"]["dtype"] == "bfloat16" + assert "H100-80G" in recipe.hardware + assert "A100-80G" in recipe.hardware + assert "bf16" in recipe.hardware["H100-80G"] + assert "vllm" in recipe.hardware["H100-80G"]["bf16"] + assert recipe.hardware["H100-80G"]["bf16"]["vllm"]["dtype"] == "bfloat16" assert ( - recipe.profiles["H100-80G"]["bf16"]["vllm"]["gpu_memory_utilization"] + recipe.hardware["H100-80G"]["bf16"]["vllm"]["gpu_memory_utilization"] == 0.95 ) - assert recipe.profiles["A100-80G"]["fp8"]["vllm"]["dtype"] == "fp8" + assert recipe.hardware["A100-80G"]["fp8"]["vllm"]["dtype"] == "fp8" - def test_parse_profiles_empty(self) -> None: - """No profiles → empty dict.""" + def test_parse_hardware_empty(self) -> None: + """No hardware layer → empty dict.""" raw: dict[str, object] = {"size_range": [6, 12]} recipe = _parse_recipe("test-8b", raw) - assert recipe.profiles == {} + assert recipe.hardware == {} - def test_parse_profiles_preserves_other_fields(self) -> None: - """known_issues and tested_versions survive alongside profiles.""" + def test_parse_preserves_other_fields(self) -> None: + """known_issues and tested_versions survive alongside hardware.""" raw = { - "profiles": { + "hardware": { "H100-80G": { "bf16": {"vllm": {"dtype": "bfloat16"}}, }, @@ -159,7 +161,7 @@ def test_parse_profiles_preserves_other_fields(self) -> None: assert recipe.known_issues == ["Some known issue"] assert recipe.tested_versions == {"vllm": [">=0.8.0"]} assert recipe.size_range == (6.0, 12.0) - assert "H100-80G" in recipe.profiles + assert "H100-80G" in recipe.hardware def test_old_fields_ignored(self) -> None: """Old frameworks/hw_overrides/precision_overrides silently ignored.""" @@ -171,7 +173,7 @@ def test_old_fields_ignored(self) -> None: "precision_overrides": { "fp8": {"A100-80G": {"vllm": {"max_model_len": 65536}}}, }, - "profiles": { + "hardware": { "H100-80G": { "bf16": {"vllm": {"dtype": "bfloat16"}}, }, @@ -182,18 +184,18 @@ def test_old_fields_ignored(self) -> None: assert not hasattr(recipe, "frameworks") assert not hasattr(recipe, "hardware_overrides") assert not hasattr(recipe, "precision_overrides") - # Profiles should be parsed normally - assert "H100-80G" in recipe.profiles + # The hardware layer should be parsed normally + assert "H100-80G" in recipe.hardware -class TestResolveProfile: - """Tests for resolve_profile.""" +class TestResolveHardwareProfile: + """Tests for resolve_hardware_profile.""" @pytest.fixture def sample_recipe(self) -> Recipe: return Recipe( name="test-model", - profiles={ + hardware={ "H100-80G": { "bf16": { "vllm": { @@ -219,7 +221,9 @@ def sample_recipe(self) -> Recipe: def test_exact_match(self, sample_recipe: Recipe) -> None: """hw+prec+fw all match → correct params.""" - result = resolve_profile(sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm" + ) assert result is not None assert result["dtype"] == "bfloat16" assert result["gpu_memory_utilization"] == 0.95 @@ -227,7 +231,7 @@ def test_exact_match(self, sample_recipe: Recipe) -> None: def test_fuzzy_gpu_match(self, sample_recipe: Recipe) -> None: """'NVIDIA H100-SXM5-80GB' matches profile key 'H100-80G'.""" - result = resolve_profile( + result = resolve_hardware_profile( sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "sglang" ) assert result is not None @@ -235,36 +239,44 @@ def test_fuzzy_gpu_match(self, sample_recipe: Recipe) -> None: def test_precision_not_found_returns_none(self, sample_recipe: Recipe) -> None: """fp8 on H100 (only bf16 defined) → None.""" - result = resolve_profile(sample_recipe, "NVIDIA H100-SXM5-80GB", "fp8", "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA H100-SXM5-80GB", "fp8", "vllm" + ) assert result is None def test_gpu_not_found_returns_none(self, sample_recipe: Recipe) -> None: """Unknown GPU → None.""" - result = resolve_profile(sample_recipe, "NVIDIA V100-SXM2-32GB", "bf16", "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA V100-SXM2-32GB", "bf16", "vllm" + ) assert result is None def test_gpu_none_returns_none(self, sample_recipe: Recipe) -> None: """gpu_model=None → None.""" - result = resolve_profile(sample_recipe, None, "bf16", "vllm") + result = resolve_hardware_profile(sample_recipe, None, "bf16", "vllm") assert result is None def test_precision_none_defaults_to_bf16(self, sample_recipe: Recipe) -> None: """precision=None → uses 'bf16'.""" - result = resolve_profile(sample_recipe, "NVIDIA H100-SXM5-80GB", None, "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA H100-SXM5-80GB", None, "vllm" + ) assert result is not None assert result["dtype"] == "bfloat16" def test_framework_not_in_profile_returns_none(self, sample_recipe: Recipe) -> None: """'tensorrt' not in any profile → None.""" - result = resolve_profile( + result = resolve_hardware_profile( sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "tensorrt" ) assert result is None - def test_empty_profiles_returns_none(self) -> None: - """Empty profiles → None.""" - recipe = Recipe(name="empty", profiles={}) - result = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") + def test_empty_hardware_returns_none(self) -> None: + """Empty hardware layer → None.""" + recipe = Recipe(name="empty", hardware={}) + result = resolve_hardware_profile( + recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm" + ) assert result is None @@ -308,7 +320,7 @@ def test_tested_versions_not_in_profiles(self) -> None: _coerce_param would mangle lists into strings. """ recipe = load_recipe("qwen3-4b") - for _hw_key, prec_map in recipe.profiles.items(): + for _hw_key, prec_map in recipe.hardware.items(): for _prec_key, fw_map in prec_map.items(): for fw_params in fw_map.values(): assert "tested_versions" not in fw_params @@ -399,7 +411,7 @@ def registry_with_issue(tmp_path, monkeypatch): "zz-test-1b": { "size_range": [1.0, 2.0], "known_issues": [_MARKER], - "profiles": { + "hardware": { "H100-80G": { "bf16": { "vllm": {"dtype": "bfloat16", "max_model_len": 4096}, @@ -440,3 +452,127 @@ def test_load_family_recipes_does_not_emit(self, registry_with_issue) -> None: def test_list_recipes_does_not_emit(self, registry_with_issue) -> None: output = _capture_logs(lambda: registry_with_issue.list_recipes()) assert _MARKER not in output + + +class TestLegacyProfilesRejected: + """The pre-split ``profiles`` key must fail loudly, not be read silently.""" + + def test_profiles_key_rejected(self) -> None: + raw = { + "size_range": [6, 12], + "profiles": {"H100-80G": {"bf16": {"vllm": {"dtype": "bfloat16"}}}}, + } + with pytest.raises(ValueError, match="removed 'profiles' key"): + _parse_recipe("test-8b", raw) + + def test_error_names_both_replacement_layers(self) -> None: + """The migration hint must say where the two halves go.""" + with pytest.raises(ValueError) as exc: + _parse_recipe("test-8b", {"profiles": {}}) + assert "hardware" in str(exc.value) + assert "capabilities" in str(exc.value) + + def test_unknown_capability_model_type_rejected(self) -> None: + raw = {"capabilities": {"chat": {"vllm": {"tool_call_parser": "hermes"}}}} + with pytest.raises(ValueError, match="unknown capability model type"): + _parse_recipe("test-8b", raw) + + +class TestCapabilityModelType: + """Eval-config ``type`` → recipe capability key.""" + + def test_gen_maps_to_base(self) -> None: + assert capability_model_type("gen") == "base" + + def test_chat_maps_to_instruct(self) -> None: + assert capability_model_type("chat") == "instruct" + + def test_none_defaults_to_instruct(self) -> None: + """Matches the eval config's own default for an undeclared type.""" + assert capability_model_type(None) == "instruct" + + +class TestResolveCapabilityProfile: + @pytest.fixture + def sample_recipe(self) -> Recipe: + return _parse_recipe( + "test-model", + { + "hardware": {"H100-80G": {"bf16": {"vllm": {"dtype": "bfloat16"}}}}, + "capabilities": { + "instruct": { + "vllm": { + "enable_auto_tool_choice": True, + "tool_call_parser": "hermes", + }, + "sglang": {"tool_call_parser": "qwen"}, + }, + "base": {"vllm": {}, "sglang": {}}, + }, + }, + ) + + def test_instruct_returns_parsers(self, sample_recipe: Recipe) -> None: + assert resolve_capability_profile(sample_recipe, "instruct", "vllm") == { + "enable_auto_tool_choice": True, + "tool_call_parser": "hermes", + } + + def test_base_returns_empty(self, sample_recipe: Recipe) -> None: + assert resolve_capability_profile(sample_recipe, "base", "vllm") == {} + + def test_unknown_framework_returns_empty(self, sample_recipe: Recipe) -> None: + assert resolve_capability_profile(sample_recipe, "instruct", "tensorrt") == {} + + def test_undeclared_model_type_returns_empty(self, sample_recipe: Recipe) -> None: + assert resolve_capability_profile(sample_recipe, "nonexistent", "vllm") == {} + + def test_returns_a_copy(self, sample_recipe: Recipe) -> None: + """Mutating the result must not corrupt the loaded recipe.""" + result = resolve_capability_profile(sample_recipe, "instruct", "vllm") + result["tool_call_parser"] = "mutated" + assert sample_recipe.capabilities["instruct"]["vllm"]["tool_call_parser"] == ( + "hermes" + ) + + +class TestShippedRecipeCapabilities: + """The split must actually withhold instruct behavior from base models.""" + + def test_instruct_gets_tool_call_parser(self) -> None: + recipe = load_recipe("qwen2.5-72b") + assert resolve_capability_profile(recipe, "instruct", "vllm") == { + "enable_auto_tool_choice": True, + "tool_call_parser": "hermes", + } + + def test_base_gets_nothing(self) -> None: + """Qwen2.5-72B-Base is the ARC / HellaSwag ppl comparison target.""" + recipe = load_recipe("qwen2.5-72b") + assert resolve_capability_profile(recipe, "base", "vllm") == {} + assert resolve_capability_profile(recipe, "base", "sglang") == {} + + def test_hardware_layer_carries_no_capability_params(self) -> None: + """Parsers must not be left behind in any shipped hardware leaf.""" + capability_keys = { + "tool_call_parser", + "enable_auto_tool_choice", + "reasoning_parser", + } + for name in list_recipes(): + recipe = load_recipe(name) + for hw_key, prec_map in recipe.hardware.items(): + for precision, fw_map in prec_map.items(): + for framework, params in fw_map.items(): + leaked = capability_keys & set(params) + assert not leaked, ( + f"{name} {hw_key}/{precision}/{framework} " + f"still carries {leaked}" + ) + + def test_every_shipped_recipe_declares_both_model_types(self) -> None: + """A missing `base` key would silently fall back to no capabilities, + making an authoring omission indistinguishable from an intentional one.""" + for name in list_recipes(): + recipe = load_recipe(name) + assert set(recipe.capabilities) == {"instruct", "base"}, name From 23f12b15f107f587f11e26685d9221426dc21e74 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 8 Aug 2026 17:06:36 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix(recipes):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20pin=20merge=20order,=20validate=20model=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the hardware/capability split. The golden fixture called the two layer helpers itself, so it pinned a reimplementation of the merge rather than the production one: swapping the `merge_params` argument order in either `_resolve_recipe_params` or `auto_resolve_plan` left the whole suite green while the key order in `infer_plans.yaml` moved — the exact thing `--resume` compares byte-for-byte. Both paths now assert hardware-before-capabilities on a resolved plan, and both mutations fail. `auto_resolve_plan` merged the two layers with a dict literal, which the `merge_params` docstring calls out as the anti-pattern it exists to replace: it skips key normalization and the dash/underscore collision check. It also collapsed a present-but-empty hardware profile to `None`; presence is preserved. `capability_model_type` mapped everything that was not "gen" onto "instruct". Since this change makes "base" a first-class recipe word while the config keyword stays "gen", `type: base` was a likely mistake that silently selected the opposite layer — and the eval session only rejects it after the engine is launched. Unknown types now raise where the typo is still cheap. `derive_model_type` is reached from the infer layer before an `EvalSession` exists, and full validation only runs under `--dry-run`, so a list-shaped `tasks:` surfaced as an `AttributeError` from inside recipe resolution. The session's shape check is now shared, and both entry points report the same error. Also: record why `base:` is declared explicitly empty rather than omitted, why `derive_model_type` stays in session.py, and why `sieval infer start` on a bare checkpoint keeps the instruct default (no task context; the flags are inert on a base model, not a startup failure). Pin `_capture_golden.py` to the pre-split ref, since `origin/main` stops carrying `profiles` once this merges. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/cli/infer/commands.py | 16 +++++- sieval/cli/leaderboard/session.py | 53 +++++++++++++------ sieval/infer/recipes/gpt_oss.yaml | 6 +++ sieval/infer/recipes/qwen2_5.yaml | 6 +++ sieval/infer/recipes/qwen3.yaml | 6 +++ sieval/infer/recipes/registry.py | 24 +++++++-- sieval/infer/topology/resolver.py | 10 ++-- tests/unit/cli/infer/test_resolve.py | 29 +++++++++++ tests/unit/cli/leaderboard/test_session.py | 19 +++++++ tests/unit/infer/recipes/_capture_golden.py | 7 ++- tests/unit/infer/test_recipes.py | 15 ++++++ tests/unit/infer/topology/test_resolver.py | 56 +++++++++++++++++++++ 12 files changed, 223 insertions(+), 24 deletions(-) diff --git a/sieval/cli/infer/commands.py b/sieval/cli/infer/commands.py index 69e44317..cded8df6 100644 --- a/sieval/cli/infer/commands.py +++ b/sieval/cli/infer/commands.py @@ -238,7 +238,21 @@ async def _resolve_yaml() -> ResolvedInferConfig: assignments=(new_a,) + plan.assignments[1:], ) else: - # Auto-resolve mode: target is a checkpoint path + # Auto-resolve mode: target is a checkpoint path. + # + # `model_type` is deliberately left at its "instruct" default here, and + # this is the one path that does not follow the eval session. A bare + # checkpoint carries no task context, so the derivation the YAML leg and + # `sieval run` use (tasks → chat/gen) has nothing to read. Serving a base + # checkpoint through this path therefore still resolves the instruct + # capability params — which are inert on it: the parser/tool-choice flags + # are accepted and unused, not a startup failure. They do land in the + # persisted plan, so the params recorded for such a launch overstate what + # the engine actually used. Deriving the type from the checkpoint itself + # (an absent `chat_template` in `tokenizer_config.json` marks a base + # model) would close the gap; it is not done here because introspection + # currently reads only `config.json` and a dead flag does not justify + # widening it. async def _resolve() -> ResolveResult: return await auto_resolve_plan( target, diff --git a/sieval/cli/leaderboard/session.py b/sieval/cli/leaderboard/session.py index 11c1484f..19aa2f53 100644 --- a/sieval/cli/leaderboard/session.py +++ b/sieval/cli/leaderboard/session.py @@ -633,6 +633,33 @@ def resolve_task_class(class_spec: str) -> type: ) +def _validate_named_config_map( + section_name: str, + section_cfg: Any, +) -> dict[str, dict[str, Any]]: + """Validate a config section is a ``name -> dict`` mapping, and return it. + + Shared so every entry point that reads a section reports the same error for + the same malformed config. ``derive_model_type`` is now reached from the + infer layer *before* an ``EvalSession`` exists, and full config validation + only runs under ``--dry-run``, so without this a list-shaped ``tasks:`` + surfaced as an ``AttributeError`` from inside recipe resolution. + """ + if not isinstance(section_cfg, dict): + raise ValueError( + f"'{section_name}' configuration must be a dictionary " + "mapping names to config" + ) + + for item_name, item_cfg in section_cfg.items(): + if not isinstance(item_cfg, dict): + raise ValueError( + f"'{section_name}.{item_name}' configuration must be a dictionary" + ) + + return section_cfg + + def derive_model_type( model_name: str, explicit_type: str | None, @@ -651,6 +678,10 @@ def derive_model_type( ``type`` twice. Explicit-only would silently mean "instruct" for every config in the wild, since ``type`` is normally left to this inference. + It stays in this module, despite the caller now being the infer CLI, because + the inference step *is* task-class resolution — moving it would drag + :func:`resolve_task_class` along or split the two apart. + Args: model_name: Name of the model in config. explicit_type: Explicitly specified type from config, if any. @@ -660,13 +691,15 @@ def derive_model_type( Model type: ``"chat"`` or ``"gen"``. Raises: - ValueError: If tasks pointing at this model require conflicting types. + ValueError: If tasks pointing at this model require conflicting types, + or if ``tasks_cfg`` is not a ``name -> dict`` mapping. """ # 1. User explicitly specified if explicit_type is not None: return explicit_type # 2. Infer from tasks + tasks_cfg = _validate_named_config_map("tasks", tasks_cfg) required_types: set[tuple[str, str]] = set() for task_name, task_cfg in tasks_cfg.items(): @@ -925,20 +958,10 @@ def _init_runner(self) -> None: def _get_named_config_map(self, section_name: str) -> dict[str, dict[str, Any]]: """Get a config section and validate it is a name -> dict mapping.""" - section_cfg = self.config.get(section_name, {}) - if not isinstance(section_cfg, dict): - raise ValueError( - f"'{section_name}' configuration must be a dictionary " - "mapping names to config" - ) - - for item_name, item_cfg in section_cfg.items(): - if not isinstance(item_cfg, dict): - raise ValueError( - f"'{section_name}.{item_name}' configuration must be a dictionary" - ) - - return section_cfg + return _validate_named_config_map( + section_name, + self.config.get(section_name, {}), + ) @staticmethod def _normalize_dict(value: Any, field_name: str) -> dict[str, Any]: diff --git a/sieval/infer/recipes/gpt_oss.yaml b/sieval/infer/recipes/gpt_oss.yaml index 6d76882d..4d7aae36 100644 --- a/sieval/infer/recipes/gpt_oss.yaml +++ b/sieval/infer/recipes/gpt_oss.yaml @@ -17,6 +17,12 @@ # hardware[gpu][precision][framework] → perf/memory/parallelism/context # capabilities[model_type][framework] → behavior (parsers, tool choice) # +# `base:` is declared explicitly empty rather than omitted. Resolution returns +# {} for an absent key either way, so the empty block is documentation, not +# behavior: it records that the base checkpoint was considered and needs none +# of these params. A sweep test asserts every entry declares both model types, +# so a new entry cannot skip that decision without someone noticing. +# # Perf flags (max_num_batched_tokens, max_cudagraph_capture_size, # stream_interval, no_enable_prefix_caching) are copied verbatim from # vllm-project/recipes OpenAI/GPT-OSS_Hopper.yaml. diff --git a/sieval/infer/recipes/qwen2_5.yaml b/sieval/infer/recipes/qwen2_5.yaml index 6b878aee..fab73fb5 100644 --- a/sieval/infer/recipes/qwen2_5.yaml +++ b/sieval/infer/recipes/qwen2_5.yaml @@ -8,6 +8,12 @@ # gen→base) and merges it over the hardware layer. Within a layer there is no # inheritance or merge — what you see is what you get. # +# `base:` is declared explicitly empty rather than omitted. Resolution returns +# {} for an absent key either way, so the empty block is documentation, not +# behavior: it records that the base checkpoint was considered and needs none +# of these params. A sweep test asserts every entry declares both model types, +# so a new entry cannot skip that decision without someone noticing. +# # The base layer matters here: Qwen2.5-72B-Base is the comparison target for # the ARC and HellaSwag ppl/clp tasks, and it resolves to the same size bucket # as Qwen2.5-72B-Instruct. diff --git a/sieval/infer/recipes/qwen3.yaml b/sieval/infer/recipes/qwen3.yaml index 80be0e61..12035aea 100644 --- a/sieval/infer/recipes/qwen3.yaml +++ b/sieval/infer/recipes/qwen3.yaml @@ -8,6 +8,12 @@ # gen→base) and merges it over the hardware layer. Within a layer there is no # inheritance or merge — what you see is what you get. # +# `base:` is declared explicitly empty rather than omitted. Resolution returns +# {} for an absent key either way, so the empty block is documentation, not +# behavior: it records that the base checkpoint was considered and needs none +# of these params. A sweep test asserts every entry declares both model types, +# so a new entry cannot skip that decision without someone noticing. +# # Hardware notes: # A100-40G: 40GB HBM2e, Ampere, no native FP8 (bf16 only) # H100-80G: 80GB HBM3, Hopper, native FP8, NVLink 4 (900GB/s) diff --git a/sieval/infer/recipes/registry.py b/sieval/infer/recipes/registry.py index 070b0f50..aab79b29 100644 --- a/sieval/infer/recipes/registry.py +++ b/sieval/infer/recipes/registry.py @@ -51,11 +51,27 @@ def _coerce_param(value: object) -> ParamValue: def capability_model_type(model_type: str | None) -> str: """Map an eval config's model ``type`` onto a recipe capability key. - ``"chat"`` → ``"instruct"``, ``"gen"`` → ``"base"``. Anything else - (including ``None``) falls back to ``"instruct"``, matching the eval - config's own default for an undeclared ``type``. + ``"chat"`` → ``"instruct"``, ``"gen"`` → ``"base"``. ``None`` → ``"instruct"``, + matching the eval config's own default for an undeclared ``type``. + + Anything else raises. The config vocabulary (``chat``/``gen``) and the recipe + vocabulary (``instruct``/``base``) differ, so writing the recipe's word into + a model config is an easy mistake — and ``type: base`` silently defaulting + would select the *opposite* capability layer. The eval session rejects an + unknown ``type`` too, but only once it builds the model, which is after the + engine has been launched; failing here keeps the typo cheap. + + Raises: + ValueError: If ``model_type`` is neither ``"chat"``, ``"gen"`` nor ``None``. """ - return "base" if model_type == "gen" else "instruct" + if model_type is None or model_type == "chat": + return "instruct" + if model_type == "gen": + return "base" + raise ValueError( + f"Unknown model type {model_type!r}; expected 'chat' or 'gen'. " + f"('instruct' / 'base' are recipe capability keys, not config types.)" + ) @dataclass diff --git a/sieval/infer/topology/resolver.py b/sieval/infer/topology/resolver.py index dc3d8268..b6cf3a38 100644 --- a/sieval/infer/topology/resolver.py +++ b/sieval/infer/topology/resolver.py @@ -424,9 +424,13 @@ async def auto_resolve_plan( prec_key = precision_key(identity) profile = resolve_hardware_profile(recipe, gpu.model, prec_key, backend) capabilities = resolve_capability_profile(recipe, model_type, backend) - merged = {**(profile or {}), **capabilities} - if merged: - recipe_params = merged + # Capabilities last, matching `_resolve_recipe_params`; both sites feed + # the same `infer_plans.yaml`, whose key order `--resume` compares + # byte-for-byte. `merge_params` (not a dict literal) so a dash/underscore + # collision across the two layers is reported here rather than silently + # picking a winner. + if profile is not None or capabilities: + recipe_params = merge_params(profile or {}, capabilities) # 4. Build user hints from overrides; non-topology keys become engine params hints: UserHints | None = None diff --git a/tests/unit/cli/infer/test_resolve.py b/tests/unit/cli/infer/test_resolve.py index 58fbb96a..686be09e 100644 --- a/tests/unit/cli/infer/test_resolve.py +++ b/tests/unit/cli/infer/test_resolve.py @@ -824,6 +824,35 @@ async def test_gen_inferred_from_task_omits_parsers(self, tmp_path: Path) -> Non for key in self._CAPABILITY_KEYS: assert key not in params, f"{key} leaked into an inferred-gen model" + @pytest.mark.anyio + async def test_capabilities_merge_after_hardware(self, tmp_path: Path) -> None: + """Capability params must land *after* hardware params in the plan. + + Key order reaches ``infer_plans.yaml``, which ``--resume`` compares + byte-for-byte, so the argument order of the ``merge_params`` call in + ``_resolve_recipe_params`` is part of the on-disk contract rather than + an implementation detail. The golden fixture cannot pin this: it calls + the two layer helpers itself, so it would stay green while the + production merge reordered underneath it. + """ + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg(tmp_path, model_dir, {"type": "chat"}) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + keys = list(plan.assignments[0].engine_params) + hardware_keys = {"dtype", "mem_fraction_static", "context_length"} + capability_keys = {"reasoning_parser", "tool_call_parser"} + assert hardware_keys <= set(keys), keys + assert capability_keys <= set(keys), keys + last_hardware = max(keys.index(k) for k in hardware_keys) + first_capability = min(keys.index(k) for k in capability_keys) + assert last_hardware < first_capability, ( + f"capability params must merge after hardware params; got {keys}" + ) + @pytest.mark.anyio async def test_chat_task_still_gets_parsers(self, tmp_path: Path) -> None: """The inference must not flip instruct models to base.""" diff --git a/tests/unit/cli/leaderboard/test_session.py b/tests/unit/cli/leaderboard/test_session.py index c7a15622..a505c242 100644 --- a/tests/unit/cli/leaderboard/test_session.py +++ b/tests/unit/cli/leaderboard/test_session.py @@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +import yaml from sieval.cli.leaderboard.session import ( _NONMATCH_RUNNER_KEYS, @@ -624,6 +625,24 @@ def test_explicit_type_wins(self): def test_defaults_to_chat_with_no_tasks(self): assert derive_model_type("m", None, {}) == "chat" + def test_rejects_non_mapping_tasks_section(self): + """A list-shaped `tasks:` must not surface as an AttributeError. + + The infer layer reaches this before an EvalSession exists, and full + config validation only runs under `--dry-run`, so this is the first + code to touch the section on a normal `sieval run`. The shapes come + from `yaml.safe_load` rather than a literal because that is how an + untyped config actually reaches the annotated parameter. + """ + tasks_cfg = yaml.safe_load("tasks:\n - arc\n - hellaswag\n")["tasks"] + with pytest.raises(ValueError, match="'tasks' configuration must be"): + derive_model_type("m", None, tasks_cfg) + + def test_rejects_non_mapping_task_entry(self): + tasks_cfg = yaml.safe_load("tasks:\n arc: ARCEasyFewShotPplTask\n")["tasks"] + with pytest.raises(ValueError, match="'tasks.arc' configuration must be"): + derive_model_type("m", None, tasks_cfg) + def test_infers_gen_from_task_without_explicit_type(self): """The case explicit-only reading would miss: no `type:` in config.""" diff --git a/tests/unit/infer/recipes/_capture_golden.py b/tests/unit/infer/recipes/_capture_golden.py index 821b1da1..61625db5 100644 --- a/tests/unit/infer/recipes/_capture_golden.py +++ b/tests/unit/infer/recipes/_capture_golden.py @@ -24,6 +24,11 @@ _OUT = Path(__file__).parent / "golden_recipe_profiles.json" _RECIPES = ("gpt_oss.yaml", "qwen2_5.yaml", "qwen3.yaml") +# Last commit on main still carrying the pre-split ``profiles`` key — this +# branch's base. Not ``origin/main``: once the split merges, that ref no longer +# has the schema this script reads, and the capture would come back empty. +_PRE_SPLIT_REF = "1c15c00c" + def _load_at_ref(ref: str, path: str) -> dict: proc = subprocess.run( @@ -68,7 +73,7 @@ def capture(ref: str) -> list[dict]: if __name__ == "__main__": - ref = sys.argv[1] if len(sys.argv) > 1 else "origin/main" + ref = sys.argv[1] if len(sys.argv) > 1 else _PRE_SPLIT_REF records = capture(ref) if not records: raise SystemExit(f"No pre-split recipes found at {ref!r}") diff --git a/tests/unit/infer/test_recipes.py b/tests/unit/infer/test_recipes.py index 177e5250..66f6f7b0 100644 --- a/tests/unit/infer/test_recipes.py +++ b/tests/unit/infer/test_recipes.py @@ -491,6 +491,21 @@ def test_none_defaults_to_instruct(self) -> None: """Matches the eval config's own default for an undeclared type.""" assert capability_model_type(None) == "instruct" + def test_recipe_vocabulary_is_rejected(self) -> None: + """`type: base` must not silently select the *instruct* layer. + + The config vocabulary is chat/gen and the recipe's is instruct/base, so + writing the recipe's word into a model config is the likely mistake — + and defaulting it would pick the opposite layer of the one intended. + """ + for bad in ("base", "instruct"): + with pytest.raises(ValueError, match="expected 'chat' or 'gen'"): + capability_model_type(bad) + + def test_unknown_type_is_rejected(self) -> None: + with pytest.raises(ValueError, match="Unknown model type"): + capability_model_type("cht") + class TestResolveCapabilityProfile: @pytest.fixture diff --git a/tests/unit/infer/topology/test_resolver.py b/tests/unit/infer/topology/test_resolver.py index 097597ef..64ffdebb 100644 --- a/tests/unit/infer/topology/test_resolver.py +++ b/tests/unit/infer/topology/test_resolver.py @@ -655,6 +655,62 @@ async def test_basic_auto_resolve(self, tmp_path): assert result.plan.checkpoint == str(checkpoint) assert result.plan.validate() == [] + @pytest.mark.anyio + async def test_capabilities_merge_after_hardware(self, tmp_path): + """The hardware layer must precede the capability layer in the plan. + + Same on-disk contract as the CLI path: engine_params key order reaches + ``infer_plans.yaml`` and ``--resume`` compares it byte-for-byte. This is + the only test that sees the order ``auto_resolve_plan`` actually + produces, so a swapped merge here would otherwise go unnoticed. + """ + import json + from unittest.mock import AsyncMock, patch + + from sieval.infer.introspect import GPUInfo + from sieval.infer.topology.resolver import auto_resolve_plan + + checkpoint = tmp_path / "Qwen3-4B" + checkpoint.mkdir() + (checkpoint / "config.json").write_text( + json.dumps( + { + "architectures": ["Qwen3ForCausalLM"], + "model_type": "qwen3", + "vocab_size": 151936, + "hidden_size": 2560, + "num_hidden_layers": 36, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "intermediate_size": 9216, + "torch_dtype": "bfloat16", + } + ) + ) + + gpu = GPUInfo(model="NVIDIA H100-SXM5-80GB", count=1, memory_mib=81920) + with patch( + "sieval.infer.topology.resolver.detect_local_gpu", + new_callable=AsyncMock, + return_value=gpu, + ): + result = await auto_resolve_plan( + str(checkpoint), + backend="sglang", + model_type="instruct", + ) + + keys = list(result.plan.assignments[0].engine_params) + hardware_keys = {"dtype", "mem_fraction_static", "context_length"} + capability_keys = {"reasoning_parser", "tool_call_parser"} + assert hardware_keys <= set(keys), keys + assert capability_keys <= set(keys), keys + last_hardware = max(keys.index(k) for k in hardware_keys) + first_capability = min(keys.index(k) for k in capability_keys) + assert last_hardware < first_capability, ( + f"capability params must merge after hardware params; got {keys}" + ) + @pytest.mark.anyio async def test_no_gpu_raises(self, tmp_path): """auto_resolve_plan with no GPU should raise RuntimeError.""" From 17b91fec8c0a13e0178d2a9c6dfaaa55a541aecd Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 8 Aug 2026 22:04:18 +0800 Subject: [PATCH 4/5] fix(recipes): validate the capability key and name it for its vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the hardware/capability split. `resolve_capability_profile` returned `{}` for any key it did not recognize, and `auto_resolve_plan` exposed that key as a public `model_type` kwarg with no validation. Passing a *config* type — `"chat"`, `"gen"` — or a typo therefore resolved to no capability params at all: silently base-like serving, the exact inverse of what this split exists to do, on the one path an outside caller can reach. The lookup now rejects anything outside `CAPABILITY_MODEL_TYPES`, while still returning `{}` for a valid key a recipe simply does not declare. The recipe-side parameter is renamed `model_type` -> `capability` throughout (`resolve_capability_profile`, `_resolve_recipe_params`, `_resolve_with_recipe`, `_try_auto_resolve_recipe`, `auto_resolve_plan`). One name was carrying two vocabularies one call apart — `chat`/`gen` out of `derive_model_type`, `instruct`/`base` into the recipe layer — which is what made the above look legal at the call site. `capability_model_type` keeps its `model_type` param; it is the translator and takes the config's word by definition. `sieval run`'s path-only leg had no test asserting the translated value: every existing test patches `auto_resolve_plan` wholesale, so dropping the `capability_model_type` call there left all 1101 cli+infer tests green while an instruct model lost its parsers. Three tests now pin it, including the load-bearing case (no `type:` in config, gen task => base). Also, from the same review: * `sieval infer start` gains `--model-type`. It was the one entry point that could not say what it was serving — no config, no task context — so a base checkpoint recorded instruct params in its plan. `auto_resolve_plan` already took the key, so this needed no checkpoint introspection. Rejected in YAML mode rather than silently overriding the config. * The `merge_params` comment in `auto_resolve_plan` claimed it reports a dash/underscore collision *across* the two layers. It does not — collision detection is within a single source; across sources it normalizes and the later source wins. Restated as what it does buy: one canonical key form at both merge sites. Verified: `merge_params({'max-model-len': 1}, {'max_model_len': 2})` returns `{'max_model_len': 2}` without raising. * `RoleAssignment`'s docstring still described `engine_params` as coming from "recipe profiles", the key this PR removed. * `_capture_golden.py` pins the full pre-split SHA rather than an abbreviation, and fails with the ref named instead of writing an empty fixture when it is unreachable. Regenerating from the pinned ref reproduces the committed fixture byte-for-byte. * `docs/guide/infer.md` documents the capability layer and the new flag. Co-Authored-By: Claude Opus 5 (1M context) --- docs/guide/infer.md | 22 +++ sieval/cli/infer/commands.py | 52 +++++-- sieval/cli/infer/recipe.py | 23 +-- sieval/cli/run.py | 2 +- sieval/infer/recipes/registry.py | 30 +++- sieval/infer/topology/models.py | 5 +- sieval/infer/topology/resolver.py | 20 ++- tests/unit/cli/infer/test_commands.py | 148 ++++++++++++++++++++ tests/unit/cli/infer/test_resolve.py | 4 +- tests/unit/cli/test_run.py | 88 ++++++++++++ tests/unit/infer/recipes/_capture_golden.py | 22 ++- tests/unit/infer/test_recipes.py | 23 ++- tests/unit/infer/topology/test_resolver.py | 2 +- 13 files changed, 394 insertions(+), 47 deletions(-) create mode 100644 tests/unit/cli/infer/test_commands.py diff --git a/docs/guide/infer.md b/docs/guide/infer.md index dc946910..de77e69b 100644 --- a/docs/guide/infer.md +++ b/docs/guide/infer.md @@ -29,8 +29,30 @@ sieval infer start /path/to/Qwen3-8B -- --served-model-name my-model # Detach: return immediately without waiting for ready sieval infer start /path/to/Qwen3-8B --detach + +# Serve a base checkpoint: skip the recipe's instruct-only serving params +sieval infer start /path/to/Qwen3-8B-Base --model-type gen ``` +### Model type and the capability layer + +A recipe splits its params into a **hardware** layer (dtype, memory, +parallelism, context) and a **capability** layer (reasoning parser, tool-call +parser, tool choice). The capability layer is selected by model type: `chat` +resolves the instruct params, `gen` resolves none — a base checkpoint has no +chat template and no tool-calling surface for those flags to act on. + +In YAML mode the type comes from the config: the model's `type:` if declared, +otherwise it is inferred from the tasks pointing at that model (a PPL or CLP +task requires `gen`). `sieval run` does the same. Checkpoint mode has no config +and no task context, so `--model-type` declares it there; unset, it defaults to +`chat`. Passing `--model-type` in YAML mode is rejected rather than silently +overriding the config. + +The instruct flags are inert on a base checkpoint — the engine accepts and +ignores them — so this affects what `infer_plans.yaml` records as the params +used, not whether the service starts. + ## YAML Infer Configuration Models with a `path` field (and no `api_base`) or an `infer` section in the YAML config are automatically launched by `sieval run` and stopped after evaluation completes. diff --git a/sieval/cli/infer/commands.py b/sieval/cli/infer/commands.py index cded8df6..c34e9fd9 100644 --- a/sieval/cli/infer/commands.py +++ b/sieval/cli/infer/commands.py @@ -34,6 +34,7 @@ ) from sieval.infer.deployer import DeployError, DeployTimeoutError from sieval.infer.params import merge_params +from sieval.infer.recipes import capability_model_type from sieval.infer.topology.models import ( ResolveResult, RoleAssignment, @@ -196,6 +197,18 @@ def infer_start( ), ), ] = None, + model_type: Annotated[ + str | None, + typer.Option( + "--model-type", + help=( + "'chat' or 'gen', for checkpoint (auto-resolve) mode only. " + "Selects the recipe's capability layer: 'gen' serves a base " + "checkpoint without the instruct parser/tool-choice params. " + "Rejected in YAML mode, which reads the type from the config." + ), + ), + ] = None, ) -> None: """Launch an inference service.\n 1. Auto-resolve (recommended):\n @@ -214,7 +227,17 @@ def infer_start( # Decide mode: YAML file vs checkpoint directory if target_path.suffix in (".yaml", ".yml") and target_path.is_file(): - # YAML mode + # YAML mode. The config already carries the model type (declared, or + # derived from the tasks), so accepting the flag here would let the two + # disagree silently — reject it rather than pick a winner. + if model_type is not None: + raise typer.BadParameter( + "--model-type applies to checkpoint mode only; in YAML mode the " + "model type comes from the config's `type:` or is derived from " + "the tasks using the model.", + param_hint="--model-type", + ) + async def _resolve_yaml() -> ResolvedInferConfig: return await resolve_infer_config(target_path, model) @@ -240,24 +263,25 @@ async def _resolve_yaml() -> ResolvedInferConfig: else: # Auto-resolve mode: target is a checkpoint path. # - # `model_type` is deliberately left at its "instruct" default here, and - # this is the one path that does not follow the eval session. A bare - # checkpoint carries no task context, so the derivation the YAML leg and - # `sieval run` use (tasks → chat/gen) has nothing to read. Serving a base - # checkpoint through this path therefore still resolves the instruct - # capability params — which are inert on it: the parser/tool-choice flags - # are accepted and unused, not a startup failure. They do land in the - # persisted plan, so the params recorded for such a launch overstate what - # the engine actually used. Deriving the type from the checkpoint itself - # (an absent `chat_template` in `tokenizer_config.json` marks a base - # model) would close the gap; it is not done here because introspection - # currently reads only `config.json` and a dead flag does not justify - # widening it. + # This is the one path with no task context, so the derivation the YAML + # leg and `sieval run` use (tasks → chat/gen) has nothing to read. The + # operator declares it instead, via `--model-type`; unset keeps the + # instruct default. Introspection is deliberately not widened to guess + # it (an absent `chat_template` in `tokenizer_config.json` marks a base + # model, but introspection reads only `config.json` today) — an explicit + # flag is both cheaper and unambiguous. Left unset on a base checkpoint, + # the instruct parser/tool-choice params still resolve; they are inert + # on it — accepted and unused, not a startup failure — but they do land + # in the persisted plan, so the recorded params overstate what the + # engine applied. + capability = capability_model_type(model_type) + async def _resolve() -> ResolveResult: return await auto_resolve_plan( target, backend=backend, overrides=engine_overrides or None, + capability=capability, ) resolve_result = anyio.run(_resolve) diff --git a/sieval/cli/infer/recipe.py b/sieval/cli/infer/recipe.py index f237d4b1..08e7ec8a 100644 --- a/sieval/cli/infer/recipe.py +++ b/sieval/cli/infer/recipe.py @@ -114,7 +114,7 @@ async def resolve_infer_config( # Which capability layer to serve. Derived from the same config the eval # session reads, so a base checkpoint gets base capabilities even when the # config leaves `type` to task inference (the normal case). - model_type = capability_model_type( + capability = capability_model_type( derive_model_type(model_name, mcfg.get("type"), cfg.get("tasks") or {}) ) @@ -130,7 +130,7 @@ async def resolve_infer_config( checkpoint, backend_name, overrides, - model_type, + capability, ) elif checkpoint: # Case 2: no recipe, but checkpoint available → try auto-resolve @@ -139,7 +139,7 @@ async def resolve_infer_config( backend_name=backend_name, overrides=overrides, model_name=model_name, - model_type=model_type, + capability=capability, ) elif overrides: # Case 3: no recipe, no checkpoint, but overrides → use as-is @@ -244,7 +244,7 @@ async def _resolve_recipe_params( recipe: Recipe, backend_name: str, overrides: dict[str, ParamValue], - model_type: str, + capability: str, ) -> dict[str, ParamValue]: """Resolve engine params for a recipe. @@ -255,9 +255,10 @@ async def _resolve_recipe_params( eliminate duplication. Args: - model_type: Recipe capability key (``"instruct"`` / ``"base"``), which + capability: Recipe capability key (``"instruct"`` / ``"base"``), which selects the capability layer. A base checkpoint resolves to no - parser or tool-choice params. + parser or tool-choice params. Recipe vocabulary, not a config + ``type:`` — map one with :func:`capability_model_type`. """ # Normalize overrides once up front so the dtype check below and the # final merge operate on a single canonical key form. @@ -283,7 +284,7 @@ async def _resolve_recipe_params( prec_key = precision_key(identity) gpu_model = gpu.model if gpu else None profile = resolve_hardware_profile(recipe, gpu_model, prec_key, backend_name) - capabilities = resolve_capability_profile(recipe, model_type, backend_name) + capabilities = resolve_capability_profile(recipe, capability, backend_name) if profile is None and identity.dtype and "dtype" not in overrides: # No hardware profile → fall back to the model's intrinsic dtype @@ -304,7 +305,7 @@ async def _resolve_with_recipe( checkpoint: str, backend_name: str, overrides: dict[str, ParamValue], - model_type: str, + capability: str, ) -> dict[str, ParamValue] | None: """Merge params for an already-loaded recipe via shared merge logic. @@ -330,7 +331,7 @@ async def _resolve_with_recipe( recipe, backend_name, overrides, - model_type, + capability, ) else: # No identity — can only use overrides @@ -372,7 +373,7 @@ async def _try_auto_resolve_recipe( backend_name: str, overrides: dict[str, ParamValue], model_name: str, - model_type: str, + capability: str, ) -> dict[str, ParamValue] | None: """Attempt to auto-resolve a recipe from checkpoint introspection. @@ -414,7 +415,7 @@ async def _try_auto_resolve_recipe( recipe, backend_name, overrides, - model_type, + capability, ) family_recipes = load_family_recipes(identity.family) diff --git a/sieval/cli/run.py b/sieval/cli/run.py index e967093a..5df47901 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -103,7 +103,7 @@ async def _run_all( checkpoint = model_config["path"] result = await auto_resolve_plan( checkpoint=checkpoint, - model_type=capability_model_type( + capability=capability_model_type( derive_model_type( model_name, model_config.get("type"), diff --git a/sieval/infer/recipes/registry.py b/sieval/infer/recipes/registry.py index aab79b29..6cb84fc2 100644 --- a/sieval/infer/recipes/registry.py +++ b/sieval/infer/recipes/registry.py @@ -399,24 +399,42 @@ def resolve_hardware_profile( def resolve_capability_profile( recipe: Recipe, - model_type: str, + capability: str, framework: str, ) -> dict[str, ParamValue]: - """Look up behavior params for a model type + framework. + """Look up behavior params for a capability layer + framework. + + The parameter is named ``capability``, not ``model_type``, because it takes + the *recipe* vocabulary (``instruct``/``base``) rather than the config's + (``chat``/``gen``). The two coexist one call apart — see + :func:`capability_model_type`, which translates — and an unrecognized key + would otherwise resolve to ``{}``, i.e. silently to base-like serving. Args: recipe: Typed Recipe with a ``capabilities`` field. - model_type: A recipe capability key — ``"instruct"`` or ``"base"``. + capability: A recipe capability key — ``"instruct"`` or ``"base"``. Use :func:`capability_model_type` to map an eval config's ``type`` onto it. framework: Framework name (e.g. ``"vllm"``, ``"sglang"``). Returns: - A shallow copy of the matched params, or ``{}`` when the - ``model_type``/``framework`` entry is absent — the expected case for a + A shallow copy of the matched params, or ``{}`` when the recipe declares + no entry for this ``capability``/``framework`` — the expected case for a base checkpoint, which declares no capability params. + + Raises: + ValueError: If ``capability`` is not one of + :data:`CAPABILITY_MODEL_TYPES`. A key outside that set can only be + a caller bug, and returning ``{}`` for it would withhold the very + params this layer exists to supply. """ - return dict(recipe.capabilities.get(model_type, {}).get(framework, {})) + if capability not in CAPABILITY_MODEL_TYPES: + raise ValueError( + f"Unknown recipe capability {capability!r}; expected one of " + f"{', '.join(CAPABILITY_MODEL_TYPES)}. (Config model types are " + f"'chat' / 'gen' — map them with capability_model_type() first.)" + ) + return dict(recipe.capabilities.get(capability, {}).get(framework, {})) def load_recipe(name: str) -> Recipe: diff --git a/sieval/infer/topology/models.py b/sieval/infer/topology/models.py index 7c83544c..a5eb27dd 100644 --- a/sieval/infer/topology/models.py +++ b/sieval/infer/topology/models.py @@ -116,8 +116,9 @@ class ScalingPolicy: class RoleAssignment: """Complete deployment description for one role. - engine_params carries non-parallel parameters from recipe profiles - (dtype, max_model_len, gpu_memory_utilization, etc.). Each role can + engine_params carries non-parallel parameters merged from a recipe's + hardware and capability layers (dtype, max_model_len, + gpu_memory_utilization, tool_call_parser, etc.). Each role can have different engine_params -- e.g. PD split with different hardware or different memory budgets per role. The translator appends these as CLI flags alongside the parallel topology flags. diff --git a/sieval/infer/topology/resolver.py b/sieval/infer/topology/resolver.py index b6cf3a38..c8e617e6 100644 --- a/sieval/infer/topology/resolver.py +++ b/sieval/infer/topology/resolver.py @@ -375,7 +375,7 @@ async def auto_resolve_plan( *, backend: str = "sglang", overrides: dict[str, ParamValue] | None = None, - model_type: str = "instruct", + capability: str = "instruct", ) -> ResolveResult: """Auto-resolve a DeploymentPlan from a checkpoint path. @@ -383,9 +383,13 @@ async def auto_resolve_plan( ResolveResult instead of InferConfig. Args: - model_type: Recipe capability key (``"instruct"`` / ``"base"``). Base + capability: Recipe capability key (``"instruct"`` / ``"base"``). Base checkpoints resolve without parser or tool-choice params. Defaults to ``"instruct"`` for standalone callers with no task context. + This is the *recipe* vocabulary, not a config ``type:`` — pass a + config type through + :func:`sieval.infer.recipes.capability_model_type` first. An + unrecognized value raises rather than resolving to no capabilities. """ # Normalize keys at the entry: the TOPO_KEYS filter (step 4) and # _overrides_to_hints both look up specific underscore-form keys, so a @@ -423,12 +427,16 @@ async def auto_resolve_plan( if recipe is not None: prec_key = precision_key(identity) profile = resolve_hardware_profile(recipe, gpu.model, prec_key, backend) - capabilities = resolve_capability_profile(recipe, model_type, backend) + capabilities = resolve_capability_profile(recipe, capability, backend) # Capabilities last, matching `_resolve_recipe_params`; both sites feed # the same `infer_plans.yaml`, whose key order `--resume` compares - # byte-for-byte. `merge_params` (not a dict literal) so a dash/underscore - # collision across the two layers is reported here rather than silently - # picking a winner. + # byte-for-byte. `merge_params` rather than a dict literal so both sites + # emit one canonical key form — it normalizes each source, so a + # dash-form recipe key cannot reach the plan through only one of them. + # Note it does *not* flag a dash/underscore clash *across* sources: it + # normalizes and the later source wins. Recipe YAML is underscore-only + # by convention (`sieval/infer/params.py`), and a within-layer clash + # still raises. if profile is not None or capabilities: recipe_params = merge_params(profile or {}, capabilities) diff --git a/tests/unit/cli/infer/test_commands.py b/tests/unit/cli/infer/test_commands.py new file mode 100644 index 00000000..d0ab5b06 --- /dev/null +++ b/tests/unit/cli/infer/test_commands.py @@ -0,0 +1,148 @@ +"""Tests for `sieval infer start`'s capability-layer selection. + +Checkpoint mode is the one entry point with no task context to derive the model +type from, so the operator declares it with ``--model-type``. These tests pin +the translation into the recipe's own vocabulary and the refusal to accept the +flag in YAML mode, where the config already answers the question. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +import yaml +from typer.testing import CliRunner + +from sieval.cli.infer import infer_app +from sieval.infer.topology.models import ( + DeploymentPlan, + DeviceGroup, + ParallelTopology, + RoleAssignment, + WellKnownRole, +) +from sieval.infer.topology.resolver import ResolveResult + +runner = CliRunner() + + +def _fake_plan() -> DeploymentPlan: + return DeploymentPlan( + checkpoint="/ckpt", + backend="sglang", + assignments=( + RoleAssignment( + role=WellKnownRole.FULL, + devices=DeviceGroup(count=1, gpu_model="H100"), + topology=ParallelTopology(tp=1, dp=1, pp=1), + engine_params={"context_length": 32768}, + ), + ), + ) + + +def _write_checkpoint(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + (path / "config.json").write_text( + json.dumps({"architectures": ["Qwen3ForCausalLM"], "model_type": "qwen3"}) + ) + return path + + +class TestInferStartModelType: + """`--model-type` selects the recipe capability layer in checkpoint mode.""" + + def _capability_for(self, tmp_path: Path, args: list[str]) -> str: + """Run `infer start --dry-run` and return the resolved capability.""" + resolve = AsyncMock(return_value=ResolveResult(plan=_fake_plan(), steps=())) + translator = MagicMock() + cmd = MagicMock() + cmd.cli_args = ["sglang", "--model-path", "/ckpt"] + cmd.health_url = "http://localhost:8000/health" + cmd.env = {} + translator.translate.return_value = [cmd] + + with ( + patch("sieval.cli.infer.commands.auto_resolve_plan", new=resolve), + patch( + "sieval.cli.infer.commands.get_translator", + return_value=translator, + ), + patch("sieval.cli.infer.commands.validate_plan", return_value=[]), + ): + result = runner.invoke(infer_app, args) + + assert result.exit_code == 0, result.output + resolve.assert_awaited_once() + call = resolve.await_args + assert call is not None + return call.kwargs["capability"] + + def test_default_is_instruct(self, tmp_path: Path): + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B") + assert ( + self._capability_for(tmp_path, ["start", str(ckpt), "--dry-run"]) + == "instruct" + ) + + def test_gen_selects_the_base_layer(self, tmp_path: Path): + """The point of the flag: serve a base checkpoint without instruct params.""" + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B-Base") + capability = self._capability_for( + tmp_path, + ["start", str(ckpt), "--dry-run", "--model-type", "gen"], + ) + assert capability == "base" + + def test_chat_selects_the_instruct_layer(self, tmp_path: Path): + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B") + capability = self._capability_for( + tmp_path, + ["start", str(ckpt), "--dry-run", "--model-type", "chat"], + ) + assert capability == "instruct" + + @pytest.mark.parametrize("bad", ["base", "instruct", "cht"]) + def test_recipe_vocabulary_and_typos_are_rejected(self, tmp_path: Path, bad: str): + """The flag takes config words; `--model-type base` must not silently pass. + + `base` is the recipe layer's own name for what `gen` selects, so it is + the likely slip — and accepting it as an unknown default would pick the + *instruct* layer, the opposite of what was asked for. + """ + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B") + result = runner.invoke( + infer_app, + ["start", str(ckpt), "--dry-run", "--model-type", bad], + ) + assert result.exit_code != 0 + assert "expected 'chat' or 'gen'" in str(result.exception or result.output) + + def test_rejected_in_yaml_mode(self, tmp_path: Path): + """YAML already carries the type; accepting the flag would let them differ.""" + cfg = tmp_path / "cfg.yaml" + cfg.write_text( + yaml.safe_dump( + { + "models": { + "m": { + "type": "chat", + "infer": {"backend": "sglang", "recipe": "qwen3-4b"}, + } + } + } + ) + ) + + result = runner.invoke( + infer_app, + ["start", str(cfg), "--dry-run", "--model-type", "gen"], + ) + + assert result.exit_code != 0 + # Typer renders the message inside a wrapped Rich panel, so match a + # fragment short enough to survive line breaking. + assert "applies to checkpoint mode" in result.output diff --git a/tests/unit/cli/infer/test_resolve.py b/tests/unit/cli/infer/test_resolve.py index 686be09e..972e5281 100644 --- a/tests/unit/cli/infer/test_resolve.py +++ b/tests/unit/cli/infer/test_resolve.py @@ -693,7 +693,7 @@ async def test_user_override_dash_form_wins_over_profile_underscore( recipe, backend_name="vllm", overrides={"foo-bar": 42}, - model_type="instruct", + capability="instruct", ) assert result == {"foo_bar": 42} @@ -726,7 +726,7 @@ async def test_dash_form_override_is_normalized_when_fallback_engages( recipe, backend_name="vllm", overrides={"max-model-len": 4096}, - model_type="instruct", + capability="instruct", ) # Fallback injects identity.dtype; dash-form override normalizes to underscore. diff --git a/tests/unit/cli/test_run.py b/tests/unit/cli/test_run.py index a9eb6371..83612aaa 100644 --- a/tests/unit/cli/test_run.py +++ b/tests/unit/cli/test_run.py @@ -368,6 +368,94 @@ async def test_path_only_model_inherits_yaml_deterministic(self, tmp_path: Path) assert translated_plans[0].deterministic is True +class TestPathOnlyCapabilityLayer: + """Path-only mode must translate the config model type into the recipe's + capability vocabulary before handing it to ``auto_resolve_plan``. + + The two vocabularies sit one call apart (``chat``/``gen`` in the config, + ``instruct``/``base`` in the recipe). Every other test here patches + ``auto_resolve_plan`` wholesale, so without these assertions dropping the + ``capability_model_type`` translation — passing a raw ``"chat"`` straight + through — leaves the whole suite green while an instruct model silently + loses its parser params. + """ + + async def _capability_for(self, tmp_path: Path, config: dict) -> str: + from sieval.cli.run import _run_all + + config_path = tmp_path / "cfg.yaml" + config_path.write_text(yaml.safe_dump(config)) + + resolve = AsyncMock(return_value=ResolveResult(plan=_fake_plan(), steps=())) + mock_translator = MagicMock() + mock_translator.translate.side_effect = _make_translate_capture()[1] + mock_handle = MagicMock() + mock_handle.endpoint = "http://localhost:8000/v1" + + with ( + patch("sieval.cli.run.auto_resolve_plan", new=resolve), + patch("sieval.cli.run.get_translator", return_value=mock_translator), + patch( + "sieval.cli.run.launch_model", + new=AsyncMock(return_value=([mock_handle], None)), + ), + patch("sieval.cli.run.cleanup_model", new=AsyncMock()), + patch("sieval.infer.topology.validator.validate_plan", return_value=[]), + patch( + "sieval.cli.leaderboard.session.arun_session", + new=AsyncMock(return_value={}), + ), + ): + await _run_all(config_path=config_path, verbose=False, resume=False) + + resolve.assert_awaited_once() + call = resolve.await_args + assert call is not None + return call.kwargs["capability"] + + @pytest.mark.anyio + async def test_chat_config_resolves_instruct_layer(self, tmp_path: Path): + capability = await self._capability_for( + tmp_path, + { + "models": {"model_a": {"path": "/tmp/ckpt", "type": "chat"}}, + "result_dir": str(tmp_path / "out"), + "tasks": {}, + }, + ) + assert capability == "instruct" + + @pytest.mark.anyio + async def test_gen_config_resolves_base_layer(self, tmp_path: Path): + capability = await self._capability_for( + tmp_path, + { + "models": {"model_a": {"path": "/tmp/ckpt", "type": "gen"}}, + "result_dir": str(tmp_path / "out"), + "tasks": {}, + }, + ) + assert capability == "base" + + @pytest.mark.anyio + async def test_undeclared_type_is_derived_from_the_tasks(self, tmp_path: Path): + """The load-bearing case: no `type:` in the config, a gen task on it.""" + capability = await self._capability_for( + tmp_path, + { + "models": {"model_a": {"path": "/tmp/ckpt"}}, + "result_dir": str(tmp_path / "out"), + "tasks": { + "arc_ppl": { + "model": "model_a", + "class": "ARCEasyFewShotPplTask", + }, + }, + }, + ) + assert capability == "base" + + class TestDeterministicPassedToSession: """`_run_all` forwards the raw CLI ``deterministic`` value to ``arun_session``; EvalSession computes the monotone OR with YAML diff --git a/tests/unit/infer/recipes/_capture_golden.py b/tests/unit/infer/recipes/_capture_golden.py index 61625db5..4cc2448a 100644 --- a/tests/unit/infer/recipes/_capture_golden.py +++ b/tests/unit/infer/recipes/_capture_golden.py @@ -9,6 +9,14 @@ pre-split schema no longer exists in the code: pre-split ``resolve_profile`` returned the YAML leaf dict verbatim, so the leaf order *is* the resolved order. +The committed fixture — not this script — is the durable artifact. What it pins +is a historical fact that cannot change, so nothing in the test suite needs to +regenerate it; the script exists to show where those numbers came from and to +let a reviewer reproduce them. That also bounds the cost of the git dependency: +if ``_PRE_SPLIT_REF`` ever becomes unreachable (a shallow clone, or a re-cut +history), this script stops working while the tests keep passing. It fails with +the ref named rather than writing an empty fixture. + Usage: pdm run python tests/unit/infer/recipes/_capture_golden.py [git-ref] AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) @@ -27,7 +35,9 @@ # Last commit on main still carrying the pre-split ``profiles`` key — this # branch's base. Not ``origin/main``: once the split merges, that ref no longer # has the schema this script reads, and the capture would come back empty. -_PRE_SPLIT_REF = "1c15c00c" +# Full SHA, not abbreviated: an abbreviation is only unambiguous against the +# history that existed when it was written. +_PRE_SPLIT_REF = "1c15c00c499b640808db03a64f73591ac9733c06" def _load_at_ref(ref: str, path: str) -> dict: @@ -35,8 +45,16 @@ def _load_at_ref(ref: str, path: str) -> dict: ["git", "show", f"{ref}:{path}"], capture_output=True, text=True, - check=True, ) + if proc.returncode != 0: + raise SystemExit( + f"Cannot read {path!r} at {ref!r}: {proc.stderr.strip()}\n" + "The pre-split ref is unreachable here (shallow clone, or the " + "history no longer contains it). The committed fixture remains " + "valid — it pins a fact that cannot change — so the test suite is " + "unaffected; only regeneration is. Pass a reachable ref that still " + "carries the `profiles` schema if you need to re-derive it." + ) return yaml.safe_load(proc.stdout) or {} diff --git a/tests/unit/infer/test_recipes.py b/tests/unit/infer/test_recipes.py index 66f6f7b0..f18adbb9 100644 --- a/tests/unit/infer/test_recipes.py +++ b/tests/unit/infer/test_recipes.py @@ -539,8 +539,27 @@ def test_base_returns_empty(self, sample_recipe: Recipe) -> None: def test_unknown_framework_returns_empty(self, sample_recipe: Recipe) -> None: assert resolve_capability_profile(sample_recipe, "instruct", "tensorrt") == {} - def test_undeclared_model_type_returns_empty(self, sample_recipe: Recipe) -> None: - assert resolve_capability_profile(sample_recipe, "nonexistent", "vllm") == {} + def test_valid_but_undeclared_capability_returns_empty( + self, sample_recipe: Recipe + ) -> None: + """A recipe may omit a layer entirely; that is not a caller error.""" + recipe = _parse_recipe( + "instruct-only", + {"capabilities": {"instruct": {"vllm": {"tool_call_parser": "hermes"}}}}, + ) + assert resolve_capability_profile(recipe, "base", "vllm") == {} + + def test_unknown_capability_rejected(self, sample_recipe: Recipe) -> None: + """A key outside CAPABILITY_MODEL_TYPES is a caller bug, not a base model. + + Returning ``{}`` would make every typo — and every config-vocabulary + word reaching the recipe layer by mistake — resolve to *no* capability + params, i.e. silently to base-like serving. `auto_resolve_plan` takes + this key as a public kwarg, so the mistake is reachable from outside. + """ + for bad in ("nonexistent", "chat", "gen", ""): + with pytest.raises(ValueError, match="Unknown recipe capability"): + resolve_capability_profile(sample_recipe, bad, "vllm") def test_returns_a_copy(self, sample_recipe: Recipe) -> None: """Mutating the result must not corrupt the loaded recipe.""" diff --git a/tests/unit/infer/topology/test_resolver.py b/tests/unit/infer/topology/test_resolver.py index 64ffdebb..e03fec3e 100644 --- a/tests/unit/infer/topology/test_resolver.py +++ b/tests/unit/infer/topology/test_resolver.py @@ -697,7 +697,7 @@ async def test_capabilities_merge_after_hardware(self, tmp_path): result = await auto_resolve_plan( str(checkpoint), backend="sglang", - model_type="instruct", + capability="instruct", ) keys = list(result.plan.assignments[0].engine_params) From bac695a04a73f3f4c5f0e2dbfbec0b8c4c5ca3f8 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 8 Aug 2026 22:50:29 +0800 Subject: [PATCH 5/5] test(infer): assert the CLI failure message wherever it surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--model-type`'s rejection tests read the message out of `result.exception or result.output`, which encodes *how* the CLI currently reports a failure rather than what it reports. An escaping `ValueError` lands in `result.exception` today, so the assertion works — but #85 funnels command failures through `render()`, after which the same message arrives on stdout with `result.exception` set to `SystemExit(1)`, and the `or` short-circuits on the truthy `SystemExit` to give `'1'`. Verified by trial-merging #85 into this branch: 3 of these fail there, purely on the assertion. The behavior after #85 is better, not worse — a structured error result instead of a raw traceback — so the test is what needs fixing. `_failure_text()` joins stdout and the exception repr, so the assertions are about the message and pass under either plumbing. Confirmed both ways: 7 passed on this branch alone, and 1141 passed across tests/unit/cli + tests/unit/infer on the trial merge with #85 (previously 3 failed). This removes the ordering hazard from this side. The two PRs still conflict textually in `sieval/cli/run.py` — both edit the same import block, #59 adding `derive_model_type` and #85 swapping `cli_error_message` for `cli_command` — which whichever lands second resolves by taking both. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/cli/infer/test_commands.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/unit/cli/infer/test_commands.py b/tests/unit/cli/infer/test_commands.py index d0ab5b06..36a42ea3 100644 --- a/tests/unit/cli/infer/test_commands.py +++ b/tests/unit/cli/infer/test_commands.py @@ -29,6 +29,20 @@ runner = CliRunner() +def _failure_text(result) -> str: + """Everywhere a failed command's message can surface, joined. + + A raised exception and rendered stdout are not interchangeable, and which + one carries the message is a property of the CLI's error plumbing, not of + the behavior under test. Today an escaping `ValueError` reaches + `result.exception`; once command failures are funnelled through `render()` + (scitix/sieval#85) the same message arrives on stdout with + `result.exception` set to `SystemExit`. Asserting on both keeps these tests + about the message, so they do not depend on which change lands first. + """ + return f"{result.output}\n{result.exception!r}" + + def _fake_plan() -> DeploymentPlan: return DeploymentPlan( checkpoint="/ckpt", @@ -119,7 +133,7 @@ def test_recipe_vocabulary_and_typos_are_rejected(self, tmp_path: Path, bad: str ["start", str(ckpt), "--dry-run", "--model-type", bad], ) assert result.exit_code != 0 - assert "expected 'chat' or 'gen'" in str(result.exception or result.output) + assert "expected 'chat' or 'gen'" in _failure_text(result) def test_rejected_in_yaml_mode(self, tmp_path: Path): """YAML already carries the type; accepting the flag would let them differ.""" @@ -145,4 +159,4 @@ def test_rejected_in_yaml_mode(self, tmp_path: Path): assert result.exit_code != 0 # Typer renders the message inside a wrapped Rich panel, so match a # fragment short enough to survive line breaking. - assert "applies to checkpoint mode" in result.output + assert "applies to checkpoint mode" in _failure_text(result)