diff --git a/docs/guide/infer.md b/docs/guide/infer.md index dc946910..de77e69b 100644 --- a/docs/guide/infer.md +++ b/docs/guide/infer.md @@ -29,8 +29,30 @@ sieval infer start /path/to/Qwen3-8B -- --served-model-name my-model # Detach: return immediately without waiting for ready sieval infer start /path/to/Qwen3-8B --detach + +# Serve a base checkpoint: skip the recipe's instruct-only serving params +sieval infer start /path/to/Qwen3-8B-Base --model-type gen ``` +### Model type and the capability layer + +A recipe splits its params into a **hardware** layer (dtype, memory, +parallelism, context) and a **capability** layer (reasoning parser, tool-call +parser, tool choice). The capability layer is selected by model type: `chat` +resolves the instruct params, `gen` resolves none — a base checkpoint has no +chat template and no tool-calling surface for those flags to act on. + +In YAML mode the type comes from the config: the model's `type:` if declared, +otherwise it is inferred from the tasks pointing at that model (a PPL or CLP +task requires `gen`). `sieval run` does the same. Checkpoint mode has no config +and no task context, so `--model-type` declares it there; unset, it defaults to +`chat`. Passing `--model-type` in YAML mode is rejected rather than silently +overriding the config. + +The instruct flags are inert on a base checkpoint — the engine accepts and +ignores them — so this affects what `infer_plans.yaml` records as the params +used, not whether the service starts. + ## YAML Infer Configuration Models with a `path` field (and no `api_base`) or an `infer` section in the YAML config are automatically launched by `sieval run` and stopped after evaluation completes. diff --git a/sieval/cli/infer/commands.py b/sieval/cli/infer/commands.py index 69e44317..c34e9fd9 100644 --- a/sieval/cli/infer/commands.py +++ b/sieval/cli/infer/commands.py @@ -34,6 +34,7 @@ ) from sieval.infer.deployer import DeployError, DeployTimeoutError from sieval.infer.params import merge_params +from sieval.infer.recipes import capability_model_type from sieval.infer.topology.models import ( ResolveResult, RoleAssignment, @@ -196,6 +197,18 @@ def infer_start( ), ), ] = None, + model_type: Annotated[ + str | None, + typer.Option( + "--model-type", + help=( + "'chat' or 'gen', for checkpoint (auto-resolve) mode only. " + "Selects the recipe's capability layer: 'gen' serves a base " + "checkpoint without the instruct parser/tool-choice params. " + "Rejected in YAML mode, which reads the type from the config." + ), + ), + ] = None, ) -> None: """Launch an inference service.\n 1. Auto-resolve (recommended):\n @@ -214,7 +227,17 @@ def infer_start( # Decide mode: YAML file vs checkpoint directory if target_path.suffix in (".yaml", ".yml") and target_path.is_file(): - # YAML mode + # YAML mode. The config already carries the model type (declared, or + # derived from the tasks), so accepting the flag here would let the two + # disagree silently — reject it rather than pick a winner. + if model_type is not None: + raise typer.BadParameter( + "--model-type applies to checkpoint mode only; in YAML mode the " + "model type comes from the config's `type:` or is derived from " + "the tasks using the model.", + param_hint="--model-type", + ) + async def _resolve_yaml() -> ResolvedInferConfig: return await resolve_infer_config(target_path, model) @@ -238,12 +261,27 @@ async def _resolve_yaml() -> ResolvedInferConfig: assignments=(new_a,) + plan.assignments[1:], ) else: - # Auto-resolve mode: target is a checkpoint path + # Auto-resolve mode: target is a checkpoint path. + # + # This is the one path with no task context, so the derivation the YAML + # leg and `sieval run` use (tasks → chat/gen) has nothing to read. The + # operator declares it instead, via `--model-type`; unset keeps the + # instruct default. Introspection is deliberately not widened to guess + # it (an absent `chat_template` in `tokenizer_config.json` marks a base + # model, but introspection reads only `config.json` today) — an explicit + # flag is both cheaper and unambiguous. Left unset on a base checkpoint, + # the instruct parser/tool-choice params still resolve; they are inert + # on it — accepted and unused, not a startup failure — but they do land + # in the persisted plan, so the recorded params overstate what the + # engine applied. + capability = capability_model_type(model_type) + async def _resolve() -> ResolveResult: return await auto_resolve_plan( target, backend=backend, overrides=engine_overrides or None, + capability=capability, ) resolve_result = anyio.run(_resolve) diff --git a/sieval/cli/infer/recipe.py b/sieval/cli/infer/recipe.py index 68935130..08e7ec8a 100644 --- a/sieval/cli/infer/recipe.py +++ b/sieval/cli/infer/recipe.py @@ -15,6 +15,7 @@ import yaml from loguru import logger +from sieval.cli.leaderboard.session import derive_model_type from sieval.infer.config import ParamValue from sieval.infer.introspect import ( GPUInfo, @@ -26,11 +27,13 @@ from sieval.infer.params import merge_params from sieval.infer.recipes import ( Recipe, + capability_model_type, list_recipes, load_family_recipes, load_recipe, match_recipe, - resolve_profile, + resolve_capability_profile, + resolve_hardware_profile, ) from sieval.infer.topology.models import ( CP_KEYS, @@ -108,6 +111,13 @@ async def resolve_infer_config( raw_env = infer_dict.get("env") or {} user_env: dict[str, str] = {k: str(v) for k, v in raw_env.items()} + # Which capability layer to serve. Derived from the same config the eval + # session reads, so a base checkpoint gets base capabilities even when the + # config leaves `type` to task inference (the normal case). + capability = capability_model_type( + derive_model_type(model_name, mcfg.get("type"), cfg.get("tasks") or {}) + ) + # Recipe resolution recipe_params: dict[str, ParamValue] | None = None recipe_name = infer_dict.get("recipe") @@ -120,6 +130,7 @@ async def resolve_infer_config( checkpoint, backend_name, overrides, + capability, ) elif checkpoint: # Case 2: no recipe, but checkpoint available → try auto-resolve @@ -128,6 +139,7 @@ async def resolve_infer_config( backend_name=backend_name, overrides=overrides, model_name=model_name, + capability=capability, ) elif overrides: # Case 3: no recipe, no checkpoint, but overrides → use as-is @@ -232,13 +244,21 @@ async def _resolve_recipe_params( recipe: Recipe, backend_name: str, overrides: dict[str, ParamValue], + capability: str, ) -> dict[str, ParamValue]: """Resolve engine params for a recipe. - Pipeline: formula TP/DP → profile → overrides → safety check. + Pipeline: formula TP/DP → hardware profile → capability profile → + overrides → safety check. Extracted from _resolve_with_recipe / _try_auto_resolve_recipe to eliminate duplication. + + Args: + capability: Recipe capability key (``"instruct"`` / ``"base"``), which + selects the capability layer. A base checkpoint resolves to no + parser or tool-choice params. Recipe vocabulary, not a config + ``type:`` — map one with :func:`capability_model_type`. """ # Normalize overrides once up front so the dtype check below and the # final merge operate on a single canonical key form. @@ -259,16 +279,19 @@ async def _resolve_recipe_params( if dp > 1: params[dp_key] = dp - # Profile overrides formula + # Recipe layers override the formula: hardware first, then the capability + # layer for this model type (a base checkpoint contributes nothing). prec_key = precision_key(identity) gpu_model = gpu.model if gpu else None - profile = resolve_profile(recipe, gpu_model, prec_key, backend_name) + profile = resolve_hardware_profile(recipe, gpu_model, prec_key, backend_name) + capabilities = resolve_capability_profile(recipe, capability, backend_name) if profile is None and identity.dtype and "dtype" not in overrides: - # No profile → fallback to model's intrinsic dtype (unless user overrode). + # No hardware profile → fall back to the model's intrinsic dtype + # (unless the user overrode it). params["dtype"] = identity.dtype - params = merge_params(params, profile or {}, overrides) + params = merge_params(params, profile or {}, capabilities, overrides) # Safety check if gpu: @@ -282,6 +305,7 @@ async def _resolve_with_recipe( checkpoint: str, backend_name: str, overrides: dict[str, ParamValue], + capability: str, ) -> dict[str, ParamValue] | None: """Merge params for an already-loaded recipe via shared merge logic. @@ -302,7 +326,13 @@ async def _resolve_with_recipe( ) if identity is not None: - return await _resolve_recipe_params(identity, recipe, backend_name, overrides) + return await _resolve_recipe_params( + identity, + recipe, + backend_name, + overrides, + capability, + ) else: # No identity — can only use overrides return dict(overrides) if overrides else None @@ -343,6 +373,7 @@ async def _try_auto_resolve_recipe( backend_name: str, overrides: dict[str, ParamValue], model_name: str, + capability: str, ) -> dict[str, ParamValue] | None: """Attempt to auto-resolve a recipe from checkpoint introspection. @@ -379,7 +410,13 @@ async def _try_auto_resolve_recipe( if recipe is not None: # Matched — use shared merge logic - params = await _resolve_recipe_params(identity, recipe, backend_name, overrides) + params = await _resolve_recipe_params( + identity, + recipe, + backend_name, + overrides, + capability, + ) family_recipes = load_family_recipes(identity.family) family_names = [r.name for r in family_recipes] diff --git a/sieval/cli/leaderboard/session.py b/sieval/cli/leaderboard/session.py index 72f736ca..19aa2f53 100644 --- a/sieval/cli/leaderboard/session.py +++ b/sieval/cli/leaderboard/session.py @@ -633,6 +633,127 @@ def resolve_task_class(class_spec: str) -> type: ) +def _validate_named_config_map( + section_name: str, + section_cfg: Any, +) -> dict[str, dict[str, Any]]: + """Validate a config section is a ``name -> dict`` mapping, and return it. + + Shared so every entry point that reads a section reports the same error for + the same malformed config. ``derive_model_type`` is now reached from the + infer layer *before* an ``EvalSession`` exists, and full config validation + only runs under ``--dry-run``, so without this a list-shaped ``tasks:`` + surfaced as an ``AttributeError`` from inside recipe resolution. + """ + if not isinstance(section_cfg, dict): + raise ValueError( + f"'{section_name}' configuration must be a dictionary " + "mapping names to config" + ) + + for item_name, item_cfg in section_cfg.items(): + if not isinstance(item_cfg, dict): + raise ValueError( + f"'{section_name}.{item_name}' configuration must be a dictionary" + ) + + return section_cfg + + +def derive_model_type( + model_name: str, + explicit_type: str | None, + tasks_cfg: Mapping[str, Mapping[str, Any]], +) -> str: + """Decide whether a config's model is a ``"chat"`` or ``"gen"`` model. + + Priority: + 1. The config's explicit ``type``. + 2. The ``model_type`` declared by the tasks pointing at this model. + 3. Default to ``"chat"``. + + Both the eval session (which model class to construct) and infer recipe + resolution (which capability layer to serve) must reach the same answer for + the same model, so the derivation lives here rather than being read off + ``type`` twice. Explicit-only would silently mean "instruct" for every + config in the wild, since ``type`` is normally left to this inference. + + It stays in this module, despite the caller now being the infer CLI, because + the inference step *is* task-class resolution — moving it would drag + :func:`resolve_task_class` along or split the two apart. + + Args: + model_name: Name of the model in config. + explicit_type: Explicitly specified type from config, if any. + tasks_cfg: The config's ``tasks`` mapping, used for inference. + + Returns: + Model type: ``"chat"`` or ``"gen"``. + + Raises: + ValueError: If tasks pointing at this model require conflicting types, + or if ``tasks_cfg`` is not a ``name -> dict`` mapping. + """ + # 1. User explicitly specified + if explicit_type is not None: + return explicit_type + + # 2. Infer from tasks + tasks_cfg = _validate_named_config_map("tasks", tasks_cfg) + required_types: set[tuple[str, str]] = set() + + for task_name, task_cfg in tasks_cfg.items(): + if task_cfg.get("model") != model_name: + continue + + # Resolve task class to check its model_type attribute + task_class_spec = task_cfg.get("class") + if not task_class_spec: + continue + + try: + task_class = resolve_task_class(task_class_spec) + task_model_type = getattr(task_class, "model_type", None) + + if task_model_type is not None: + required_types.add((task_name, task_model_type)) + except (ImportError, AttributeError): + # If we can't resolve the task class yet, skip it + # Validation will catch issues later + continue + + # Check for conflicts + unique_types = {t for _, t in required_types} + + if len(unique_types) > 1: + # Conflicting requirements + conflict_info = "\n".join( + f" - {task_name} requires '{model_type}'" + for task_name, model_type in sorted(required_types) + ) + raise ValueError( + f"Model '{model_name}' is used by tasks requiring different types:\n" + f"{conflict_info}\n" + f"Please either:\n" + f" 1. Explicitly specify 'type: chat' or 'type: gen' in model config\n" + f" 2. Use separate models for different types" + ) + + if len(unique_types) == 1: + # All tasks agree on the same type + inferred_type = unique_types.pop() + logger.info( + "Inferred model '{}' type as '{}' from task requirements", + model_name, + inferred_type, + ) + return inferred_type + + # 3. Default to "chat" + logger.info("Using default type 'chat' for model '{}'", model_name) + return "chat" + + def _guess_submodule_names(class_name: str) -> list[str]: """ Guess possible submodule names from a class name. @@ -837,20 +958,10 @@ def _init_runner(self) -> None: def _get_named_config_map(self, section_name: str) -> dict[str, dict[str, Any]]: """Get a config section and validate it is a name -> dict mapping.""" - section_cfg = self.config.get(section_name, {}) - if not isinstance(section_cfg, dict): - raise ValueError( - f"'{section_name}' configuration must be a dictionary " - "mapping names to config" - ) - - for item_name, item_cfg in section_cfg.items(): - if not isinstance(item_cfg, dict): - raise ValueError( - f"'{section_name}.{item_name}' configuration must be a dictionary" - ) - - return section_cfg + return _validate_named_config_map( + section_name, + self.config.get(section_name, {}), + ) @staticmethod def _normalize_dict(value: Any, field_name: str) -> dict[str, Any]: @@ -871,82 +982,12 @@ def _normalize_list(value: Any, field_name: str) -> list[Any]: return value def _infer_model_type(self, model_name: str, explicit_type: str | None) -> str: - """ - Infer the model type based on task requirements. - - Priority: - 1. User explicitly specifies type in config - 2. Infer from tasks that use this model - 3. Default to "chat" - - Args: - model_name: Name of the model in config - explicit_type: Explicitly specified type from config (if any) - - Returns: - Model type: "chat" or "gen" - - Raises: - ValueError: If tasks require conflicting model types - """ - # 1. User explicitly specified - if explicit_type is not None: - return explicit_type - - # 2. Infer from tasks - tasks_cfg = self._get_named_config_map("tasks") - required_types: set[tuple[str, str]] = set() - - for task_name, task_cfg in tasks_cfg.items(): - if task_cfg.get("model") != model_name: - continue - - # Resolve task class to check its model_type attribute - task_class_spec = task_cfg.get("class") - if not task_class_spec: - continue - - try: - task_class = resolve_task_class(task_class_spec) - task_model_type = getattr(task_class, "model_type", None) - - if task_model_type is not None: - required_types.add((task_name, task_model_type)) - except (ImportError, AttributeError): - # If we can't resolve the task class yet, skip it - # Validation will catch issues later - continue - - # Check for conflicts - unique_types = {t for _, t in required_types} - - if len(unique_types) > 1: - # Conflicting requirements - conflict_info = "\n".join( - f" - {task_name} requires '{model_type}'" - for task_name, model_type in sorted(required_types) - ) - raise ValueError( - f"Model '{model_name}' is used by tasks requiring different types:\n" - f"{conflict_info}\n" - f"Please either:\n" - f" 1. Explicitly specify 'type: chat' or 'type: gen' in model config\n" - f" 2. Use separate models for different types" - ) - - if len(unique_types) == 1: - # All tasks agree on the same type - inferred_type = unique_types.pop() - logger.info( - "Inferred model '{}' type as '{}' from task requirements", - model_name, - inferred_type, - ) - return inferred_type - - # 3. Default to "chat" - logger.info("Using default type 'chat' for model '{}'", model_name) - return "chat" + """Infer this session's model type — see :func:`derive_model_type`.""" + return derive_model_type( + model_name, + explicit_type, + self._get_named_config_map("tasks"), + ) def _setup_models(self) -> None: """Initialize all models from config.""" diff --git a/sieval/cli/run.py b/sieval/cli/run.py index 14ce4a44..5df47901 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -17,13 +17,18 @@ from loguru import logger from sieval.cli.infer import cleanup_model, launch_model, resolve_infer_config -from sieval.cli.leaderboard.session import resolve_deterministic, unwrap_proxies +from sieval.cli.leaderboard.session import ( + derive_model_type, + resolve_deterministic, + unwrap_proxies, +) from sieval.cli.output import CommandResult, OutputFormat, cli_command, render from sieval.core.utils.logging import configure_logging, log_user from sieval.infer.backends import get_translator from sieval.infer.backends.translator import inject_user_env from sieval.infer.config import InferHandle from sieval.infer.deployer import LocalDeployer +from sieval.infer.recipes import capability_model_type from sieval.infer.topology.resolver import auto_resolve_plan @@ -92,10 +97,19 @@ async def _run_all( model_name, ) else: - # Path-only mode: auto-resolve from checkpoint + # Path-only mode: auto-resolve from checkpoint. The capability + # layer follows the same model type the eval session will use, + # which is usually inferred from the tasks rather than declared. checkpoint = model_config["path"] result = await auto_resolve_plan( checkpoint=checkpoint, + capability=capability_model_type( + derive_model_type( + model_name, + model_config.get("type"), + config.get("tasks") or {}, + ) + ), ) plan = result.plan user_env = {} # path-only mode has no YAML env section diff --git a/sieval/infer/recipes/__init__.py b/sieval/infer/recipes/__init__.py index 6f1578c6..3af2311c 100644 --- a/sieval/infer/recipes/__init__.py +++ b/sieval/infer/recipes/__init__.py @@ -1,19 +1,25 @@ from sieval.infer.recipes.registry import ( + CAPABILITY_MODEL_TYPES, Recipe, + capability_model_type, check_tested_versions, list_recipes, load_family_recipes, load_recipe, match_recipe, - resolve_profile, + resolve_capability_profile, + resolve_hardware_profile, ) __all__ = [ + "CAPABILITY_MODEL_TYPES", "Recipe", + "capability_model_type", "check_tested_versions", "list_recipes", "load_family_recipes", "load_recipe", "match_recipe", - "resolve_profile", + "resolve_capability_profile", + "resolve_hardware_profile", ] diff --git a/sieval/infer/recipes/gpt_oss.yaml b/sieval/infer/recipes/gpt_oss.yaml index da3c3de3..4d7aae36 100644 --- a/sieval/infer/recipes/gpt_oss.yaml +++ b/sieval/infer/recipes/gpt_oss.yaml @@ -13,9 +13,24 @@ # gpt-oss-120b H100-80G profile assumes TP>=2. Single-card H100-80G OOMs on # defaults; override gpu_memory_utilization=0.95 and max_num_batched_tokens=1024. # +# Structure: +# hardware[gpu][precision][framework] → perf/memory/parallelism/context +# capabilities[model_type][framework] → behavior (parsers, tool choice) +# +# `base:` is declared explicitly empty rather than omitted. Resolution returns +# {} for an absent key either way, so the empty block is documentation, not +# behavior: it records that the base checkpoint was considered and needs none +# of these params. A sweep test asserts every entry declares both model types, +# so a new entry cannot skip that decision without someone noticing. +# # Perf flags (max_num_batched_tokens, max_cudagraph_capture_size, # stream_interval, no_enable_prefix_caching) are copied verbatim from # vllm-project/recipes OpenAI/GPT-OSS_Hopper.yaml. +# +# `no_enable_prefix_caching` is a model-intrinsic recommendation rather than a +# hardware property, so it does not really belong in the hardware layer. It +# stays there for now: prefix caching is being modelled as an engine-level +# constraint (scitix/sieval#47), and moving the flag first would migrate it twice. _family: gpt-oss @@ -26,15 +41,12 @@ gpt-oss-20b: sglang: [">=0.5.2"] known_issues: - "Deterministic inference: use vLLM (gpt-oss batch-invariance fixed in vllm#35404). SGLang's --enable-deterministic-inference is unreliable on gpt-oss (sglang#19431 tracks 120b; no direct 20b repro)." - profiles: + hardware: H100-80G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.90 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -42,16 +54,11 @@ gpt-oss-20b: sglang: context_length: 131072 mem_fraction_static: 0.85 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss H200-141G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.90 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -59,8 +66,18 @@ gpt-oss-20b: sglang: context_length: 131072 mem_fraction_static: 0.85 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss + capabilities: + instruct: + vllm: + reasoning_parser: openai + tool_call_parser: openai + enable_auto_tool_choice: true + sglang: + reasoning_parser: gpt-oss + tool_call_parser: gpt-oss + base: + vllm: {} + sglang: {} gpt-oss-120b: size_range: [100, 150] @@ -69,15 +86,12 @@ gpt-oss-120b: sglang: [">=0.5.2"] known_issues: - "Deterministic inference: use vLLM (gpt-oss batch-invariance fixed in vllm#35404). SGLang's --enable-deterministic-inference diverges at large batch sizes on gpt-oss-120b (sglang#19431, open)." - profiles: + hardware: H100-80G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.85 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -85,16 +99,11 @@ gpt-oss-120b: sglang: context_length: 131072 mem_fraction_static: 0.82 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss H200-141G: mxfp4: vllm: max_model_len: 131072 gpu_memory_utilization: 0.90 - reasoning_parser: openai - tool_call_parser: openai - enable_auto_tool_choice: true max_num_batched_tokens: 8192 max_cudagraph_capture_size: 2048 stream_interval: 20 @@ -102,5 +111,15 @@ gpt-oss-120b: sglang: context_length: 131072 mem_fraction_static: 0.85 - reasoning_parser: gpt-oss - tool_call_parser: gpt-oss + capabilities: + instruct: + vllm: + reasoning_parser: openai + tool_call_parser: openai + enable_auto_tool_choice: true + sglang: + reasoning_parser: gpt-oss + tool_call_parser: gpt-oss + base: + vllm: {} + sglang: {} diff --git a/sieval/infer/recipes/qwen2_5.yaml b/sieval/infer/recipes/qwen2_5.yaml index 52936f00..fab73fb5 100644 --- a/sieval/infer/recipes/qwen2_5.yaml +++ b/sieval/infer/recipes/qwen2_5.yaml @@ -1,7 +1,22 @@ -# Qwen2.5 family recipes — profile-based structure +# Qwen2.5 family recipes — two-layer structure # -# Each profile: profiles[hardware][precision][framework] → complete params -# No inheritance or merge — what you see is what you get. +# hardware[gpu][precision][framework] → perf/memory/parallelism/context +# capabilities[model_type][framework] → behavior (parsers, tool choice) +# +# One entry serves both the instruct and the base checkpoint of a family/size; +# resolution picks the capability layer from the model's `type` (chat→instruct, +# gen→base) and merges it over the hardware layer. Within a layer there is no +# inheritance or merge — what you see is what you get. +# +# `base:` is declared explicitly empty rather than omitted. Resolution returns +# {} for an absent key either way, so the empty block is documentation, not +# behavior: it records that the base checkpoint was considered and needs none +# of these params. A sweep test asserts every entry declares both model types, +# so a new entry cannot skip that decision without someone noticing. +# +# The base layer matters here: Qwen2.5-72B-Base is the comparison target for +# the ARC and HellaSwag ppl/clp tasks, and it resolves to the same size bucket +# as Qwen2.5-72B-Instruct. # # Hardware notes: # A100-40G: 40GB HBM2e, Ampere, no native FP8 (bf16 only) @@ -44,459 +59,424 @@ qwen2.5-0.5b: tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-1.5b: size_range: [1, 2] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-3b: size_range: [2, 5] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-7b: size_range: [5, 10] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-14b: size_range: [10, 20] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-32b: size_range: [20, 40] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen2.5-72b: size_range: [60, 80] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + tool_call_parser: qwen + base: + vllm: {} + sglang: {} diff --git a/sieval/infer/recipes/qwen3.yaml b/sieval/infer/recipes/qwen3.yaml index 8422519c..12035aea 100644 --- a/sieval/infer/recipes/qwen3.yaml +++ b/sieval/infer/recipes/qwen3.yaml @@ -1,7 +1,18 @@ -# Qwen3 family recipes — profile-based structure +# Qwen3 family recipes — two-layer structure # -# Each profile: profiles[hardware][precision][framework] → complete params -# No inheritance or merge — what you see is what you get. +# hardware[gpu][precision][framework] → perf/memory/parallelism/context +# capabilities[model_type][framework] → behavior (parsers, tool choice) +# +# One entry serves both the instruct and the base checkpoint of a family/size; +# resolution picks the capability layer from the model's `type` (chat→instruct, +# gen→base) and merges it over the hardware layer. Within a layer there is no +# inheritance or merge — what you see is what you get. +# +# `base:` is declared explicitly empty rather than omitted. Resolution returns +# {} for an absent key either way, so the empty block is documentation, not +# behavior: it records that the base checkpoint was considered and needs none +# of these params. A sweep test asserts every entry declares both model types, +# so a new entry cannot skip that decision without someone noticing. # # Hardware notes: # A100-40G: 40GB HBM2e, Ampere, no native FP8 (bf16 only) @@ -40,681 +51,564 @@ qwen3-0.6b: tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-1.7b: size_range: [1, 3] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.90 max_model_len: 40960 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.85 context_length: 40960 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-4b: size_range: [3, 6] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-8b: size_range: [6, 12] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-14b: size_range: [12, 20] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-30b-a3b: size_range: [20, 32] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-32b: size_range: [32, 40] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-72b: size_range: [60, 80] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} qwen3-235b-a22b: size_range: [200, 250] tested_versions: vllm: [">=0.9.0"] sglang: [">=0.4.6.post1", "==0.0.0.dev1"] - profiles: + hardware: A100-40G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 8192 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 8192 - reasoning_parser: qwen3 - tool_call_parser: qwen H100-80G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 16384 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 16384 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen H200-141G: bf16: vllm: dtype: bfloat16 gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: dtype: bfloat16 mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen fp8: vllm: gpu_memory_utilization: 0.95 max_model_len: 32768 - reasoning_parser: qwen3 - enable_auto_tool_choice: true - tool_call_parser: hermes sglang: mem_fraction_static: 0.90 context_length: 32768 - reasoning_parser: qwen3 - tool_call_parser: qwen + capabilities: + instruct: + vllm: + reasoning_parser: qwen3 + enable_auto_tool_choice: true + tool_call_parser: hermes + sglang: + reasoning_parser: qwen3 + tool_call_parser: qwen + base: + vllm: {} + sglang: {} diff --git a/sieval/infer/recipes/registry.py b/sieval/infer/recipes/registry.py index 61b6d13c..6cb84fc2 100644 --- a/sieval/infer/recipes/registry.py +++ b/sieval/infer/recipes/registry.py @@ -5,8 +5,14 @@ 1. Family match: model architecture → recipe YAML file (e.g. qwen3 → qwen.yaml) 2. Size match: parameter count → size bucket within that file (e.g. ~4B → qwen3-4b) -Profile resolution uses fuzzy GPU matching to look up a complete, self-contained -parameter set from ``profiles[hardware][precision][framework]``. +A matched recipe carries two independent param layers, resolved separately and +merged by the caller (capabilities last): + +* ``hardware[gpu][precision][framework]`` — perf/memory/parallelism/context, + looked up by fuzzy GPU matching. +* ``capabilities[model_type][framework]`` — behavior (parsers, tool choice), + looked up by model type, so one entry serves a family/size's instruct **and** + base checkpoint without the base one inheriting instruct-only params. AI-Generated Code - Claude Opus 4.6 (Anthropic) """ @@ -38,34 +44,96 @@ def _coerce_param(value: object) -> ParamValue: return str(value) +CAPABILITY_MODEL_TYPES: tuple[str, ...] = ("instruct", "base") +"""Model-type keys a recipe's ``capabilities`` layer may declare.""" + + +def capability_model_type(model_type: str | None) -> str: + """Map an eval config's model ``type`` onto a recipe capability key. + + ``"chat"`` → ``"instruct"``, ``"gen"`` → ``"base"``. ``None`` → ``"instruct"``, + matching the eval config's own default for an undeclared ``type``. + + Anything else raises. The config vocabulary (``chat``/``gen``) and the recipe + vocabulary (``instruct``/``base``) differ, so writing the recipe's word into + a model config is an easy mistake — and ``type: base`` silently defaulting + would select the *opposite* capability layer. The eval session rejects an + unknown ``type`` too, but only once it builds the model, which is after the + engine has been launched; failing here keeps the typo cheap. + + Raises: + ValueError: If ``model_type`` is neither ``"chat"``, ``"gen"`` nor ``None``. + """ + if model_type is None or model_type == "chat": + return "instruct" + if model_type == "gen": + return "base" + raise ValueError( + f"Unknown model type {model_type!r}; expected 'chat' or 'gen'. " + f"('instruct' / 'base' are recipe capability keys, not config types.)" + ) + + @dataclass class Recipe: """Typed representation of a model infer recipe. + A recipe entry serves both the instruct and the base checkpoint of a + family/size, so serving params are split by who owns them: + Fields: name: Recipe name (e.g. "qwen3-8b") size_range: [min_b, max_b) parameter count range in billions - profiles: Per-hardware, per-precision, per-framework params, e.g. - {"H100-80G": {"bf16": {"vllm": {"dtype": "bfloat16", ...}}}} + hardware: Per-hardware, per-precision, per-framework perf/memory + params, e.g. {"H100-80G": {"bf16": {"vllm": {"dtype": ...}}}} + capabilities: Per-model-type, per-framework behavior params (parsers, + tool choice), e.g. {"instruct": {"vllm": {"tool_call_parser": ...}}} known_issues: Human-readable issue descriptions tested_versions: Per-framework version specifiers """ name: str = "" size_range: tuple[float, float] = (0.0, float("inf")) - profiles: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = field( + hardware: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = field( + default_factory=dict, + ) + capabilities: dict[str, dict[str, dict[str, ParamValue]]] = field( default_factory=dict, ) known_issues: list[str] = field(default_factory=list) tested_versions: dict[str, list[str]] = field(default_factory=dict) +def _coerce_params(raw: object) -> dict[str, ParamValue]: + """Coerce a raw YAML mapping into a flat framework param dict.""" + if not isinstance(raw, dict): + return {} + return {str(k): _coerce_param(v) for k, v in raw.items()} + + +def _coerce_framework_map(raw: object) -> dict[str, dict[str, ParamValue]]: + """Coerce a raw YAML mapping of ``framework -> params``.""" + if not isinstance(raw, dict): + return {} + return {str(fw): _coerce_params(params) for fw, params in raw.items()} + + def _parse_recipe(name: str, raw: dict[str, object]) -> Recipe: """Parse a raw YAML dict into a typed Recipe. Old fields (``frameworks``, ``hardware_overrides``, ``precision_overrides``) - are silently ignored if present — only ``profiles`` is parsed. + are silently ignored if present. The pre-split ``profiles`` key is rejected + outright — recipe files ship in-repo, so there is no legacy on-disk state to + support, and silently reading it would hand base checkpoints the parser + params the split exists to withhold. """ + if "profiles" in raw: + raise ValueError( + f"Recipe {name!r} uses the removed 'profiles' key. Split it into " + "'hardware' (hw -> precision -> framework: dtype, memory, " + "parallelism, context) and 'capabilities' (instruct|base -> " + "framework: parsers, tool choice)." + ) # size_range (optional) raw_range = raw.get("size_range") if isinstance(raw_range, list) and len(raw_range) == 2: @@ -77,24 +145,30 @@ def _parse_recipe(name: str, raw: dict[str, object]) -> Recipe: else: size_range = (0.0, float("inf")) - # profiles (optional) — hw_key → prec_key → framework → params - raw_profiles = raw.get("profiles", {}) - profiles: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = {} - if isinstance(raw_profiles, dict): - for hw_key, prec_map in raw_profiles.items(): - if isinstance(prec_map, dict): - prec_dict: dict[str, dict[str, dict[str, ParamValue]]] = {} - for prec_key, fw_map in prec_map.items(): - if isinstance(fw_map, dict): - fw_dict: dict[str, dict[str, ParamValue]] = {} - for fw_name, fw_params in fw_map.items(): - if isinstance(fw_params, dict): - typed: dict[str, ParamValue] = {} - for k, v in fw_params.items(): - typed[str(k)] = _coerce_param(v) - fw_dict[str(fw_name)] = typed - prec_dict[str(prec_key)] = fw_dict - profiles[str(hw_key)] = prec_dict + # hardware (optional) — hw_key → prec_key → framework → params + raw_hardware = raw.get("hardware", {}) + hardware: dict[str, dict[str, dict[str, dict[str, ParamValue]]]] = {} + if isinstance(raw_hardware, dict): + for hw_key, prec_map in raw_hardware.items(): + if not isinstance(prec_map, dict): + continue + hardware[str(hw_key)] = { + str(prec_key): _coerce_framework_map(fw_map) + for prec_key, fw_map in prec_map.items() + } + + # capabilities (optional) — model_type → framework → params + raw_caps = raw.get("capabilities", {}) + capabilities: dict[str, dict[str, dict[str, ParamValue]]] = {} + if isinstance(raw_caps, dict): + for model_type, fw_map in raw_caps.items(): + if str(model_type) not in CAPABILITY_MODEL_TYPES: + raise ValueError( + f"Recipe {name!r} declares unknown capability model type " + f"{model_type!r}; expected one of " + f"{', '.join(CAPABILITY_MODEL_TYPES)}." + ) + capabilities[str(model_type)] = _coerce_framework_map(fw_map) # known_issues (optional) raw_issues = raw.get("known_issues", []) @@ -115,7 +189,8 @@ def _parse_recipe(name: str, raw: dict[str, object]) -> Recipe: return Recipe( name=name, size_range=size_range, - profiles=profiles, + hardware=hardware, + capabilities=capabilities, known_issues=known_issues, tested_versions=tested_versions, ) @@ -254,21 +329,21 @@ def match_recipe(family: str, param_billions: float) -> Recipe | None: return None -def resolve_profile( +def resolve_hardware_profile( recipe: Recipe, gpu_model: str | None, precision: str | None, framework: str, ) -> dict[str, ParamValue] | None: - """Look up a complete parameter set from the recipe's profiles. + """Look up the perf/memory parameter set from the recipe's hardware layer. - Uses fuzzy GPU matching: splits the profile hardware key on ``[-_\\s]+`` - and checks that every resulting token appears in ``gpu_model`` (case- - insensitive). For example, key ``"H100-80G"`` matches GPU string + Uses fuzzy GPU matching: splits the hardware key on ``[-_\\s]+`` and checks + that every resulting token appears in ``gpu_model`` (case-insensitive). + For example, key ``"H100-80G"`` matches GPU string ``"NVIDIA H100-SXM5-80GB"``. Args: - recipe: Typed Recipe with a ``profiles`` field. + recipe: Typed Recipe with a ``hardware`` field. gpu_model: Detected GPU name (e.g. ``"NVIDIA A100-SXM4-80GB"``). ``None`` → return ``None`` immediately. precision: Precision key (e.g. ``"bf16"``, ``"fp8"``). @@ -285,17 +360,17 @@ def resolve_profile( if precision is None: precision = "bf16" - if not recipe.profiles: + if not recipe.hardware: return None gpu_lower = gpu_model.lower() - for hw_key, prec_map in recipe.profiles.items(): + for hw_key, prec_map in recipe.hardware.items(): key_tokens = re.split(r"[-_\s]+", hw_key.lower()) if all(token in gpu_lower for token in key_tokens): - logger.info("GPU {!r} matched profile hardware key {!r}", gpu_model, hw_key) + logger.info("GPU {!r} matched hardware key {!r}", gpu_model, hw_key) if precision not in prec_map: logger.warning( - "Precision {!r} not in profile for hw {!r} (available: {})", + "Precision {!r} not in hardware {!r} (available: {})", precision, hw_key, ", ".join(prec_map), @@ -304,7 +379,7 @@ def resolve_profile( fw_map = prec_map[precision] if framework not in fw_map: logger.info( - "Framework {!r} not in profile {}[{}] (available: {})", + "Framework {!r} not in hardware {}[{}] (available: {})", framework, hw_key, precision, @@ -314,15 +389,54 @@ def resolve_profile( return dict(fw_map[framework]) logger.info( - "GPU {!r} did not match any profile hardware key " - "in recipe {!r} (available: {})", + "GPU {!r} did not match any hardware key in recipe {!r} (available: {})", gpu_model, recipe.name, - ", ".join(recipe.profiles), + ", ".join(recipe.hardware), ) return None +def resolve_capability_profile( + recipe: Recipe, + capability: str, + framework: str, +) -> dict[str, ParamValue]: + """Look up behavior params for a capability layer + framework. + + The parameter is named ``capability``, not ``model_type``, because it takes + the *recipe* vocabulary (``instruct``/``base``) rather than the config's + (``chat``/``gen``). The two coexist one call apart — see + :func:`capability_model_type`, which translates — and an unrecognized key + would otherwise resolve to ``{}``, i.e. silently to base-like serving. + + Args: + recipe: Typed Recipe with a ``capabilities`` field. + capability: A recipe capability key — ``"instruct"`` or ``"base"``. + Use :func:`capability_model_type` to map an eval config's + ``type`` onto it. + framework: Framework name (e.g. ``"vllm"``, ``"sglang"``). + + Returns: + A shallow copy of the matched params, or ``{}`` when the recipe declares + no entry for this ``capability``/``framework`` — the expected case for a + base checkpoint, which declares no capability params. + + Raises: + ValueError: If ``capability`` is not one of + :data:`CAPABILITY_MODEL_TYPES`. A key outside that set can only be + a caller bug, and returning ``{}`` for it would withhold the very + params this layer exists to supply. + """ + if capability not in CAPABILITY_MODEL_TYPES: + raise ValueError( + f"Unknown recipe capability {capability!r}; expected one of " + f"{', '.join(CAPABILITY_MODEL_TYPES)}. (Config model types are " + f"'chat' / 'gen' — map them with capability_model_type() first.)" + ) + return dict(recipe.capabilities.get(capability, {}).get(framework, {})) + + def load_recipe(name: str) -> Recipe: """Load a recipe by exact name from the registry. diff --git a/sieval/infer/topology/models.py b/sieval/infer/topology/models.py index 7c83544c..a5eb27dd 100644 --- a/sieval/infer/topology/models.py +++ b/sieval/infer/topology/models.py @@ -116,8 +116,9 @@ class ScalingPolicy: class RoleAssignment: """Complete deployment description for one role. - engine_params carries non-parallel parameters from recipe profiles - (dtype, max_model_len, gpu_memory_utilization, etc.). Each role can + engine_params carries non-parallel parameters merged from a recipe's + hardware and capability layers (dtype, max_model_len, + gpu_memory_utilization, tool_call_parser, etc.). Each role can have different engine_params -- e.g. PD split with different hardware or different memory budgets per role. The translator appends these as CLI flags alongside the parallel topology flags. diff --git a/sieval/infer/topology/resolver.py b/sieval/infer/topology/resolver.py index ffdc3118..c8e617e6 100644 --- a/sieval/infer/topology/resolver.py +++ b/sieval/infer/topology/resolver.py @@ -20,7 +20,11 @@ introspect_checkpoint_with_config, ) from sieval.infer.params import merge_params -from sieval.infer.recipes import match_recipe, resolve_profile +from sieval.infer.recipes import ( + match_recipe, + resolve_capability_profile, + resolve_hardware_profile, +) from sieval.infer.topology.models import ( TOPO_KEYS, DeploymentPlan, @@ -371,11 +375,21 @@ async def auto_resolve_plan( *, backend: str = "sglang", overrides: dict[str, ParamValue] | None = None, + capability: str = "instruct", ) -> ResolveResult: """Auto-resolve a DeploymentPlan from a checkpoint path. Replaces resolve.auto_resolve(). Same 5-step flow but returns ResolveResult instead of InferConfig. + + Args: + capability: Recipe capability key (``"instruct"`` / ``"base"``). Base + checkpoints resolve without parser or tool-choice params. Defaults + to ``"instruct"`` for standalone callers with no task context. + This is the *recipe* vocabulary, not a config ``type:`` — pass a + config type through + :func:`sieval.infer.recipes.capability_model_type` first. An + unrecognized value raises rather than resolving to no capabilities. """ # Normalize keys at the entry: the TOPO_KEYS filter (step 4) and # _overrides_to_hints both look up specific underscore-form keys, so a @@ -407,14 +421,24 @@ async def auto_resolve_plan( gpu_memory_mib=gpu.memory_mib, ) - # 3. Match recipe → resolve profile → recipe_params + # 3. Match recipe → resolve hardware + capability layers → recipe_params recipe = match_recipe(identity.family, identity.param_billions) recipe_params: dict[str, ParamValue] | None = None if recipe is not None: prec_key = precision_key(identity) - profile = resolve_profile(recipe, gpu.model, prec_key, backend) - if profile is not None: - recipe_params = profile + profile = resolve_hardware_profile(recipe, gpu.model, prec_key, backend) + capabilities = resolve_capability_profile(recipe, capability, backend) + # Capabilities last, matching `_resolve_recipe_params`; both sites feed + # the same `infer_plans.yaml`, whose key order `--resume` compares + # byte-for-byte. `merge_params` rather than a dict literal so both sites + # emit one canonical key form — it normalizes each source, so a + # dash-form recipe key cannot reach the plan through only one of them. + # Note it does *not* flag a dash/underscore clash *across* sources: it + # normalizes and the later source wins. Recipe YAML is underscore-only + # by convention (`sieval/infer/params.py`), and a within-layer clash + # still raises. + if profile is not None or capabilities: + recipe_params = merge_params(profile or {}, capabilities) # 4. Build user hints from overrides; non-topology keys become engine params hints: UserHints | None = None diff --git a/tests/unit/cli/infer/test_commands.py b/tests/unit/cli/infer/test_commands.py new file mode 100644 index 00000000..36a42ea3 --- /dev/null +++ b/tests/unit/cli/infer/test_commands.py @@ -0,0 +1,162 @@ +"""Tests for `sieval infer start`'s capability-layer selection. + +Checkpoint mode is the one entry point with no task context to derive the model +type from, so the operator declares it with ``--model-type``. These tests pin +the translation into the recipe's own vocabulary and the refusal to accept the +flag in YAML mode, where the config already answers the question. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +import yaml +from typer.testing import CliRunner + +from sieval.cli.infer import infer_app +from sieval.infer.topology.models import ( + DeploymentPlan, + DeviceGroup, + ParallelTopology, + RoleAssignment, + WellKnownRole, +) +from sieval.infer.topology.resolver import ResolveResult + +runner = CliRunner() + + +def _failure_text(result) -> str: + """Everywhere a failed command's message can surface, joined. + + A raised exception and rendered stdout are not interchangeable, and which + one carries the message is a property of the CLI's error plumbing, not of + the behavior under test. Today an escaping `ValueError` reaches + `result.exception`; once command failures are funnelled through `render()` + (scitix/sieval#85) the same message arrives on stdout with + `result.exception` set to `SystemExit`. Asserting on both keeps these tests + about the message, so they do not depend on which change lands first. + """ + return f"{result.output}\n{result.exception!r}" + + +def _fake_plan() -> DeploymentPlan: + return DeploymentPlan( + checkpoint="/ckpt", + backend="sglang", + assignments=( + RoleAssignment( + role=WellKnownRole.FULL, + devices=DeviceGroup(count=1, gpu_model="H100"), + topology=ParallelTopology(tp=1, dp=1, pp=1), + engine_params={"context_length": 32768}, + ), + ), + ) + + +def _write_checkpoint(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + (path / "config.json").write_text( + json.dumps({"architectures": ["Qwen3ForCausalLM"], "model_type": "qwen3"}) + ) + return path + + +class TestInferStartModelType: + """`--model-type` selects the recipe capability layer in checkpoint mode.""" + + def _capability_for(self, tmp_path: Path, args: list[str]) -> str: + """Run `infer start --dry-run` and return the resolved capability.""" + resolve = AsyncMock(return_value=ResolveResult(plan=_fake_plan(), steps=())) + translator = MagicMock() + cmd = MagicMock() + cmd.cli_args = ["sglang", "--model-path", "/ckpt"] + cmd.health_url = "http://localhost:8000/health" + cmd.env = {} + translator.translate.return_value = [cmd] + + with ( + patch("sieval.cli.infer.commands.auto_resolve_plan", new=resolve), + patch( + "sieval.cli.infer.commands.get_translator", + return_value=translator, + ), + patch("sieval.cli.infer.commands.validate_plan", return_value=[]), + ): + result = runner.invoke(infer_app, args) + + assert result.exit_code == 0, result.output + resolve.assert_awaited_once() + call = resolve.await_args + assert call is not None + return call.kwargs["capability"] + + def test_default_is_instruct(self, tmp_path: Path): + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B") + assert ( + self._capability_for(tmp_path, ["start", str(ckpt), "--dry-run"]) + == "instruct" + ) + + def test_gen_selects_the_base_layer(self, tmp_path: Path): + """The point of the flag: serve a base checkpoint without instruct params.""" + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B-Base") + capability = self._capability_for( + tmp_path, + ["start", str(ckpt), "--dry-run", "--model-type", "gen"], + ) + assert capability == "base" + + def test_chat_selects_the_instruct_layer(self, tmp_path: Path): + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B") + capability = self._capability_for( + tmp_path, + ["start", str(ckpt), "--dry-run", "--model-type", "chat"], + ) + assert capability == "instruct" + + @pytest.mark.parametrize("bad", ["base", "instruct", "cht"]) + def test_recipe_vocabulary_and_typos_are_rejected(self, tmp_path: Path, bad: str): + """The flag takes config words; `--model-type base` must not silently pass. + + `base` is the recipe layer's own name for what `gen` selects, so it is + the likely slip — and accepting it as an unknown default would pick the + *instruct* layer, the opposite of what was asked for. + """ + ckpt = _write_checkpoint(tmp_path / "Qwen3-4B") + result = runner.invoke( + infer_app, + ["start", str(ckpt), "--dry-run", "--model-type", bad], + ) + assert result.exit_code != 0 + assert "expected 'chat' or 'gen'" in _failure_text(result) + + def test_rejected_in_yaml_mode(self, tmp_path: Path): + """YAML already carries the type; accepting the flag would let them differ.""" + cfg = tmp_path / "cfg.yaml" + cfg.write_text( + yaml.safe_dump( + { + "models": { + "m": { + "type": "chat", + "infer": {"backend": "sglang", "recipe": "qwen3-4b"}, + } + } + } + ) + ) + + result = runner.invoke( + infer_app, + ["start", str(cfg), "--dry-run", "--model-type", "gen"], + ) + + assert result.exit_code != 0 + # Typer renders the message inside a wrapped Rich panel, so match a + # fragment short enough to survive line breaking. + assert "applies to checkpoint mode" in _failure_text(result) diff --git a/tests/unit/cli/infer/test_resolve.py b/tests/unit/cli/infer/test_resolve.py index df976822..972e5281 100644 --- a/tests/unit/cli/infer/test_resolve.py +++ b/tests/unit/cli/infer/test_resolve.py @@ -684,7 +684,7 @@ async def test_user_override_dash_form_wins_over_profile_underscore( new=AsyncMock(return_value=None), ), patch( - "sieval.cli.infer.recipe.resolve_profile", + "sieval.cli.infer.recipe.resolve_hardware_profile", return_value={"foo_bar": 0}, ), ): @@ -693,6 +693,7 @@ async def test_user_override_dash_form_wins_over_profile_underscore( recipe, backend_name="vllm", overrides={"foo-bar": 42}, + capability="instruct", ) assert result == {"foo_bar": 42} @@ -716,7 +717,7 @@ async def test_dash_form_override_is_normalized_when_fallback_engages( new=AsyncMock(return_value=None), ), patch( - "sieval.cli.infer.recipe.resolve_profile", + "sieval.cli.infer.recipe.resolve_hardware_profile", return_value=None, # profile-less → dtype fallback engaged ), ): @@ -725,7 +726,149 @@ async def test_dash_form_override_is_normalized_when_fallback_engages( recipe, backend_name="vllm", overrides={"max-model-len": 4096}, + capability="instruct", ) # Fallback injects identity.dtype; dash-form override normalizes to underscore. assert result == {"dtype": "bfloat16", "max_model_len": 4096} + + +# --------------------------------------------------------------------------- +# Capability layer: which params a model type resolves to +# --------------------------------------------------------------------------- + + +class TestCapabilityLayerResolution: + """`resolve_infer_config` must pick the capability layer from the model type + the eval session will use — which is normally inferred from the tasks, not + declared as `type:` in the config.""" + + _CAPABILITY_KEYS = ("reasoning_parser", "tool_call_parser") + + def _write_cfg( + self, + tmp_path: Path, + model_dir: Path, + model_cfg_extra: dict, + tasks_cfg: dict | None = None, + ) -> Path: + cfg = { + "models": { + "mymodel": { + "infer": { + "backend": "sglang", + "recipe": "qwen3-4b", + "checkpoint": str(model_dir), + }, + **model_cfg_extra, + }, + }, + } + if tasks_cfg is not None: + cfg["tasks"] = tasks_cfg + yaml_path = tmp_path / "config.yaml" + yaml_path.write_text(yaml.dump(cfg)) + return yaml_path + + @pytest.mark.anyio + async def test_instruct_model_gets_parsers(self, tmp_path: Path) -> None: + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg(tmp_path, model_dir, {"type": "chat"}) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + assert params["reasoning_parser"] == "qwen3" + assert params["tool_call_parser"] == "qwen" + + @pytest.mark.anyio + async def test_explicit_gen_type_omits_parsers(self, tmp_path: Path) -> None: + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg(tmp_path, model_dir, {"type": "gen"}) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + for key in self._CAPABILITY_KEYS: + assert key not in params + # The hardware half must survive. + assert params["context_length"] == 32768 + + @pytest.mark.anyio + async def test_gen_inferred_from_task_omits_parsers(self, tmp_path: Path) -> None: + """No `type:` in the config — the base path must still be chosen. + + This is the realistic shape: no config in the repo declares `type:`, so + reading the explicit field alone would resolve every base checkpoint to + instruct capabilities. + """ + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg( + tmp_path, + model_dir, + {}, + tasks_cfg={ + "arc_ppl": {"model": "mymodel", "class": "ARCEasyFewShotPplTask"}, + }, + ) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + for key in self._CAPABILITY_KEYS: + assert key not in params, f"{key} leaked into an inferred-gen model" + + @pytest.mark.anyio + async def test_capabilities_merge_after_hardware(self, tmp_path: Path) -> None: + """Capability params must land *after* hardware params in the plan. + + Key order reaches ``infer_plans.yaml``, which ``--resume`` compares + byte-for-byte, so the argument order of the ``merge_params`` call in + ``_resolve_recipe_params`` is part of the on-disk contract rather than + an implementation detail. The golden fixture cannot pin this: it calls + the two layer helpers itself, so it would stay green while the + production merge reordered underneath it. + """ + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg(tmp_path, model_dir, {"type": "chat"}) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + keys = list(plan.assignments[0].engine_params) + hardware_keys = {"dtype", "mem_fraction_static", "context_length"} + capability_keys = {"reasoning_parser", "tool_call_parser"} + assert hardware_keys <= set(keys), keys + assert capability_keys <= set(keys), keys + last_hardware = max(keys.index(k) for k in hardware_keys) + first_capability = min(keys.index(k) for k in capability_keys) + assert last_hardware < first_capability, ( + f"capability params must merge after hardware params; got {keys}" + ) + + @pytest.mark.anyio + async def test_chat_task_still_gets_parsers(self, tmp_path: Path) -> None: + """The inference must not flip instruct models to base.""" + model_dir = tmp_path / "Qwen3-4B" + _write_qwen3_4b_checkpoint(model_dir) + yaml_path = self._write_cfg( + tmp_path, + model_dir, + {}, + tasks_cfg={ + "aime": {"model": "mymodel", "class": "AIME2024ZeroShotGenTask"}, + }, + ) + + with patch(_GPU_PATCH, new_callable=AsyncMock, return_value=_MOCK_GPU): + _, plan, _env = await resolve_infer_config(yaml_path) + + params = _get_all_params(plan) + assert params["tool_call_parser"] == "qwen" diff --git a/tests/unit/cli/leaderboard/test_session.py b/tests/unit/cli/leaderboard/test_session.py index 1292ee87..a505c242 100644 --- a/tests/unit/cli/leaderboard/test_session.py +++ b/tests/unit/cli/leaderboard/test_session.py @@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +import yaml from sieval.cli.leaderboard.session import ( _NONMATCH_RUNNER_KEYS, @@ -34,6 +35,7 @@ _strip_header, _strip_noncomparable_fields, arun_session, + derive_model_type, load_class_from_name, load_class_from_path, resolve_class, @@ -612,6 +614,69 @@ def mock_resolve(_spec): runner._infer_model_type("m", None) +class TestDeriveModelType: + """`derive_model_type` is shared by the eval session and recipe resolution, + so both reach the same answer for one model. Tested directly because + `sieval run` calls the function, not the session method.""" + + def test_explicit_type_wins(self): + assert derive_model_type("m", "gen", {}) == "gen" + + def test_defaults_to_chat_with_no_tasks(self): + assert derive_model_type("m", None, {}) == "chat" + + def test_rejects_non_mapping_tasks_section(self): + """A list-shaped `tasks:` must not surface as an AttributeError. + + The infer layer reaches this before an EvalSession exists, and full + config validation only runs under `--dry-run`, so this is the first + code to touch the section on a normal `sieval run`. The shapes come + from `yaml.safe_load` rather than a literal because that is how an + untyped config actually reaches the annotated parameter. + """ + tasks_cfg = yaml.safe_load("tasks:\n - arc\n - hellaswag\n")["tasks"] + with pytest.raises(ValueError, match="'tasks' configuration must be"): + derive_model_type("m", None, tasks_cfg) + + def test_rejects_non_mapping_task_entry(self): + tasks_cfg = yaml.safe_load("tasks:\n arc: ARCEasyFewShotPplTask\n")["tasks"] + with pytest.raises(ValueError, match="'tasks.arc' configuration must be"): + derive_model_type("m", None, tasks_cfg) + + def test_infers_gen_from_task_without_explicit_type(self): + """The case explicit-only reading would miss: no `type:` in config.""" + + class FakeTask: + model_type = "gen" + + tasks_cfg = {"t1": {"model": "m", "class": "fake.FakeTask"}} + with patch( + "sieval.cli.leaderboard.session.resolve_task_class", + return_value=FakeTask, + ): + assert derive_model_type("m", None, tasks_cfg) == "gen" + + def test_ignores_tasks_pointing_at_other_models(self): + class FakeTask: + model_type = "gen" + + tasks_cfg = {"t1": {"model": "other", "class": "fake.FakeTask"}} + with patch( + "sieval.cli.leaderboard.session.resolve_task_class", + return_value=FakeTask, + ): + assert derive_model_type("m", None, tasks_cfg) == "chat" + + def test_unresolvable_task_class_is_skipped(self): + """Validation reports import errors; derivation must not raise here.""" + tasks_cfg = {"t1": {"model": "m", "class": "missing.Task"}} + with patch( + "sieval.cli.leaderboard.session.resolve_task_class", + side_effect=ImportError("nope"), + ): + assert derive_model_type("m", None, tasks_cfg) == "chat" + + class TestSetupModelsEngine: """`engine` field dispatches a gen model to GenModel vs SglangGenModel.""" diff --git a/tests/unit/cli/test_run.py b/tests/unit/cli/test_run.py index a9eb6371..83612aaa 100644 --- a/tests/unit/cli/test_run.py +++ b/tests/unit/cli/test_run.py @@ -368,6 +368,94 @@ async def test_path_only_model_inherits_yaml_deterministic(self, tmp_path: Path) assert translated_plans[0].deterministic is True +class TestPathOnlyCapabilityLayer: + """Path-only mode must translate the config model type into the recipe's + capability vocabulary before handing it to ``auto_resolve_plan``. + + The two vocabularies sit one call apart (``chat``/``gen`` in the config, + ``instruct``/``base`` in the recipe). Every other test here patches + ``auto_resolve_plan`` wholesale, so without these assertions dropping the + ``capability_model_type`` translation — passing a raw ``"chat"`` straight + through — leaves the whole suite green while an instruct model silently + loses its parser params. + """ + + async def _capability_for(self, tmp_path: Path, config: dict) -> str: + from sieval.cli.run import _run_all + + config_path = tmp_path / "cfg.yaml" + config_path.write_text(yaml.safe_dump(config)) + + resolve = AsyncMock(return_value=ResolveResult(plan=_fake_plan(), steps=())) + mock_translator = MagicMock() + mock_translator.translate.side_effect = _make_translate_capture()[1] + mock_handle = MagicMock() + mock_handle.endpoint = "http://localhost:8000/v1" + + with ( + patch("sieval.cli.run.auto_resolve_plan", new=resolve), + patch("sieval.cli.run.get_translator", return_value=mock_translator), + patch( + "sieval.cli.run.launch_model", + new=AsyncMock(return_value=([mock_handle], None)), + ), + patch("sieval.cli.run.cleanup_model", new=AsyncMock()), + patch("sieval.infer.topology.validator.validate_plan", return_value=[]), + patch( + "sieval.cli.leaderboard.session.arun_session", + new=AsyncMock(return_value={}), + ), + ): + await _run_all(config_path=config_path, verbose=False, resume=False) + + resolve.assert_awaited_once() + call = resolve.await_args + assert call is not None + return call.kwargs["capability"] + + @pytest.mark.anyio + async def test_chat_config_resolves_instruct_layer(self, tmp_path: Path): + capability = await self._capability_for( + tmp_path, + { + "models": {"model_a": {"path": "/tmp/ckpt", "type": "chat"}}, + "result_dir": str(tmp_path / "out"), + "tasks": {}, + }, + ) + assert capability == "instruct" + + @pytest.mark.anyio + async def test_gen_config_resolves_base_layer(self, tmp_path: Path): + capability = await self._capability_for( + tmp_path, + { + "models": {"model_a": {"path": "/tmp/ckpt", "type": "gen"}}, + "result_dir": str(tmp_path / "out"), + "tasks": {}, + }, + ) + assert capability == "base" + + @pytest.mark.anyio + async def test_undeclared_type_is_derived_from_the_tasks(self, tmp_path: Path): + """The load-bearing case: no `type:` in the config, a gen task on it.""" + capability = await self._capability_for( + tmp_path, + { + "models": {"model_a": {"path": "/tmp/ckpt"}}, + "result_dir": str(tmp_path / "out"), + "tasks": { + "arc_ppl": { + "model": "model_a", + "class": "ARCEasyFewShotPplTask", + }, + }, + }, + ) + assert capability == "base" + + class TestDeterministicPassedToSession: """`_run_all` forwards the raw CLI ``deterministic`` value to ``arun_session``; EvalSession computes the monotone OR with YAML diff --git a/tests/unit/infer/recipes/_capture_golden.py b/tests/unit/infer/recipes/_capture_golden.py new file mode 100644 index 00000000..4cc2448a --- /dev/null +++ b/tests/unit/infer/recipes/_capture_golden.py @@ -0,0 +1,99 @@ +"""Regenerate the recipe-resolution golden fixture from the pre-split recipes. + +The fixture pins what every ``(recipe, hardware, precision, framework)`` triple +resolved to **before** ``profiles`` was split into ``hardware`` + +``capabilities``, with parameter **order** preserved — ``infer_plans.yaml`` is +compared byte-for-byte under ``--resume``, so ordering is part of the contract. + +It is captured from a git ref rather than from the working tree, because the +pre-split schema no longer exists in the code: pre-split ``resolve_profile`` +returned the YAML leaf dict verbatim, so the leaf order *is* the resolved order. + +The committed fixture — not this script — is the durable artifact. What it pins +is a historical fact that cannot change, so nothing in the test suite needs to +regenerate it; the script exists to show where those numbers came from and to +let a reviewer reproduce them. That also bounds the cost of the git dependency: +if ``_PRE_SPLIT_REF`` ever becomes unreachable (a shallow clone, or a re-cut +history), this script stops working while the tests keep passing. It fails with +the ref named rather than writing an empty fixture. + +Usage: pdm run python tests/unit/infer/recipes/_capture_golden.py [git-ref] + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +import subprocess +import sys +from pathlib import Path + +import yaml + +_OUT = Path(__file__).parent / "golden_recipe_profiles.json" +_RECIPES = ("gpt_oss.yaml", "qwen2_5.yaml", "qwen3.yaml") + +# Last commit on main still carrying the pre-split ``profiles`` key — this +# branch's base. Not ``origin/main``: once the split merges, that ref no longer +# has the schema this script reads, and the capture would come back empty. +# Full SHA, not abbreviated: an abbreviation is only unambiguous against the +# history that existed when it was written. +_PRE_SPLIT_REF = "1c15c00c499b640808db03a64f73591ac9733c06" + + +def _load_at_ref(ref: str, path: str) -> dict: + proc = subprocess.run( + ["git", "show", f"{ref}:{path}"], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + raise SystemExit( + f"Cannot read {path!r} at {ref!r}: {proc.stderr.strip()}\n" + "The pre-split ref is unreachable here (shallow clone, or the " + "history no longer contains it). The committed fixture remains " + "valid — it pins a fact that cannot change — so the test suite is " + "unaffected; only regeneration is. Pass a reachable ref that still " + "carries the `profiles` schema if you need to re-derive it." + ) + return yaml.safe_load(proc.stdout) or {} + + +def capture(ref: str) -> list[dict]: + """Record every pre-split profile leaf, preserving parameter order.""" + records: list[dict] = [] + for name in _RECIPES: + data = _load_at_ref(ref, f"sieval/infer/recipes/{name}") + for entry, raw in data.items(): + if entry.startswith("_") or not isinstance(raw, dict): + continue + for hw_key, prec_map in (raw.get("profiles") or {}).items(): + for precision, fw_map in prec_map.items(): + for framework, params in fw_map.items(): + records.append( + { + "recipe": entry, + "hardware_key": hw_key, + "precision": precision, + "framework": framework, + # List of pairs, not a dict: order is asserted. + "params": [[k, v] for k, v in params.items()], + } + ) + records.sort( + key=lambda r: ( + r["recipe"], + r["hardware_key"], + r["precision"], + r["framework"], + ) + ) + return records + + +if __name__ == "__main__": + ref = sys.argv[1] if len(sys.argv) > 1 else _PRE_SPLIT_REF + records = capture(ref) + if not records: + raise SystemExit(f"No pre-split recipes found at {ref!r}") + _OUT.write_text(json.dumps(records, indent=2) + "\n") + print(f"Wrote {len(records)} records from {ref} to {_OUT}") diff --git a/tests/unit/infer/recipes/golden_recipe_profiles.json b/tests/unit/infer/recipes/golden_recipe_profiles.json new file mode 100644 index 00000000..f324c374 --- /dev/null +++ b/tests/unit/infer/recipes/golden_recipe_profiles.json @@ -0,0 +1,4538 @@ +[ + { + "recipe": "gpt-oss-120b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.82 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-120b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.85 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "gpt-oss-120b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-120b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H100-80G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "sglang", + "params": [ + [ + "context_length", + 131072 + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "reasoning_parser", + "gpt-oss" + ], + [ + "tool_call_parser", + "gpt-oss" + ] + ] + }, + { + "recipe": "gpt-oss-20b", + "hardware_key": "H200-141G", + "precision": "mxfp4", + "framework": "vllm", + "params": [ + [ + "max_model_len", + 131072 + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "reasoning_parser", + "openai" + ], + [ + "tool_call_parser", + "openai" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "max_num_batched_tokens", + 8192 + ], + [ + "max_cudagraph_capture_size", + 2048 + ], + [ + "stream_interval", + 20 + ], + [ + "no_enable_prefix_caching", + true + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-0.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-1.5b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen2.5-7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-0.6b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.85 + ], + [ + "context_length", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-1.7b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.9 + ], + [ + "max_model_len", + 40960 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-14b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 8192 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 8192 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-235b-a22b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-30b-a3b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-32b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-4b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 16384 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-72b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "A100-40G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H100-80G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "sglang", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "bf16", + "framework": "vllm", + "params": [ + [ + "dtype", + "bfloat16" + ], + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "sglang", + "params": [ + [ + "mem_fraction_static", + 0.9 + ], + [ + "context_length", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "tool_call_parser", + "qwen" + ] + ] + }, + { + "recipe": "qwen3-8b", + "hardware_key": "H200-141G", + "precision": "fp8", + "framework": "vllm", + "params": [ + [ + "gpu_memory_utilization", + 0.95 + ], + [ + "max_model_len", + 32768 + ], + [ + "reasoning_parser", + "qwen3" + ], + [ + "enable_auto_tool_choice", + true + ], + [ + "tool_call_parser", + "hermes" + ] + ] + } +] diff --git a/tests/unit/infer/recipes/test_golden.py b/tests/unit/infer/recipes/test_golden.py new file mode 100644 index 00000000..b5ab8f41 --- /dev/null +++ b/tests/unit/infer/recipes/test_golden.py @@ -0,0 +1,112 @@ +"""Characterization tests: the capability split must not move instruct output. + +The fixture (``golden_recipe_profiles.json``, regenerated by +``_capture_golden.py``) records what every profile leaf resolved to before +``profiles`` was split into ``hardware`` + ``capabilities``. An instruct model +must still resolve to exactly the same parameters, because a resolved plan +reaches ``infer_plans.yaml`` and ``--resume`` compares that file byte-for-byte. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +from pathlib import Path + +from sieval.infer.params import merge_params +from sieval.infer.recipes import ( + load_recipe, + resolve_capability_profile, + resolve_hardware_profile, +) + +_GOLDEN = Path(__file__).parent / "golden_recipe_profiles.json" + +# Recipes whose capability params sat *between* hardware params in the pre-split +# leaves, so hoisting them into their own layer necessarily moves them to the +# end. Content is unchanged; only key order differs. This is reachable only by +# resuming a pre-split run, which the resume version gate already rejects across +# the minor boundary this ships in — but the exception is pinned here so a +# future reorder cannot slip in unnoticed. +_KNOWN_REORDERED: frozenset[str] = frozenset({"gpt-oss-20b", "gpt-oss-120b"}) + + +def _golden_records() -> list[dict]: + return json.loads(_GOLDEN.read_text()) + + +def _resolve_instruct(record: dict) -> dict: + """Resolve a golden record's triple the way the instruct path now does.""" + recipe = load_recipe(record["recipe"]) + hardware = resolve_hardware_profile( + recipe, + record["hardware_key"], + record["precision"], + record["framework"], + ) + capabilities = resolve_capability_profile(recipe, "instruct", record["framework"]) + return merge_params(hardware or {}, capabilities) + + +def test_golden_fixture_is_populated(): + records = _golden_records() + assert len(records) == 168, "fixture drifted; regenerate with _capture_golden.py" + assert all(r["params"] for r in records) + + +def test_instruct_resolution_preserves_pre_split_content(): + """Every triple resolves to the same parameters as before the split.""" + for record in _golden_records(): + expected = dict(record["params"]) + actual = _resolve_instruct(record) + assert actual == expected, ( + f"{record['recipe']} {record['hardware_key']} " + f"{record['precision']} {record['framework']}" + ) + + +def test_instruct_resolution_preserves_pre_split_order(): + """Key order is preserved except for the recipes documented above.""" + reordered: set[str] = set() + for record in _golden_records(): + expected = [k for k, _ in record["params"]] + actual = list(_resolve_instruct(record)) + if actual != expected: + assert sorted(actual) == sorted(expected), ( + f"{record['recipe']}: order test saw a content diff" + ) + reordered.add(record["recipe"]) + + assert reordered == _KNOWN_REORDERED, ( + "set of reordered recipes changed; a resolved plan's key order moved, " + "which breaks --resume byte matching" + ) + + +def test_base_resolution_omits_capability_params(): + """A base checkpoint inherits hardware params but no instruct behavior.""" + capability_keys = { + "tool_call_parser", + "enable_auto_tool_choice", + "reasoning_parser", + } + checked = 0 + for record in _golden_records(): + recipe = load_recipe(record["recipe"]) + hardware = resolve_hardware_profile( + recipe, + record["hardware_key"], + record["precision"], + record["framework"], + ) + base = merge_params( + hardware or {}, + resolve_capability_profile(recipe, "base", record["framework"]), + ) + assert not (capability_keys & set(base)), ( + f"{record['recipe']} leaked instruct capabilities into the base path" + ) + # The hardware half must still be there — the split withholds behavior, + # not memory/context sizing. + assert base == (hardware or {}) + checked += 1 + assert checked == 168 diff --git a/tests/unit/infer/recipes/test_gpt_oss.py b/tests/unit/infer/recipes/test_gpt_oss.py index 75d967c0..6ff4f23d 100644 --- a/tests/unit/infer/recipes/test_gpt_oss.py +++ b/tests/unit/infer/recipes/test_gpt_oss.py @@ -3,10 +3,29 @@ AI-Generated Code - Claude Opus 4.7 (Anthropic) """ -from sieval.infer.recipes import Recipe, load_recipe, resolve_profile +from sieval.infer.config import ParamValue +from sieval.infer.recipes import ( + Recipe, + load_recipe, + resolve_capability_profile, + resolve_hardware_profile, +) from sieval.infer.recipes.registry import load_family_recipes +def _instruct_params( + recipe: Recipe, + gpu_model: str, + precision: str, + framework: str, +) -> dict[str, ParamValue] | None: + """Compose the two layers the way an instruct model resolves them.""" + hardware = resolve_hardware_profile(recipe, gpu_model, precision, framework) + if hardware is None: + return None + return {**hardware, **resolve_capability_profile(recipe, "instruct", framework)} + + class TestGptOssRecipeShape: def test_family_loads_two_buckets(self) -> None: names = {r.name for r in load_family_recipes("gpt-oss")} @@ -22,22 +41,22 @@ def test_120b_size_range(self) -> None: def test_20b_has_both_hardware_tiers(self) -> None: recipe = load_recipe("gpt-oss-20b") - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware def test_120b_has_both_hardware_tiers(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware def test_only_mxfp4_precision(self) -> None: - h100 = load_recipe("gpt-oss-120b").profiles["H100-80G"] + h100 = load_recipe("gpt-oss-120b").hardware["H100-80G"] assert "mxfp4" in h100 assert "bf16" not in h100 assert "fp8" not in h100 def test_both_frameworks_per_profile(self) -> None: - h100_mxfp4 = load_recipe("gpt-oss-120b").profiles["H100-80G"]["mxfp4"] + h100_mxfp4 = load_recipe("gpt-oss-120b").hardware["H100-80G"]["mxfp4"] assert "vllm" in h100_mxfp4 assert "sglang" in h100_mxfp4 @@ -61,7 +80,7 @@ def test_20b_known_issues(self) -> None: class TestGptOss120bProfiles: def test_h100_vllm_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") assert params is not None assert params["max_model_len"] == 131072 assert params["gpu_memory_utilization"] == 0.85 @@ -75,7 +94,7 @@ def test_h100_vllm_profile(self) -> None: def test_h100_sglang_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") assert params is not None assert params["context_length"] == 131072 assert params["mem_fraction_static"] == 0.82 @@ -84,7 +103,7 @@ def test_h100_sglang_profile(self) -> None: def test_h200_vllm_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H200-141GB", "mxfp4", "vllm") + params = _instruct_params(recipe, "NVIDIA H200-141GB", "mxfp4", "vllm") assert params is not None assert params["gpu_memory_utilization"] == 0.90 assert params["max_num_batched_tokens"] == 8192 @@ -93,7 +112,7 @@ def test_h200_vllm_profile(self) -> None: def test_h200_sglang_profile(self) -> None: recipe = load_recipe("gpt-oss-120b") - params = resolve_profile(recipe, "NVIDIA H200-141GB", "mxfp4", "sglang") + params = _instruct_params(recipe, "NVIDIA H200-141GB", "mxfp4", "sglang") assert params is not None assert params["mem_fraction_static"] == 0.85 @@ -101,7 +120,7 @@ def test_h200_sglang_profile(self) -> None: class TestGptOss20bProfiles: def test_h100_vllm_profile(self) -> None: recipe = load_recipe("gpt-oss-20b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm") assert params is not None assert params["max_model_len"] == 131072 assert params["gpu_memory_utilization"] == 0.90 @@ -109,20 +128,45 @@ def test_h100_vllm_profile(self) -> None: def test_h100_sglang_profile(self) -> None: recipe = load_recipe("gpt-oss-20b") - params = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") + params = _instruct_params(recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "sglang") assert params is not None assert params["mem_fraction_static"] == 0.85 +class TestGptOssBasePath: + """A gpt-oss base checkpoint keeps the perf flags but loses the parsers.""" + + def test_base_omits_parsers_but_keeps_prefix_cache_flag(self) -> None: + recipe = load_recipe("gpt-oss-120b") + hardware = resolve_hardware_profile( + recipe, "NVIDIA H100-SXM5-80GB", "mxfp4", "vllm" + ) + assert hardware is not None + base = {**hardware, **resolve_capability_profile(recipe, "base", "vllm")} + assert "reasoning_parser" not in base + assert "tool_call_parser" not in base + assert "enable_auto_tool_choice" not in base + # Model-intrinsic, currently declared on the hardware layer, so it must + # survive the base path too. + assert base["no_enable_prefix_caching"] is True + assert base["max_model_len"] == 131072 + + class TestGptOssNegativeLookups: def test_bf16_not_defined(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") is None + assert ( + resolve_hardware_profile(recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") + is None + ) def test_a100_not_in_scope(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert resolve_profile(recipe, "NVIDIA A100-SXM4-80GB", "mxfp4", "vllm") is None + assert ( + resolve_hardware_profile(recipe, "NVIDIA A100-SXM4-80GB", "mxfp4", "vllm") + is None + ) def test_b200_not_in_scope(self) -> None: recipe = load_recipe("gpt-oss-120b") - assert resolve_profile(recipe, "NVIDIA B200", "mxfp4", "vllm") is None + assert resolve_hardware_profile(recipe, "NVIDIA B200", "mxfp4", "vllm") is None diff --git a/tests/unit/infer/test_recipes.py b/tests/unit/infer/test_recipes.py index b177f51e..f18adbb9 100644 --- a/tests/unit/infer/test_recipes.py +++ b/tests/unit/infer/test_recipes.py @@ -1,4 +1,4 @@ -"""Tests for sieval.infer.recipes.registry — recipe loading and profile resolution. +"""Tests for sieval.infer.recipes.registry — recipe loading and layer resolution. AI-Generated Code - Claude Opus 4.6 (Anthropic) """ @@ -10,10 +10,12 @@ from sieval.infer.recipes import ( Recipe, + capability_model_type, check_tested_versions, list_recipes, load_recipe, - resolve_profile, + resolve_capability_profile, + resolve_hardware_profile, ) from sieval.infer.recipes.registry import _parse_recipe @@ -28,17 +30,17 @@ def test_list_recipes(self) -> None: class TestLoadRecipe: def test_load_recipe(self) -> None: - """Verify loading qwen3-8b returns expected profile-based structure.""" + """Verify loading qwen3-8b returns the expected two-layer structure.""" recipe = load_recipe("qwen3-8b") assert isinstance(recipe, Recipe) assert recipe.known_issues == [] # Hardware tiers present - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware # H100-80G / bf16 / vllm has correct params - h100_bf16_vllm = recipe.profiles["H100-80G"]["bf16"]["vllm"] + h100_bf16_vllm = recipe.hardware["H100-80G"]["bf16"]["vllm"] assert h100_bf16_vllm["dtype"] == "bfloat16" assert h100_bf16_vllm["gpu_memory_utilization"] == 0.95 assert h100_bf16_vllm["max_model_len"] == 32768 @@ -48,8 +50,8 @@ def test_load_recipe_qwen3_06b(self) -> None: recipe = load_recipe("qwen3-0.6b") assert isinstance(recipe, Recipe) assert recipe.size_range == (0.3, 1.0) - assert "H100-80G" in recipe.profiles - assert "H200-141G" in recipe.profiles + assert "H100-80G" in recipe.hardware + assert "H200-141G" in recipe.hardware def test_load_recipe_not_found(self) -> None: """Verify LookupError for nonexistent recipe.""" @@ -79,7 +81,7 @@ def test_load_recipe_underscore_prefix_rejected(self) -> None: def test_load_recipe_qwen3_235b_a22b_has_fp8_profile(self) -> None: """Verify qwen3-235b-a22b H100 has fp8 with correct max_model_len.""" recipe = load_recipe("qwen3-235b-a22b") - fp8_vllm = recipe.profiles["H100-80G"]["fp8"]["vllm"] + fp8_vllm = recipe.hardware["H100-80G"]["fp8"]["vllm"] assert fp8_vllm["max_model_len"] == 32768 def test_hardware_overrides_removed(self) -> None: @@ -90,13 +92,13 @@ def test_hardware_overrides_removed(self) -> None: assert not hasattr(recipe, "precision_overrides") -class TestParseRecipeProfiles: - """Tests for _parse_recipe with the new profiles structure.""" +class TestParseRecipeLayers: + """Tests for _parse_recipe with the hardware + capabilities structure.""" - def test_parse_profiles_structure(self) -> None: + def test_parse_hardware_structure(self) -> None: """Verify 4-level nested dict parsed correctly.""" raw = { - "profiles": { + "hardware": { "H100-80G": { "bf16": { "vllm": { @@ -126,27 +128,27 @@ def test_parse_profiles_structure(self) -> None: }, } recipe = _parse_recipe("test-model", raw) - assert "H100-80G" in recipe.profiles - assert "A100-80G" in recipe.profiles - assert "bf16" in recipe.profiles["H100-80G"] - assert "vllm" in recipe.profiles["H100-80G"]["bf16"] - assert recipe.profiles["H100-80G"]["bf16"]["vllm"]["dtype"] == "bfloat16" + assert "H100-80G" in recipe.hardware + assert "A100-80G" in recipe.hardware + assert "bf16" in recipe.hardware["H100-80G"] + assert "vllm" in recipe.hardware["H100-80G"]["bf16"] + assert recipe.hardware["H100-80G"]["bf16"]["vllm"]["dtype"] == "bfloat16" assert ( - recipe.profiles["H100-80G"]["bf16"]["vllm"]["gpu_memory_utilization"] + recipe.hardware["H100-80G"]["bf16"]["vllm"]["gpu_memory_utilization"] == 0.95 ) - assert recipe.profiles["A100-80G"]["fp8"]["vllm"]["dtype"] == "fp8" + assert recipe.hardware["A100-80G"]["fp8"]["vllm"]["dtype"] == "fp8" - def test_parse_profiles_empty(self) -> None: - """No profiles → empty dict.""" + def test_parse_hardware_empty(self) -> None: + """No hardware layer → empty dict.""" raw: dict[str, object] = {"size_range": [6, 12]} recipe = _parse_recipe("test-8b", raw) - assert recipe.profiles == {} + assert recipe.hardware == {} - def test_parse_profiles_preserves_other_fields(self) -> None: - """known_issues and tested_versions survive alongside profiles.""" + def test_parse_preserves_other_fields(self) -> None: + """known_issues and tested_versions survive alongside hardware.""" raw = { - "profiles": { + "hardware": { "H100-80G": { "bf16": {"vllm": {"dtype": "bfloat16"}}, }, @@ -159,7 +161,7 @@ def test_parse_profiles_preserves_other_fields(self) -> None: assert recipe.known_issues == ["Some known issue"] assert recipe.tested_versions == {"vllm": [">=0.8.0"]} assert recipe.size_range == (6.0, 12.0) - assert "H100-80G" in recipe.profiles + assert "H100-80G" in recipe.hardware def test_old_fields_ignored(self) -> None: """Old frameworks/hw_overrides/precision_overrides silently ignored.""" @@ -171,7 +173,7 @@ def test_old_fields_ignored(self) -> None: "precision_overrides": { "fp8": {"A100-80G": {"vllm": {"max_model_len": 65536}}}, }, - "profiles": { + "hardware": { "H100-80G": { "bf16": {"vllm": {"dtype": "bfloat16"}}, }, @@ -182,18 +184,18 @@ def test_old_fields_ignored(self) -> None: assert not hasattr(recipe, "frameworks") assert not hasattr(recipe, "hardware_overrides") assert not hasattr(recipe, "precision_overrides") - # Profiles should be parsed normally - assert "H100-80G" in recipe.profiles + # The hardware layer should be parsed normally + assert "H100-80G" in recipe.hardware -class TestResolveProfile: - """Tests for resolve_profile.""" +class TestResolveHardwareProfile: + """Tests for resolve_hardware_profile.""" @pytest.fixture def sample_recipe(self) -> Recipe: return Recipe( name="test-model", - profiles={ + hardware={ "H100-80G": { "bf16": { "vllm": { @@ -219,7 +221,9 @@ def sample_recipe(self) -> Recipe: def test_exact_match(self, sample_recipe: Recipe) -> None: """hw+prec+fw all match → correct params.""" - result = resolve_profile(sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm" + ) assert result is not None assert result["dtype"] == "bfloat16" assert result["gpu_memory_utilization"] == 0.95 @@ -227,7 +231,7 @@ def test_exact_match(self, sample_recipe: Recipe) -> None: def test_fuzzy_gpu_match(self, sample_recipe: Recipe) -> None: """'NVIDIA H100-SXM5-80GB' matches profile key 'H100-80G'.""" - result = resolve_profile( + result = resolve_hardware_profile( sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "sglang" ) assert result is not None @@ -235,36 +239,44 @@ def test_fuzzy_gpu_match(self, sample_recipe: Recipe) -> None: def test_precision_not_found_returns_none(self, sample_recipe: Recipe) -> None: """fp8 on H100 (only bf16 defined) → None.""" - result = resolve_profile(sample_recipe, "NVIDIA H100-SXM5-80GB", "fp8", "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA H100-SXM5-80GB", "fp8", "vllm" + ) assert result is None def test_gpu_not_found_returns_none(self, sample_recipe: Recipe) -> None: """Unknown GPU → None.""" - result = resolve_profile(sample_recipe, "NVIDIA V100-SXM2-32GB", "bf16", "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA V100-SXM2-32GB", "bf16", "vllm" + ) assert result is None def test_gpu_none_returns_none(self, sample_recipe: Recipe) -> None: """gpu_model=None → None.""" - result = resolve_profile(sample_recipe, None, "bf16", "vllm") + result = resolve_hardware_profile(sample_recipe, None, "bf16", "vllm") assert result is None def test_precision_none_defaults_to_bf16(self, sample_recipe: Recipe) -> None: """precision=None → uses 'bf16'.""" - result = resolve_profile(sample_recipe, "NVIDIA H100-SXM5-80GB", None, "vllm") + result = resolve_hardware_profile( + sample_recipe, "NVIDIA H100-SXM5-80GB", None, "vllm" + ) assert result is not None assert result["dtype"] == "bfloat16" def test_framework_not_in_profile_returns_none(self, sample_recipe: Recipe) -> None: """'tensorrt' not in any profile → None.""" - result = resolve_profile( + result = resolve_hardware_profile( sample_recipe, "NVIDIA H100-SXM5-80GB", "bf16", "tensorrt" ) assert result is None - def test_empty_profiles_returns_none(self) -> None: - """Empty profiles → None.""" - recipe = Recipe(name="empty", profiles={}) - result = resolve_profile(recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm") + def test_empty_hardware_returns_none(self) -> None: + """Empty hardware layer → None.""" + recipe = Recipe(name="empty", hardware={}) + result = resolve_hardware_profile( + recipe, "NVIDIA H100-SXM5-80GB", "bf16", "vllm" + ) assert result is None @@ -308,7 +320,7 @@ def test_tested_versions_not_in_profiles(self) -> None: _coerce_param would mangle lists into strings. """ recipe = load_recipe("qwen3-4b") - for _hw_key, prec_map in recipe.profiles.items(): + for _hw_key, prec_map in recipe.hardware.items(): for _prec_key, fw_map in prec_map.items(): for fw_params in fw_map.values(): assert "tested_versions" not in fw_params @@ -399,7 +411,7 @@ def registry_with_issue(tmp_path, monkeypatch): "zz-test-1b": { "size_range": [1.0, 2.0], "known_issues": [_MARKER], - "profiles": { + "hardware": { "H100-80G": { "bf16": { "vllm": {"dtype": "bfloat16", "max_model_len": 4096}, @@ -440,3 +452,161 @@ def test_load_family_recipes_does_not_emit(self, registry_with_issue) -> None: def test_list_recipes_does_not_emit(self, registry_with_issue) -> None: output = _capture_logs(lambda: registry_with_issue.list_recipes()) assert _MARKER not in output + + +class TestLegacyProfilesRejected: + """The pre-split ``profiles`` key must fail loudly, not be read silently.""" + + def test_profiles_key_rejected(self) -> None: + raw = { + "size_range": [6, 12], + "profiles": {"H100-80G": {"bf16": {"vllm": {"dtype": "bfloat16"}}}}, + } + with pytest.raises(ValueError, match="removed 'profiles' key"): + _parse_recipe("test-8b", raw) + + def test_error_names_both_replacement_layers(self) -> None: + """The migration hint must say where the two halves go.""" + with pytest.raises(ValueError) as exc: + _parse_recipe("test-8b", {"profiles": {}}) + assert "hardware" in str(exc.value) + assert "capabilities" in str(exc.value) + + def test_unknown_capability_model_type_rejected(self) -> None: + raw = {"capabilities": {"chat": {"vllm": {"tool_call_parser": "hermes"}}}} + with pytest.raises(ValueError, match="unknown capability model type"): + _parse_recipe("test-8b", raw) + + +class TestCapabilityModelType: + """Eval-config ``type`` → recipe capability key.""" + + def test_gen_maps_to_base(self) -> None: + assert capability_model_type("gen") == "base" + + def test_chat_maps_to_instruct(self) -> None: + assert capability_model_type("chat") == "instruct" + + def test_none_defaults_to_instruct(self) -> None: + """Matches the eval config's own default for an undeclared type.""" + assert capability_model_type(None) == "instruct" + + def test_recipe_vocabulary_is_rejected(self) -> None: + """`type: base` must not silently select the *instruct* layer. + + The config vocabulary is chat/gen and the recipe's is instruct/base, so + writing the recipe's word into a model config is the likely mistake — + and defaulting it would pick the opposite layer of the one intended. + """ + for bad in ("base", "instruct"): + with pytest.raises(ValueError, match="expected 'chat' or 'gen'"): + capability_model_type(bad) + + def test_unknown_type_is_rejected(self) -> None: + with pytest.raises(ValueError, match="Unknown model type"): + capability_model_type("cht") + + +class TestResolveCapabilityProfile: + @pytest.fixture + def sample_recipe(self) -> Recipe: + return _parse_recipe( + "test-model", + { + "hardware": {"H100-80G": {"bf16": {"vllm": {"dtype": "bfloat16"}}}}, + "capabilities": { + "instruct": { + "vllm": { + "enable_auto_tool_choice": True, + "tool_call_parser": "hermes", + }, + "sglang": {"tool_call_parser": "qwen"}, + }, + "base": {"vllm": {}, "sglang": {}}, + }, + }, + ) + + def test_instruct_returns_parsers(self, sample_recipe: Recipe) -> None: + assert resolve_capability_profile(sample_recipe, "instruct", "vllm") == { + "enable_auto_tool_choice": True, + "tool_call_parser": "hermes", + } + + def test_base_returns_empty(self, sample_recipe: Recipe) -> None: + assert resolve_capability_profile(sample_recipe, "base", "vllm") == {} + + def test_unknown_framework_returns_empty(self, sample_recipe: Recipe) -> None: + assert resolve_capability_profile(sample_recipe, "instruct", "tensorrt") == {} + + def test_valid_but_undeclared_capability_returns_empty( + self, sample_recipe: Recipe + ) -> None: + """A recipe may omit a layer entirely; that is not a caller error.""" + recipe = _parse_recipe( + "instruct-only", + {"capabilities": {"instruct": {"vllm": {"tool_call_parser": "hermes"}}}}, + ) + assert resolve_capability_profile(recipe, "base", "vllm") == {} + + def test_unknown_capability_rejected(self, sample_recipe: Recipe) -> None: + """A key outside CAPABILITY_MODEL_TYPES is a caller bug, not a base model. + + Returning ``{}`` would make every typo — and every config-vocabulary + word reaching the recipe layer by mistake — resolve to *no* capability + params, i.e. silently to base-like serving. `auto_resolve_plan` takes + this key as a public kwarg, so the mistake is reachable from outside. + """ + for bad in ("nonexistent", "chat", "gen", ""): + with pytest.raises(ValueError, match="Unknown recipe capability"): + resolve_capability_profile(sample_recipe, bad, "vllm") + + def test_returns_a_copy(self, sample_recipe: Recipe) -> None: + """Mutating the result must not corrupt the loaded recipe.""" + result = resolve_capability_profile(sample_recipe, "instruct", "vllm") + result["tool_call_parser"] = "mutated" + assert sample_recipe.capabilities["instruct"]["vllm"]["tool_call_parser"] == ( + "hermes" + ) + + +class TestShippedRecipeCapabilities: + """The split must actually withhold instruct behavior from base models.""" + + def test_instruct_gets_tool_call_parser(self) -> None: + recipe = load_recipe("qwen2.5-72b") + assert resolve_capability_profile(recipe, "instruct", "vllm") == { + "enable_auto_tool_choice": True, + "tool_call_parser": "hermes", + } + + def test_base_gets_nothing(self) -> None: + """Qwen2.5-72B-Base is the ARC / HellaSwag ppl comparison target.""" + recipe = load_recipe("qwen2.5-72b") + assert resolve_capability_profile(recipe, "base", "vllm") == {} + assert resolve_capability_profile(recipe, "base", "sglang") == {} + + def test_hardware_layer_carries_no_capability_params(self) -> None: + """Parsers must not be left behind in any shipped hardware leaf.""" + capability_keys = { + "tool_call_parser", + "enable_auto_tool_choice", + "reasoning_parser", + } + for name in list_recipes(): + recipe = load_recipe(name) + for hw_key, prec_map in recipe.hardware.items(): + for precision, fw_map in prec_map.items(): + for framework, params in fw_map.items(): + leaked = capability_keys & set(params) + assert not leaked, ( + f"{name} {hw_key}/{precision}/{framework} " + f"still carries {leaked}" + ) + + def test_every_shipped_recipe_declares_both_model_types(self) -> None: + """A missing `base` key would silently fall back to no capabilities, + making an authoring omission indistinguishable from an intentional one.""" + for name in list_recipes(): + recipe = load_recipe(name) + assert set(recipe.capabilities) == {"instruct", "base"}, name diff --git a/tests/unit/infer/topology/test_resolver.py b/tests/unit/infer/topology/test_resolver.py index 097597ef..e03fec3e 100644 --- a/tests/unit/infer/topology/test_resolver.py +++ b/tests/unit/infer/topology/test_resolver.py @@ -655,6 +655,62 @@ async def test_basic_auto_resolve(self, tmp_path): assert result.plan.checkpoint == str(checkpoint) assert result.plan.validate() == [] + @pytest.mark.anyio + async def test_capabilities_merge_after_hardware(self, tmp_path): + """The hardware layer must precede the capability layer in the plan. + + Same on-disk contract as the CLI path: engine_params key order reaches + ``infer_plans.yaml`` and ``--resume`` compares it byte-for-byte. This is + the only test that sees the order ``auto_resolve_plan`` actually + produces, so a swapped merge here would otherwise go unnoticed. + """ + import json + from unittest.mock import AsyncMock, patch + + from sieval.infer.introspect import GPUInfo + from sieval.infer.topology.resolver import auto_resolve_plan + + checkpoint = tmp_path / "Qwen3-4B" + checkpoint.mkdir() + (checkpoint / "config.json").write_text( + json.dumps( + { + "architectures": ["Qwen3ForCausalLM"], + "model_type": "qwen3", + "vocab_size": 151936, + "hidden_size": 2560, + "num_hidden_layers": 36, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "intermediate_size": 9216, + "torch_dtype": "bfloat16", + } + ) + ) + + gpu = GPUInfo(model="NVIDIA H100-SXM5-80GB", count=1, memory_mib=81920) + with patch( + "sieval.infer.topology.resolver.detect_local_gpu", + new_callable=AsyncMock, + return_value=gpu, + ): + result = await auto_resolve_plan( + str(checkpoint), + backend="sglang", + capability="instruct", + ) + + keys = list(result.plan.assignments[0].engine_params) + hardware_keys = {"dtype", "mem_fraction_static", "context_length"} + capability_keys = {"reasoning_parser", "tool_call_parser"} + assert hardware_keys <= set(keys), keys + assert capability_keys <= set(keys), keys + last_hardware = max(keys.index(k) for k in hardware_keys) + first_capability = min(keys.index(k) for k in capability_keys) + assert last_hardware < first_capability, ( + f"capability params must merge after hardware params; got {keys}" + ) + @pytest.mark.anyio async def test_no_gpu_raises(self, tmp_path): """auto_resolve_plan with no GPU should raise RuntimeError."""