From 5821214c71785fcce5b2e46ec03c1f9037292c95 Mon Sep 17 00:00:00 2001 From: Noridom1 Date: Thu, 29 Oct 2026 16:23:49 +0700 Subject: [PATCH 1/2] feat(serving): speedbench scenario family (SPEED-Bench acceptance length) Port of the SPEED-Bench collectors to the serving architecture: one request per row of the fully resolved Green-Lab/speed-bench-prepared files (pinned per config in the dataset lock), first turn only as the aiperf/vLLM loaders send it, golden-matrix sampling defaults, thinking switch via chat_template_kwargs. Shares specdec (spec_decode block), arms: and the summary speedup column with greenspec; the data is public, so greenspec's upload guard is not applied. --- .../single_node/serving/datasets/lock.yaml | 33 ++++ .../serving/scenarios/speedbench.yaml | 31 ++++ .../gemma4-speedbench-qualitative.yaml | 40 +++++ inferencex-e2e/docs/serving/ARCHITECTURE.md | 13 +- inferencex-e2e/infx/serving/client.py | 33 +++- inferencex-e2e/infx/serving/greenspec.py | 1 + inferencex-e2e/infx/serving/scenarios.py | 24 ++- inferencex-e2e/infx/serving/schema.py | 1 + inferencex-e2e/infx/serving/speedbench.py | 170 ++++++++++++++++++ .../infx/tests/serving/test_configs.py | 4 +- .../infx/tests/serving/test_datasets.py | 6 +- .../infx/tests/serving/test_scenarios.py | 24 +++ .../infx/tests/serving/test_speedbench.py | 127 +++++++++++++ 13 files changed, 493 insertions(+), 14 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/serving/scenarios/speedbench.yaml create mode 100644 inferencex-e2e/configs/serving/gemma4-speedbench-qualitative.yaml create mode 100644 inferencex-e2e/infx/serving/speedbench.py create mode 100644 inferencex-e2e/infx/tests/serving/test_speedbench.py diff --git a/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml b/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml index 04e4307bc4..e2b98df150 100644 --- a/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml +++ b/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml @@ -50,3 +50,36 @@ greenspec-v1.1.0: revision: f76978679a941e21eb701e7e046a0038a94efa7b file: greenspec.jsonl sha256: 8c1a8dad3bd1afbe6155e57287e533cb50beaced482074ded4b34c87c7789fe4 +# SPEED-Bench, fully resolved: every row the public release masks is refetched from its source +# (infx.serving.speedbench selects rows from the config's file). The prompts are public data, but the +# resolved copy lives in a private repo: GREENLAB_HF_TOKEN must be able to read it. +speedbench-qualitative: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: qualitative.jsonl + sha256: e9bbbe998472b79501ed79ba2bf126e58b63e153a6deef70ce2b63ea7a4de845 +speedbench-throughput-1k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_1k.jsonl + sha256: 0b5665f61dcbf52dd357d515fd45f9e4c17c68eb902c788c81a4c9271b217377 +speedbench-throughput-2k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_2k.jsonl + sha256: be247367a3ddd3c74b72d03122818d89c6e2e2fe956e6d66099d7ae0c45402a6 +speedbench-throughput-8k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_8k.jsonl + sha256: 94f3ab2d3cd456654ee234faf888c459c5451069d6cfab6e051591b288e71739 +speedbench-throughput-16k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_16k.jsonl + sha256: 77cbee1b4abc6b205ee9ad918bd8e1290a6514f4cf0ff9dc7db497afd1fffca2 +speedbench-throughput-32k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_32k.jsonl + sha256: 04148f06e19c490856dcb9efc1b1fd4593354d4abd4bf07c6f5dc4ce91348c0b diff --git a/inferencex-e2e/benchmarks/single_node/serving/scenarios/speedbench.yaml b/inferencex-e2e/benchmarks/single_node/serving/scenarios/speedbench.yaml new file mode 100644 index 0000000000..6230e4be00 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/serving/scenarios/speedbench.yaml @@ -0,0 +1,31 @@ +# SpeedBench family: SPEED-Bench acceptance-length prompts (nvidia/SPEED-Bench, fully resolved copy +# Green-Lab/speed-bench-prepared; datasets/lock.yaml pins one jsonl per config). +# infx.serving.speedbench selects the rows (overrides: config, category) and builds one request body per row +# (first turn only, temperature/top_p/max_tokens/chat_template_kwargs from overrides) and writes them as the +# raw_payload file IX_CUSTOM_DATASET_FILE points to; AgentX sends each body as is. Every request streams and +# completes on its own (no ignore_eos): acceptance only means something on natural output. `requests` is the +# selection size unless overrides.requests lowers it; the client sets it once it has counted the selection, +# so the 1 below is only a placeholder. The dataset name below is a placeholder too: scenarios.py sets it +# from overrides.config. +schemaVersion: "2.0" +benchmark: + unsafeOverride: true + endpoint: + type: chat + path: /v1/chat/completions + streaming: true + useServerTokenCount: true + datasets: + - name: speedbench-qualitative + type: file + format: raw_payload + path: ${IX_CUSTOM_DATASET_FILE} + sampling: sequential + randomSeed: 42 + phases: + - name: profiling + kind: profiling + type: concurrency + requests: 1 + failedRequestThreshold: 0.1 +randomSeed: 42 diff --git a/inferencex-e2e/configs/serving/gemma4-speedbench-qualitative.yaml b/inferencex-e2e/configs/serving/gemma4-speedbench-qualitative.yaml new file mode 100644 index 0000000000..d162d9fd3c --- /dev/null +++ b/inferencex-e2e/configs/serving/gemma4-speedbench-qualitative.yaml @@ -0,0 +1,40 @@ +# Pattern: SPEED-Bench acceptance length, thinking off, one category per job (speedbench scenario). +# Gemma-4-31B FP8-block, vLLM, 1 GPU, one request in flight (the SPEED-Bench collectors' CONCURRENCY=1), MTP draft +# depth 6 against the base engine in one run (arms:). Sampling is the golden AL matrices' (temperature 1.0, +# top_p 0.95, 4096 output tokens); the thinking switch is chat_template_kwargs. Without `category:` the run +# is all 880 prompts of the qualitative config (11 categories x 80); `config: throughput_8k` selects the +# entropy tiers at 8k input. The run is one request per row of the selection (no duration): the results carry +# spec_decode.acceptance_length. The prompts are public; the resolved copy is read with GREENLAB_HF_TOKEN. +name: gemma4-speedbench-qualitative +runner: cluster:hcm-hgx02 +image: vllm/vllm-openai:v0.30.0 +model: RedHatAI/gemma-4-31B-it-FP8-block +gpus: 1 +precision: fp8block +env: + NCCL_P2P_LEVEL: NVL + VLLM_DISABLE_COMPILE_CACHE: "1" + VLLM_ATTENTION_BACKEND: FLASHINFER +serve: | + vllm serve RedHatAI/gemma-4-31B-it-FP8-block + --served-model-name RedHatAI/gemma-4-31B-it-FP8-block + --trust-remote-code + --tensor-parallel-size 1 + --gpu-memory-utilization 0.92 + --max-model-len 16384 + --max-num-seqs 64 + --no-enable-prefix-caching + --enable-auto-tool-choice + --tool-call-parser gemma4 + --reasoning-parser gemma4 +arms: + base: {} + mtp6: + append: | + --speculative-config '{"method": "mtp", "model": "google/gemma-4-31B-it-assistant", "num_speculative_tokens": 6}' +scenario: speedbench +overrides: + category: coding + chat_template_kwargs: + enable_thinking: false +conc: [1] diff --git a/inferencex-e2e/docs/serving/ARCHITECTURE.md b/inferencex-e2e/docs/serving/ARCHITECTURE.md index f7de365813..741eb51fbf 100644 --- a/inferencex-e2e/docs/serving/ARCHITECTURE.md +++ b/inferencex-e2e/docs/serving/ARCHITECTURE.md @@ -212,7 +212,7 @@ that several nodes need belong in a registry, where every node caches its own sq ## 6. Scenarios -Seven base files in `benchmarks/single_node/serving/scenarios/`: +Eight base files in `benchmarks/single_node/serving/scenarios/`: | Family | Workload | Dataset | |---|---|---| @@ -223,6 +223,7 @@ Seven base files in `benchmarks/single_node/serving/scenarios/`: | `hidden-prefix-e3` | prefixes reused without being visible to the router | pinned file, Mooncake trace format | | `synthetic` | generated ISL/OSL with a shared-prefix share | synthetic | | `greenspec` | speculative-decoding bench cut from production chat traffic, one request per row | pinned private file (`Green-Lab/GreenSpec`), `raw_payload` format | +| `speedbench` | SPEED-Bench acceptance-length prompts, one request per row | pinned files (`Green-Lab/speed-bench-prepared`, a private copy of public data), `raw_payload` format | Pinned files are listed in `benchmarks/single_node/serving/datasets/lock.yaml` with their URL and sha256; the client downloads and checks them before the run. @@ -247,6 +248,16 @@ Three things differ from the other families: the aggregate result (`infx/serving/specdec.py`). A point without speculative decoding has no such block. - The runner needs `GREENLAB_HF_TOKEN` in `actions-runner/.env`: the release is not public. +`speedbench` follows the same pattern (no duration, the `spec_decode` block, `GREENLAB_HF_TOKEN`) with its own +keys: `config` (`qualitative`, default, or `throughput_1k` ... `throughput_32k`), `category` (one of +that config's categories; default all), `max_tokens` (default 4096), `temperature` (1.0), `top_p` (0.95), +`chat_template_kwargs` (the thinking switch, e.g. `enable_thinking: false`) and `requests`. The defaults are the +golden AL matrices'. A request is the first turn of a row alone, as the aiperf and vLLM SPEED-Bench loaders send +it. The result label carries every non-default key (`speedbench-qualitative-coding-ctk-enable_thinking-false`), +so runs that differ in thinking mode never share a summary row. The files are the fully resolved copy of +`nvidia/SPEED-Bench`: the public release masks about 60% of the qualitative rows. The prompts are public, so +none of GreenSpec's guard applies: console, server log and results upload as for any other family. + A base arm and a speculative arm are the `arms:` of one serving YAML (below). The `summary` job of `gn-benchmark.yml` prints one table of all points of a run on the run page, with the speedup of each speculative point over the base point of the same scenario and concurrency. diff --git a/inferencex-e2e/infx/serving/client.py b/inferencex-e2e/infx/serving/client.py index 5d4a2956ab..2aec602267 100644 --- a/inferencex-e2e/infx/serving/client.py +++ b/inferencex-e2e/infx/serving/client.py @@ -20,10 +20,11 @@ import urllib.request from collections.abc import Mapping from pathlib import Path +from typing import Any from infx.bench import proc from infx.bench.agentic.venv import Runtime, bootstrap -from infx.serving import greenspec, specdec +from infx.serving import greenspec, specdec, speedbench OFFLINE = ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE") DATASET_PATH = ".dataset_path" @@ -186,6 +187,18 @@ def with_token_source(scenario_yaml: str, source: str) -> str: return yaml.safe_dump(doc, sort_keys=False) +def selection_family(env: Mapping[str, str]) -> tuple[Any, Any] | None: + """The (module, Settings) of the selection family the job's serving YAML runs, else None. + + greenspec and speedbench replay a selection of a pinned file, one request per row, and report + acceptance length. Only greenspec's file is internal: it alone goes through ``guard``. + """ + for module in (greenspec, speedbench): + if (chosen := module.settings(env)) is not None: + return module, chosen + return None + + def with_requests(scenario_yaml: str, requests: int) -> str: """The scenario with its profiling phase set to ``requests`` requests (the size of the selection).""" import yaml # in the AgentX venv @@ -215,13 +228,14 @@ def prepare(env: Mapping[str, str], runtime: Runtime) -> int: try: path = fetch(env["SERVING_DATASET"]) - if chosen := greenspec.settings(env): + if found := selection_family(env): + module, chosen = found selection = Path(SELECTION_DIR) / "selection.jsonl" - count = greenspec.write_selection(path, chosen, served_model(env), selection) - print(f"GreenSpec selection {greenspec.label(chosen)}: {count} requests", flush=True) + count = module.write_selection(path, chosen, served_model(env), selection) + print(f"{module.TAG} selection {module.label(chosen)}: {count} requests", flush=True) path = selection (result_dir / DATASET_PATH).write_text(str(path)) - except (DatasetError, greenspec.GreenSpecError) as error: + except (DatasetError, greenspec.GreenSpecError, speedbench.SpeedBenchError) as error: print(f"ERROR {error}", file=sys.stderr) return 1 if url := env.get("SERVING_REMOTE_URL"): @@ -256,7 +270,9 @@ def run(env: Mapping[str, str], runtime: Runtime) -> int: scenario = result_dir / "scenario.yaml" token_marker = result_dir / COUNT_SOURCE_MARKER source = token_marker.read_text().strip() if token_marker.exists() else None - chosen = greenspec.settings(env) + found = selection_family(env) + chosen = found[1] if found else None + internal = found is not None and found[0] is greenspec scenario_yaml = with_token_source(env["SERVING_SCENARIO_YAML"], source or "server") child = online(env) | { "AIPERF_DATASET_CONFIGURATION_TIMEOUT": "1800", @@ -280,7 +296,7 @@ def run(env: Mapping[str, str], runtime: Runtime) -> int: metrics_urls = [url for url in env.get("AIPERF_SERVER_METRICS_URLS", "").split(",") if url] if chosen: specdec.save(result_dir / "spec_metrics_before.json", specdec.snapshot(metrics_urls)) - replay = greenspec.tee_filtered if chosen else proc.tee + replay = greenspec.tee_filtered if internal else proc.tee replay_rc = replay(argv, result_dir / "benchmark.log", child) if chosen: specdec.save(result_dir / "spec_metrics_after.json", specdec.snapshot(metrics_urls)) @@ -302,7 +318,8 @@ def run(env: Mapping[str, str], runtime: Runtime) -> int: file=sys.stderr) else: specdec.merge(aggregate, spec) - guard_rc = guard(env, result_dir, aggregate, output_dir) + if internal: + guard_rc = guard(env, result_dir, aggregate, output_dir) return replay_rc or aggregate_rc or validate_rc or guard_rc diff --git a/inferencex-e2e/infx/serving/greenspec.py b/inferencex-e2e/infx/serving/greenspec.py index 73da91f3a6..b2bbaf7c74 100644 --- a/inferencex-e2e/infx/serving/greenspec.py +++ b/inferencex-e2e/infx/serving/greenspec.py @@ -31,6 +31,7 @@ from typing import Any FAMILY = "greenspec" +TAG = "GreenSpec" # how the family names itself in the job log META_KEYS = ("question_id", "category", "bucket", "lite", "isl", "osl", "thinking_src", "recorded") MAX_TOKEN_KEYS = ("max_tokens", "max_completion_tokens") SUBSETS = ("diverse", "agentic_coding", "agentic_tools", "llm_task") diff --git a/inferencex-e2e/infx/serving/scenarios.py b/inferencex-e2e/infx/serving/scenarios.py index 8c6b348a7a..b1c276a1be 100644 --- a/inferencex-e2e/infx/serving/scenarios.py +++ b/inferencex-e2e/infx/serving/scenarios.py @@ -13,12 +13,12 @@ import yaml -from infx.serving import greenspec +from infx.serving import greenspec, speedbench from infx.serving.datasets import LockEntry, load_lock from infx.serving.slo import SLOError, parse_slo SCENARIOS_DIR = Path(__file__).resolve().parents[2] / "benchmarks/single_node/serving/scenarios" -FAMILIES = ("agentx", "tau2", "livekit", "shared-prefix-e2", "hidden-prefix-e3", "synthetic", "greenspec") +FAMILIES = ("agentx", "tau2", "livekit", "shared-prefix-e2", "hidden-prefix-e3", "synthetic", "greenspec", "speedbench") OVERRIDE_KEYS = ("duration", "dataset", "max_context_length", "generation", "trace_idle_gap_cap") SYNTHETIC_OVERRIDES = ("duration",) GENERATION_KEYS = ("top_p", "temperature") @@ -189,6 +189,24 @@ def _greenspec(overrides: dict[str, Any], point: Point | None, search: bool, slo return Rendered(doc, greenspec.label(chosen), "file", name, False, 0.0) +def _speedbench(overrides: dict[str, Any], point: Point | None, search: bool, slo: str | None, + lock: dict[str, LockEntry]) -> Rendered: + """The speedbench scenario with its selection; the run length is the selection, so there is no duration.""" + if point is not None or search or slo: + raise ScenarioError("workload/slo: only for scenario synthetic, not speedbench") + try: + chosen = speedbench.normalize(overrides) + except speedbench.SpeedBenchError as error: + raise ScenarioError(str(error)) from None + if chosen.dataset not in lock: + raise ScenarioError(f"scenario: dataset {chosen.dataset} of speedbench is not in the dataset lock") + doc = load_base("speedbench") + doc["benchmark"]["datasets"][0]["name"] = chosen.dataset + if chosen.requests: + _profiling(doc)["requests"] = chosen.requests + return Rendered(doc, speedbench.label(chosen), "file", chosen.dataset, False, 0.0) + + def render(family: str, overrides: dict[str, Any] | None = None, point: Point | None = None, search: bool = False, slo: str | None = None, lock: dict[str, LockEntry] | None = None) -> Rendered: @@ -197,6 +215,8 @@ def render(family: str, overrides: dict[str, Any] | None = None, point: Point | lock = load_lock() if lock is None else lock if family == "greenspec": return _greenspec(overrides, point, search, slo, lock) + if family == "speedbench": + return _speedbench(overrides, point, search, slo, lock) if family == "synthetic": if point is None: raise ScenarioError("workload: synthetic needs isl, osl and cache") diff --git a/inferencex-e2e/infx/serving/schema.py b/inferencex-e2e/infx/serving/schema.py index b8ed679b5c..53e77b4f4b 100644 --- a/inferencex-e2e/infx/serving/schema.py +++ b/inferencex-e2e/infx/serving/schema.py @@ -229,6 +229,7 @@ class ServingSpec(BaseModel): "hidden-prefix-e3", "synthetic", "greenspec", + "speedbench", ] overrides: dict[str, Any] = Field(default_factory=dict) workload: Workload | None = None diff --git a/inferencex-e2e/infx/serving/speedbench.py b/inferencex-e2e/infx/serving/speedbench.py new file mode 100644 index 0000000000..934fa76cdc --- /dev/null +++ b/inferencex-e2e/infx/serving/speedbench.py @@ -0,0 +1,170 @@ +"""SPEED-Bench on the serving client: row selection and request payloads. + +SPEED-Bench (nvidia/SPEED-Bench) measures speculative-decoding acceptance on a fixed prompt set. The +public release masks about 60% of its qualitative rows, so the scenario reads the fully resolved copy +``Green-Lab/speed-bench-prepared`` (private; each config is one jsonl, pinned in datasets/lock.yaml). +Like ``greenspec`` it replays one selection through AgentX as a ``raw_payload`` file. SPEED-Bench is a +public set, so unlike greenspec nothing is filtered or withheld: the console, server log and result files +are uploaded as for any other scenario. + +``settings`` the selection and sampling a serving YAML asks for (overrides:). +``write_selection`` select the rows and write each as a ready request body. + +Each request is the first turn of a row alone, as the aiperf and vLLM SPEED-Bench loaders send it; +later turns of a multi-turn row are dropped. Every request streams and completes on its own (no +ignore_eos): acceptance only means something on natural output. Sampling defaults are the golden AL +matrices' (temperature 1.0, top_p 0.95, 4096 output tokens); ``chat_template_kwargs`` carries the +thinking switch of the old per-mode collectors. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from infx.serving.greenspec import iter_rows # one jsonl line per row, same as the greenspec release + +FAMILY = "speedbench" +TAG = "SpeedBench" +QUALITATIVE = "qualitative" +THROUGHPUTS = ("throughput_1k", "throughput_2k", "throughput_8k", "throughput_16k", "throughput_32k") +CONFIGS = (QUALITATIVE, *THROUGHPUTS) +CATEGORIES = { + QUALITATIVE: ("coding", "humanities", "math", "multilingual", "qa", "rag", "reasoning", "roleplay", "stem", + "summarization", "writing"), + **dict.fromkeys(THROUGHPUTS, ("low_entropy", "mixed", "high_entropy")), +} +OVERRIDES = ("config", "category", "max_tokens", "temperature", "top_p", "chat_template_kwargs", "requests") +DEFAULT_MAX_TOKENS = 4096 +DEFAULT_TEMPERATURE = 1.0 +DEFAULT_TOP_P = 0.95 + + +class SpeedBenchError(ValueError): + """A selection problem, as one line.""" + + +@dataclass(frozen=True) +class Settings: + config: str = QUALITATIVE + category: str | None = None # None: every category of the config + max_tokens: int = DEFAULT_MAX_TOKENS + temperature: float = DEFAULT_TEMPERATURE + top_p: float = DEFAULT_TOP_P + chat_template_kwargs: tuple[tuple[str, Any], ...] = () # sorted items, so Settings stays hashable + requests: int | None = None # None: the whole selection + + @property + def dataset(self) -> str: + """The name of the config's file in datasets/lock.yaml.""" + return f"speedbench-{self.config.replace('_', '-')}" + + +def _number(overrides: Mapping[str, Any], key: str, default: float, low: float, high: float) -> float: + value = overrides.get(key, default) + if isinstance(value, bool) or not isinstance(value, (int, float)) or not low <= value <= high: + raise SpeedBenchError(f"overrides.{key}: a number from {low} to {high}, got {value!r}") + return float(value) + + +def normalize(overrides: Mapping[str, Any]) -> Settings: + """The overrides of a speedbench serving YAML as ``Settings``; one line per problem.""" + if unknown := sorted(set(overrides) - set(OVERRIDES)): + raise SpeedBenchError(f"overrides: speedbench takes {', '.join(OVERRIDES)}, not {', '.join(unknown)}") + config = overrides.get("config", QUALITATIVE) + if config not in CONFIGS: + raise SpeedBenchError(f"overrides.config: {config!r} is not one of {', '.join(CONFIGS)}") + category = overrides.get("category") + if category is not None and category not in CATEGORIES[config]: + raise SpeedBenchError(f"overrides.category: {category!r} is not one of {', '.join(CATEGORIES[config])} " + f"(the categories of {config})") + max_tokens, requests = overrides.get("max_tokens", DEFAULT_MAX_TOKENS), overrides.get("requests") + for field, value in (("max_tokens", max_tokens), ("requests", requests)): + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value < 1): + raise SpeedBenchError(f"overrides.{field}: a positive integer, got {value!r}") + kwargs = overrides.get("chat_template_kwargs") or {} + if not isinstance(kwargs, dict) or not all(isinstance(k, str) for k in kwargs): + raise SpeedBenchError(f"overrides.chat_template_kwargs: a mapping with string keys, got {kwargs!r}") + return Settings( + config, category, max_tokens, + _number(overrides, "temperature", DEFAULT_TEMPERATURE, 0.0, 2.0), + _number(overrides, "top_p", DEFAULT_TOP_P, 0.0, 1.0), + tuple(sorted(kwargs.items())), requests, + ) + + +def settings(env: Mapping[str, str]) -> Settings | None: + """The job's ``Settings`` when its serving YAML runs the speedbench scenario, else None.""" + import yaml # not imported at the top: the client's first pass runs before the AgentX venv exists + + try: + raw = yaml.safe_load(env.get("SERVING_YAML") or "") + except yaml.YAMLError: + return None + if not isinstance(raw, dict) or raw.get("scenario") != FAMILY: + return None + return normalize(raw.get("overrides") or {}) + + +def label(chosen: Settings) -> str: + """The result label of a selection: speedbench-qualitative, speedbench-throughput_8k-mixed-n20, ... + + Every non-default knob is in it (ctk-enable_thinking-false, mt2048, t0.6, p1): the run summary lines up + results by label and concurrency. + """ + parts = [FAMILY, chosen.config] + if chosen.category: + parts.append(chosen.category) + for key, value in chosen.chat_template_kwargs: + parts.append("ctk-" + re.sub(r"[^0-9A-Za-z_.]+", "-", f"{key}-{json.dumps(value)}").strip("-")) + if chosen.max_tokens != DEFAULT_MAX_TOKENS: + parts.append(f"mt{chosen.max_tokens}") + if chosen.temperature != DEFAULT_TEMPERATURE: + parts.append(f"t{chosen.temperature:g}") + if chosen.top_p != DEFAULT_TOP_P: + parts.append(f"p{chosen.top_p:g}") + if chosen.requests: + parts.append(f"n{chosen.requests}") + return "-".join(parts) + + +def prompt(row: dict[str, Any]) -> str: + """The first turn of a row, stripped; empty when the row has none (such a row is skipped).""" + messages = row.get("messages") + if not isinstance(messages, list) or not messages or not isinstance(messages[0], dict): + return "" + return str(messages[0].get("content") or "").strip() + + +def select(path: Path, chosen: Settings) -> list[dict[str, Any]]: + return [row for row in iter_rows(path) + if (chosen.category is None or row.get("category") == chosen.category) and prompt(row)] + + +def payload(row: dict[str, Any], model: str, chosen: Settings) -> dict[str, Any]: + """One row -> the request body the client sends for it.""" + body: dict[str, Any] = { + "model": model, + "messages": [{"role": "user", "content": prompt(row)}], + "max_tokens": chosen.max_tokens, + "temperature": chosen.temperature, + "top_p": chosen.top_p, + } + if chosen.chat_template_kwargs: + body["chat_template_kwargs"] = dict(chosen.chat_template_kwargs) + return {**body, "stream": True, "stream_options": {"include_usage": True}} + + +def write_selection(dataset: Path, chosen: Settings, model: str, out: Path) -> int: + """Write the selected rows as one request body per line to ``out``; return how many.""" + rows = select(dataset, chosen) + if not rows: + raise SpeedBenchError(f"selection {label(chosen)}: no row of {chosen.config} matches") + out.parent.mkdir(parents=True, exist_ok=True) + with open(out, "w", encoding="utf-8") as f: + f.writelines(json.dumps(payload(row, model, chosen), ensure_ascii=False) + "\n" for row in rows) + return len(rows) diff --git a/inferencex-e2e/infx/tests/serving/test_configs.py b/inferencex-e2e/infx/tests/serving/test_configs.py index ca9808719d..00bb2bd5af 100644 --- a/inferencex-e2e/infx/tests/serving/test_configs.py +++ b/inferencex-e2e/infx/tests/serving/test_configs.py @@ -7,8 +7,8 @@ CONFIGS = sorted((Path(__file__).resolve().parents[3] / "configs/serving").glob("*.yaml")) -def test_thirteen_examples_one_per_pattern(): - assert len(CONFIGS) == 13 +def test_fourteen_examples_one_per_pattern(): + assert len(CONFIGS) == 14 @pytest.mark.parametrize("path", CONFIGS, ids=[p.stem for p in CONFIGS]) diff --git a/inferencex-e2e/infx/tests/serving/test_datasets.py b/inferencex-e2e/infx/tests/serving/test_datasets.py index 37e1a19f8c..b2baddc730 100644 --- a/inferencex-e2e/infx/tests/serving/test_datasets.py +++ b/inferencex-e2e/infx/tests/serving/test_datasets.py @@ -7,12 +7,16 @@ from infx.serving.datasets import DatasetError, LockEntry, fetch, load_lock -def test_lock_pins_seven_datasets_greenspec_without_entries(): +def test_lock_pins_greenspec_and_speedbench_without_entries(): lock = load_lock() assert set(lock) == { "tau2-cs", "shared-prefix-e2", "hidden-prefix-e3", "livekit-111868-v1", "livekit-111868-v2", "livekit-111868-v1.1-warm", "greenspec-v1.1.0", + "speedbench-qualitative", "speedbench-throughput-1k", "speedbench-throughput-2k", + "speedbench-throughput-8k", "speedbench-throughput-16k", "speedbench-throughput-32k", } + assert {e.repo for n, e in lock.items() if n.startswith("speedbench-")} == {"Green-Lab/speed-bench-prepared"} + assert lock["speedbench-qualitative"].file == "qualitative.jsonl" and lock["speedbench-qualitative"].entries is None assert lock["greenspec-v1.1.0"].repo == "Green-Lab/GreenSpec" and lock["greenspec-v1.1.0"].entries is None assert lock["livekit-111868-v1.1-warm"].entries == 671 assert lock["livekit-111868-v1"].entries == 1217 diff --git a/inferencex-e2e/infx/tests/serving/test_scenarios.py b/inferencex-e2e/infx/tests/serving/test_scenarios.py index bd60f6048a..222fa29b21 100644 --- a/inferencex-e2e/infx/tests/serving/test_scenarios.py +++ b/inferencex-e2e/infx/tests/serving/test_scenarios.py @@ -108,6 +108,30 @@ def test_bad_greenspec_variants_fail_with_one_line(args, message): render(*args) +def test_speedbench_renders_the_config_dataset_and_has_no_duration(): + rendered = render("speedbench", {"config": "throughput_8k", "category": "mixed", "requests": 30}) + dataset = rendered.doc["benchmark"]["datasets"][0] + assert (rendered.kind, rendered.dataset, rendered.label) == ( + "file", "speedbench-throughput-8k", "speedbench-throughput_8k-mixed-n30") + assert dataset["name"] == "speedbench-throughput-8k" + assert dataset["format"] == "raw_payload" and dataset["path"] == "${IX_CUSTOM_DATASET_FILE}" + [phase] = rendered.doc["benchmark"]["phases"] + assert phase["requests"] == 30 and "duration" not in phase and rendered.duration == 0.0 + assert "ignore_eos" not in rendered.doc["benchmark"]["endpoint"].get("extra", {}) + default = render("speedbench") + assert (default.label, default.dataset) == ("speedbench-qualitative", "speedbench-qualitative") + + +@pytest.mark.parametrize("args, message", [ + (("speedbench", {"category": "mixed"}), "overrides.category"), + (("speedbench", {"subset": "diverse"}), "speedbench takes"), + (("speedbench", {}, Point("8k", "1k", 0)), "workload/slo: only for scenario synthetic, not speedbench"), +]) +def test_bad_speedbench_variants_fail_with_one_line(args, message): + with pytest.raises(ScenarioError, match=message): + render(*args) + + def test_families_have_base_files(): from infx.serving.scenarios import SCENARIOS_DIR assert {path.stem for path in SCENARIOS_DIR.glob("*.yaml")} == set(FAMILIES) diff --git a/inferencex-e2e/infx/tests/serving/test_speedbench.py b/inferencex-e2e/infx/tests/serving/test_speedbench.py new file mode 100644 index 0000000000..1e9db97b7a --- /dev/null +++ b/inferencex-e2e/infx/tests/serving/test_speedbench.py @@ -0,0 +1,127 @@ +import json +from pathlib import Path + +import pytest + +from infx.serving import client +from infx.serving.speedbench import ( + Settings, + SpeedBenchError, + label, + normalize, + payload, + prompt, + select, + settings, + write_selection, +) + + +def _row(qid: str, category: str, *turns: str) -> dict: + return {"question_id": qid, "category": category, "messages": [{"role": "user", "content": t} for t in turns]} + + +ROWS = [ + _row("a" * 32, "coding", " write fizzbuzz ", "now in Rust"), + _row("b" * 32, "coding", "reverse a list"), + _row("c" * 32, "math", "integrate x"), + _row("d" * 32, "math", " "), # empty first turn: skipped, as the aiperf loader does + {"question_id": "e" * 32, "category": "math", "messages": []}, +] + + +@pytest.fixture +def dataset(tmp_path: Path) -> Path: + path = tmp_path / "qualitative.jsonl" + path.write_text("".join(json.dumps(r) + "\n" for r in ROWS)) + return path + + +@pytest.mark.parametrize("chosen, expected", [ + (Settings(), ["a" * 32, "b" * 32, "c" * 32]), + (Settings(category="coding"), ["a" * 32, "b" * 32]), + (Settings(category="math"), ["c" * 32]), + (Settings(category="qa"), []), +]) +def test_select_skips_rows_without_a_first_turn(dataset, chosen, expected): + assert [r["question_id"] for r in select(dataset, chosen)] == expected + + +def test_prompt_is_the_stripped_first_turn_only(): + assert prompt(ROWS[0]) == "write fizzbuzz" + assert prompt({"messages": None}) == "" and prompt({}) == "" and prompt({"messages": ["x"]}) == "" + + +def test_payload_defaults_match_the_golden_matrices(): + body = payload(ROWS[0], "m", Settings()) + assert body == { + "model": "m", "messages": [{"role": "user", "content": "write fizzbuzz"}], + "max_tokens": 4096, "temperature": 1.0, "top_p": 0.95, + "stream": True, "stream_options": {"include_usage": True}, + } + assert "ignore_eos" not in body + + +def test_payload_carries_the_thinking_switch(): + chosen = normalize({"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 2048, "temperature": 0}) + body = payload(ROWS[1], "m", chosen) + assert body["chat_template_kwargs"] == {"enable_thinking": False} + assert (body["max_tokens"], body["temperature"]) == (2048, 0.0) + + +def test_write_selection(dataset, tmp_path): + out = tmp_path / "sel" / "selection.jsonl" + assert write_selection(dataset, Settings(category="coding"), "m", out) == 2 + lines = [json.loads(line) for line in out.read_text().splitlines()] + assert [line["messages"][0]["content"] for line in lines] == ["write fizzbuzz", "reverse a list"] + with pytest.raises(SpeedBenchError, match="no row of qualitative matches"): + write_selection(dataset, Settings(category="qa"), "m", out) + + +@pytest.mark.parametrize("overrides, expected", [ + ({}, "speedbench-qualitative"), + ({"category": "coding"}, "speedbench-qualitative-coding"), + ({"config": "throughput_8k", "category": "mixed", "requests": 20}, "speedbench-throughput_8k-mixed-n20"), + ({"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 1024}, + "speedbench-qualitative-ctk-enable_thinking-false-mt1024"), + ({"temperature": 0.6, "top_p": 1}, "speedbench-qualitative-t0.6-p1"), +]) +def test_label_names_every_non_default_knob(overrides, expected): + assert label(normalize(overrides)) == expected + + +def test_dataset_name_is_the_lock_key(): + assert Settings().dataset == "speedbench-qualitative" + assert Settings(config="throughput_16k").dataset == "speedbench-throughput-16k" + + +@pytest.mark.parametrize("overrides, message", [ + ({"config": "throughput_4k"}, "overrides.config"), + ({"category": "mixed"}, "overrides.category"), # a throughput category on qualitative + ({"config": "throughput_2k", "category": "coding"}, "overrides.category"), + ({"max_tokens": 0}, "overrides.max_tokens"), + ({"requests": True}, "overrides.requests"), + ({"temperature": -1}, "overrides.temperature"), + ({"top_p": 1.5}, "overrides.top_p"), + ({"top_p": True}, "overrides.top_p"), + ({"chat_template_kwargs": ["x"]}, "overrides.chat_template_kwargs"), + ({"subset": "diverse"}, "speedbench takes"), +]) +def test_bad_overrides(overrides, message): + with pytest.raises(SpeedBenchError, match=message): + normalize(overrides) + + +def test_settings_only_for_a_speedbench_yaml(): + yaml_text = "scenario: speedbench\noverrides:\n category: math\n" + assert settings({"SERVING_YAML": yaml_text}) == Settings(category="math") + assert settings({"SERVING_YAML": "scenario: greenspec\n"}) is None + assert settings({"SERVING_YAML": ": : ["}) is None and settings({}) is None + + +def test_selection_family_picks_the_module_of_the_yaml(): + from infx.serving import greenspec, speedbench + + assert client.selection_family({"SERVING_YAML": "scenario: speedbench\n"})[0] is speedbench + assert client.selection_family({"SERVING_YAML": "scenario: greenspec\n"})[0] is greenspec + assert client.selection_family({"SERVING_YAML": "scenario: agentx\n"}) is None From 18725964cf9832e2d957f9690e1fb3da54be3796 Mon Sep 17 00:00:00 2001 From: Noridom1 Date: Fri, 30 Oct 2026 09:54:31 +0700 Subject: [PATCH 2/2] tests: keep the speedbench test files in the repo's unformatted test style (ruff excludes tests) --- .../infx/tests/serving/test_datasets.py | 54 ++-- .../infx/tests/serving/test_scenarios.py | 233 ++++-------------- .../infx/tests/serving/test_speedbench.py | 95 +++---- 3 files changed, 97 insertions(+), 285 deletions(-) diff --git a/inferencex-e2e/infx/tests/serving/test_datasets.py b/inferencex-e2e/infx/tests/serving/test_datasets.py index 6b60687d72..b2baddc730 100644 --- a/inferencex-e2e/infx/tests/serving/test_datasets.py +++ b/inferencex-e2e/infx/tests/serving/test_datasets.py @@ -10,51 +10,25 @@ def test_lock_pins_greenspec_and_speedbench_without_entries(): lock = load_lock() assert set(lock) == { - "tau2-cs", - "shared-prefix-e2", - "hidden-prefix-e3", - "livekit-111868-v1", - "livekit-111868-v2", - "livekit-111868-v1.1-warm", - "greenspec-v1.1.0", - "speedbench-qualitative", - "speedbench-throughput-1k", - "speedbench-throughput-2k", - "speedbench-throughput-8k", - "speedbench-throughput-16k", - "speedbench-throughput-32k", + "tau2-cs", "shared-prefix-e2", "hidden-prefix-e3", + "livekit-111868-v1", "livekit-111868-v2", "livekit-111868-v1.1-warm", "greenspec-v1.1.0", + "speedbench-qualitative", "speedbench-throughput-1k", "speedbench-throughput-2k", + "speedbench-throughput-8k", "speedbench-throughput-16k", "speedbench-throughput-32k", } - assert {e.repo for n, e in lock.items() if n.startswith("speedbench-")} == { - "Green-Lab/speed-bench-prepared" - } - assert ( - lock["speedbench-qualitative"].file == "qualitative.jsonl" - and lock["speedbench-qualitative"].entries is None - ) - assert ( - lock["greenspec-v1.1.0"].repo == "Green-Lab/GreenSpec" - and lock["greenspec-v1.1.0"].entries is None - ) + assert {e.repo for n, e in lock.items() if n.startswith("speedbench-")} == {"Green-Lab/speed-bench-prepared"} + assert lock["speedbench-qualitative"].file == "qualitative.jsonl" and lock["speedbench-qualitative"].entries is None + assert lock["greenspec-v1.1.0"].repo == "Green-Lab/GreenSpec" and lock["greenspec-v1.1.0"].entries is None assert lock["livekit-111868-v1.1-warm"].entries == 671 assert lock["livekit-111868-v1"].entries == 1217 -@pytest.mark.parametrize( - "body, message", - [ - ("x: {repo: a/b}\n", "x needs exactly repo, revision, file, sha256"), - ("x: {repo: a/b, revision: abc, file: f, sha256: " + "b" * 64 + "}\n", "x.revision"), - # all-digit hashes would load as YAML ints, hence a/b - ( - "x: {repo: a/b, revision: " - + "a" * 40 - + ", file: f, sha256: " - + "b" * 64 - + ", entries: 0}\n", - "x.entries must be a positive integer", - ), - ], -) +@pytest.mark.parametrize("body, message", [ + ("x: {repo: a/b}\n", "x needs exactly repo, revision, file, sha256"), + ("x: {repo: a/b, revision: abc, file: f, sha256: " + "b" * 64 + "}\n", "x.revision"), + # all-digit hashes would load as YAML ints, hence a/b + ("x: {repo: a/b, revision: " + "a" * 40 + ", file: f, sha256: " + "b" * 64 + ", entries: 0}\n", + "x.entries must be a positive integer"), +]) def test_bad_lock_is_one_line_error(tmp_path, body, message): path = tmp_path / "lock.yaml" path.write_text(body) diff --git a/inferencex-e2e/infx/tests/serving/test_scenarios.py b/inferencex-e2e/infx/tests/serving/test_scenarios.py index 5ad600c753..222fa29b21 100644 --- a/inferencex-e2e/infx/tests/serving/test_scenarios.py +++ b/inferencex-e2e/infx/tests/serving/test_scenarios.py @@ -12,135 +12,34 @@ CASES = [ ("agentx-cc-062126", "agentx", {}, None, False, "agentx-cc-062126"), ("agentx-cc-062126-smoke", "agentx", {"duration": 180}, None, False, "agentx-cc-062126-smoke"), - ( - "agentx-cc-062126-256k", - "agentx", - {"dataset": "semianalysis_cc_traces_weka_062126_256k"}, - None, - False, - "agentx-cc-062126-256k", - ), - ( - "agentx-cc-062126-1m", - "agentx", - {"max_context_length": None}, - None, - False, - "agentx-cc-062126-1m", - ), - ( - "agentx-cc-062126-topp95", - "agentx", - {"generation": GEN}, - None, - False, - "agentx-cc-062126-topp95", - ), + ("agentx-cc-062126-256k", "agentx", {"dataset": "semianalysis_cc_traces_weka_062126_256k"}, None, False, + "agentx-cc-062126-256k"), + ("agentx-cc-062126-1m", "agentx", {"max_context_length": None}, None, False, "agentx-cc-062126-1m"), + ("agentx-cc-062126-topp95", "agentx", {"generation": GEN}, None, False, "agentx-cc-062126-topp95"), ("custom-agentic-tau2", "tau2", {}, None, False, "custom-agentic-tau2"), - ( - "custom-agentic-tau2-smoke", - "tau2", - {"duration": 180}, - None, - False, - "custom-agentic-tau2-smoke", - ), - ( - "custom-agentic-livekit111868-v1.1-warm", - "livekit", - {}, - None, - False, - "custom-agentic-livekit111868-v1.1-warm", - ), - ( - "custom-agentic-livekit111868-v1", - "livekit", - {"dataset": "livekit-111868-v1"}, - None, - False, - "custom-agentic-livekit111868-v1", - ), - ( - "custom-agentic-livekit111868-v2", - "livekit", - {"dataset": "livekit-111868-v2"}, - None, - False, - "custom-agentic-livekit111868-v2", - ), - ( - "custom-agentic-livekit111868-smoke", - "livekit", - {"dataset": "livekit-111868-v2", "duration": 180}, - None, - False, - "custom-agentic-livekit111868-v2-smoke", - ), + ("custom-agentic-tau2-smoke", "tau2", {"duration": 180}, None, False, "custom-agentic-tau2-smoke"), + ("custom-agentic-livekit111868-v1.1-warm", "livekit", {}, None, False, + "custom-agentic-livekit111868-v1.1-warm"), + ("custom-agentic-livekit111868-v1", "livekit", {"dataset": "livekit-111868-v1"}, None, False, + "custom-agentic-livekit111868-v1"), + ("custom-agentic-livekit111868-v2", "livekit", {"dataset": "livekit-111868-v2"}, None, False, + "custom-agentic-livekit111868-v2"), + ("custom-agentic-livekit111868-smoke", "livekit", {"dataset": "livekit-111868-v2", "duration": 180}, None, + False, "custom-agentic-livekit111868-v2-smoke"), ("agentx-shared-prefix-e2", "shared-prefix-e2", {}, None, False, "agentx-shared-prefix-e2"), ("agentx-hidden-prefix-e3", "hidden-prefix-e3", {}, None, False, "agentx-hidden-prefix-e3"), - ( - "agentx-hidden-prefix-e3-smoke", - "hidden-prefix-e3", - {"duration": 180}, - None, - False, - "agentx-hidden-prefix-e3-smoke", - ), - ( - "sweep-isl8k-osl512-cache50", - "synthetic", - {}, - Point("8k", "512", 50), - True, - "sweep-isl8k-osl512-cache50", - ), - ( - "sweep-isl8k-osl1k-cache90", - "synthetic", - {}, - Point("8k", "1k", 90), - True, - "sweep-isl8k-osl1k-cache90", - ), - ( - "sweep-isl128k-osl512-cache0", - "synthetic", - {}, - Point("128k", "512", 0), - True, - "sweep-isl128k-osl512-cache0", - ), - ( - "sweep-isl128k-osl1k-cache0", - "synthetic", - {}, - Point("128k", "1k", 0), - True, - "sweep-isl128k-osl1k-cache0", - ), - ( - "fixed-isl8k-osl1k-cache50", - "synthetic", - {}, - Point("8k", "1k", 50), - False, - "fixed-isl8k-osl1k-cache50", - ), - ( - "fixed-isl8k-osl1k-cache0", - "synthetic", - {}, - Point("8k", "1k", 0), - False, - "fixed-isl8k-osl1k-cache0", - ), + ("agentx-hidden-prefix-e3-smoke", "hidden-prefix-e3", {"duration": 180}, None, False, + "agentx-hidden-prefix-e3-smoke"), + ("sweep-isl8k-osl512-cache50", "synthetic", {}, Point("8k", "512", 50), True, "sweep-isl8k-osl512-cache50"), + ("sweep-isl8k-osl1k-cache90", "synthetic", {}, Point("8k", "1k", 90), True, "sweep-isl8k-osl1k-cache90"), + ("sweep-isl128k-osl512-cache0", "synthetic", {}, Point("128k", "512", 0), True, "sweep-isl128k-osl512-cache0"), + ("sweep-isl128k-osl1k-cache0", "synthetic", {}, Point("128k", "1k", 0), True, "sweep-isl128k-osl1k-cache0"), + ("fixed-isl8k-osl1k-cache50", "synthetic", {}, Point("8k", "1k", 50), False, "fixed-isl8k-osl1k-cache50"), + ("fixed-isl8k-osl1k-cache0", "synthetic", {}, Point("8k", "1k", 0), False, "fixed-isl8k-osl1k-cache0"), ] -@pytest.mark.parametrize( - "legacy, family, overrides, point, search, label", CASES, ids=[c[0] for c in CASES] -) +@pytest.mark.parametrize("legacy, family, overrides, point, search, label", CASES, ids=[c[0] for c in CASES]) def test_base_plus_variant_equals_the_legacy_file(legacy, family, overrides, point, search, label): rendered = render(family, overrides, point, search) assert rendered.doc == yaml.safe_load((LEGACY / f"{legacy}.yaml").read_text()) @@ -162,40 +61,27 @@ def test_kinds_and_datasets(): def test_slo_replaces_the_sweep_filters(): - doc = render( - "synthetic", - point=Point("8k", "1k", 0), - search=True, - slo="time_to_first_token:p95:lt:500;inter_token_latency:p50:lt:50", - ).doc + doc = render("synthetic", point=Point("8k", "1k", 0), search=True, + slo="time_to_first_token:p95:lt:500;inter_token_latency:p50:lt:50").doc assert doc["sweep"]["sla_filters"] == [ {"metric_tag": "time_to_first_token", "stat": "p95", "op": "lt", "threshold": 500.0}, {"metric_tag": "inter_token_latency", "stat": "p50", "op": "lt", "threshold": 50.0}, ] -@pytest.mark.parametrize( - "args, message", - [ - (("agentx", {"warmup": 1}), "overrides: unknown key"), - (("agentx", {"dataset": "tau2-cs"}), "overrides.dataset: agentx takes a public"), - (("tau2", {"dataset": "semianalysis_cc_traces_weka_062126"}), "is not in the dataset lock"), - (("agentx", {"duration": 0}), "overrides.duration"), - (("agentx", {"generation": {"top_k": 5}}), "overrides.generation"), - (("synthetic", {}), "workload: synthetic needs"), - ( - ("synthetic", {"dataset": "x"}, Point("8k", "1k", 0)), - "overrides: synthetic only takes duration", - ), - (("synthetic", {}, Point("16k", "1k", 0)), "workload.isl"), - ( - ("synthetic", {}, Point("8k", "1k", 0), False, "time_to_first_token:avg:lt:1"), - "slo: only applies", - ), - (("agentx", {}, Point("8k", "1k", 0)), "workload/slo: only for scenario synthetic"), - (("nope",), "scenario: 'nope' is not one of"), - ], -) +@pytest.mark.parametrize("args, message", [ + (("agentx", {"warmup": 1}), "overrides: unknown key"), + (("agentx", {"dataset": "tau2-cs"}), "overrides.dataset: agentx takes a public"), + (("tau2", {"dataset": "semianalysis_cc_traces_weka_062126"}), "is not in the dataset lock"), + (("agentx", {"duration": 0}), "overrides.duration"), + (("agentx", {"generation": {"top_k": 5}}), "overrides.generation"), + (("synthetic", {}), "workload: synthetic needs"), + (("synthetic", {"dataset": "x"}, Point("8k", "1k", 0)), "overrides: synthetic only takes duration"), + (("synthetic", {}, Point("16k", "1k", 0)), "workload.isl"), + (("synthetic", {}, Point("8k", "1k", 0), False, "time_to_first_token:avg:lt:1"), "slo: only applies"), + (("agentx", {}, Point("8k", "1k", 0)), "workload/slo: only for scenario synthetic"), + (("nope",), "scenario: 'nope' is not one of"), +]) def test_bad_variants_fail_with_one_line(args, message): with pytest.raises(ScenarioError, match=message): render(*args) @@ -204,11 +90,7 @@ def test_bad_variants_fail_with_one_line(args, message): def test_greenspec_renders_the_selection_and_has_no_duration(): rendered = render("greenspec", {"subset": "agentic_tools", "lite": True, "requests": 50}) dataset = rendered.doc["benchmark"]["datasets"][0] - assert (rendered.kind, rendered.dataset, rendered.label) == ( - "file", - "greenspec-v1.1.0", - "greenspec-agentic_tools-lite-n50", - ) + assert (rendered.kind, rendered.dataset, rendered.label) == ("file", "greenspec-v1.1.0", "greenspec-agentic_tools-lite-n50") assert dataset["format"] == "raw_payload" and dataset["path"] == "${IX_CUSTOM_DATASET_FILE}" [phase] = rendered.doc["benchmark"]["phases"] assert phase["requests"] == 50 and "duration" not in phase and rendered.duration == 0.0 @@ -216,29 +98,21 @@ def test_greenspec_renders_the_selection_and_has_no_duration(): assert render("greenspec").label == "greenspec-diverse" -@pytest.mark.parametrize( - "args, message", - [ - (("greenspec", {"subset": "coding"}), "overrides.subset"), - (("greenspec", {"duration": 180}), "greenspec takes"), - (("greenspec", {}, Point("8k", "1k", 0)), "workload/slo: only for scenario synthetic"), - ], -) +@pytest.mark.parametrize("args, message", [ + (("greenspec", {"subset": "coding"}), "overrides.subset"), + (("greenspec", {"duration": 180}), "greenspec takes"), + (("greenspec", {}, Point("8k", "1k", 0)), "workload/slo: only for scenario synthetic"), +]) def test_bad_greenspec_variants_fail_with_one_line(args, message): with pytest.raises(ScenarioError, match=message): render(*args) def test_speedbench_renders_the_config_dataset_and_has_no_duration(): - rendered = render( - "speedbench", {"config": "throughput_8k", "category": "mixed", "requests": 30} - ) + rendered = render("speedbench", {"config": "throughput_8k", "category": "mixed", "requests": 30}) dataset = rendered.doc["benchmark"]["datasets"][0] assert (rendered.kind, rendered.dataset, rendered.label) == ( - "file", - "speedbench-throughput-8k", - "speedbench-throughput_8k-mixed-n30", - ) + "file", "speedbench-throughput-8k", "speedbench-throughput_8k-mixed-n30") assert dataset["name"] == "speedbench-throughput-8k" assert dataset["format"] == "raw_payload" and dataset["path"] == "${IX_CUSTOM_DATASET_FILE}" [phase] = rendered.doc["benchmark"]["phases"] @@ -248,17 +122,11 @@ def test_speedbench_renders_the_config_dataset_and_has_no_duration(): assert (default.label, default.dataset) == ("speedbench-qualitative", "speedbench-qualitative") -@pytest.mark.parametrize( - "args, message", - [ - (("speedbench", {"category": "mixed"}), "overrides.category"), - (("speedbench", {"subset": "diverse"}), "speedbench takes"), - ( - ("speedbench", {}, Point("8k", "1k", 0)), - "workload/slo: only for scenario synthetic, not speedbench", - ), - ], -) +@pytest.mark.parametrize("args, message", [ + (("speedbench", {"category": "mixed"}), "overrides.category"), + (("speedbench", {"subset": "diverse"}), "speedbench takes"), + (("speedbench", {}, Point("8k", "1k", 0)), "workload/slo: only for scenario synthetic, not speedbench"), +]) def test_bad_speedbench_variants_fail_with_one_line(args, message): with pytest.raises(ScenarioError, match=message): render(*args) @@ -266,5 +134,4 @@ def test_bad_speedbench_variants_fail_with_one_line(args, message): def test_families_have_base_files(): from infx.serving.scenarios import SCENARIOS_DIR - assert {path.stem for path in SCENARIOS_DIR.glob("*.yaml")} == set(FAMILIES) diff --git a/inferencex-e2e/infx/tests/serving/test_speedbench.py b/inferencex-e2e/infx/tests/serving/test_speedbench.py index 6166459ded..1e9db97b7a 100644 --- a/inferencex-e2e/infx/tests/serving/test_speedbench.py +++ b/inferencex-e2e/infx/tests/serving/test_speedbench.py @@ -18,11 +18,7 @@ def _row(qid: str, category: str, *turns: str) -> dict: - return { - "question_id": qid, - "category": category, - "messages": [{"role": "user", "content": t} for t in turns], - } + return {"question_id": qid, "category": category, "messages": [{"role": "user", "content": t} for t in turns]} ROWS = [ @@ -41,44 +37,33 @@ def dataset(tmp_path: Path) -> Path: return path -@pytest.mark.parametrize( - "chosen, expected", - [ - (Settings(), ["a" * 32, "b" * 32, "c" * 32]), - (Settings(category="coding"), ["a" * 32, "b" * 32]), - (Settings(category="math"), ["c" * 32]), - (Settings(category="qa"), []), - ], -) +@pytest.mark.parametrize("chosen, expected", [ + (Settings(), ["a" * 32, "b" * 32, "c" * 32]), + (Settings(category="coding"), ["a" * 32, "b" * 32]), + (Settings(category="math"), ["c" * 32]), + (Settings(category="qa"), []), +]) def test_select_skips_rows_without_a_first_turn(dataset, chosen, expected): assert [r["question_id"] for r in select(dataset, chosen)] == expected def test_prompt_is_the_stripped_first_turn_only(): assert prompt(ROWS[0]) == "write fizzbuzz" - assert ( - prompt({"messages": None}) == "" and prompt({}) == "" and prompt({"messages": ["x"]}) == "" - ) + assert prompt({"messages": None}) == "" and prompt({}) == "" and prompt({"messages": ["x"]}) == "" def test_payload_defaults_match_the_golden_matrices(): body = payload(ROWS[0], "m", Settings()) assert body == { - "model": "m", - "messages": [{"role": "user", "content": "write fizzbuzz"}], - "max_tokens": 4096, - "temperature": 1.0, - "top_p": 0.95, - "stream": True, - "stream_options": {"include_usage": True}, + "model": "m", "messages": [{"role": "user", "content": "write fizzbuzz"}], + "max_tokens": 4096, "temperature": 1.0, "top_p": 0.95, + "stream": True, "stream_options": {"include_usage": True}, } assert "ignore_eos" not in body def test_payload_carries_the_thinking_switch(): - chosen = normalize( - {"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 2048, "temperature": 0} - ) + chosen = normalize({"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 2048, "temperature": 0}) body = payload(ROWS[1], "m", chosen) assert body["chat_template_kwargs"] == {"enable_thinking": False} assert (body["max_tokens"], body["temperature"]) == (2048, 0.0) @@ -88,30 +73,19 @@ def test_write_selection(dataset, tmp_path): out = tmp_path / "sel" / "selection.jsonl" assert write_selection(dataset, Settings(category="coding"), "m", out) == 2 lines = [json.loads(line) for line in out.read_text().splitlines()] - assert [line["messages"][0]["content"] for line in lines] == [ - "write fizzbuzz", - "reverse a list", - ] + assert [line["messages"][0]["content"] for line in lines] == ["write fizzbuzz", "reverse a list"] with pytest.raises(SpeedBenchError, match="no row of qualitative matches"): write_selection(dataset, Settings(category="qa"), "m", out) -@pytest.mark.parametrize( - "overrides, expected", - [ - ({}, "speedbench-qualitative"), - ({"category": "coding"}, "speedbench-qualitative-coding"), - ( - {"config": "throughput_8k", "category": "mixed", "requests": 20}, - "speedbench-throughput_8k-mixed-n20", - ), - ( - {"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 1024}, - "speedbench-qualitative-ctk-enable_thinking-false-mt1024", - ), - ({"temperature": 0.6, "top_p": 1}, "speedbench-qualitative-t0.6-p1"), - ], -) +@pytest.mark.parametrize("overrides, expected", [ + ({}, "speedbench-qualitative"), + ({"category": "coding"}, "speedbench-qualitative-coding"), + ({"config": "throughput_8k", "category": "mixed", "requests": 20}, "speedbench-throughput_8k-mixed-n20"), + ({"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 1024}, + "speedbench-qualitative-ctk-enable_thinking-false-mt1024"), + ({"temperature": 0.6, "top_p": 1}, "speedbench-qualitative-t0.6-p1"), +]) def test_label_names_every_non_default_knob(overrides, expected): assert label(normalize(overrides)) == expected @@ -121,21 +95,18 @@ def test_dataset_name_is_the_lock_key(): assert Settings(config="throughput_16k").dataset == "speedbench-throughput-16k" -@pytest.mark.parametrize( - "overrides, message", - [ - ({"config": "throughput_4k"}, "overrides.config"), - ({"category": "mixed"}, "overrides.category"), # a throughput category on qualitative - ({"config": "throughput_2k", "category": "coding"}, "overrides.category"), - ({"max_tokens": 0}, "overrides.max_tokens"), - ({"requests": True}, "overrides.requests"), - ({"temperature": -1}, "overrides.temperature"), - ({"top_p": 1.5}, "overrides.top_p"), - ({"top_p": True}, "overrides.top_p"), - ({"chat_template_kwargs": ["x"]}, "overrides.chat_template_kwargs"), - ({"subset": "diverse"}, "speedbench takes"), - ], -) +@pytest.mark.parametrize("overrides, message", [ + ({"config": "throughput_4k"}, "overrides.config"), + ({"category": "mixed"}, "overrides.category"), # a throughput category on qualitative + ({"config": "throughput_2k", "category": "coding"}, "overrides.category"), + ({"max_tokens": 0}, "overrides.max_tokens"), + ({"requests": True}, "overrides.requests"), + ({"temperature": -1}, "overrides.temperature"), + ({"top_p": 1.5}, "overrides.top_p"), + ({"top_p": True}, "overrides.top_p"), + ({"chat_template_kwargs": ["x"]}, "overrides.chat_template_kwargs"), + ({"subset": "diverse"}, "speedbench takes"), +]) def test_bad_overrides(overrides, message): with pytest.raises(SpeedBenchError, match=message): normalize(overrides)