diff --git a/.agents/skills/serving-bench/SKILL.md b/.agents/skills/serving-bench/SKILL.md index 192ebf811c..05db79ed03 100644 --- a/.agents/skills/serving-bench/SKILL.md +++ b/.agents/skills/serving-bench/SKILL.md @@ -33,7 +33,7 @@ storage: # optional: a non-GPU sidecar started first arms: # optional: engine variants in one run, e.g. base vs speculative (see ARCHITECTURE.md, Arms) base: {} mtp6: {append: "--speculative-config '...'"} -scenario: agentx # agentx | tau2 | livekit | shared-prefix-e2 | hidden-prefix-e3 | synthetic | greenspec +scenario: agentx # agentx | tau2 | livekit | shared-prefix-e2 | hidden-prefix-e3 | synthetic | greenspec | speedbench overrides: {duration: 180} # optional variant, see the table below conc: [4, 8] # one benchmark job per value ``` @@ -50,6 +50,7 @@ Working examples, one per pattern: `inferencex-e2e/configs/serving/*.yaml`. They | `shared-prefix-e2` / `hidden-prefix-e3` | prefix-cache A/B traces (HiCache L2 vs Mooncake L3) | 1800 s | | `synthetic` | synthetic ISL/OSL with a shared-prefix cache hit, needs `workload:` | 300 s per point | | `greenspec` | speculative-decoding bench from production chat traffic (private; result has `spec_decode.acceptance_length`) | the whole selection, no duration | +| `speedbench` | SPEED-Bench acceptance-length prompts (private copy of public data; result has `spec_decode.acceptance_length`) | the whole selection, no duration | Variants (`overrides:`, only these keys): @@ -62,6 +63,7 @@ Variants (`overrides:`, only these keys): | LiveKit V1 / V2 trace | `overrides: {dataset: livekit-111868-v1}` (or `-v2`) | | a different trace idle-gap cap | `overrides: {trace_idle_gap_cap: 1}` | | GreenSpec subset, lite, bucket, cap | `scenario: greenspec` + `overrides: {subset: agentic_tools, lite: true, max_tokens: 4096, requests: 200}` (only these keys; no `duration`) | +| SPEED-Bench config, category, cap | `scenario: speedbench` + `overrides: {config: qualitative, category: coding, requests: 40}` (config: `qualitative` or `throughput_{1k,2k,8k,16k,32k}`; no `duration`) | Synthetic grid, one job per point, all in one run: diff --git a/.agents/skills/serving-runner-setup/SKILL.md b/.agents/skills/serving-runner-setup/SKILL.md index 05e35b53a4..6881972178 100644 --- a/.agents/skills/serving-runner-setup/SKILL.md +++ b/.agents/skills/serving-runner-setup/SKILL.md @@ -73,7 +73,7 @@ for d in $BASE/gharunner0*/actions-runner; do (cd "$d" && sudo ./svc.sh install The registration token comes from the settings page or `gh api -X POST repos/vngcloud/InferenceX/actions/runners/registration-token --jq .token` (valid exactly 1 hour, not configurable; mint it right before running `setup.sh`). `setup.sh` prints `All runners configured!` even when registration failed: check that every slot printed its own `[] Done.` line. -Every `datasets/lock.yaml` entry marked `private: true` (GreenSpec) needs `GREENLAB_HF_TOKEN`; the other repos are public. `infx/serving/datasets.py` downloads anonymously when `GREENLAB_HF_TOKEN` is unset, so a runner that is to run `scenario: greenspec` needs `GREENLAB_HF_TOKEN=…` in `actions-runner/.env`. Other scenarios need no `.env`. +Every `datasets/lock.yaml` entry marked `private: true` (GreenSpec, SPEED-Bench) needs `GREENLAB_HF_TOKEN`; the other repos are public. `infx/serving/datasets.py` downloads anonymously when `GREENLAB_HF_TOKEN` is unset, so a runner that is to run `scenario: greenspec` or `scenario: speedbench` needs `GREENLAB_HF_TOKEN=…` in `actions-runner/.env`. Other scenarios need no `.env`. ## 3. Add the cluster record diff --git a/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml b/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml index dcf76d53ab..e08e27dba2 100644 --- a/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml +++ b/inferencex-e2e/benchmarks/single_node/serving/datasets/lock.yaml @@ -52,3 +52,42 @@ greenspec-v1.1.1: file: greenspec.jsonl sha256: 6a26785d07c235446e57ba34463a1ae9e2dcff938fec6246881492e511fb0f43 private: true +# SPEED-Bench, fully resolved: every row the public release masks is refetched from its source +# (infx.serving.speedbench selects rows from the config's file). The prompts are public data, but the +# resolved copy lives in a private repo: GREENLAB_HF_TOKEN must be able to read it. +speedbench-qualitative: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: qualitative.jsonl + sha256: e9bbbe998472b79501ed79ba2bf126e58b63e153a6deef70ce2b63ea7a4de845 + private: true +speedbench-throughput-1k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_1k.jsonl + sha256: 0b5665f61dcbf52dd357d515fd45f9e4c17c68eb902c788c81a4c9271b217377 + private: true +speedbench-throughput-2k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_2k.jsonl + sha256: be247367a3ddd3c74b72d03122818d89c6e2e2fe956e6d66099d7ae0c45402a6 + private: true +speedbench-throughput-8k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_8k.jsonl + sha256: 94f3ab2d3cd456654ee234faf888c459c5451069d6cfab6e051591b288e71739 + private: true +speedbench-throughput-16k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_16k.jsonl + sha256: 77cbee1b4abc6b205ee9ad918bd8e1290a6514f4cf0ff9dc7db497afd1fffca2 + private: true +speedbench-throughput-32k: + repo: Green-Lab/speed-bench-prepared + revision: 184d7ed9fae19efa9d4d4e9959ac83cca83b3415 + file: throughput_32k.jsonl + sha256: 04148f06e19c490856dcb9efc1b1fd4593354d4abd4bf07c6f5dc4ce91348c0b + private: true diff --git a/inferencex-e2e/benchmarks/single_node/serving/scenarios/speedbench.yaml b/inferencex-e2e/benchmarks/single_node/serving/scenarios/speedbench.yaml new file mode 100644 index 0000000000..6230e4be00 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/serving/scenarios/speedbench.yaml @@ -0,0 +1,31 @@ +# SpeedBench family: SPEED-Bench acceptance-length prompts (nvidia/SPEED-Bench, fully resolved copy +# Green-Lab/speed-bench-prepared; datasets/lock.yaml pins one jsonl per config). +# infx.serving.speedbench selects the rows (overrides: config, category) and builds one request body per row +# (first turn only, temperature/top_p/max_tokens/chat_template_kwargs from overrides) and writes them as the +# raw_payload file IX_CUSTOM_DATASET_FILE points to; AgentX sends each body as is. Every request streams and +# completes on its own (no ignore_eos): acceptance only means something on natural output. `requests` is the +# selection size unless overrides.requests lowers it; the client sets it once it has counted the selection, +# so the 1 below is only a placeholder. The dataset name below is a placeholder too: scenarios.py sets it +# from overrides.config. +schemaVersion: "2.0" +benchmark: + unsafeOverride: true + endpoint: + type: chat + path: /v1/chat/completions + streaming: true + useServerTokenCount: true + datasets: + - name: speedbench-qualitative + type: file + format: raw_payload + path: ${IX_CUSTOM_DATASET_FILE} + sampling: sequential + randomSeed: 42 + phases: + - name: profiling + kind: profiling + type: concurrency + requests: 1 + failedRequestThreshold: 0.1 +randomSeed: 42 diff --git a/inferencex-e2e/configs/serving/gemma4-speedbench-qualitative.yaml b/inferencex-e2e/configs/serving/gemma4-speedbench-qualitative.yaml new file mode 100644 index 0000000000..d162d9fd3c --- /dev/null +++ b/inferencex-e2e/configs/serving/gemma4-speedbench-qualitative.yaml @@ -0,0 +1,40 @@ +# Pattern: SPEED-Bench acceptance length, thinking off, one category per job (speedbench scenario). +# Gemma-4-31B FP8-block, vLLM, 1 GPU, one request in flight (the SPEED-Bench collectors' CONCURRENCY=1), MTP draft +# depth 6 against the base engine in one run (arms:). Sampling is the golden AL matrices' (temperature 1.0, +# top_p 0.95, 4096 output tokens); the thinking switch is chat_template_kwargs. Without `category:` the run +# is all 880 prompts of the qualitative config (11 categories x 80); `config: throughput_8k` selects the +# entropy tiers at 8k input. The run is one request per row of the selection (no duration): the results carry +# spec_decode.acceptance_length. The prompts are public; the resolved copy is read with GREENLAB_HF_TOKEN. +name: gemma4-speedbench-qualitative +runner: cluster:hcm-hgx02 +image: vllm/vllm-openai:v0.30.0 +model: RedHatAI/gemma-4-31B-it-FP8-block +gpus: 1 +precision: fp8block +env: + NCCL_P2P_LEVEL: NVL + VLLM_DISABLE_COMPILE_CACHE: "1" + VLLM_ATTENTION_BACKEND: FLASHINFER +serve: | + vllm serve RedHatAI/gemma-4-31B-it-FP8-block + --served-model-name RedHatAI/gemma-4-31B-it-FP8-block + --trust-remote-code + --tensor-parallel-size 1 + --gpu-memory-utilization 0.92 + --max-model-len 16384 + --max-num-seqs 64 + --no-enable-prefix-caching + --enable-auto-tool-choice + --tool-call-parser gemma4 + --reasoning-parser gemma4 +arms: + base: {} + mtp6: + append: | + --speculative-config '{"method": "mtp", "model": "google/gemma-4-31B-it-assistant", "num_speculative_tokens": 6}' +scenario: speedbench +overrides: + category: coding + chat_template_kwargs: + enable_thinking: false +conc: [1] diff --git a/inferencex-e2e/docs/serving/ARCHITECTURE.md b/inferencex-e2e/docs/serving/ARCHITECTURE.md index 7f07e9f81d..6bdfa930b5 100644 --- a/inferencex-e2e/docs/serving/ARCHITECTURE.md +++ b/inferencex-e2e/docs/serving/ARCHITECTURE.md @@ -214,7 +214,7 @@ that several nodes need belong in a registry, where every node caches its own sq ## 6. Scenarios -Seven base files in `benchmarks/single_node/serving/scenarios/`: +Eight base files in `benchmarks/single_node/serving/scenarios/`: | Family | Workload | Dataset | |---|---|---| @@ -225,6 +225,7 @@ Seven base files in `benchmarks/single_node/serving/scenarios/`: | `hidden-prefix-e3` | prefixes reused without being visible to the router | pinned file, Mooncake trace format | | `synthetic` | generated ISL/OSL with a shared-prefix share | synthetic | | `greenspec` | speculative-decoding bench cut from production chat traffic, one request per row | pinned private file (`Green-Lab/GreenSpec`), `raw_payload` format | +| `speedbench` | SPEED-Bench acceptance-length prompts, one request per row | pinned files (`Green-Lab/speed-bench-prepared`, a private copy of public data), `raw_payload` format | Pinned files are listed in `benchmarks/single_node/serving/datasets/lock.yaml` with their URL and sha256; the client downloads and checks them before the run. @@ -249,6 +250,16 @@ Three things differ from the other families: the aggregate result (`infx/serving/specdec.py`). A point without speculative decoding has no such block. - The runner needs `GREENLAB_HF_TOKEN` in `actions-runner/.env`: the release is not public. +`speedbench` follows the same pattern (no duration, the `spec_decode` block, `GREENLAB_HF_TOKEN`) with its own +keys: `config` (`qualitative`, default, or `throughput_1k` ... `throughput_32k`), `category` (one of +that config's categories; default all), `max_tokens` (default 4096), `temperature` (1.0), `top_p` (0.95), +`chat_template_kwargs` (the thinking switch, e.g. `enable_thinking: false`) and `requests`. The defaults are the +golden AL matrices'. A request is the first turn of a row alone, as the aiperf and vLLM SPEED-Bench loaders send +it. The result label carries every non-default key (`speedbench-qualitative-coding-ctk-enable_thinking-false`), +so runs that differ in thinking mode never share a summary row. The files are the fully resolved copy of +`nvidia/SPEED-Bench`: the public release masks about 60% of the qualitative rows. The prompts are public, so +none of GreenSpec's guard applies: console, server log and results upload as for any other family. + A base arm and a speculative arm are the `arms:` of one serving YAML (below). The `summary` job of `gn-benchmark.yml` prints one table of all points of a run on the run page, with the speedup of each speculative point over the base point of the same scenario and concurrency. diff --git a/inferencex-e2e/infx/serving/client.py b/inferencex-e2e/infx/serving/client.py index 4d28348298..399d15e496 100644 --- a/inferencex-e2e/infx/serving/client.py +++ b/inferencex-e2e/infx/serving/client.py @@ -20,10 +20,11 @@ import urllib.request from collections.abc import Mapping from pathlib import Path +from typing import Any from infx.bench import proc from infx.bench.agentic.venv import Runtime, bootstrap -from infx.serving import greenspec, specdec +from infx.serving import greenspec, specdec, speedbench OFFLINE = ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE") DATASET_PATH = ".dataset_path" @@ -193,6 +194,18 @@ def with_token_source(scenario_yaml: str, source: str) -> str: return yaml.safe_dump(doc, sort_keys=False) +def selection_family(env: Mapping[str, str]) -> tuple[Any, Any] | None: + """The (module, Settings) of the selection family the job's serving YAML runs, else None. + + greenspec and speedbench replay a selection of a pinned file, one request per row, and report + acceptance length. Only greenspec's file is internal: it alone goes through ``guard``. + """ + for module in (greenspec, speedbench): + if (chosen := module.settings(env)) is not None: + return module, chosen + return None + + def with_requests(scenario_yaml: str, requests: int) -> str: """The scenario with its profiling phase set to ``requests`` requests (the size of the selection).""" import yaml # in the AgentX venv @@ -222,15 +235,16 @@ def prepare(env: Mapping[str, str], runtime: Runtime) -> int: try: path = fetch(env["SERVING_DATASET"]) - if chosen := greenspec.settings(env): + if found := selection_family(env): + module, chosen = found selection = Path(SELECTION_DIR) / "selection.jsonl" - count = greenspec.write_selection(path, chosen, served_model(env), selection) + count = module.write_selection(path, chosen, served_model(env), selection) print( - f"GreenSpec selection {greenspec.label(chosen)}: {count} requests", flush=True + f"{module.TAG} selection {module.label(chosen)}: {count} requests", flush=True ) path = selection (result_dir / DATASET_PATH).write_text(str(path)) - except (DatasetError, greenspec.GreenSpecError) as error: + except (DatasetError, greenspec.GreenSpecError, speedbench.SpeedBenchError) as error: print(f"ERROR {error}", file=sys.stderr) return 1 if url := env.get("SERVING_REMOTE_URL"): @@ -265,7 +279,9 @@ def run(env: Mapping[str, str], runtime: Runtime) -> int: scenario = result_dir / "scenario.yaml" token_marker = result_dir / COUNT_SOURCE_MARKER source = token_marker.read_text().strip() if token_marker.exists() else None - chosen = greenspec.settings(env) + found = selection_family(env) + chosen = found[1] if found else None + internal = found is not None and found[0] is greenspec scenario_yaml = with_token_source(env["SERVING_SCENARIO_YAML"], source or "server") child = online(env) | { "AIPERF_DATASET_CONFIGURATION_TIMEOUT": "1800", @@ -290,7 +306,7 @@ def run(env: Mapping[str, str], runtime: Runtime) -> int: metrics_urls = [url for url in env.get("AIPERF_SERVER_METRICS_URLS", "").split(",") if url] if chosen: specdec.save(result_dir / "spec_metrics_before.json", specdec.snapshot(metrics_urls)) - replay = greenspec.tee_filtered if chosen else proc.tee + replay = greenspec.tee_filtered if internal else proc.tee replay_rc = replay(argv, result_dir / "benchmark.log", child) if chosen: specdec.save(result_dir / "spec_metrics_after.json", specdec.snapshot(metrics_urls)) @@ -334,7 +350,8 @@ def run(env: Mapping[str, str], runtime: Runtime) -> int: ) else: specdec.merge(aggregate, spec) - guard_rc = guard(env, result_dir, aggregate, output_dir) + if internal: + guard_rc = guard(env, result_dir, aggregate, output_dir) return replay_rc or aggregate_rc or validate_rc or guard_rc diff --git a/inferencex-e2e/infx/serving/greenspec.py b/inferencex-e2e/infx/serving/greenspec.py index f4ceac121f..9e6bc1c6ab 100644 --- a/inferencex-e2e/infx/serving/greenspec.py +++ b/inferencex-e2e/infx/serving/greenspec.py @@ -31,6 +31,7 @@ from typing import Any FAMILY = "greenspec" +TAG = "GreenSpec" # how the family names itself in the job log META_KEYS = ("question_id", "category", "bucket", "lite", "isl", "osl", "thinking_src", "recorded") MAX_TOKEN_KEYS = ("max_tokens", "max_completion_tokens") SUBSETS = ("diverse", "agentic_coding", "agentic_tools", "llm_task") diff --git a/inferencex-e2e/infx/serving/scenarios.py b/inferencex-e2e/infx/serving/scenarios.py index c76b23abb1..f3b0c6dc1e 100644 --- a/inferencex-e2e/infx/serving/scenarios.py +++ b/inferencex-e2e/infx/serving/scenarios.py @@ -13,7 +13,7 @@ import yaml -from infx.serving import greenspec +from infx.serving import greenspec, speedbench from infx.serving.datasets import LockEntry, load_lock from infx.serving.slo import SLOError, parse_slo @@ -26,6 +26,7 @@ "hidden-prefix-e3", "synthetic", "greenspec", + "speedbench", ) OVERRIDE_KEYS = ( "duration", @@ -239,6 +240,31 @@ def _greenspec( return Rendered(doc, greenspec.label(chosen), "file", name, False, 0.0) +def _speedbench( + overrides: dict[str, Any], + point: Point | None, + search: bool, + slo: str | None, + lock: dict[str, LockEntry], +) -> Rendered: + """The speedbench scenario with its selection; the run length is the selection, so there is no duration.""" + if point is not None or search or slo: + raise ScenarioError("workload/slo: only for scenario synthetic, not speedbench") + try: + chosen = speedbench.normalize(overrides) + except speedbench.SpeedBenchError as error: + raise ScenarioError(str(error)) from None + if chosen.dataset not in lock: + raise ScenarioError( + f"scenario: dataset {chosen.dataset} of speedbench is not in the dataset lock" + ) + doc = load_base("speedbench") + doc["benchmark"]["datasets"][0]["name"] = chosen.dataset + if chosen.requests: + _profiling(doc)["requests"] = chosen.requests + return Rendered(doc, speedbench.label(chosen), "file", chosen.dataset, False, 0.0) + + def render( family: str, overrides: dict[str, Any] | None = None, @@ -252,6 +278,8 @@ def render( lock = load_lock() if lock is None else lock if family == "greenspec": return _greenspec(overrides, point, search, slo, lock) + if family == "speedbench": + return _speedbench(overrides, point, search, slo, lock) if family == "synthetic": if point is None: raise ScenarioError("workload: synthetic needs isl, osl and cache") diff --git a/inferencex-e2e/infx/serving/schema.py b/inferencex-e2e/infx/serving/schema.py index 90a7b771f6..9bcb04b1c2 100644 --- a/inferencex-e2e/infx/serving/schema.py +++ b/inferencex-e2e/infx/serving/schema.py @@ -229,6 +229,7 @@ class ServingSpec(BaseModel): "hidden-prefix-e3", "synthetic", "greenspec", + "speedbench", ] overrides: dict[str, Any] = Field(default_factory=dict) workload: Workload | None = None diff --git a/inferencex-e2e/infx/serving/speedbench.py b/inferencex-e2e/infx/serving/speedbench.py new file mode 100644 index 0000000000..6b75ece225 --- /dev/null +++ b/inferencex-e2e/infx/serving/speedbench.py @@ -0,0 +1,222 @@ +"""SPEED-Bench on the serving client: row selection and request payloads. + +SPEED-Bench (nvidia/SPEED-Bench) measures speculative-decoding acceptance on a fixed prompt set. The +public release masks about 60% of its qualitative rows, so the scenario reads the fully resolved copy +``Green-Lab/speed-bench-prepared`` (private; each config is one jsonl, pinned in datasets/lock.yaml). +Like ``greenspec`` it replays one selection through AgentX as a ``raw_payload`` file. SPEED-Bench is a +public set, so unlike greenspec nothing is filtered or withheld: the console, server log and result files +are uploaded as for any other scenario. + +``settings`` the selection and sampling a serving YAML asks for (overrides:). +``write_selection`` select the rows and write each as a ready request body. + +Each request is the first turn of a row alone, as the aiperf and vLLM SPEED-Bench loaders send it; +later turns of a multi-turn row are dropped. Every request streams and completes on its own (no +ignore_eos): acceptance only means something on natural output. Sampling defaults are the golden AL +matrices' (temperature 1.0, top_p 0.95, 4096 output tokens); ``chat_template_kwargs`` carries the +thinking switch of the old per-mode collectors. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from infx.serving.greenspec import ( + iter_rows, +) # one jsonl line per row, same as the greenspec release + +FAMILY = "speedbench" +TAG = "SpeedBench" +QUALITATIVE = "qualitative" +THROUGHPUTS = ( + "throughput_1k", + "throughput_2k", + "throughput_8k", + "throughput_16k", + "throughput_32k", +) +CONFIGS = (QUALITATIVE, *THROUGHPUTS) +CATEGORIES = { + QUALITATIVE: ( + "coding", + "humanities", + "math", + "multilingual", + "qa", + "rag", + "reasoning", + "roleplay", + "stem", + "summarization", + "writing", + ), + **dict.fromkeys(THROUGHPUTS, ("low_entropy", "mixed", "high_entropy")), +} +OVERRIDES = ( + "config", + "category", + "max_tokens", + "temperature", + "top_p", + "chat_template_kwargs", + "requests", +) +DEFAULT_MAX_TOKENS = 4096 +DEFAULT_TEMPERATURE = 1.0 +DEFAULT_TOP_P = 0.95 + + +class SpeedBenchError(ValueError): + """A selection problem, as one line.""" + + +@dataclass(frozen=True) +class Settings: + config: str = QUALITATIVE + category: str | None = None # None: every category of the config + max_tokens: int = DEFAULT_MAX_TOKENS + temperature: float = DEFAULT_TEMPERATURE + top_p: float = DEFAULT_TOP_P + chat_template_kwargs: tuple[ + tuple[str, Any], ... + ] = () # sorted items, so Settings stays hashable + requests: int | None = None # None: the whole selection + + @property + def dataset(self) -> str: + """The name of the config's file in datasets/lock.yaml.""" + return f"speedbench-{self.config.replace('_', '-')}" + + +def _number( + overrides: Mapping[str, Any], key: str, default: float, low: float, high: float +) -> float: + value = overrides.get(key, default) + if isinstance(value, bool) or not isinstance(value, (int, float)) or not low <= value <= high: + raise SpeedBenchError(f"overrides.{key}: a number from {low} to {high}, got {value!r}") + return float(value) + + +def normalize(overrides: Mapping[str, Any]) -> Settings: + """The overrides of a speedbench serving YAML as ``Settings``; one line per problem.""" + if unknown := sorted(set(overrides) - set(OVERRIDES)): + raise SpeedBenchError( + f"overrides: speedbench takes {', '.join(OVERRIDES)}, not {', '.join(unknown)}" + ) + config = overrides.get("config", QUALITATIVE) + if config not in CONFIGS: + raise SpeedBenchError(f"overrides.config: {config!r} is not one of {', '.join(CONFIGS)}") + category = overrides.get("category") + if category is not None and category not in CATEGORIES[config]: + raise SpeedBenchError( + f"overrides.category: {category!r} is not one of {', '.join(CATEGORIES[config])} " + f"(the categories of {config})" + ) + max_tokens, requests = ( + overrides.get("max_tokens", DEFAULT_MAX_TOKENS), + overrides.get("requests"), + ) + for field, value in (("max_tokens", max_tokens), ("requests", requests)): + if value is not None and ( + isinstance(value, bool) or not isinstance(value, int) or value < 1 + ): + raise SpeedBenchError(f"overrides.{field}: a positive integer, got {value!r}") + kwargs = overrides.get("chat_template_kwargs") or {} + if not isinstance(kwargs, dict) or not all(isinstance(k, str) for k in kwargs): + raise SpeedBenchError( + f"overrides.chat_template_kwargs: a mapping with string keys, got {kwargs!r}" + ) + return Settings( + config, + category, + max_tokens, + _number(overrides, "temperature", DEFAULT_TEMPERATURE, 0.0, 2.0), + _number(overrides, "top_p", DEFAULT_TOP_P, 0.0, 1.0), + tuple(sorted(kwargs.items())), + requests, + ) + + +def settings(env: Mapping[str, str]) -> Settings | None: + """The job's ``Settings`` when its serving YAML runs the speedbench scenario, else None.""" + import yaml # not imported at the top: the client's first pass runs before the AgentX venv exists + + try: + raw = yaml.safe_load(env.get("SERVING_YAML") or "") + except yaml.YAMLError: + return None + if not isinstance(raw, dict) or raw.get("scenario") != FAMILY: + return None + return normalize(raw.get("overrides") or {}) + + +def label(chosen: Settings) -> str: + """The result label of a selection: speedbench-qualitative, speedbench-throughput_8k-mixed-n20, ... + + Every non-default knob is in it (ctk-enable_thinking-false, mt2048, t0.6, p1): the run summary lines up + results by label and concurrency. + """ + parts = [FAMILY, chosen.config] + if chosen.category: + parts.append(chosen.category) + for key, value in chosen.chat_template_kwargs: + parts.append( + "ctk-" + re.sub(r"[^0-9A-Za-z_.]+", "-", f"{key}-{json.dumps(value)}").strip("-") + ) + if chosen.max_tokens != DEFAULT_MAX_TOKENS: + parts.append(f"mt{chosen.max_tokens}") + if chosen.temperature != DEFAULT_TEMPERATURE: + parts.append(f"t{chosen.temperature:g}") + if chosen.top_p != DEFAULT_TOP_P: + parts.append(f"p{chosen.top_p:g}") + if chosen.requests: + parts.append(f"n{chosen.requests}") + return "-".join(parts) + + +def prompt(row: dict[str, Any]) -> str: + """The first turn of a row, stripped; empty when the row has none (such a row is skipped).""" + messages = row.get("messages") + if not isinstance(messages, list) or not messages or not isinstance(messages[0], dict): + return "" + return str(messages[0].get("content") or "").strip() + + +def select(path: Path, chosen: Settings) -> list[dict[str, Any]]: + return [ + row + for row in iter_rows(path) + if (chosen.category is None or row.get("category") == chosen.category) and prompt(row) + ] + + +def payload(row: dict[str, Any], model: str, chosen: Settings) -> dict[str, Any]: + """One row -> the request body the client sends for it.""" + body: dict[str, Any] = { + "model": model, + "messages": [{"role": "user", "content": prompt(row)}], + "max_tokens": chosen.max_tokens, + "temperature": chosen.temperature, + "top_p": chosen.top_p, + } + if chosen.chat_template_kwargs: + body["chat_template_kwargs"] = dict(chosen.chat_template_kwargs) + return {**body, "stream": True, "stream_options": {"include_usage": True}} + + +def write_selection(dataset: Path, chosen: Settings, model: str, out: Path) -> int: + """Write the selected rows as one request body per line to ``out``; return how many.""" + rows = select(dataset, chosen) + if not rows: + raise SpeedBenchError(f"selection {label(chosen)}: no row of {chosen.config} matches") + out.parent.mkdir(parents=True, exist_ok=True) + with open(out, "w", encoding="utf-8") as f: + f.writelines( + json.dumps(payload(row, model, chosen), ensure_ascii=False) + "\n" for row in rows + ) + return len(rows) diff --git a/inferencex-e2e/infx/tests/serving/test_scenarios.py b/inferencex-e2e/infx/tests/serving/test_scenarios.py index 83836cac40..3ef9177976 100644 --- a/inferencex-e2e/infx/tests/serving/test_scenarios.py +++ b/inferencex-e2e/infx/tests/serving/test_scenarios.py @@ -108,6 +108,30 @@ def test_bad_greenspec_variants_fail_with_one_line(args, message): render(*args) +def test_speedbench_renders_the_config_dataset_and_has_no_duration(): + rendered = render("speedbench", {"config": "throughput_8k", "category": "mixed", "requests": 30}) + dataset = rendered.doc["benchmark"]["datasets"][0] + assert (rendered.kind, rendered.dataset, rendered.label) == ( + "file", "speedbench-throughput-8k", "speedbench-throughput_8k-mixed-n30") + assert dataset["name"] == "speedbench-throughput-8k" + assert dataset["format"] == "raw_payload" and dataset["path"] == "${IX_CUSTOM_DATASET_FILE}" + [phase] = rendered.doc["benchmark"]["phases"] + assert phase["requests"] == 30 and "duration" not in phase and rendered.duration == 0.0 + assert "ignore_eos" not in rendered.doc["benchmark"]["endpoint"].get("extra", {}) + default = render("speedbench") + assert (default.label, default.dataset) == ("speedbench-qualitative", "speedbench-qualitative") + + +@pytest.mark.parametrize("args, message", [ + (("speedbench", {"category": "mixed"}), "overrides.category"), + (("speedbench", {"subset": "diverse"}), "speedbench takes"), + (("speedbench", {}, Point("8k", "1k", 0)), "workload/slo: only for scenario synthetic, not speedbench"), +]) +def test_bad_speedbench_variants_fail_with_one_line(args, message): + with pytest.raises(ScenarioError, match=message): + render(*args) + + def test_warmup_requests_per_lane_override(): def profiling(doc): return next(p for p in doc["benchmark"]["phases"] if p.get("kind") == "profiling") diff --git a/inferencex-e2e/infx/tests/serving/test_speedbench.py b/inferencex-e2e/infx/tests/serving/test_speedbench.py new file mode 100644 index 0000000000..1e9db97b7a --- /dev/null +++ b/inferencex-e2e/infx/tests/serving/test_speedbench.py @@ -0,0 +1,127 @@ +import json +from pathlib import Path + +import pytest + +from infx.serving import client +from infx.serving.speedbench import ( + Settings, + SpeedBenchError, + label, + normalize, + payload, + prompt, + select, + settings, + write_selection, +) + + +def _row(qid: str, category: str, *turns: str) -> dict: + return {"question_id": qid, "category": category, "messages": [{"role": "user", "content": t} for t in turns]} + + +ROWS = [ + _row("a" * 32, "coding", " write fizzbuzz ", "now in Rust"), + _row("b" * 32, "coding", "reverse a list"), + _row("c" * 32, "math", "integrate x"), + _row("d" * 32, "math", " "), # empty first turn: skipped, as the aiperf loader does + {"question_id": "e" * 32, "category": "math", "messages": []}, +] + + +@pytest.fixture +def dataset(tmp_path: Path) -> Path: + path = tmp_path / "qualitative.jsonl" + path.write_text("".join(json.dumps(r) + "\n" for r in ROWS)) + return path + + +@pytest.mark.parametrize("chosen, expected", [ + (Settings(), ["a" * 32, "b" * 32, "c" * 32]), + (Settings(category="coding"), ["a" * 32, "b" * 32]), + (Settings(category="math"), ["c" * 32]), + (Settings(category="qa"), []), +]) +def test_select_skips_rows_without_a_first_turn(dataset, chosen, expected): + assert [r["question_id"] for r in select(dataset, chosen)] == expected + + +def test_prompt_is_the_stripped_first_turn_only(): + assert prompt(ROWS[0]) == "write fizzbuzz" + assert prompt({"messages": None}) == "" and prompt({}) == "" and prompt({"messages": ["x"]}) == "" + + +def test_payload_defaults_match_the_golden_matrices(): + body = payload(ROWS[0], "m", Settings()) + assert body == { + "model": "m", "messages": [{"role": "user", "content": "write fizzbuzz"}], + "max_tokens": 4096, "temperature": 1.0, "top_p": 0.95, + "stream": True, "stream_options": {"include_usage": True}, + } + assert "ignore_eos" not in body + + +def test_payload_carries_the_thinking_switch(): + chosen = normalize({"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 2048, "temperature": 0}) + body = payload(ROWS[1], "m", chosen) + assert body["chat_template_kwargs"] == {"enable_thinking": False} + assert (body["max_tokens"], body["temperature"]) == (2048, 0.0) + + +def test_write_selection(dataset, tmp_path): + out = tmp_path / "sel" / "selection.jsonl" + assert write_selection(dataset, Settings(category="coding"), "m", out) == 2 + lines = [json.loads(line) for line in out.read_text().splitlines()] + assert [line["messages"][0]["content"] for line in lines] == ["write fizzbuzz", "reverse a list"] + with pytest.raises(SpeedBenchError, match="no row of qualitative matches"): + write_selection(dataset, Settings(category="qa"), "m", out) + + +@pytest.mark.parametrize("overrides, expected", [ + ({}, "speedbench-qualitative"), + ({"category": "coding"}, "speedbench-qualitative-coding"), + ({"config": "throughput_8k", "category": "mixed", "requests": 20}, "speedbench-throughput_8k-mixed-n20"), + ({"chat_template_kwargs": {"enable_thinking": False}, "max_tokens": 1024}, + "speedbench-qualitative-ctk-enable_thinking-false-mt1024"), + ({"temperature": 0.6, "top_p": 1}, "speedbench-qualitative-t0.6-p1"), +]) +def test_label_names_every_non_default_knob(overrides, expected): + assert label(normalize(overrides)) == expected + + +def test_dataset_name_is_the_lock_key(): + assert Settings().dataset == "speedbench-qualitative" + assert Settings(config="throughput_16k").dataset == "speedbench-throughput-16k" + + +@pytest.mark.parametrize("overrides, message", [ + ({"config": "throughput_4k"}, "overrides.config"), + ({"category": "mixed"}, "overrides.category"), # a throughput category on qualitative + ({"config": "throughput_2k", "category": "coding"}, "overrides.category"), + ({"max_tokens": 0}, "overrides.max_tokens"), + ({"requests": True}, "overrides.requests"), + ({"temperature": -1}, "overrides.temperature"), + ({"top_p": 1.5}, "overrides.top_p"), + ({"top_p": True}, "overrides.top_p"), + ({"chat_template_kwargs": ["x"]}, "overrides.chat_template_kwargs"), + ({"subset": "diverse"}, "speedbench takes"), +]) +def test_bad_overrides(overrides, message): + with pytest.raises(SpeedBenchError, match=message): + normalize(overrides) + + +def test_settings_only_for_a_speedbench_yaml(): + yaml_text = "scenario: speedbench\noverrides:\n category: math\n" + assert settings({"SERVING_YAML": yaml_text}) == Settings(category="math") + assert settings({"SERVING_YAML": "scenario: greenspec\n"}) is None + assert settings({"SERVING_YAML": ": : ["}) is None and settings({}) is None + + +def test_selection_family_picks_the_module_of_the_yaml(): + from infx.serving import greenspec, speedbench + + assert client.selection_family({"SERVING_YAML": "scenario: speedbench\n"})[0] is speedbench + assert client.selection_family({"SERVING_YAML": "scenario: greenspec\n"})[0] is greenspec + assert client.selection_family({"SERVING_YAML": "scenario: agentx\n"}) is None