diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 4119913b1fa4..1a0cac76f959 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -872,6 +872,11 @@ def __init__( self.model_path = "" self.dataset_file = client_config_data.get("dataset_file", "") self.use_nv_sa_benchmark = client_config_data.get("use_nv_sa_benchmark", False) + # Derived from lane identity only (gen_only + concurrency == 1); do not + # set this from lane YAML. warmup is intentionally not a baseline match + # key, which is only sound while its value stays fully determined by + # benchmark_mode and concurrency. + self.warmup = client_config_data.get("warmup", False) self.env_vars = env_vars # spec_decoding flag is retained for DB matching (b_eos column). --ignore-eos # is now always passed; output-length stability with spec decoding comes from @@ -950,11 +955,15 @@ def _to_default_benchmark_cmd(self) -> List[str]: str(self.concurrency * self.iterations), "--max-concurrency", str(self.concurrency), - "--no-test-input", "--percentile-metrics", "ttft,tpot,itl,e2el", "--ignore-eos", ] + # benchmark_serving's initial single-prompt test run (excluded from + # metrics) doubles as a warmup request; keep it disabled unless the + # lane requests one. + if not self.warmup: + benchmark_cmd.append("--no-test-input") if dataset_path: benchmark_cmd.append("--dataset-name") benchmark_cmd.append("trtllm_custom") @@ -1012,6 +1021,7 @@ def to_db_data(self) -> dict: "b_trust_remote_code": self.trust_remote_code, "b_use_nv_sa_benchmark": self.use_nv_sa_benchmark, "b_eos": self.spec_decoding, + "b_warmup": self.warmup, "s_client_log_link": "", "s_client_env_vars": self.env_vars, } @@ -1978,6 +1988,15 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): "use_nv_sa_benchmark": use_nv_sa_benchmark, "accuracy_config": accuracy_data, "only_run_accuracy": only_run_accuracy, + # gen_only measures a single round (iterations forced to 1 + # above), so one-time costs like the cache transceiver's lazy + # connection setup would otherwise land entirely on the + # measured TTFT. Scoped to concurrency == 1: the gen executor's + # fill gate (TLLM_BENCHMARK_REQ_QUEUES_SIZE) only opens once + # `concurrency` requests are queued, so a lone warmup request + # would deadlock higher-concurrency lanes — which amortize the + # cold start anyway. + "warmup": benchmark_mode == "gen_only" and concurrency == 1, } client_config = ClientConfig( client_config_data,