diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index cd782b4a98..e5d56b3214 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -9,14 +9,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + container: "ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + image: "ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h" frameworks: dynamo: "1.4.0" @@ -79,7 +79,7 @@ backend: enable_offload: false vllm_config: prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" @@ -102,7 +102,7 @@ backend: tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" + moe-backend: "deep_gemm_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: @@ -130,7 +130,7 @@ backend: tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" + moe-backend: "deep_gemm_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] prefill_environment: @@ -207,6 +207,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:prompt_tokens_cached_by_source" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" # Avoid concurrent readers observing a mismatched mmap data/index pair. AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6ab7aee031..0938a6dcd9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8049,7 +8049,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f + image: ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv @@ -8063,108 +8063,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - - spec-decoding: mtp - conc-list: [1152] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 12 - ep: 12 - dp-attn: true - - spec-decoding: mtp - conc-list: [1024] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: mtp - conc-list: [256] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: mtp - conc-list: [512] - router: { name: dynamo-router, version: "1.4.0" } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: mtp - conc-list: [4] - router: { name: dynamo-router, version: "1.4.0" } - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [128] - router: { name: dynamo-router, version: "1.4.0" } - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - spec-decoding: mtp conc-list: [256] router: { name: dynamo-router, version: "1.4.0" } @@ -8482,7 +8380,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 8 @@ -8499,7 +8397,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 6 tp: 4 @@ -8516,7 +8414,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 32 @@ -8533,7 +8431,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 32 @@ -8550,7 +8448,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -8567,7 +8465,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -9337,7 +9235,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: dp-attn: false additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" # GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. # Six topology variants use NIXL KV transfer and MTP3/MTP5 decoding. glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: @@ -9366,7 +9264,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 4 @@ -9383,7 +9281,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 8 @@ -9400,7 +9298,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 4 @@ -9417,7 +9315,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -9434,7 +9332,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -9451,7 +9349,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 8 diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 6db5d351e8..833a411b77 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -286,6 +286,8 @@ Topology rules: - Disaggregated: inspect every prefill backend, every decode backend, and the frontend/router. A healthy decode pool does not prove prefill/KV transfer health. - Confirm the AIPerf command includes all `AIPERF_SERVER_METRICS_URLS`. Missing endpoints produce falsely healthy partial evidence. +On GB300's pinned srt-slurm v1.0.36 custom AgentX path, set `benchmark.env.AIPERF_REQUIRED_SERVER_METRIC_PREFIX` to opt into worker endpoint discovery and require that metric prefix in the exported artifacts. The launcher applies a scoped backport; without the opt-in, the custom command otherwise discovers only the frontend. + Read each endpoint directly when summaries are ambiguous: ```bash diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 6e8c8e03c8..11c97bab6f 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -286,6 +286,8 @@ rg -n -i 'Phase |warmup|profiling|returned=|in_flight=|queue=|kv_usage=|prefix_c - Disaggregated：检查每个 prefill backend、每个 decode backend 以及 frontend/router。Decode pool 健康不能证明 prefill/KV transfer 健康。 - 确认 AIPerf 命令包含所有 `AIPERF_SERVER_METRICS_URLS`；缺少 endpoint 会产生片面而虚假的健康证据。 +在 GB300 固定使用的 srt-slurm v1.0.36 自定义 AgentX 路径中，设置 `benchmark.env.AIPERF_REQUIRED_SERVER_METRIC_PREFIX` 可启用 worker endpoint 发现，并要求导出的 artifact 包含该指标前缀。Launcher 会应用限定范围的兼容补丁；未启用时，自定义命令默认只发现 frontend。 + Summary 不明确时直接读取每个 endpoint： ```bash diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 53a2ba7e75..0fa0f4c34f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6831,3 +6831,86 @@ - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Validate source-labeled prompt-token cache metrics on the proven GB300 vLLM disaggregated AgentX topology at concurrency 256." + - "Use one TP4 prefill worker and one TP16 decode worker with NIXL KV transfer; transferred prompt-token hits must export source=p2p." + - "Run the Python-only metric overlay built from official vLLM nightly 7c5dc571 and the HIT_PENDING fix from vLLM PR #53087." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Use the validation image's upstream deep_gemm_mega_moe backend for both prefill and decode workers; the older recipe's deep_gemm_amxf4_mega_moe backend name is not supported by this image." + - "Preserve the proven DEP4/DEP16 topology and single concurrency-256 AgentX point with its eval companion." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Collect every prefill and decode worker's Prometheus endpoint in the custom AgentX command using srt-slurm's existing endpoint discovery." + - "Require the cached-token-by-source metric in exported artifacts; router-only metrics must not pass validation. Preserve the image, topology, concurrency, and eval coverage." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Pin the AgentX client fix that tags automatic warmup baseline scrapes as warmup, preventing warmup counter growth from entering profiling-only token-source totals." + - "Preserve the proven DEP4/DEP16 topology, concurrency 256, patched vLLM image, full eval, and canonical warmup and one-hour profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Backport vLLM PR #46906 so Mooncake's external store retains sliding-window prefix checkpoints independently of HBM retention." + - "Use the Python-only token-source validation image b7fc8985 while preserving the proven DEP4/DEP16 topology, concurrency 256, full eval, and canonical warmup and one-hour profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Use the 4d3c4a23 Python-only validation image with upstream PR #50388 hybrid KV-load recovery, so a missing Mooncake object does not crash DeepSeek-V4's multi-group scheduler." + - "Keep failed loads out of cached-token source totals and include the async-load queue-progress fix already validated by the H100 offload sweep." + - "Preserve the existing DEP4/DEP16 topology, single concurrency-256 point, real model eval, canonical warmup, and one-hour profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Enable recompute for the prefill MultiConnector's optional Mooncake cache loads, using the validation image's tested hybrid recovery when a key disappears after lookup." + - "Keep decode-side NIXL transfer failures fail-closed, with unchanged topology, concurrency, eval, warmup, and profiling duration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Standardize all remaining GB300 partition overrides in this validation branch on batch_1, including the DeepSeek-V4 and GLM-5.2 TensorRT-LLM configs. Preserve historical changelog entries." + - "Keep the official validation sweep limited to the existing vLLM P/D concurrency-256 point and its full evaluation; its launcher already defaults to batch_1. No image, topology, warmup, or profiling changes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Use the 0d8739d0 Python-only validation image with upstream PR #54853 complete connector block tables and PR #54643 rejected Mooncake-load handling, fixing the missing-current-block-table assertion observed in the P/D warmup." + - "Preserve the metric implementation, DEP4/DEP16 topology, single concurrency-256 point, full evaluation, canonical warmup, and one-hour profile." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 7b5f70380e..68fa3c3b38 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -299,6 +299,9 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then # Mooncake compatibility. Keep it pinned so sweeps are reproducible. git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 + # v1.0.36 omits worker metrics from custom AgentX commands. Opt in only + # for recipes requiring backend metrics, using its existing URL discovery. + git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-custom-metrics.patch" || exit 1 mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ diff --git a/runners/patches/srt-slurm-custom-metrics.patch b/runners/patches/srt-slurm-custom-metrics.patch new file mode 100644 index 0000000000..9fb23d6c8f --- /dev/null +++ b/runners/patches/srt-slurm-custom-metrics.patch @@ -0,0 +1,16 @@ +diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py +--- a/src/srtctl/cli/mixins/benchmark_stage.py ++++ b/src/srtctl/cli/mixins/benchmark_stage.py +@@ -468,7 +468,10 @@ +- # Add AIPerf-specific env vars for AIPerf-driven benchmarks only +- if isinstance(runner, AIPerfBenchmarkRunner): ++ # Custom AgentX commands opt into worker metrics through their artifact contract. ++ if isinstance(runner, AIPerfBenchmarkRunner) or ( ++ runner.name == "Custom" ++ and self.config.benchmark.env.get("AIPERF_REQUIRED_SERVER_METRIC_PREFIX") ++ ): + env.update(self._get_aiperf_server_metrics_env()) + if self.config.benchmark.aiperf_package: + env["AIPERF_PACKAGE"] = self.config.benchmark.aiperf_package + + return env diff --git a/runners/test_srt_custom_metrics_patch.py b/runners/test_srt_custom_metrics_patch.py new file mode 100644 index 0000000000..4d6d843c64 --- /dev/null +++ b/runners/test_srt_custom_metrics_patch.py @@ -0,0 +1,64 @@ +import runpy +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +PATCH = Path(__file__).parent / "patches/srt-slurm-custom-metrics.patch" +TARGET = "src/srtctl/cli/mixins/benchmark_stage.py" +ORIGINAL = '''class Stage: + def get_env(self, runner): + env = {} + # Add AIPerf-specific env vars for AIPerf-driven benchmarks only + if isinstance(runner, AIPerfBenchmarkRunner): + env.update(self._get_aiperf_server_metrics_env()) + if self.config.benchmark.aiperf_package: + env["AIPERF_PACKAGE"] = self.config.benchmark.aiperf_package + + return env +''' + + +class AIPerfRunner: + name = "AIPerf" + + +@pytest.mark.parametrize( + "runner,required,expected", + [ + (AIPerfRunner(), "", True), + (SimpleNamespace(name="Custom"), "vllm:prompt_tokens_cached_by_source", True), + (SimpleNamespace(name="Custom"), "", False), + (SimpleNamespace(name="Other"), "vllm:", False), + ], +) +def test_custom_metrics_opt_in(tmp_path, runner, required, expected): + target = tmp_path / TARGET + target.parent.mkdir(parents=True) + target.write_text(ORIGINAL) + subprocess.run(["git", "apply", str(PATCH)], cwd=tmp_path, check=True) + symbols = runpy.run_path( + str(target), init_globals={"AIPerfBenchmarkRunner": AIPerfRunner} + ) + stage = symbols["Stage"]() + stage.config = SimpleNamespace( + benchmark=SimpleNamespace( + env={"AIPERF_REQUIRED_SERVER_METRIC_PREFIX": required}, + aiperf_package=None, + ) + ) + urls = "http://prefill:7500/metrics,http://decode:7501/metrics" + stage._get_aiperf_server_metrics_env = lambda: {"AIPERF_SERVER_METRICS_URLS": urls} + assert stage.get_env(runner) == ( + {"AIPERF_SERVER_METRICS_URLS": urls} if expected else {} + ) + + +def test_custom_metrics_patch_rejects_unexpected_source(tmp_path): + target = tmp_path / TARGET + target.parent.mkdir(parents=True) + target.write_text("# Different upstream implementation\n") + result = subprocess.run(["git", "apply", str(PATCH)], cwd=tmp_path, capture_output=True) + assert result.returncode != 0 + assert target.read_text() == "# Different upstream implementation\n" diff --git a/utils/aiperf b/utils/aiperf index 754356e9a3..dcbd942654 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 754356e9a39acc6cc6afb242d123bb57c3fb6f75 +Subproject commit dcbd942654ef87b357c7722837855dbccedc2353