From b3f8ba49fc5455c8e341c8bae362d4f707cae2fd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 10:20:07 -0500 Subject: [PATCH 01/17] Add H100 MiniMax-M3 disaggregated source validation --- .../disagg-1p1d-tp8-tp8-c8-agentic.yaml | 128 ++++++++++++++++++ configs/nvidia-master.yaml | 27 ++++ runners/launch_h100-dgxc-slurm.sh | 33 ++++- .../test_generate_sweep_configs.py | 4 +- 4 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml new file mode 100644 index 0000000000..171eacf265 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml @@ -0,0 +1,128 @@ +name: "minimax-m3-vllm-disagg-h100-1p1d-tp8-tp8-c8-fp8-agentic" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +slurm: + time_limit: "5:00:00" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + env: + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: "vllm" + connector: null + kv_events_config: true + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_LOG_STATS_INTERVAL: "1" + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + + decode_environment: *worker-environment + + vllm_config: + prefill: &worker-config + served-model-name: "MiniMaxAI/MiniMax-M3-MXFP8" + tensor-parallel-size: 8 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + cpu-offload-gb: 26 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + attention-backend: "TRITON_ATTN" + max-num-seqs: 16 + max-cudagraph-capture-size: 32 + max-num-batched-tokens: 16384 + safetensors-load-strategy: "lazy" + enable-prompt-tokens-details: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: *worker-config + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: "custom" + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 481db4fcef..bbef99b0db 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7356,6 +7356,33 @@ minimaxm3-fp8-h100-vllm-agentic-mtp: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] } +minimaxm3-fp8-h100-dynamo-vllm-agentic-disagg: + image: ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: cluster:h100-dgxc + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.4.0.dev20260730" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: none + kv-offloading: none + conc-list: [8] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml" + decode: { num-worker: 1, tp: 8, ep: 1, dp-attn: false } + minimaxm3-fp8-h200-vllm-agentic-mtp: image: vllm/vllm-openai:v0.27.1 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 50c6a8ad1f..eebbd58f68 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -4,6 +4,8 @@ set -e # System-specific configuration for H100 DGXC Slurm cluster SLURM_PARTITION="hpc-gpu-1" SLURM_ACCOUNT="customer" +VLLM_AGENTIC_SRT_TAG="v1.0.50" +VLLM_AGENTIC_SRT_PIN="e4019633c9e2bc25f38c44b81edf52bb0504d937" # Route spec-decoding=mtp configs to the _mtp benchmark script (parity with # the h200 launchers, which have carried SPEC_SUFFIX since #392). @@ -33,8 +35,16 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" exit 1 fi + elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then + if [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/models--MiniMaxAI--MiniMax-M3-MXFP8/snapshots/c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" + else + echo "Unsupported model prefix/precision for dynamo-vllm: $MODEL_PREFIX/$PRECISION" + exit 1 + fi else - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, dynamo-vllm" exit 1 fi @@ -45,8 +55,19 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi + if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then + git clone --branch "$VLLM_AGENTIC_SRT_TAG" --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + test "$(git rev-parse HEAD)" = "$VLLM_AGENTIC_SRT_PIN" || { + echo "Error: srt-slurm $VLLM_AGENTIC_SRT_TAG resolved to an unexpected commit" >&2 + exit 1 + } + mkdir -p recipes/vllm/minimax-m3/h100-fp8/agentic + cp -rT \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic" \ + recipes/vllm/minimax-m3/h100-fp8/agentic # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then + elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" else @@ -93,6 +114,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') SQUASH_FILE="/mnt/nfs/sa-shared/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" + elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then + CONTAINER_KEY="$IMAGE" + SQUASH_FILE="/mnt/nfs/lustre/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + if ! unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then + echo "Error: expected pre-imported vLLM image at $SQUASH_FILE" >&2 + exit 1 + fi fi export ISL="$ISL" @@ -120,6 +148,7 @@ model_paths: containers: dynamo-trtllm: "${SQUASH_FILE}" dynamo-sglang: "${SQUASH_FILE}" + dynamo-vllm: "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" latest: "${SQUASH_FILE}" "${CONTAINER_KEY}": "${SQUASH_FILE}" diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 520e4105e0..ae70ccffbf 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2359,9 +2359,9 @@ def test_kimi_minimax_trimmed_eval_matrix_covers_current_configs( json.dumps(manifest, separators=(',', ':')).encode() ).hexdigest() - assert len(manifest) == 63 + assert len(manifest) == 64 assert manifest_digest == ( - '1630cdd6fbf77302ee3286b118710576aa090e1502b3e5ec482b7f537a3f1132' + '06d825174c26317f0299837a535d50352a1fd871fb655f5bc4136cf5a0f3cd8a' ), manifest_digest for row in rows: if isinstance(row['conc'], list): From cf67f7d9c6bf48ec050fbe6bc70cbbe331286d41 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 10:20:47 -0500 Subject: [PATCH 02/17] Schedule H100 disaggregated source validation --- perf-changelog.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3dd2322a81..0b44fb5ee8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6741,3 +6741,14 @@ - "Configure model-specific tool and reasoning parsers for the Kimi-K3 and MiniMax-M3 serving paths so opt-in verifier requests produce structured OpenAI tool calls." - "Limit shared AgentX readiness and synthetic-acceptance rewrites to eval-only runs so default throughput recipe rendering remains unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - minimaxm3-fp8-h100-dynamo-vllm-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add one H100 MiniMax-M3 MXFP8 AgentX disaggregated validation point at concurrency 8 with one TP8 prefill worker and one TP8 decode worker." + - "Transfer KV directly with NixlConnector so the patched vLLM image must export transferred cached prompt tokens with source=p2p." + - "Pin NVIDIA/srt-slurm v1.0.50 and collect the Prometheus endpoints of both logical vLLM workers through AIPerf." + - "Use ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h, matching the DRAM, NVMe, and tiered validation sweep in PR #2796." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From 572b08768e44beac521f997ea4fcb4a1676d7c41 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 10:31:53 -0500 Subject: [PATCH 03/17] Allow cold H100 disaggregated container startup --- runners/launch_h100-dgxc-slurm.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index eebbd58f68..5412b4296b 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -62,6 +62,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Error: srt-slurm $VLLM_AGENTIC_SRT_TAG resolved to an unexpected commit" >&2 exit 1 } + # The 18 GB validation image can take more than srt-slurm's default + # five-minute window to unpack on a cold H100 node. + sed -i 's/timeout=300/timeout=900/g' src/srtctl/cli/do_sweep.py mkdir -p recipes/vllm/minimax-m3/h100-fp8/agentic cp -rT \ "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic" \ From 0bb3a24c210677e81ce2ea826bcf6055b2f6c8fa Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 10:47:23 -0500 Subject: [PATCH 04/17] Revert "Allow cold H100 disaggregated container startup" This reverts commit 572b08768e44beac521f997ea4fcb4a1676d7c41. --- runners/launch_h100-dgxc-slurm.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 5412b4296b..eebbd58f68 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -62,9 +62,6 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Error: srt-slurm $VLLM_AGENTIC_SRT_TAG resolved to an unexpected commit" >&2 exit 1 } - # The 18 GB validation image can take more than srt-slurm's default - # five-minute window to unpack on a cold H100 node. - sed -i 's/timeout=300/timeout=900/g' src/srtctl/cli/do_sweep.py mkdir -p recipes/vllm/minimax-m3/h100-fp8/agentic cp -rT \ "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic" \ From 1f500885e400b407672ad15a0ef5fa812ad7a067 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 10:47:24 -0500 Subject: [PATCH 05/17] Revert "Schedule H100 disaggregated source validation" This reverts commit cf67f7d9c6bf48ec050fbe6bc70cbbe331286d41. --- perf-changelog.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0b44fb5ee8..3dd2322a81 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6741,14 +6741,3 @@ - "Configure model-specific tool and reasoning parsers for the Kimi-K3 and MiniMax-M3 serving paths so opt-in verifier requests produce structured OpenAI tool calls." - "Limit shared AgentX readiness and synthetic-acceptance rewrites to eval-only runs so default throughput recipe rendering remains unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 - -- config-keys: - - minimaxm3-fp8-h100-dynamo-vllm-agentic-disagg - scenario-type: - - agentic-coding - description: - - "Add one H100 MiniMax-M3 MXFP8 AgentX disaggregated validation point at concurrency 8 with one TP8 prefill worker and one TP8 decode worker." - - "Transfer KV directly with NixlConnector so the patched vLLM image must export transferred cached prompt tokens with source=p2p." - - "Pin NVIDIA/srt-slurm v1.0.50 and collect the Prometheus endpoints of both logical vLLM workers through AIPerf." - - "Use ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h, matching the DRAM, NVMe, and tiered validation sweep in PR #2796." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From 4308bee22cd76590938d1af262e106c4197ce547 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 10:47:25 -0500 Subject: [PATCH 06/17] Revert "Add H100 MiniMax-M3 disaggregated source validation" This reverts commit b3f8ba49fc5455c8e341c8bae362d4f707cae2fd. --- .../disagg-1p1d-tp8-tp8-c8-agentic.yaml | 128 ------------------ configs/nvidia-master.yaml | 27 ---- runners/launch_h100-dgxc-slurm.sh | 33 +---- .../test_generate_sweep_configs.py | 4 +- 4 files changed, 4 insertions(+), 188 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml deleted file mode 100644 index 171eacf265..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "minimax-m3-vllm-disagg-h100-1p1d-tp8-tp8-c8-fp8-agentic" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -slurm: - time_limit: "5:00:00" - -resources: - gpu_type: "h100" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-reset-states: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - env: - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: "vllm" - connector: null - kv_events_config: true - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_LOG_STATS_INTERVAL: "1" - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - - decode_environment: *worker-environment - - vllm_config: - prefill: &worker-config - served-model-name: "MiniMaxAI/MiniMax-M3-MXFP8" - tensor-parallel-size: 8 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - cpu-offload-gb: 26 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - attention-backend: "TRITON_ATTN" - max-num-seqs: 16 - max-cudagraph-capture-size: 32 - max-num-batched-tokens: 16384 - safetensors-load-strategy: "lazy" - enable-prompt-tokens-details: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: *worker-config - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: "custom" - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bbef99b0db..481db4fcef 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7356,33 +7356,6 @@ minimaxm3-fp8-h100-vllm-agentic-mtp: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] } -minimaxm3-fp8-h100-dynamo-vllm-agentic-disagg: - image: ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:h100-dgxc - precision: fp8 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.4.0.dev20260730" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: none - kv-offloading: none - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/h100-fp8/agentic/disagg-1p1d-tp8-tp8-c8-agentic.yaml" - decode: { num-worker: 1, tp: 8, ep: 1, dp-attn: false } - minimaxm3-fp8-h200-vllm-agentic-mtp: image: vllm/vllm-openai:v0.27.1 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index eebbd58f68..50c6a8ad1f 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -4,8 +4,6 @@ set -e # System-specific configuration for H100 DGXC Slurm cluster SLURM_PARTITION="hpc-gpu-1" SLURM_ACCOUNT="customer" -VLLM_AGENTIC_SRT_TAG="v1.0.50" -VLLM_AGENTIC_SRT_PIN="e4019633c9e2bc25f38c44b81edf52bb0504d937" # Route spec-decoding=mtp configs to the _mtp benchmark script (parity with # the h200 launchers, which have carried SPEC_SUFFIX since #392). @@ -35,16 +33,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" exit 1 fi - elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then - if [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/models--MiniMaxAI--MiniMax-M3-MXFP8/snapshots/c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" - else - echo "Unsupported model prefix/precision for dynamo-vllm: $MODEL_PREFIX/$PRECISION" - exit 1 - fi else - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, dynamo-vllm" + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" exit 1 fi @@ -55,19 +45,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - git clone --branch "$VLLM_AGENTIC_SRT_TAG" --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - test "$(git rev-parse HEAD)" = "$VLLM_AGENTIC_SRT_PIN" || { - echo "Error: srt-slurm $VLLM_AGENTIC_SRT_TAG resolved to an unexpected commit" >&2 - exit 1 - } - mkdir -p recipes/vllm/minimax-m3/h100-fp8/agentic - cp -rT \ - "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/h100-fp8/agentic" \ - recipes/vllm/minimax-m3/h100-fp8/agentic # TODO(CJQ): make first class upon srt-slurm upstream refactor - elif [[ "$IS_AGENTIC" == "1" ]]; then + if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" else @@ -114,13 +93,6 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') SQUASH_FILE="/mnt/nfs/sa-shared/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" - elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then - CONTAINER_KEY="$IMAGE" - SQUASH_FILE="/mnt/nfs/lustre/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - if ! unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then - echo "Error: expected pre-imported vLLM image at $SQUASH_FILE" >&2 - exit 1 - fi fi export ISL="$ISL" @@ -148,7 +120,6 @@ model_paths: containers: dynamo-trtllm: "${SQUASH_FILE}" dynamo-sglang: "${SQUASH_FILE}" - dynamo-vllm: "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" latest: "${SQUASH_FILE}" "${CONTAINER_KEY}": "${SQUASH_FILE}" diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index ae70ccffbf..520e4105e0 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2359,9 +2359,9 @@ def test_kimi_minimax_trimmed_eval_matrix_covers_current_configs( json.dumps(manifest, separators=(',', ':')).encode() ).hexdigest() - assert len(manifest) == 64 + assert len(manifest) == 63 assert manifest_digest == ( - '06d825174c26317f0299837a535d50352a1fd871fb655f5bc4136cf5a0f3cd8a' + '1630cdd6fbf77302ee3286b118710576aa090e1502b3e5ec482b7f537a3f1132' ), manifest_digest for row in rows: if isinstance(row['conc'], list): From c221f799761e8a7cbe63725d5c3e679cc9506b14 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 10:52:12 -0500 Subject: [PATCH 07/17] Use proven GB300 disaggregated validation point --- configs/nvidia-master.yaml | 104 +------------------------------------ perf-changelog.yaml | 10 ++++ 2 files changed, 11 insertions(+), 103 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 481db4fcef..f1caf39962 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7891,7 +7891,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f + image: ttl.sh/cquil11-vllm-tier-7c5dc571-pr53087-d9b6d498-arm64-20260901:24h model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv @@ -7905,40 +7905,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - - spec-decoding: mtp - conc-list: [1152] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 12 - ep: 12 - dp-attn: true - - spec-decoding: mtp - conc-list: [1024] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - spec-decoding: mtp conc-list: [256] router: { name: dynamo-router, version: "1.3.0.dev20260720" } @@ -7956,74 +7922,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 8 ep: 8 dp-attn: true - - spec-decoding: mtp - conc-list: [512] - router: { name: dynamo-router, version: "1.4.0" } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: mtp - conc-list: [4] - router: { name: dynamo-router, version: "1.4.0" } - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [128] - router: { name: dynamo-router, version: "1.4.0" } - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: mtp - conc-list: [256] - router: { name: dynamo-router, version: "1.4.0" } - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true # GB200 mirror of the dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-{agg,disagg} # tuning above (PR #2571). Any worker that used 4 GPUs on GB300 uses 8 on diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3dd2322a81..c118ed19e7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6741,3 +6741,13 @@ - "Configure model-specific tool and reasoning parsers for the Kimi-K3 and MiniMax-M3 serving paths so opt-in verifier requests produce structured OpenAI tool calls." - "Limit shared AgentX readiness and synthetic-acceptance rewrites to eval-only runs so default throughput recipe rendering remains unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Validate source-labeled prompt-token cache metrics on the proven GB300 vLLM disaggregated AgentX topology at concurrency 256." + - "Use one TP4 prefill worker and one TP8 decode worker with NIXL KV transfer; transferred prompt-token hits must export source=p2p." + - "Run the Python-only metric overlay built from official vLLM nightly 7c5dc571 and the HIT_PENDING fix from vLLM PR #53087." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From 66e7bf46090c50ccddc3242fac7f0a98dd14003d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 11:04:17 -0500 Subject: [PATCH 08/17] Match proven GB300 c256 topology --- configs/nvidia-master.yaml | 8 ++++---- perf-changelog.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f1caf39962..8d6cc41444 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7907,7 +7907,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp conc-list: [256] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 tp: 4 @@ -7916,11 +7916,11 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true # GB200 mirror of the dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-{agg,disagg} diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c118ed19e7..1716f3d055 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6748,6 +6748,6 @@ - agentic-coding description: - "Validate source-labeled prompt-token cache metrics on the proven GB300 vLLM disaggregated AgentX topology at concurrency 256." - - "Use one TP4 prefill worker and one TP8 decode worker with NIXL KV transfer; transferred prompt-token hits must export source=p2p." + - "Use one TP4 prefill worker and one TP16 decode worker with NIXL KV transfer; transferred prompt-token hits must export source=p2p." - "Run the Python-only metric overlay built from official vLLM nightly 7c5dc571 and the HIT_PENDING fix from vLLM PR #53087." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From 3057a73ebbf8382f801a443f86c71d3615977501 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 11:17:40 -0500 Subject: [PATCH 09/17] Pin validation image in GB300 recipe --- .../disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index cd782b4a98..74310eae20 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -9,14 +9,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + container: "ttl.sh/cquil11-vllm-tier-7c5dc571-pr53087-d9b6d498-arm64-20260901:24h" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + image: "ttl.sh/cquil11-vllm-tier-7c5dc571-pr53087-d9b6d498-arm64-20260901:24h" frameworks: dynamo: "1.4.0" From cb93ced68562bbfd28374c11b8f44bdbd166eff6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 11:34:23 -0500 Subject: [PATCH 10/17] fix: align disaggregated validation with upstream MoE backend --- .../disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml | 4 ++-- perf-changelog.yaml | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index 74310eae20..eab3e1d7b7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -102,7 +102,7 @@ backend: tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" + moe-backend: "deep_gemm_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: @@ -130,7 +130,7 @@ backend: tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" + moe-backend: "deep_gemm_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] prefill_environment: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1716f3d055..d810e77915 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6751,3 +6751,12 @@ - "Use one TP4 prefill worker and one TP16 decode worker with NIXL KV transfer; transferred prompt-token hits must export source=p2p." - "Run the Python-only metric overlay built from official vLLM nightly 7c5dc571 and the HIT_PENDING fix from vLLM PR #53087." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Use the validation image's upstream deep_gemm_mega_moe backend for both prefill and decode workers; the older recipe's deep_gemm_amxf4_mega_moe backend name is not supported by this image." + - "Preserve the proven DEP4/DEP16 topology and single concurrency-256 AgentX point with its eval companion." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From 7e709c0cdbf026476b75de9c860fa37710ef62fb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 12:09:34 -0500 Subject: [PATCH 11/17] fix: collect worker metrics for custom AgentX validation --- ...b300-1p1d-dep4-dep16-c256-mtp-agentic.yaml | 1 + docs/eval-agentx-procedures.md | 2 + docs/eval-agentx-procedures_zh.md | 2 + perf-changelog.yaml | 9 +++ runners/launch_gb300-nv.sh | 3 + .../patches/srt-slurm-custom-metrics.patch | 16 +++++ runners/test_srt_custom_metrics_patch.py | 64 +++++++++++++++++++ 7 files changed, 97 insertions(+) create mode 100644 runners/patches/srt-slurm-custom-metrics.patch create mode 100644 runners/test_srt_custom_metrics_patch.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index eab3e1d7b7..2046a5fa92 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -207,6 +207,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:prompt_tokens_cached_by_source" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" # Avoid concurrent readers observing a mismatched mmap data/index pair. AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 6db5d351e8..833a411b77 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -286,6 +286,8 @@ Topology rules: - Disaggregated: inspect every prefill backend, every decode backend, and the frontend/router. A healthy decode pool does not prove prefill/KV transfer health. - Confirm the AIPerf command includes all `AIPERF_SERVER_METRICS_URLS`. Missing endpoints produce falsely healthy partial evidence. +On GB300's pinned srt-slurm v1.0.36 custom AgentX path, set `benchmark.env.AIPERF_REQUIRED_SERVER_METRIC_PREFIX` to opt into worker endpoint discovery and require that metric prefix in the exported artifacts. The launcher applies a scoped backport; without the opt-in, the custom command otherwise discovers only the frontend. + Read each endpoint directly when summaries are ambiguous: ```bash diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 6e8c8e03c8..11c97bab6f 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -286,6 +286,8 @@ rg -n -i 'Phase |warmup|profiling|returned=|in_flight=|queue=|kv_usage=|prefix_c - Disaggregated:检查每个 prefill backend、每个 decode backend 以及 frontend/router。Decode pool 健康不能证明 prefill/KV transfer 健康。 - 确认 AIPerf 命令包含所有 `AIPERF_SERVER_METRICS_URLS`;缺少 endpoint 会产生片面而虚假的健康证据。 +在 GB300 固定使用的 srt-slurm v1.0.36 自定义 AgentX 路径中,设置 `benchmark.env.AIPERF_REQUIRED_SERVER_METRIC_PREFIX` 可启用 worker endpoint 发现,并要求导出的 artifact 包含该指标前缀。Launcher 会应用限定范围的兼容补丁;未启用时,自定义命令默认只发现 frontend。 + Summary 不明确时直接读取每个 endpoint: ```bash diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d810e77915..9f40af853e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6760,3 +6760,12 @@ - "Use the validation image's upstream deep_gemm_mega_moe backend for both prefill and decode workers; the older recipe's deep_gemm_amxf4_mega_moe backend name is not supported by this image." - "Preserve the proven DEP4/DEP16 topology and single concurrency-256 AgentX point with its eval companion." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Collect every prefill and decode worker's Prometheus endpoint in the custom AgentX command using srt-slurm's existing endpoint discovery." + - "Require the cached-token-by-source metric in exported artifacts; router-only metrics must not pass validation. Preserve the image, topology, concurrency, and eval coverage." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 91fb3b4918..ab0f04fd16 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -299,6 +299,9 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then # Mooncake compatibility. Keep it pinned so sweeps are reproducible. git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 + # v1.0.36 omits worker metrics from custom AgentX commands. Opt in only + # for recipes requiring backend metrics, using its existing URL discovery. + git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-custom-metrics.patch" || exit 1 mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ diff --git a/runners/patches/srt-slurm-custom-metrics.patch b/runners/patches/srt-slurm-custom-metrics.patch new file mode 100644 index 0000000000..9fb23d6c8f --- /dev/null +++ b/runners/patches/srt-slurm-custom-metrics.patch @@ -0,0 +1,16 @@ +diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py +--- a/src/srtctl/cli/mixins/benchmark_stage.py ++++ b/src/srtctl/cli/mixins/benchmark_stage.py +@@ -468,7 +468,10 @@ +- # Add AIPerf-specific env vars for AIPerf-driven benchmarks only +- if isinstance(runner, AIPerfBenchmarkRunner): ++ # Custom AgentX commands opt into worker metrics through their artifact contract. ++ if isinstance(runner, AIPerfBenchmarkRunner) or ( ++ runner.name == "Custom" ++ and self.config.benchmark.env.get("AIPERF_REQUIRED_SERVER_METRIC_PREFIX") ++ ): + env.update(self._get_aiperf_server_metrics_env()) + if self.config.benchmark.aiperf_package: + env["AIPERF_PACKAGE"] = self.config.benchmark.aiperf_package + + return env diff --git a/runners/test_srt_custom_metrics_patch.py b/runners/test_srt_custom_metrics_patch.py new file mode 100644 index 0000000000..4d6d843c64 --- /dev/null +++ b/runners/test_srt_custom_metrics_patch.py @@ -0,0 +1,64 @@ +import runpy +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +PATCH = Path(__file__).parent / "patches/srt-slurm-custom-metrics.patch" +TARGET = "src/srtctl/cli/mixins/benchmark_stage.py" +ORIGINAL = '''class Stage: + def get_env(self, runner): + env = {} + # Add AIPerf-specific env vars for AIPerf-driven benchmarks only + if isinstance(runner, AIPerfBenchmarkRunner): + env.update(self._get_aiperf_server_metrics_env()) + if self.config.benchmark.aiperf_package: + env["AIPERF_PACKAGE"] = self.config.benchmark.aiperf_package + + return env +''' + + +class AIPerfRunner: + name = "AIPerf" + + +@pytest.mark.parametrize( + "runner,required,expected", + [ + (AIPerfRunner(), "", True), + (SimpleNamespace(name="Custom"), "vllm:prompt_tokens_cached_by_source", True), + (SimpleNamespace(name="Custom"), "", False), + (SimpleNamespace(name="Other"), "vllm:", False), + ], +) +def test_custom_metrics_opt_in(tmp_path, runner, required, expected): + target = tmp_path / TARGET + target.parent.mkdir(parents=True) + target.write_text(ORIGINAL) + subprocess.run(["git", "apply", str(PATCH)], cwd=tmp_path, check=True) + symbols = runpy.run_path( + str(target), init_globals={"AIPerfBenchmarkRunner": AIPerfRunner} + ) + stage = symbols["Stage"]() + stage.config = SimpleNamespace( + benchmark=SimpleNamespace( + env={"AIPERF_REQUIRED_SERVER_METRIC_PREFIX": required}, + aiperf_package=None, + ) + ) + urls = "http://prefill:7500/metrics,http://decode:7501/metrics" + stage._get_aiperf_server_metrics_env = lambda: {"AIPERF_SERVER_METRICS_URLS": urls} + assert stage.get_env(runner) == ( + {"AIPERF_SERVER_METRICS_URLS": urls} if expected else {} + ) + + +def test_custom_metrics_patch_rejects_unexpected_source(tmp_path): + target = tmp_path / TARGET + target.parent.mkdir(parents=True) + target.write_text("# Different upstream implementation\n") + result = subprocess.run(["git", "apply", str(PATCH)], cwd=tmp_path, capture_output=True) + assert result.returncode != 0 + assert target.read_text() == "# Different upstream implementation\n" From edc96a6d14a089c8cdcde90d105025de6bc6e8ae Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 15:35:38 -0500 Subject: [PATCH 12/17] fix(agentx): isolate P/D profiling metrics from warmup --- perf-changelog.yaml | 9 +++++++++ utils/aiperf | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 71fd22866e..6dac5ad99d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6801,3 +6801,12 @@ - "Collect every prefill and decode worker's Prometheus endpoint in the custom AgentX command using srt-slurm's existing endpoint discovery." - "Require the cached-token-by-source metric in exported artifacts; router-only metrics must not pass validation. Preserve the image, topology, concurrency, and eval coverage." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Pin the AgentX client fix that tags automatic warmup baseline scrapes as warmup, preventing warmup counter growth from entering profiling-only token-source totals." + - "Preserve the proven DEP4/DEP16 topology, concurrency 256, patched vLLM image, full eval, and canonical warmup and one-hour profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 diff --git a/utils/aiperf b/utils/aiperf index 754356e9a3..dcbd942654 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 754356e9a39acc6cc6afb242d123bb57c3fb6f75 +Subproject commit dcbd942654ef87b357c7722837855dbccedc2353 From ef79a0b7765b0f12fe702edaebf64f82d3364f31 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 15:51:03 -0500 Subject: [PATCH 13/17] fix: retain Mooncake store checkpoints in the P/D validation image --- .../disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index 2046a5fa92..eafab72c24 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -9,14 +9,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" model: path: "deepseek-v4-pro" - container: "ttl.sh/cquil11-vllm-tier-7c5dc571-pr53087-d9b6d498-arm64-20260901:24h" + container: "ttl.sh/cquil11-vllm-tier-pd-retention-b7fc8985-arm64-20260901:24h" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "ttl.sh/cquil11-vllm-tier-7c5dc571-pr53087-d9b6d498-arm64-20260901:24h" + image: "ttl.sh/cquil11-vllm-tier-pd-retention-b7fc8985-arm64-20260901:24h" frameworks: dynamo: "1.4.0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b5d6b6dbec..b905532c1e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8049,7 +8049,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: ttl.sh/cquil11-vllm-tier-7c5dc571-pr53087-d9b6d498-arm64-20260901:24h + image: ttl.sh/cquil11-vllm-tier-pd-retention-b7fc8985-arm64-20260901:24h model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6dac5ad99d..d87e4cc64a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6810,3 +6810,12 @@ - "Pin the AgentX client fix that tags automatic warmup baseline scrapes as warmup, preventing warmup counter growth from entering profiling-only token-source totals." - "Preserve the proven DEP4/DEP16 topology, concurrency 256, patched vLLM image, full eval, and canonical warmup and one-hour profiling." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Backport vLLM PR #46906 so Mooncake's external store retains sliding-window prefix checkpoints independently of HBM retention." + - "Use the Python-only token-source validation image b7fc8985 while preserving the proven DEP4/DEP16 topology, concurrency 256, full eval, and canonical warmup and one-hour profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From 0fcc29d28679f43138de4f9740f890567e1ff10f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 20:39:41 -0500 Subject: [PATCH 14/17] fix(agentic): recover hybrid KV cache misses in P/D validation image Use supporting image 4d3c4a23 with upstream hybrid load-failure recovery and the already validated async queue-progress fix. Preserve the single DEP4/DEP16 c256 point, full eval, and canonical warmup/profile. The previous run crashed on a Mooncake OBJECT_NOT_FOUND response followed by single-group tuple unpack in the scheduler. AI assistance used for diagnosis and validation. --- .../disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index eafab72c24..c3052eb473 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -9,14 +9,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" model: path: "deepseek-v4-pro" - container: "ttl.sh/cquil11-vllm-tier-pd-retention-b7fc8985-arm64-20260901:24h" + container: "ttl.sh/cquil11-vllm-tier-pd-recovery-4d3c4a23-arm64-20260903:24h" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "ttl.sh/cquil11-vllm-tier-pd-retention-b7fc8985-arm64-20260901:24h" + image: "ttl.sh/cquil11-vllm-tier-pd-recovery-4d3c4a23-arm64-20260903:24h" frameworks: dynamo: "1.4.0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7acfc86dca..18b01bd957 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8049,7 +8049,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: ttl.sh/cquil11-vllm-tier-pd-retention-b7fc8985-arm64-20260901:24h + image: ttl.sh/cquil11-vllm-tier-pd-recovery-4d3c4a23-arm64-20260903:24h model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5b42deff27..66ef476b90 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6867,3 +6867,13 @@ - "Backport vLLM PR #46906 so Mooncake's external store retains sliding-window prefix checkpoints independently of HBM retention." - "Use the Python-only token-source validation image b7fc8985 while preserving the proven DEP4/DEP16 topology, concurrency 256, full eval, and canonical warmup and one-hour profiling." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Use the 4d3c4a23 Python-only validation image with upstream PR #50388 hybrid KV-load recovery, so a missing Mooncake object does not crash DeepSeek-V4's multi-group scheduler." + - "Keep failed loads out of cached-token source totals and include the async-load queue-progress fix already validated by the H100 offload sweep." + - "Preserve the existing DEP4/DEP16 topology, single concurrency-256 point, real model eval, canonical warmup, and one-hour profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From dab55b2c772368bc0d537fceff3a8443a5b8b61c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 20:41:07 -0500 Subject: [PATCH 15/17] fix(agentic): recompute missing prefill cache loads Set the prefill MultiConnector failure policy to recompute so optional Mooncake cache eviction races recover through the tested hybrid path. Preserve fail-closed NIXL decode transfers. AI assistance used for diagnosis and validation. --- .../disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml | 2 +- perf-changelog.yaml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index c3052eb473..a03716c7a3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -79,7 +79,7 @@ backend: enable_offload: false vllm_config: prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 66ef476b90..6a66d98599 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6877,3 +6877,12 @@ - "Keep failed loads out of cached-token source totals and include the async-load queue-progress fix already validated by the H100 offload sweep." - "Preserve the existing DEP4/DEP16 topology, single concurrency-256 point, real model eval, canonical warmup, and one-hour profiling." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Enable recompute for the prefill MultiConnector's optional Mooncake cache loads, using the validation image's tested hybrid recovery when a key disappears after lookup." + - "Keep decode-side NIXL transfer failures fail-closed, with unchanged topology, concurrency, eval, warmup, and profiling duration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From 7187c077abaee172e938aa5061fb64f504e55f03 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 23:53:39 -0500 Subject: [PATCH 16/17] fix(config): route GB300 partition overrides to batch_1 --- configs/nvidia-master.yaml | 26 +++++++++++++------------- perf-changelog.yaml | 9 +++++++++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4f0a817160..f9296a585e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8380,7 +8380,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 8 @@ -8397,7 +8397,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 6 tp: 4 @@ -8414,7 +8414,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 32 @@ -8431,7 +8431,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 32 @@ -8448,7 +8448,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -8465,7 +8465,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -9235,7 +9235,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: dp-attn: false additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" # GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. # Six topology variants use NIXL KV transfer and MTP3/MTP5 decoding. glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: @@ -9264,7 +9264,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 4 @@ -9281,7 +9281,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 8 @@ -9298,7 +9298,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 4 tp: 4 @@ -9315,7 +9315,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -9332,7 +9332,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 16 @@ -9349,7 +9349,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml" - - "SLURM_PARTITION=batch_3" + - "SLURM_PARTITION=batch_1" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 97f7baf0df..33982d4e16 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6896,3 +6896,12 @@ - "Enable recompute for the prefill MultiConnector's optional Mooncake cache loads, using the validation image's tested hybrid recovery when a key disappears after lookup." - "Keep decode-side NIXL transfer failures fail-closed, with unchanged topology, concurrency, eval, warmup, and profiling duration." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Standardize all remaining GB300 partition overrides in this validation branch on batch_1, including the DeepSeek-V4 and GLM-5.2 TensorRT-LLM configs. Preserve historical changelog entries." + - "Keep the official validation sweep limited to the existing vLLM P/D concurrency-256 point and its full evaluation; its launcher already defaults to batch_1. No image, topology, warmup, or profiling changes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 From b41d05864fa0f17674b36b8eb89cfaa21c58468b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 09:47:43 -0500 Subject: [PATCH 17/17] fix(benchmark): use P/D image with complete connector block tables --- .../disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml index a03716c7a3..e5d56b3214 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml @@ -9,14 +9,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" model: path: "deepseek-v4-pro" - container: "ttl.sh/cquil11-vllm-tier-pd-recovery-4d3c4a23-arm64-20260903:24h" + container: "ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "ttl.sh/cquil11-vllm-tier-pd-recovery-4d3c4a23-arm64-20260903:24h" + image: "ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h" frameworks: dynamo: "1.4.0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f9296a585e..0938a6dcd9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8049,7 +8049,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: ttl.sh/cquil11-vllm-tier-pd-recovery-4d3c4a23-arm64-20260903:24h + image: ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 33982d4e16..0fa0f4c34f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6905,3 +6905,12 @@ - "Standardize all remaining GB300 partition overrides in this validation branch on batch_1, including the DeepSeek-V4 and GLM-5.2 TensorRT-LLM configs. Preserve historical changelog entries." - "Keep the official validation sweep limited to the existing vLLM P/D concurrency-256 point and its full evaluation; its launcher already defaults to batch_1. No image, topology, warmup, or profiling changes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Use the 0d8739d0 Python-only validation image with upstream PR #54853 complete connector block tables and PR #54643 rejected Mooncake-load handling, fixing the missing-current-block-table assertion observed in the P/D warmup." + - "Preserve the metric implementation, DEP4/DEP16 topology, single concurrency-256 point, full evaluation, canonical warmup, and one-hour profile." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797