Validate vLLM P/D cache-source metrics on GB300

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml

@@ -9,14 +9,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic"
 
 model:
   path: "deepseek-v4-pro"
-  container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
+  container: "ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h"
   precision: "fp4"
 
 identity:
   model:
     repo: "deepseek-ai/DeepSeek-V4-Pro"
   container:
-    image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
+    image: "ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h"
   frameworks:
     dynamo: "1.4.0"
 
@@ -79,7 +79,7 @@ backend:
       enable_offload: false
   vllm_config:
     prefill:
-      kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
+      kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
       served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
       safetensors-load-strategy: "prefetch"
       kv-cache-dtype: "fp8"
@@ -102,7 +102,7 @@ backend:
       tokenizer-mode: "deepseek_v4"
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
       speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
-      moe-backend: "deep_gemm_amxf4_mega_moe"
+      moe-backend: "deep_gemm_mega_moe"
       numa-bind: true
       numa-bind-nodes: [0, 0, 1, 1]
     decode:
@@ -130,7 +130,7 @@ backend:
       tokenizer-mode: "deepseek_v4"
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
       speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
-      moe-backend: "deep_gemm_amxf4_mega_moe"
+      moe-backend: "deep_gemm_mega_moe"
       numa-bind: true
       numa-bind-nodes: [0, 0, 1, 1]
   prefill_environment:
@@ -207,6 +207,7 @@ benchmark:
     PORT: "8000"
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:prompt_tokens_cached_by_source"
     AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
     # Avoid concurrent readers observing a mismatched mmap data/index pair.
     AIPERF_DATASET_MMAP_CACHE_ENABLED: "false"

configs/nvidia-master.yaml

@@ -8049,7 +8049,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg:
           - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49"
           - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml"
 dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg:
-  image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f
+  image: ttl.sh/cquil11-vllm-tier-pd-block-state-0d8739d0-arm64-20260903:24h
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:gb300-nv
@@ -8063,108 +8063,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg:
     - search-space:
       # Keep the checked-in recipes on real MTP verification. Throughput jobs
       # inject the committed golden AL at launch; EVAL_ONLY leaves them real.
-      - spec-decoding: mtp
-        conc-list: [1152]
-        router: { name: dynamo-router, version: "1.3.0.dev20260720" }
-        prefill:
-          num-worker: 2
-          tp: 8
-          ep: 8
-          dp-attn: true
-          additional-settings:
-          - "SYNTHETIC_ACCEPTANCE=true"
-          - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49"
-          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml"
-        decode:
-          num-worker: 1
-          tp: 12
-          ep: 12
-          dp-attn: true
-      - spec-decoding: mtp
-        conc-list: [1024]
-        router: { name: dynamo-router, version: "1.3.0.dev20260720" }
-        prefill:
-          num-worker: 2
-          tp: 8
-          ep: 8
-          dp-attn: true
-          additional-settings:
-          - "SYNTHETIC_ACCEPTANCE=true"
-          - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49"
-          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml"
-        decode:
-          num-worker: 1
-          tp: 16
-          ep: 16
-          dp-attn: true
-      - spec-decoding: mtp
-        conc-list: [256]
-        router: { name: dynamo-router, version: "1.3.0.dev20260720" }
-        prefill:
-          num-worker: 1
-          tp: 4
-          ep: 4
-          dp-attn: true
-          additional-settings:
-          - "SYNTHETIC_ACCEPTANCE=true"
-          - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49"
-          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml"
-        decode:
-          num-worker: 1
-          tp: 8
-          ep: 8
-          dp-attn: true
-      - spec-decoding: mtp
-        conc-list: [512]
-        router: { name: dynamo-router, version: "1.4.0" }
-        prefill:
-          num-worker: 1
-          tp: 8
-          ep: 8
-          dp-attn: true
-          additional-settings:
-          - "SYNTHETIC_ACCEPTANCE=true"
-          - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49"
-          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml"
-        decode:
-          num-worker: 1
-          tp: 16
-          ep: 16
-          dp-attn: true
-      - spec-decoding: mtp
-        conc-list: [4]
-        router: { name: dynamo-router, version: "1.4.0" }
-        prefill:
-          num-worker: 1
-          tp: 4
-          ep: 4
-          dp-attn: true
-          additional-settings:
-          - "SYNTHETIC_ACCEPTANCE=true"
-          - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49"
-          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml"
-        decode:
-          num-worker: 4
-          tp: 8
-          ep: 1
-          dp-attn: false
-      - spec-decoding: mtp
-        conc-list: [128]
-        router: { name: dynamo-router, version: "1.4.0" }
-        prefill:
-          num-worker: 1
-          tp: 4
-          ep: 4
-          dp-attn: true
-          additional-settings:
-          - "SYNTHETIC_ACCEPTANCE=true"
-          - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49"
-          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic.yaml"
-        decode:
-          num-worker: 1
-          tp: 16
-          ep: 16
-          dp-attn: true
       - spec-decoding: mtp
         conc-list: [256]
         router: { name: dynamo-router, version: "1.4.0" }
@@ -8482,7 +8380,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 4
           tp: 8
@@ -8499,7 +8397,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 6
           tp: 4
@@ -8516,7 +8414,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 32
@@ -8533,7 +8431,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 32
@@ -8550,7 +8448,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 16
@@ -8567,7 +8465,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 16
@@ -9337,7 +9235,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp:
           dp-attn: false
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
 # GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving.
 # Six topology variants use NIXL KV transfer and MTP3/MTP5 decoding.
 glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp:
@@ -9366,7 +9264,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 4
           tp: 4
@@ -9383,7 +9281,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 8
@@ -9400,7 +9298,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 4
           tp: 4
@@ -9417,7 +9315,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 16
@@ -9434,7 +9332,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 16
@@ -9451,7 +9349,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp:
           dp-attn: true
           additional-settings:
           - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml"
-          - "SLURM_PARTITION=batch_3"
+          - "SLURM_PARTITION=batch_1"
         decode:
           num-worker: 1
           tp: 8

docs/eval-agentx-procedures.md

@@ -286,6 +286,8 @@ Topology rules:
 - Disaggregated: inspect every prefill backend, every decode backend, and the frontend/router. A healthy decode pool does not prove prefill/KV transfer health.
 - Confirm the AIPerf command includes all `AIPERF_SERVER_METRICS_URLS`. Missing endpoints produce falsely healthy partial evidence.
 
+On GB300's pinned srt-slurm v1.0.36 custom AgentX path, set `benchmark.env.AIPERF_REQUIRED_SERVER_METRIC_PREFIX` to opt into worker endpoint discovery and require that metric prefix in the exported artifacts. The launcher applies a scoped backport; without the opt-in, the custom command otherwise discovers only the frontend.
+
 Read each endpoint directly when summaries are ambiguous:
 
 ```bash

docs/eval-agentx-procedures_zh.md

@@ -286,6 +286,8 @@ rg -n -i 'Phase |warmup|profiling|returned=|in_flight=|queue=|kv_usage=|prefix_c
 - Disaggregated：检查每个 prefill backend、每个 decode backend 以及 frontend/router。Decode pool 健康不能证明 prefill/KV transfer 健康。
 - 确认 AIPerf 命令包含所有 `AIPERF_SERVER_METRICS_URLS`；缺少 endpoint 会产生片面而虚假的健康证据。
 
+在 GB300 固定使用的 srt-slurm v1.0.36 自定义 AgentX 路径中，设置 `benchmark.env.AIPERF_REQUIRED_SERVER_METRIC_PREFIX` 可启用 worker endpoint 发现，并要求导出的 artifact 包含该指标前缀。Launcher 会应用限定范围的兼容补丁；未启用时，自定义命令默认只发现 frontend。
+
 Summary 不明确时直接读取每个 endpoint：
 
 ```bash

perf-changelog.yaml

@@ -6831,3 +6831,86 @@
     - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies."
     - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Validate source-labeled prompt-token cache metrics on the proven GB300 vLLM disaggregated AgentX topology at concurrency 256."
+    - "Use one TP4 prefill worker and one TP16 decode worker with NIXL KV transfer; transferred prompt-token hits must export source=p2p."
+    - "Run the Python-only metric overlay built from official vLLM nightly 7c5dc571 and the HIT_PENDING fix from vLLM PR #53087."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Use the validation image's upstream deep_gemm_mega_moe backend for both prefill and decode workers; the older recipe's deep_gemm_amxf4_mega_moe backend name is not supported by this image."
+    - "Preserve the proven DEP4/DEP16 topology and single concurrency-256 AgentX point with its eval companion."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Collect every prefill and decode worker's Prometheus endpoint in the custom AgentX command using srt-slurm's existing endpoint discovery."
+    - "Require the cached-token-by-source metric in exported artifacts; router-only metrics must not pass validation. Preserve the image, topology, concurrency, and eval coverage."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Pin the AgentX client fix that tags automatic warmup baseline scrapes as warmup, preventing warmup counter growth from entering profiling-only token-source totals."
+    - "Preserve the proven DEP4/DEP16 topology, concurrency 256, patched vLLM image, full eval, and canonical warmup and one-hour profiling."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Backport vLLM PR #46906 so Mooncake's external store retains sliding-window prefix checkpoints independently of HBM retention."
+    - "Use the Python-only token-source validation image b7fc8985 while preserving the proven DEP4/DEP16 topology, concurrency 256, full eval, and canonical warmup and one-hour profiling."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Use the 4d3c4a23 Python-only validation image with upstream PR #50388 hybrid KV-load recovery, so a missing Mooncake object does not crash DeepSeek-V4's multi-group scheduler."
+    - "Keep failed loads out of cached-token source totals and include the async-load queue-progress fix already validated by the H100 offload sweep."
+    - "Preserve the existing DEP4/DEP16 topology, single concurrency-256 point, real model eval, canonical warmup, and one-hour profiling."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Enable recompute for the prefill MultiConnector's optional Mooncake cache loads, using the validation image's tested hybrid recovery when a key disappears after lookup."
+    - "Keep decode-side NIXL transfer failures fail-closed, with unchanged topology, concurrency, eval, warmup, and profiling duration."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Standardize all remaining GB300 partition overrides in this validation branch on batch_1, including the DeepSeek-V4 and GLM-5.2 TensorRT-LLM configs. Preserve historical changelog entries."
+    - "Keep the official validation sweep limited to the existing vLLM P/D concurrency-256 point and its full evaluation; its launcher already defaults to batch_1. No image, topology, warmup, or profiling changes."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Use the 0d8739d0 Python-only validation image with upstream PR #54853 complete connector block tables and PR #54643 rejected Mooncake-load handling, fixing the missing-current-block-table assertion observed in the P/D warmup."
+    - "Preserve the metric implementation, DEP4/DEP16 topology, single concurrency-256 point, full evaluation, canonical warmup, and one-hour profile."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2797