From 420f11e346efbf63c895d3a5c757b21ee43d2668 Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Tue, 1 Sep 2026 13:32:01 -0500 Subject: [PATCH 1/7] [AMD][DSV4] Add a DP-attention arm and refresh the MI355X SGLang AgentX key Image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 -> ...-20260831. Search space: - add tp 8, ep 1, dp-attn true on the hicache host KV tier at concurrency [64, 96, 128] - drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8] - drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]; conc 16 stays on the TP8 no-offload arm Serving, split per-arm so the tensor-parallel arms keep their current settings: - shared-experts fusion stays enforced without DP and is disabled under DP attention, matching the DP baselines - swa-full-tokens-ratio stays 0.10 without DP and is 0.15 under DP attention - DP attention adds --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10 alongside the existing --enable-prefill-delayer - chunked prefill under DP scales the per-TP base (16384 at TP8, 8192 at TP4) by the DP degree rather than a fixed 8192 - mem-fraction-static 0.89 -> 0.90 Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 21 ++++++++++++------- configs/amd-master.yaml | 7 ++++--- perf-changelog.yaml | 13 ++++++++++++ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 56c522f240..f32dbf626a 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -140,12 +140,10 @@ else echo "Error: unsupported TP '$TP' (expected: 4 or 8)" >&2 exit 1 fi -# MTP adds a draft KV pool and extra graph captures on top of the spec-none -# footprint, which ran at 0.90. 0.89 recovers most of that: the DSv4 compressor -# state pools are sized from the full-attention pool and allocated after it, -# outside this budget, so the remainder has to stay large enough to cover them. -MEM_FRACTION_STATIC=0.89 +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}" PARALLEL_ARGS=(--tensor-parallel-size "$TP") +SHARED_EXPERTS_ARGS=(--enforce-shared-experts-fusion) +SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO:-0.10}" if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true @@ -158,13 +156,20 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 export GPU_MAX_HW_QUEUES=2 + SHARED_EXPERTS_ARGS=(--disable-shared-experts-fusion) + SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO_DP:-0.15}" # Chunked prefill is a whole-engine budget, so widen it by the DP degree. - CHUNKED_PREFILL_SIZE=$((8192 * TP)) + CHUNKED_PREFILL_SIZE=$((CHUNKED_PREFILL_SIZE * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer + --enable-two-batch-overlap + --enable-dp-attention-local-control-broadcast + --tokenizer-worker-num "$TP" + --stream-interval 20 + --prefill-decode-interval 10 ) fi @@ -224,9 +229,9 @@ SGLANG_CMD=( "${PARALLEL_ARGS[@]}" --attention-backend dsv4 --page-size 256 - --swa-full-tokens-ratio 0.10 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --kv-cache-dtype fp8_e4m3 - --enforce-shared-experts-fusion + "${SHARED_EXPERTS_ARGS[@]}" --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d380154a0..032b48a4d1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1742,7 +1742,7 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 + image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260831 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1753,6 +1753,7 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 96, 128], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6f59d20be3..97f767c48f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6773,3 +6773,16 @@ description: - "Enable the required DCGM power telemetry contract across all ten B200 disaggregated STP recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2560 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260831." + - "Add a DP-attention arm: tp 8, ep 1, dp-attn true, hicache host KV tier, concurrency [64, 96, 128]. It runs behind sglang-router with consistent hashing on the AIPerf correlation id so multi-turn sessions stay on the DP rank holding their prefix." + - "DP-attention serving flags: --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10, alongside the existing --enable-prefill-delayer. Chunked prefill now scales the per-TP base by the DP degree instead of a fixed 8192." + - "Shared-experts fusion is now per-arm: kept enforced on the tensor-parallel arms and disabled under DP attention, matching the DP baselines." + - "Raise mem-fraction-static from 0.89 to 0.90. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." + - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From ce19896b53f16164db8c7d3ad68af876de4fbadc Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Tue, 1 Sep 2026 13:40:53 -0500 Subject: [PATCH 2/7] Point the changelog entry at PR #2800 Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 97f767c48f..ad488b5a13 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6785,4 +6785,4 @@ - "Shared-experts fusion is now per-arm: kept enforced on the tensor-parallel arms and disabled under DP attention, matching the DP baselines." - "Raise mem-fraction-static from 0.89 to 0.90. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800 From 6eb626e142c625e21075efe007958a5f911040d4 Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Tue, 1 Sep 2026 21:22:06 -0500 Subject: [PATCH 3/7] Set GPU_MAX_HW_QUEUES per-arm: 2 without DP, 5 under DP attention The DP branch previously exported 2, and the tensor-parallel arms did not set it at all, taking the container default. Both are now explicit: 2 on TP4/TP8 and 5 under DP attention, the documented companion to two-batch overlap. Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 3 +- perf-changelog.yaml | 49 +------------------ 2 files changed, 3 insertions(+), 49 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index f32dbf626a..49851498b4 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -144,6 +144,7 @@ MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}" PARALLEL_ARGS=(--tensor-parallel-size "$TP") SHARED_EXPERTS_ARGS=(--enforce-shared-experts-fusion) SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO:-0.10}" +export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES:-2}" if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true @@ -155,7 +156,7 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=2 + export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES_DP:-5}" SHARED_EXPERTS_ARGS=(--disable-shared-experts-fusion) SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO_DP:-0.15}" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index dd7898afca..b9a6dad75d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6774,54 +6774,6 @@ - "Enable the required DCGM power telemetry contract across all ten B200 disaggregated STP recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2560 -- config-keys: - - qwen3.5-fp4-b200-sglang-mtp - description: - - "Use Triton linear attention with FlashInfer decode and prefill GDN backends for the TP2/EP2 Qwen3.5 FP4 B200 MTP points at concurrency 16, 32, and 64." - - "Keep all other topology and concurrency points on the existing backend defaults." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2790 - -- config-keys: - - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg - scenario-type: - - agentic-coding - description: - - "Refresh to collect TensorRT-LLM server metrics." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2770 - -- config-keys: - - minimaxm3-fp4-b200-trtllm-agentic-mtp - scenario-type: - - agentic-coding - description: - - "Refresh to collect TensorRT-LLM server metrics." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2771 - -- config-keys: - - minimaxm3-fp4-b300-trtllm-agentic-mtp - scenario-type: - - agentic-coding - description: - - "Refresh to collect TensorRT-LLM server metrics." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2772 - -- config-keys: - - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp - - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp - scenario-type: - - agentic-coding - description: - - "Refresh to collect TensorRT-LLM server metrics." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2773 - -- config-keys: - - dsv4-fp4-gb300-dynamo-trt-agentx - scenario-type: - - agentic-coding - description: - - "Refresh to collect TensorRT-LLM server metrics." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 - - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp scenario-type: @@ -6831,6 +6783,7 @@ - "Add a DP-attention arm: tp 8, ep 1, dp-attn true, hicache host KV tier, concurrency [64, 96, 128]. It runs behind sglang-router with consistent hashing on the AIPerf correlation id so multi-turn sessions stay on the DP rank holding their prefix." - "DP-attention serving flags: --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10, alongside the existing --enable-prefill-delayer. Chunked prefill now scales the per-TP base by the DP degree instead of a fixed 8192." - "Shared-experts fusion is now per-arm: kept enforced on the tensor-parallel arms and disabled under DP attention, matching the DP baselines." + - "GPU_MAX_HW_QUEUES is now set explicitly per-arm: 2 on the tensor-parallel arms and 5 under DP attention, the documented companion to two-batch overlap." - "Raise mem-fraction-static from 0.89 to 0.90. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800 From e3c60e13f34a14f8fabf4f945f7609d7971553ff Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Tue, 1 Sep 2026 21:25:26 -0500 Subject: [PATCH 4/7] Re-append the changelog entry so the diff is additions-only The merge resolution reordered main's trailing entries, which check-changelog rejects ("Deletions are not allowed in perf-changelog.yaml"). Restore main's file verbatim and append this entry at the end instead. Co-authored-by: Cursor --- perf-changelog.yaml | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b9a6dad75d..205683b21d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6774,6 +6774,54 @@ - "Enable the required DCGM power telemetry contract across all ten B200 disaggregated STP recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2560 +- config-keys: + - qwen3.5-fp4-b200-sglang-mtp + description: + - "Use Triton linear attention with FlashInfer decode and prefill GDN backends for the TP2/EP2 Qwen3.5 FP4 B200 MTP points at concurrency 16, 32, and 64." + - "Keep all other topology and concurrency points on the existing backend defaults." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2790 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2770 + +- config-keys: + - minimaxm3-fp4-b200-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2771 + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2772 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2773 + +- config-keys: + - dsv4-fp4-gb300-dynamo-trt-agentx + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 + - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp scenario-type: From 14f54ed6c1aa2b0f0e15f6a157487cb267550edb Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Wed, 2 Sep 2026 22:46:30 -0500 Subject: [PATCH 5/7] Set mem-fraction-static per-TP, add the FP4 indexer flag, and extend the DP arm - mem-fraction-static 0.86 on both TP8 arms, with and without DP attention; TP4 stays at 0.89 - serve with --enable-deepseek-v4-fp4-indexer - export HSA_NO_SCRATCH_RECLAIM=0 - DP arm concurrency [64, 96, 128] -> [64, 96, 128, 160] Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 5 ++++- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 15 +++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 49851498b4..217c3ceb3c 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -74,6 +74,7 @@ export SGLANG_USE_ROCM700A=0 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton export AITER_BF16_FP8_MOE_BOUND=0 export TORCH_BLAS_PREFER_HIPBLASLT=1 +export HSA_NO_SCRATCH_RECLAIM=0 # aiter batched GEMM for the absorbed MLA projections, carried by the v0.5.18 # image and off by default in environ.py. export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 @@ -134,13 +135,14 @@ SGLANG_BACKEND_PORT="$PORT" # cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. if [ "$TP" -eq 8 ]; then CHUNKED_PREFILL_SIZE=16384 + MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.86}" elif [ "$TP" -eq 4 ]; then CHUNKED_PREFILL_SIZE=8192 + MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.89}" else echo "Error: unsupported TP '$TP' (expected: 4 or 8)" >&2 exit 1 fi -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}" PARALLEL_ARGS=(--tensor-parallel-size "$TP") SHARED_EXPERTS_ARGS=(--enforce-shared-experts-fusion) SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO:-0.10}" @@ -229,6 +231,7 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --attention-backend dsv4 + --enable-deepseek-v4-fp4-indexer --page-size 256 --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --kv-cache-dtype fp8_e4m3 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 032b48a4d1..6fc95655cd 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1756,4 +1756,4 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 96, 128], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 96, 128, 160], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 205683b21d..6a50486e67 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6822,16 +6822,27 @@ - "Refresh to collect TensorRT-LLM server metrics." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 +- config-keys: + - glm5.2-fp4-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Pin the GLM-5.2 B200 SGLang AgentX curve to lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729, which includes FlashInfer 0.6.18's BF16 TRTLLM MoE allocation fix." + - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." + - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp scenario-type: - agentic-coding description: - "Update image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260831." - - "Add a DP-attention arm: tp 8, ep 1, dp-attn true, hicache host KV tier, concurrency [64, 96, 128]. It runs behind sglang-router with consistent hashing on the AIPerf correlation id so multi-turn sessions stay on the DP rank holding their prefix." + - "Add a DP-attention arm: tp 8, ep 1, dp-attn true, hicache host KV tier, concurrency [64, 96, 128, 160]. It runs behind sglang-router with consistent hashing on the AIPerf correlation id so multi-turn sessions stay on the DP rank holding their prefix." + - "Serve with --enable-deepseek-v4-fp4-indexer and set HSA_NO_SCRATCH_RECLAIM=0 on all arms." - "DP-attention serving flags: --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10, alongside the existing --enable-prefill-delayer. Chunked prefill now scales the per-TP base by the DP degree instead of a fixed 8192." - "Shared-experts fusion is now per-arm: kept enforced on the tensor-parallel arms and disabled under DP attention, matching the DP baselines." - "GPU_MAX_HW_QUEUES is now set explicitly per-arm: 2 on the tensor-parallel arms and 5 under DP attention, the documented companion to two-batch overlap." - - "Raise mem-fraction-static from 0.89 to 0.90. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." + - "mem-fraction-static becomes per-TP: 0.86 on the TP8 arms, both with and without DP attention, and 0.89 unchanged on TP4. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800 From 6fe852931debafc97466caacba4af702707010a4 Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Wed, 2 Sep 2026 22:47:07 -0500 Subject: [PATCH 6/7] Lower mem-fraction-static to 0.86 on TP4 as well Applies the same 0.86 to every arm rather than keeping TP4 at 0.89. Co-authored-by: Cursor --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 3 +-- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 217c3ceb3c..ccceb2b1d1 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -135,14 +135,13 @@ SGLANG_BACKEND_PORT="$PORT" # cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. if [ "$TP" -eq 8 ]; then CHUNKED_PREFILL_SIZE=16384 - MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.86}" elif [ "$TP" -eq 4 ]; then CHUNKED_PREFILL_SIZE=8192 - MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.89}" else echo "Error: unsupported TP '$TP' (expected: 4 or 8)" >&2 exit 1 fi +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.86}" PARALLEL_ARGS=(--tensor-parallel-size "$TP") SHARED_EXPERTS_ARGS=(--enforce-shared-experts-fusion) SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO:-0.10}" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6a50486e67..592d5a7404 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6843,6 +6843,6 @@ - "DP-attention serving flags: --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10, alongside the existing --enable-prefill-delayer. Chunked prefill now scales the per-TP base by the DP degree instead of a fixed 8192." - "Shared-experts fusion is now per-arm: kept enforced on the tensor-parallel arms and disabled under DP attention, matching the DP baselines." - "GPU_MAX_HW_QUEUES is now set explicitly per-arm: 2 on the tensor-parallel arms and 5 under DP attention, the documented companion to two-batch overlap." - - "mem-fraction-static becomes per-TP: 0.86 on the TP8 arms, both with and without DP attention, and 0.89 unchanged on TP4. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." + - "Lower mem-fraction-static from 0.89 to 0.86 on every arm: TP4, TP8 and TP8 with DP attention. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800 From 11c9a0a6b87a0c35289ad098b3f10997328cac10 Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Wed, 2 Sep 2026 22:50:49 -0500 Subject: [PATCH 7/7] Update image to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260902 Co-authored-by: Cursor --- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 6fc95655cd..76f239467e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1742,7 +1742,7 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260831 + image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260902 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 592d5a7404..05979d8dbe 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6837,7 +6837,7 @@ scenario-type: - agentic-coding description: - - "Update image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260831." + - "Update image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260902." - "Add a DP-attention arm: tp 8, ep 1, dp-attn true, hicache host KV tier, concurrency [64, 96, 128, 160]. It runs behind sglang-router with consistent hashing on the AIPerf correlation id so multi-turn sessions stay on the DP rank holding their prefix." - "Serve with --enable-deepseek-v4-fp4-indexer and set HSA_NO_SCRATCH_RECLAIM=0 on all arms." - "DP-attention serving flags: --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10, alongside the existing --enable-prefill-delayer. Chunked prefill now scales the per-TP base by the DP degree instead of a fixed 8192."