diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 56c522f24..ccceb2b1d 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -74,6 +74,7 @@ export SGLANG_USE_ROCM700A=0 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton export AITER_BF16_FP8_MOE_BOUND=0 export TORCH_BLAS_PREFER_HIPBLASLT=1 +export HSA_NO_SCRATCH_RECLAIM=0 # aiter batched GEMM for the absorbed MLA projections, carried by the v0.5.18 # image and off by default in environ.py. export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 @@ -140,12 +141,11 @@ else echo "Error: unsupported TP '$TP' (expected: 4 or 8)" >&2 exit 1 fi -# MTP adds a draft KV pool and extra graph captures on top of the spec-none -# footprint, which ran at 0.90. 0.89 recovers most of that: the DSv4 compressor -# state pools are sized from the full-attention pool and allocated after it, -# outside this budget, so the remainder has to stay large enough to cover them. -MEM_FRACTION_STATIC=0.89 +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.86}" PARALLEL_ARGS=(--tensor-parallel-size "$TP") +SHARED_EXPERTS_ARGS=(--enforce-shared-experts-fusion) +SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO:-0.10}" +export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES:-2}" if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true @@ -157,14 +157,21 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=2 + export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES_DP:-5}" + SHARED_EXPERTS_ARGS=(--disable-shared-experts-fusion) + SWA_FULL_TOKENS_RATIO="${SWA_FULL_TOKENS_RATIO_DP:-0.15}" # Chunked prefill is a whole-engine budget, so widen it by the DP degree. - CHUNKED_PREFILL_SIZE=$((8192 * TP)) + CHUNKED_PREFILL_SIZE=$((CHUNKED_PREFILL_SIZE * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer + --enable-two-batch-overlap + --enable-dp-attention-local-control-broadcast + --tokenizer-worker-num "$TP" + --stream-interval 20 + --prefill-decode-interval 10 ) fi @@ -223,10 +230,11 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --attention-backend dsv4 + --enable-deepseek-v4-fp4-indexer --page-size 256 - --swa-full-tokens-ratio 0.10 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --kv-cache-dtype fp8_e4m3 - --enforce-shared-experts-fusion + "${SHARED_EXPERTS_ARGS[@]}" --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 42567dffe..7fa80d605 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1743,7 +1743,7 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 + image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260902 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1754,6 +1754,7 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 96, 128, 160], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be72bfde7..912728b2e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6887,3 +6887,18 @@ - "Bump the MI355X MiniMax-M3 AgentX image from vllm/vllm-openai-rocm:nightly-6d4562c59b97b4e35d459ff9389e71b6fe4995de to the latest immutable nightly, vllm/vllm-openai-rocm:nightly-8a728663c1c3eeace834a95f5654fa653cc1998c (digest sha256:0d07767c0b8471eaca0f1eca97899620dfb4ad33ff2f5226789afef1a877e9bc)." - "Pick up the optimized BF16 MiniMax-M3 indexer and routed GEMM from vLLM PRs #54682 and #54845; use FULL_DECODE_ONLY CUDA graphs while breakable CUDA graphs remain disabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2825 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260902." + - "Add a DP-attention arm: tp 8, ep 1, dp-attn true, hicache host KV tier, concurrency [64, 96, 128, 160]. It runs behind sglang-router with consistent hashing on the AIPerf correlation id so multi-turn sessions stay on the DP rank holding their prefix." + - "Serve with --enable-deepseek-v4-fp4-indexer and set HSA_NO_SCRATCH_RECLAIM=0 on all arms." + - "DP-attention serving flags: --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10, alongside the existing --enable-prefill-delayer. Chunked prefill now scales the per-TP base by the DP degree instead of a fixed 8192." + - "Shared-experts fusion is now per-arm: kept enforced on the tensor-parallel arms and disabled under DP attention, matching the DP baselines." + - "GPU_MAX_HW_QUEUES is now set explicitly per-arm: 2 on the tensor-parallel arms and 5 under DP attention, the documented companion to two-batch overlap." + - "Lower mem-fraction-static from 0.89 to 0.86 on every arm: TP4, TP8 and TP8 with DP attention. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." + - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800