diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index e9e3d5eb5..93ab281be 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -90,7 +90,10 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 +# Fused AR+RMSNorm via AITER custom allreduce (replaces INT8 QuickReduce; +# the two are mutually exclusive). Measured +5.9% output throughput at +# TP2/conc4 agentic workload vs the QuickReduce baseline. +# export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 export SGLANG_TIMEOUT_KEEP_ALIVE=1800 if [ "${EVAL_ONLY:-false}" != "true" ]; then @@ -127,6 +130,7 @@ SGLANG_CMD=( --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 + --enable-aiter-allreduce-fusion --enable-metrics --enable-cache-report "${CACHE_ARGS[@]}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d380154a..f97dd977b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -372,10 +372,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 48, 56, 64] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4eb0a2caa..5ea858f85 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6821,3 +6821,19 @@ description: - "Refresh to collect TensorRT-LLM server metrics." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Enable AITER fused allreduce+RMSNorm (--enable-aiter-allreduce-fusion), replacing INT8 QuickReduce. +5.9% output throughput per GPU at TP2/conc4 agentic workload (114.1 -> 120.8 tok/s). -20.1% decode cycle time from kernel-level profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2801 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Trim concurrency sweep: remove hicache KV-offloading configs (TP4/TP2 dram), cap TP4 no-offload conc-list at 32 (was 40)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2801