From f895cef288196795e5d2b08fb9ca6fdc2f871dd9 Mon Sep 17 00:00:00 2001 From: ChangLiu0709 Date: Tue, 1 Sep 2026 23:11:13 +0000 Subject: [PATCH 1/3] [AMD][MI35X] Enable fused allreduce+RMSNorm for Qwen3.5 MXFP4 agentic-mtp Replace INT8 QuickReduce with AITER fused allreduce+RMSNorm (--enable-aiter-allreduce-fusion) on the Qwen3.5 MXFP4 MI355X agentic-coding benchmark arm. The fused kernel combines TP allreduce, residual addition, and RMSNorm into a single HIP kernel launch, cutting 256 separate kernel launches down to 134 per decode cycle. Results (TP2, conc=4, 1-hour AgentX agentic replay): - Output throughput/GPU: 114.1 -> 120.8 tok/s (+5.9%) - Decode cycle time: 14,316 -> 11,440 us (-20.1%) - MTP acceptance length: 3.39 (unchanged) - Error rate: 0/892 = 0.0% Config key: qwen3.5-fp4-mi355x-sglang-agentic-mtp Co-authored-by: Cursor --- .../single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh | 6 +++++- perf-changelog.yaml | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index e9e3d5eb5d..93ab281be3 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -90,7 +90,10 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 +# Fused AR+RMSNorm via AITER custom allreduce (replaces INT8 QuickReduce; +# the two are mutually exclusive). Measured +5.9% output throughput at +# TP2/conc4 agentic workload vs the QuickReduce baseline. +# export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 export SGLANG_TIMEOUT_KEEP_ALIVE=1800 if [ "${EVAL_ONLY:-false}" != "true" ]; then @@ -127,6 +130,7 @@ SGLANG_CMD=( --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 + --enable-aiter-allreduce-fusion --enable-metrics --enable-cache-report "${CACHE_ARGS[@]}" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4eb0a2caa6..2974b6a8e8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6821,3 +6821,10 @@ description: - "Refresh to collect TensorRT-LLM server metrics." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Enable AITER fused allreduce+RMSNorm (--enable-aiter-allreduce-fusion), replacing INT8 QuickReduce. +5.9% output throughput per GPU at TP2/conc4 agentic workload (114.1 -> 120.8 tok/s). -20.1% decode cycle time from kernel-level profiling." From 6ecec642c152e5d3d06d853d5d2cd20cfca0d8e5 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Wed, 2 Sep 2026 03:47:44 +0000 Subject: [PATCH 2/3] Update perf-changelog --- perf-changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2974b6a8e8..dced14a372 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6828,3 +6828,4 @@ - agentic-coding description: - "Enable AITER fused allreduce+RMSNorm (--enable-aiter-allreduce-fusion), replacing INT8 QuickReduce. +5.9% output throughput per GPU at TP2/conc4 agentic workload (114.1 -> 120.8 tok/s). -20.1% decode cycle time from kernel-level profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2801 From 4db558d6f8e6e337f7e9b73c5f74f06c7c782425 Mon Sep 17 00:00:00 2001 From: ChangLiu0709 Date: Thu, 3 Sep 2026 16:38:55 +0000 Subject: [PATCH 3/3] qwen3.5-fp4-mi355x: trim concurrency sweep, drop hicache configs Co-authored-by: Cursor --- configs/amd-master.yaml | 4 +--- perf-changelog.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d380154a0..f97dd977b4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -372,10 +372,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 48, 56, 64] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index dced14a372..5ea858f858 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6829,3 +6829,11 @@ description: - "Enable AITER fused allreduce+RMSNorm (--enable-aiter-allreduce-fusion), replacing INT8 QuickReduce. +5.9% output throughput per GPU at TP2/conc4 agentic workload (114.1 -> 120.8 tok/s). -20.1% decode cycle time from kernel-level profiling." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2801 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Trim concurrency sweep: remove hicache KV-offloading configs (TP4/TP2 dram), cap TP4 no-offload conc-list at 32 (was 40)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2801