From febaac40df76c80a04e048ede57b3fee897800b8 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Tue, 1 Sep 2026 07:51:33 +0000 Subject: [PATCH] [AMD][MI35X] Serve Qwen3.5 MXFP4-AttnFP8-V2 on the MI355X SGLang fixed-seq-len arms Retarget qwen3.5-fp4-mi355x-sglang and qwen3.5-fp4-mi355x-sglang-mtp from amd/Qwen3.5-397B-A17B-MXFP4 to amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2, the newer AMD checkpoint that keeps MXFP4 MoE weights, folds the shared expert into the fused MoE kernel, and runs self_attn / linear_attn in FP8. Drop the TP4 band from both arms. The checkpoint's Quark config marks linear_attn as per-channel FP8 weights with dynamic per-token activations, so QuarkW8A8Fp8 routes in_proj_ba through aiter gemm_a8w8_bpreshuffle. in_proj_ba has N=128, which TP4 shards to N=32, and the CK bpreshuffle instances are generated with GemmSpecialization::Default (NPerBlock=64, no MN padding), so IsSupportedArgument rejects it and the scheduler dies during prefill with "This GEMM is not supported!". TP2 gives N=64 and runs fine. The band comes back once SGLang falls back to torch._scaled_mm for N % 64 != 0. Image, isl/osl, runner, precision, framework and every launch flag are unchanged, and the qwen3.5-fp4-mi355x-atom arm keeps both TP bands on the older MXFP4 checkpoint. Co-Authored-By: Claude Opus 5 (1M context) --- configs/amd-master.yaml | 6 ++---- perf-changelog.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d380154a..c925c33b1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -314,7 +314,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 - model: amd/Qwen3.5-397B-A17B-MXFP4 + model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: mi355x precision: fp4 @@ -326,7 +326,6 @@ qwen3.5-fp4-mi355x-sglang: osl: 1024 search-space: - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } qwen3.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post @@ -346,7 +345,7 @@ qwen3.5-fp4-mi355x-atom: qwen3.5-fp4-mi355x-sglang-mtp: image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 - model: amd/Qwen3.5-397B-A17B-MXFP4 + model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: mi355x precision: fp4 @@ -358,7 +357,6 @@ qwen3.5-fp4-mi355x-sglang-mtp: osl: 1024 search-space: - { tp: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } qwen3.5-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4eb0a2caa..4f16e224d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6821,3 +6821,13 @@ description: - "Refresh to collect TensorRT-LLM server metrics." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Serve amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 instead of amd/Qwen3.5-397B-A17B-MXFP4, the newer AMD checkpoint that keeps MXFP4 MoE weights, quantizes the shared expert into the fused MoE kernel, and runs attention in FP8" + - "Drop the TP4 band from both arms: TP4 shards the FP8 linear_attn in_proj_ba to N=32, for which aiter's CK gemm_a8w8_bpreshuffle has no instance (NPerBlock=64, no MN padding), so the server crashes during prefill" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2793