Skip to content

perf(agentx): add Qwen3.5 GB300 SGLang Pareto curve / 性能：新增 Qwen3.5 GB300 SGLang Pareto 曲线 #2812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Ankur-singh wants to merge 2 commits into
base: main
Choose a base branch
from
Open

perf(agentx): add Qwen3.5 GB300 SGLang Pareto curve / 性能：新增 Qwen3.5 GB300 SGLang Pareto 曲线 #2812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7aa7ead
perf(agentx): add Qwen3.5 GB300 SGLang Pareto points
Ankur-singh Sep 3, 2026
60873a8
chore: link InferenceX PR 2812
Ankur-singh Sep 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions ...s/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831
model:
path: qwen3.5-fp4
container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198
precision: fp4
slurm:
time_limit: '8:00:00'
health_check:
max_attempts: 1440
interval_seconds: 10
resources:
gpu_type: gb300
gpus_per_node: 2
agg_nodes: 1
agg_workers: 1
gpus_per_agg: 2
infra:
nats_max_payload_mb: 8
frontend:
type: sglang
enable_multiple_frontends: false
backend:
type: sglang
aggregated_environment:
SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1'
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
NO_COLOR: '1'
PYTHONUNBUFFERED: '1'
PIP_BREAK_SYSTEM_PACKAGES: '1'
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
NCCL_MNNVL_ENABLE: '1'
NCCL_CUMEM_ENABLE: '1'
NCCL_NVLS_ENABLE: '0'
MC_FORCE_MNNVL: '1'
NVSHMEM_REMOTE_TRANSPORT: none
MC_TE_METRIC: 'true'
SGLANG_ENABLE_SPEC_V2: '1'
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass
SGLANG_MOE_NVFP4_DISPATCH: '1'
SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1'
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
FLASHINFER_DISABLE_VERSION_CHECK: '1'
SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache
FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
SGLANG_HEALTH_CHECK_TIMEOUT: '1800'
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache
SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1'
SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1'
SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1'
sglang_config:
aggregated:
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
enable-cache-report: true
model-path: /model/
trust-remote-code: true
quantization: modelopt_fp4
kv-cache-dtype: fp8_e4m3
tensor-parallel-size: 2
pipeline-parallel-size: 1
data-parallel-size: 1
expert-parallel-size: 1
moe-dense-tp-size: 2
enable-dp-attention: false
enable-dp-lm-head: false
moe-a2a-backend: none
load-balance-method: round_robin
mamba-radix-cache-strategy: extra_buffer
mamba-track-interval: 8192
mamba-max-states-per-path: 3
mamba-ssm-dtype: bfloat16
max-mamba-cache-size: 1536
context-length: 262144
page-size: 64
attention-backend: trtllm_mha
moe-runner-backend: flashinfer_trtllm
linear-attn-decode-backend: triton
disable-shared-experts-fusion: true
speculative-algorithm: NEXTN
speculative-num-steps: 6
speculative-eagle-topk: 1
speculative-num-draft-tokens: 7
speculative-moe-runner-backend: flashinfer_trtllm
speculative-moe-a2a-backend: none
chunked-prefill-size: 8192
max-prefill-tokens: 8192
prefill-decode-interval: 0
mem-fraction-static: 0.85
max-running-requests: 1
pp-max-micro-batch-size: 1
prefill-max-requests: 1
stream-interval: 20
decode-log-interval: 10
watchdog-timeout: 1000000
weight-loader-prefetch-checkpoints: true
weight-loader-prefetch-num-threads: 4
enable-hierarchical-cache: true
hicache-write-policy: write_back
hicache-io-backend: kernel
hicache-mem-layout: page_first_direct
hicache-size: 32
enable-metrics: true
enable-linear-replayssm-spec: true
disable-attn-tp-gather: true
cuda-graph-max-bs-decode: 1
cuda-graph-bs-decode:
- 1
sbatch_directives:
mem: '0'
cpus-per-task: '144'
srun_options:
mem: '0'
container-remap-root: ''
benchmark:
type: custom
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
RESULT_DIR: /logs/agentic
PORT: '8000'
IS_MULTINODE: 'false'
TP: '2'
WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
HF_HUB_CACHE: /hf_hub_cache
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
134 changes: 134 additions & 0 deletions ...s/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831
model:
path: qwen3.5-fp4
container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198
precision: fp4
slurm:
time_limit: '8:00:00'
health_check:
max_attempts: 1440
interval_seconds: 10
resources:
gpu_type: gb300
gpus_per_node: 2
agg_nodes: 1
agg_workers: 1
gpus_per_agg: 2
infra:
nats_max_payload_mb: 8
frontend:
type: sglang
enable_multiple_frontends: false
backend:
type: sglang
aggregated_environment:
SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1'
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
NO_COLOR: '1'
PYTHONUNBUFFERED: '1'
PIP_BREAK_SYSTEM_PACKAGES: '1'
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
NCCL_MNNVL_ENABLE: '1'
NCCL_CUMEM_ENABLE: '1'
NCCL_NVLS_ENABLE: '0'
MC_FORCE_MNNVL: '1'
NVSHMEM_REMOTE_TRANSPORT: none
MC_TE_METRIC: 'true'
SGLANG_ENABLE_SPEC_V2: '1'
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass
SGLANG_MOE_NVFP4_DISPATCH: '1'
SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1'
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1'
FLASHINFER_DISABLE_VERSION_CHECK: '1'
SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache
FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
SGLANG_HEALTH_CHECK_TIMEOUT: '1800'
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache
SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1'
SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1'
SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1'
sglang_config:
aggregated:
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
enable-cache-report: true
model-path: /model/
trust-remote-code: true
quantization: modelopt_fp4
kv-cache-dtype: fp8_e4m3
tensor-parallel-size: 2
pipeline-parallel-size: 1
data-parallel-size: 1
expert-parallel-size: 1
moe-dense-tp-size: 2
enable-dp-attention: false
enable-dp-lm-head: false
moe-a2a-backend: none
load-balance-method: round_robin
mamba-radix-cache-strategy: extra_buffer
mamba-track-interval: 8192
mamba-max-states-per-path: 3
mamba-ssm-dtype: bfloat16
max-mamba-cache-size: 1536
context-length: 262144
page-size: 64
attention-backend: trtllm_mha
moe-runner-backend: flashinfer_trtllm
linear-attn-decode-backend: triton
disable-shared-experts-fusion: true
speculative-algorithm: NEXTN
speculative-num-steps: 6
speculative-eagle-topk: 1
speculative-num-draft-tokens: 7
speculative-moe-runner-backend: flashinfer_trtllm
speculative-moe-a2a-backend: none
chunked-prefill-size: 8192
max-prefill-tokens: 8192
prefill-decode-interval: 0
mem-fraction-static: 0.85
max-running-requests: 2
pp-max-micro-batch-size: 2
prefill-max-requests: 2
cuda-graph-max-bs-decode: 2
cuda-graph-bs-decode:
- 1
- 2
stream-interval: 20
decode-log-interval: 10
watchdog-timeout: 1000000
weight-loader-prefetch-checkpoints: true
weight-loader-prefetch-num-threads: 4
enable-hierarchical-cache: true
hicache-write-policy: write_back
hicache-io-backend: kernel
hicache-mem-layout: page_first_direct
hicache-size: 128
enable-metrics: true
enable-linear-replayssm-spec: true
disable-attn-tp-gather: true
sbatch_directives:
mem: '0'
cpus-per-task: '144'
srun_options:
mem: '0'
container-remap-root: ''
benchmark:
type: custom
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
env:
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
RESULT_DIR: /logs/agentic
PORT: '8000'
IS_MULTINODE: 'false'
TP: '2'
WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
HF_HUB_CACHE: /hf_hub_cache
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
Loading