diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml new file mode 100644 index 0000000000..bb42351a74 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml @@ -0,0 +1,133 @@ +name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang + enable_multiple_frontends: false +backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml new file mode 100644 index 0000000000..5839754fb7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml @@ -0,0 +1,134 @@ +name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang + enable_multiple_frontends: false +backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 2 + pp-max-micro-batch-size: 2 + prefill-max-requests: 2 + cuda-graph-max-bs-decode: 2 + cuda-graph-bs-decode: + - 1 + - 2 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml new file mode 100644 index 0000000000..ec45c447a9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml @@ -0,0 +1,133 @@ +name: agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang + enable_multiple_frontends: false +backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/c7-mrr1/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 8 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 + disable-prefill-cuda-graph: true + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '8' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml new file mode 100644 index 0000000000..79dc96a7a5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml @@ -0,0 +1,239 @@ +name: disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +dynamo: + install: true + hash: ad6f7d54ce7b99c53576b10482755c93c921fbac +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-replica-sync: true + router-queue-threshold: None + router-temperature: '1.0' +backend: + type: sglang + prefill_environment: + # The 0831 image contains the PP+spec runtime merged by SGLang #35758, + # but still carries the pre-merge PP+spec assert in validation_hook.py. + PYTHONOPTIMIZE: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/prefill-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_PP_LAYER_PARTITION: 16,16,16,12 + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-sglang-cache + decode_environment: + SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/decode-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/decode-sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 1 + pipeline-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + disable-overlap-schedule: true + enable-dynamic-chunking: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1536 + max-running-requests: 128 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + linear-attn-decode-backend: triton + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + watchdog-timeout: 1000000 + log-level: info + nccl-port: 29500 + scheduler-recv-interval: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + moe-a2a-backend: none + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + pp-async-batch-depth: 1 + disable-cuda-graph: true + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: round_robin + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + moe-a2a-backend: flashinfer + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + speculative-draft-model-quantization: nvfp4_online + speculative-moe-runner-backend: flashinfer_trtllm_routed + speculative-moe-a2a-backend: flashinfer + chunked-prefill-size: 4096 + mem-fraction-static: 0.7 + max-mamba-cache-size: 468 + max-running-requests: 320 + cuda-graph-max-bs: 80 + disaggregation-decode-extra-slots: 2 + stream-interval: 30 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 32 + decode-log-interval: 30 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml new file mode 100644 index 0000000000..62552dc25c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml @@ -0,0 +1,239 @@ +name: disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +dynamo: + install: true + hash: ad6f7d54ce7b99c53576b10482755c93c921fbac +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 4 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-replica-sync: true + router-queue-threshold: None + router-temperature: '1.0' +backend: + type: sglang + prefill_environment: + # The 0831 image contains the PP+spec runtime merged by SGLang #35758, + # but still carries the pre-merge PP+spec assert in validation_hook.py. + PYTHONOPTIMIZE: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_PP_LAYER_PARTITION: 16,16,16,12 + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-sglang-cache + decode_environment: + SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/decode-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 1 + pipeline-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + disable-overlap-schedule: true + enable-dynamic-chunking: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1536 + max-running-requests: 128 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + speculative-algorithm: NEXTN + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + linear-attn-decode-backend: triton + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + watchdog-timeout: 1000000 + log-level: info + nccl-port: 29500 + scheduler-recv-interval: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + moe-a2a-backend: none + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + pp-async-batch-depth: 1 + disable-cuda-graph: true + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: round_robin + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + moe-a2a-backend: flashinfer + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + speculative-draft-model-quantization: nvfp4_online + speculative-moe-runner-backend: flashinfer_trtllm_routed + speculative-moe-a2a-backend: flashinfer + chunked-prefill-size: 4096 + mem-fraction-static: 0.7 + max-mamba-cache-size: 468 + max-running-requests: 320 + cuda-graph-max-bs: 80 + disaggregation-decode-extra-slots: 2 + stream-interval: 30 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 32 + decode-log-interval: 30 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6ab7aee031..567cbd4e76 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7185,6 +7185,125 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false +# Five-point Qwen3.5 AgentX Pareto refresh on the 2026-08-31 nightly. The +# throughput and high-curve points use PP4 prefill workers; the two C44 +# fingerprints and C7 point below retain their measured aggregate TP shapes. +# PR #36248 breakable prefill CUDA Graph settings are intentionally excluded. +qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: + image: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "ad6f7d54ce7b99c53576b10482755c93c921fbac" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.90 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [704] + prefill: + num-worker: 3 + tp: 1 + pp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=4.8" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml" + decode: + num-worker: 2 + tp: 4 + pp: 1 + ep: 4 + dp-attn: true + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [565] + prefill: + num-worker: 3 + tp: 1 + pp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=4.8" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml" + decode: + num-worker: 4 + tp: 4 + pp: 1 + ep: 4 + dp-attn: true + +qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: + image: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.90 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [44] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=5.04" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [44] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=5.04" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [7] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=5.04" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml" + # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- qwen3.5-fp8-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 53a2ba7e75..1b8f1dd075 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6831,3 +6831,15 @@ - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto + scenario-type: + - agentic-coding + description: + - "Add five GB300 Qwen3.5 NVFP4 AgentX Pareto points on lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198." + - "Use PP4 prefill with DEP4 decode at concurrency 704 and 565, plus two distinct TP2 aggregate concurrency-44 recipes and one TP8 aggregate concurrency-7 recipe." + - "Keep full MTP and HiCache in every recipe; throughput uses the committed synthetic acceptance lengths while EVAL_ONLY keeps real MTP behavior." + - "Run the upstream SGLang image as shipped and pin the PP recipes plus router identity to ai-dynamo/dynamo PR 14064 head ad6f7d54ce7b99c53576b10482755c93c921fbac." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2812