From 6f3bd589cec4b8828b5b7e59726d9de0d7b3aeb3 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Mon, 31 Aug 2026 10:57:48 -0700 Subject: [PATCH 1/5] feat(config): add compact GLM-5.2 B300 AgentX recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 GLM-5.2 B300 AgentX 紧凑布局配方。 --- benchmarks/multi_node/agentic_srt.sh | 10 +- .../b300-fp4/agentic/b300_ctx_hca_pin.sh | 63 +++++ ...agg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 240 +++++++++++++++++ ...gg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 242 +++++++++++++++++ ...gg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 240 +++++++++++++++++ ...gg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 242 +++++++++++++++++ ...-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 243 ++++++++++++++++++ ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 242 +++++++++++++++++ configs/nvidia-master.yaml | 104 ++++++++ perf-changelog.yaml | 7 + runners/launch_b300-nv.sh | 87 ++++++- 11 files changed, 1716 insertions(+), 4 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 0ea94e0bde..6d0d825c5b 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -24,6 +24,13 @@ fi check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION +# Use the exported srt-slurm frontend endpoint when available. Older paths keep +# benchmark_lib.sh's localhost fallback. +if [[ -n "${SRT_FRONTEND_HOST:-}" && -n "${SRT_FRONTEND_PORT:-}" ]]; then + export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" + echo "Using srt-slurm frontend endpoint: $AIPERF_SERVER_URL" +fi + BASE_RESULT_DIR="${RESULT_DIR:-/logs/agentic}" BASE_RESULT_FILENAME="$RESULT_FILENAME" read -r -a CONCURRENCIES <<< "${CONC_LIST:-$CONC}" @@ -45,7 +52,8 @@ install_agentic_deps wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" - local frontend_metrics_url="http://localhost:${PORT}/metrics" + local frontend_metrics_url="${AIPERF_SERVER_URL:-http://localhost:${PORT}}" + frontend_metrics_url="${frontend_metrics_url%/}/metrics" "$AIPERF_PYTHON" - \ "$timeout_seconds" \ diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh new file mode 100644 index 0000000000..62e44688d4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh @@ -0,0 +1,63 @@ +# Worker preamble for GLM-5.2 dynamo-trt on b300-nv. Prefill ranks select active +# HCA pairs by physical GPU; decode remains unpinned. + +unset UCX_TLS # Preserve CUDA memory registration for NIXL transfers. + +_srt_live_devices() { + set -- /sys/class/infiniband/mlx5_* + [ -e "$1" ] || { printf '%s' "$_srt_in"; return 0; } # fail open + _srt_out=""; _srt_oIFS="$IFS"; IFS=, + for _srt_d in $_srt_in; do + _srt_n="${_srt_d%%:*}" + case "$(cat "/sys/class/infiniband/$_srt_n/ports/1/state" 2>/dev/null)" in + *ACTIVE*) _srt_out="${_srt_out:+$_srt_out,}$_srt_d" ;; + esac + done + IFS="$_srt_oIFS"; printf '%s' "$_srt_out" +} + +# `symmetric` shares four rails. `bia_faithful` pins each prefill rank to its +# physical GPU's rail pair. Other values leave ranks unchanged. +case "${BASH_EXECUTION_STRING:-}" in + *SRT_FABRIC_MODE=symmetric*) + export UCX_NET_DEVICES="mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1" + echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=$UCX_NET_DEVICES" + return 0 2>/dev/null || true + ;; + *SRT_FABRIC_MODE=bia_faithful*) ;; + *) return 0 2>/dev/null || true ;; +esac + +case "${BASH_EXECUTION_STRING:-}" in + *trtllm_config_prefill*) ;; # context rank: pin below + *) return 0 2>/dev/null || true ;; # decode/frontend: unpinned +esac + +_srt_cvd=$(printf '%s' "${BASH_EXECUTION_STRING:-}" \ + | grep -oE 'CUDA_VISIBLE_DEVICES=[0-9,]+' | head -1 | cut -d= -f2) +[ -n "$_srt_cvd" ] || return 0 2>/dev/null || true + +IFS=, read -r -a _srt_g <<< "$_srt_cvd" +_srt_phys="${_srt_g[${SLURM_LOCALID:-0}]}" +case "$_srt_phys" in + 0) _srt_hca="mlx5_2:1,mlx5_3:1" ;; + 1) _srt_hca="mlx5_8:1,mlx5_9:1" ;; + 2) _srt_hca="mlx5_4:1,mlx5_5:1" ;; + 3) _srt_hca="mlx5_0:1,mlx5_1:1" ;; + 4) _srt_hca="mlx5_16:1,mlx5_17:1" ;; + 5) _srt_hca="mlx5_22:1,mlx5_23:1" ;; + 6) _srt_hca="mlx5_20:1,mlx5_21:1" ;; + 7) _srt_hca="mlx5_10:1,mlx5_11:1" ;; + *) echo "CTX_HCA_PIN: no mapping for physical GPU $_srt_phys" >&2; _srt_hca="" ;; +esac + +if [ -n "$_srt_hca" ]; then + _srt_in="$_srt_hca"; _srt_hca="$(_srt_live_devices)" +fi +if [ -n "$_srt_hca" ]; then + export UCX_NET_DEVICES="$_srt_hca" + echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=$UCX_NET_DEVICES" +else + echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=" +fi +return 0 2>/dev/null || true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml new file mode 100644 index 0000000000..76254369e4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -0,0 +1,240 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 1 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 412316860416 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '1' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p1d-compact_c1 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + orchestrator_placement: first_decode + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc22.post1 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 +name: dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml new file mode 100644 index 0000000000..7a5f072d08 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -0,0 +1,242 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 5 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 412316860416 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '20' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p1d-compact_c20 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + orchestrator_placement: first_decode + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc22.post1 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 +name: dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml new file mode 100644 index 0000000000..f3bd0cbfed --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -0,0 +1,240 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 2 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 274877906944 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '30' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p4d-compact_c30 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + orchestrator_placement: first_decode + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc22.post1 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 +name: dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5 +resources: + decode_nodes: 2 + decode_workers: 4 + gpu_type: b300 + gpus_per_decode: 4 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml new file mode 100644 index 0000000000..039dee18e6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -0,0 +1,242 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: symmetric + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: symmetric + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 5 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '60' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_3p4d-compact_c60 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + orchestrator_placement: first_decode + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc22.post1 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 +name: dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5 +resources: + decode_nodes: 2 + decode_workers: 4 + gpu_type: b300 + gpus_per_decode: 4 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 2 + prefill_workers: 3 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml new file mode 100644 index 0000000000..7b1c211676 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -0,0 +1,243 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '227' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_6p1d-compact_c227 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: true + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc22.post1 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 +name: dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 3 + prefill_workers: 6 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml new file mode 100644 index 0000000000..3f24679173 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -0,0 +1,242 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '233' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_8p2d-compact_c233 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: true + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc22.post1 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 +name: dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3 +resources: + decode_nodes: 2 + decode_workers: 2 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 4 + prefill_workers: 8 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 72adfa2fa7..f2187feee8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9871,3 +9871,107 @@ glm5.1-fp8-b200-tilert-agentic: dp-attn: false additional-settings: - "DECODE_NODES=1" +glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [20] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [30] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [60] + kv-offloading: none + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [227] + kv-offloading: none + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: mtp + conc-list: [233] + kv-offloading: none + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e75a1ffdbf..2fc2618596 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6662,3 +6662,10 @@ - "Bump image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2763 +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 NVFP4 B300 Dynamo+TensorRT-LLM AgentX recipes with compact worker packing." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 37d35c7b3d..ad407cad94 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -68,6 +68,10 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-trt" ]]; then + export SERVED_MODEL_NAME="nvidia/GLM-5.2-NVFP4" + export MODEL_PATH="/scratch/models/GLM-5.2-NVFP4" + export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5.2-NVFP4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/data/models/MiniMax-M2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" @@ -81,7 +85,7 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dy export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, glm5.2-fp4 with dynamo-trt, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" exit 1 fi @@ -107,6 +111,21 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 fi +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + # Pin v1.0.74 plus the remap-root opt-out and MPI worker srun options used by + # these recipes. + SRT_SLURM_REV="824c15e8eccd447bdf79c39d264ebafd10b1dba3" + git clone https://github.com/Thunderbeee/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + git checkout --detach "$SRT_SLURM_REV" || exit 1 + TRTLLM_RECIPES_DIR="benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" + mkdir -p "$TRTLLM_RECIPES_DIR" + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" \ + "$TRTLLM_RECIPES_DIR" + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ + -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + + fi elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -201,6 +220,29 @@ export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" +DEFAULT_MOUNTS_BLOCK='default_mounts:' +if [[ $FRAMEWORK != "dynamo-trt" || $MODEL_PREFIX != "glm5.2" ]]; then + DEFAULT_MOUNTS_BLOCK+=' + "/opt/ucx-no-ud": "/usr/local/ucx"' +else + echo "Using the container UCX for glm5.2 dynamo-trt NIXL transfers" + # The context HCA pinning preamble below is sourced by srtctl inside EVERY srun + # it launches, including the NATS/etcd step, and those run in the container. A + # host path is not visible there, so mount the directory that holds it. + CTX_HCA_PIN_HOST_DIR="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic" + DEFAULT_MOUNTS_BLOCK+=" + \"${CTX_HCA_PIN_HOST_DIR}\": \"/srt-preamble\"" +fi +if [[ "$IS_AGENTIC" == "1" ]]; then + AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/ai-perf-cache" + HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" + mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" + chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true + DEFAULT_MOUNTS_BLOCK+=" + \"${AIPERF_MMAP_CACHE_HOST_PATH}\": \"/aiperf_mmap_cache\" + \"${HF_HUB_CACHE_HOST_PATH}\": \"/hf_hub_cache\"" +fi + # Create srtslurm.yaml for srtctl SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." @@ -227,10 +269,23 @@ containers: "${IMAGE}": "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" use_exclusive_sbatch_directive: true -default_mounts: - "/opt/ucx-no-ud": "/usr/local/ucx" +${DEFAULT_MOUNTS_BLOCK} EOF +# GLM-5.2 dynamo-trt uses per-rank context-side HCA pinning. Decode remains +# unpinned; no other model or framework is affected. +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + if [[ ! -f "$CTX_HCA_PIN_HOST_DIR/b300_ctx_hca_pin.sh" ]]; then + echo "Error: context HCA pinning preamble not found at $CTX_HCA_PIN_HOST_DIR" >&2 + exit 1 + fi + # Container path, from the mount added with DEFAULT_MOUNTS_BLOCK above. The + # preamble no-ops on every launch whose command does not carry a recognised + # SRT_FABRIC_MODE, so the NATS/etcd and frontend steps are unaffected. + echo 'default_bash_preamble: ". /srt-preamble/b300_ctx_hca_pin.sh"' >> srtslurm.yaml + echo "Context HCA pinning enabled for glm5.2 dynamo-trt (/srt-preamble)" +fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then sed -i "/^ nginx-sqsh:/a\\ dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml grep -q "^ dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; } @@ -261,6 +316,16 @@ if [[ ! -f "$CONFIG_PATH" ]]; then exit 1 fi +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + sed -i "s|^ RESULT_FILENAME:.*| RESULT_FILENAME: ${RESULT_FILENAME}|" "$CONFIG_PATH" +fi + +# Eval runs execute lm-eval on the allocation head and connect over loopback. +# Throughput keeps the frontend on first_decode and the client on last_decode. +if [[ "${EVAL_ONLY:-false}" == "true" && "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + sed -i 's/^ orchestrator_placement: first_decode$/ orchestrator_placement: head/' "$CONFIG_PATH" +fi + # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" if [[ "$MODEL_PREFIX" == "minimaxm3" && -n "$MINIMAX_M3_SLURM_EXCLUDED_NODELIST" ]]; then @@ -281,6 +346,9 @@ fi if [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && "$MODEL_PATH" == /scratch/models/* ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi +if [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" && "$MODEL_PATH" == /scratch/models/* ]]; then + SRTCTL_APPLY_ARGS+=(--no-preflight) +fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") fi @@ -422,6 +490,19 @@ if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then else echo "WARNING: RUN_EVAL=true but no eval results found at $EVAL_DIR" fi + + # srt-slurm stages eval artifacts but does not write the metadata file + # consumed by score validation. Reuse the canonical metadata writer so + # topology and recipe identity stay aligned with the workflow inputs. + eval_conc_value="${EVAL_CONC:-${CONC:-1}}" + ( + export IS_MULTINODE=true + # shellcheck source=benchmarks/benchmark_lib.sh + source "$GITHUB_WORKSPACE/benchmarks/benchmark_lib.sh" + _write_lm_eval_meta_json \ + "$GITHUB_WORKSPACE/meta_env.json" "" "$eval_conc_value" + ) + echo "Wrote meta_env.json (conc=${eval_conc_value}, prefix=${MODEL_PREFIX:-unknown})" fi # Clean up srt-slurm outputs to prevent NFS silly-rename lock files From bf52d3daf60e498cae7db8db9e7e54a77f888a0f Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Mon, 31 Aug 2026 10:59:23 -0700 Subject: [PATCH 2/5] chore(changelog): link PR #2789 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在性能变更日志中补充 PR #2789 链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2fc2618596..05596226ac 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6668,4 +6668,4 @@ - agentic-coding description: - "Add GLM-5.2 NVFP4 B300 Dynamo+TensorRT-LLM AgentX recipes with compact worker packing." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2789 From f6849f1738c11ae1c032880915d8de7db19320b5 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Tue, 1 Sep 2026 15:42:28 -0700 Subject: [PATCH 3/5] fix(config): limit GLM-5.2 B300 experiment to 8p2d MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅保留 8p2d 主配置条目，并在 srt-slurm 配方中排除指定的 B300 节点。 --- ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 2 +- configs/nvidia-master.yaml | 75 ------------------- 2 files changed, 1 insertion(+), 76 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index 3f24679173..5929d4f8f0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -237,6 +237,6 @@ resources: prefill_nodes: 4 prefill_workers: 8 sbatch_directives: - exclude: b300-001,b300-005,b300-006,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 + exclude: b300-001,b300-002,b300-004,b300-005,b300-009,b300-010,b300-012,b300-013,b300-014,b300-015,b300-018 srun_options: cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f2187feee8..aa9c9eadc5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9885,81 +9885,6 @@ glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: scenarios: agentic-coding: - search-space: - - spec-decoding: mtp - conc-list: [1] - kv-offloading: none - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [20] - kv-offloading: none - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [30] - kv-offloading: none - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [60] - kv-offloading: none - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [227] - kv-offloading: none - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - spec-decoding: mtp conc-list: [233] kv-offloading: none From f88f9bb184e3fc7377ce45b500a4dc445d0ad745 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Tue, 1 Sep 2026 15:53:19 -0700 Subject: [PATCH 4/5] chore(changelog): describe GLM-5.2 8p2d experiment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新性能变更日志，说明 GLM-5.2 B300 实验仅启用 8p2d 配置并排除不可用节点。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 05596226ac..0e4be0a8e2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6667,5 +6667,5 @@ scenario-type: - agentic-coding description: - - "Add GLM-5.2 NVFP4 B300 Dynamo+TensorRT-LLM AgentX recipes with compact worker packing." + - "Add the GLM-5.2 NVFP4 B300 Dynamo+TensorRT-LLM AgentX compact 8P/2D configuration at concurrency 233, excluding unavailable B300 nodes from its Slurm allocation." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2789 From dd9c653cd4b9dfb4f775b293d9b3def46246ca9b Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Wed, 2 Sep 2026 14:27:21 -0700 Subject: [PATCH 5/5] fix(config): restore GLM-5.2 compact curve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the c1, c20, c30, c60, and c227 compact topology points and update the performance changelog to describe the full six-point curve. 中文：恢复 c1、c20、c30、c60 和 c227 紧凑拓扑点，并更新性能变更日志以描述完整的六点曲线。 --- configs/nvidia-master.yaml | 75 ++++++++++++++++++++++++++++++++++++++ perf-changelog.yaml | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b3d282e6d7..0cae2a64a2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -10043,6 +10043,81 @@ glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: scenarios: agentic-coding: - search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [20] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [30] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [60] + kv-offloading: none + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [227] + kv-offloading: none + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true - spec-decoding: mtp conc-list: [233] kv-offloading: none diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f393b32c9d..5156a0342c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6827,5 +6827,5 @@ scenario-type: - agentic-coding description: - - "Add the GLM-5.2 NVFP4 B300 Dynamo+TensorRT-LLM AgentX compact 8P/2D configuration at concurrency 233, excluding unavailable B300 nodes from its Slurm allocation." + - "Add the GLM-5.2 NVFP4 B300 Dynamo+TensorRT-LLM AgentX compact curve at concurrency 1, 20, 30, 60, 227, and 233, excluding unavailable B300 nodes from its Slurm allocations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2789