From ef327e4976d7c10912b5a7f54a98970336e2d735 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:15:32 -0700 Subject: [PATCH 1/5] Refresh the Kimi-K3 GB300 AgentX --- ...g-gb300-dcp8-dspark4-mooncake-agentic.yaml | 120 ++++++++++++++++++ ...dcp8-dspark7-maxseq2-mooncake-agentic.yaml | 57 +++------ ...gg-gb300-dcp8-nospec-mooncake-agentic.yaml | 19 ++- ...1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 88 ++++--------- ...2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 88 ++++--------- ...d-dcp8-dcp8-dspark4-mooncake-agentic.yaml} | 94 +++++--------- configs/nvidia-master.yaml | 65 +++++++--- perf-changelog.yaml | 10 ++ 8 files changed, 285 insertions(+), 256 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic.yaml => disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml} (57%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml new file mode 100644 index 0000000000..a5ffb6d0c3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml @@ -0,0 +1,120 @@ +name: "kimi-k3-vllm-agg-gb300-dcp8-dspark4-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" + WITH_NVIDIA_PEERMEM: "0" + VLLM_LOG_STATS_INTERVAL: "1" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + served-model-name: "moonshotai/Kimi-K3" + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + prefix-match-unit: 128 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml index bc7da715be..dafc046abe 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml @@ -1,19 +1,15 @@ -# p90-ITL Pareto family from OCI jobs 471335-471337 and 471341-471342: one -# aggregate TP8 x DCP8 worker with max-num-seqs 2. Drafts at DSpark K=7 to -# match the B300 low-latency arm (kimik3_fp4_b300_vllm_mtp.sh drafts at K=7 -# for conc <= 8) rather than the K=4 the measured jobs used. name: "kimi-k3-vllm-agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic" model: path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -43,7 +39,7 @@ frontend: type: dynamo enable_multiple_frontends: false args: - router-mode: "random" + router-mode: "least-loaded" router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" @@ -54,68 +50,59 @@ backend: connector: null mooncake_kv_store: master_extra_args: + - "--default_kv_lease_ttl=60000" - "--eviction_high_watermark_ratio=0.95" - "--eviction_ratio=0.10" store_config: metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" + global_segment_size: "160GB" local_buffer_size: "4GB" protocol: "rdma" - # Preserve the proven GB300 runner fabric rather than copying PR #211's - # single mlx5_0 rail, which is specific to the rail-isolated B300 fleet. device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false aggregated_environment: VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - # Use the direct DCP a2a/gather kernels rather than the collective - # fallback, matching the B300 arm (kimik3_fp4_b300_vllm_mtp.sh). VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + ETCD_LEASE_TTL: "120" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - PYTHONNOUSERSITE: "1" - TORCH_CUDA_ARCH_LIST: "10.0" - PYTHONHASHSEED: "42" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" WITH_NVIDIA_PEERMEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_LOG_STATS_INTERVAL: "1" + vllm_config: aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' served-model-name: "moonshotai/Kimi-K3" + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 tensor-parallel-size: 8 decode-context-parallel-size: 8 dcp-comm-backend: "a2a" max-num-seqs: 2 - gpu-memory-utilization: 0.92 max-num-batched-tokens: 8192 trust-remote-code: true + max-cudagraph-capture-size: 1024 + stream-interval: 10 language-model-only: true - load-format: "fastsafetensors" moe-backend: "auto" no-enable-flashinfer-autotune: true enable-cumem-allocator: true - enable-prefix-caching: true - prefix-match-unit: 128 - kv-cache-dtype: "fp8" - stream-interval: 10 attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # Throughput jobs inject the committed K=7 golden AL (3.84); EVAL_ONLY - # preserves this real target-verification configuration. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - max-cudagraph-capture-size: 1024 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + prefix-match-unit: 128 sbatch_directives: mem: "0" @@ -132,10 +119,6 @@ benchmark: command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" RESULT_DIR: "/logs/agentic" PORT: "8000" IS_MULTINODE: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml index 4dd1e931f2..9b61dd4922 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml @@ -1,17 +1,15 @@ -# Measured p90-ITL Pareto family: OCI jobs 471604 (c32), 471605 (c48), and -# 471606 (c64), using one aggregate TP8 x DCP8 worker without a draft model. name: "kimi-k3-vllm-agg-gb300-dcp8-nospec-mooncake-agentic" model: path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -41,7 +39,7 @@ frontend: type: dynamo enable_multiple_frontends: false args: - router-mode: "random" + router-mode: "least-loaded" router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" @@ -52,11 +50,12 @@ backend: connector: null mooncake_kv_store: master_extra_args: + - "--default_kv_lease_ttl=60000" - "--eviction_high_watermark_ratio=0.95" - "--eviction_ratio=0.10" store_config: metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" + global_segment_size: "160GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" @@ -71,19 +70,17 @@ backend: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + ETCD_LEASE_TTL: "120" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" + VLLM_USE_RUST_FRONTEND: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" PYTHONNOUSERSITE: "1" TORCH_CUDA_ARCH_LIST: "10.0" PYTHONHASHSEED: "42" VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" WITH_NVIDIA_PEERMEM: "0" VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" vllm_config: @@ -106,7 +103,7 @@ backend: attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' max-cudagraph-capture-size: 1024 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' sbatch_directives: mem: "0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 38475f0b22..afe0d4a653 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -1,18 +1,15 @@ -# Built vLLM image containing the Kimi-K3 DCP, DSpark, and PMU fixes. -# Measured Pareto family: OCI jobs 469790 (c32), 469796 (c48), and 469792 -# (c64), using DSpark K=4 with synthetic acceptance length 3.36. name: "kimi-k3-vllm-disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic" model: path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -45,11 +42,12 @@ frontend: type: dynamo enable_multiple_frontends: false args: - router-mode: "random" + router-mode: "least-loaded" router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" DYN_TOKENIZER_CACHE_BYTES: "8589934592" + backend: type: vllm connector: null @@ -57,24 +55,23 @@ backend: kv_events_config: prefill: true mooncake_kv_store: + master_extra_args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" store_config: metadata_server: "P2PHANDSHAKE" - # Keep the colleague's measured 150 GB Mooncake segment setting. - global_segment_size: "150GB" + global_segment_size: "160GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false prefill_environment: &kimi_env - # Use the direct DCP a2a/gather kernels rather than the collective - # fallback, matching the B300 arm (kimik3_fp4_b300_vllm_mtp.sh). VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" - # No-op in this recipe: shared-expert sequence-parallel sharding requires - # expert parallelism, which is disabled on both roles. VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" DYN_REQUEST_PLANE: "tcp" ETCD_LEASE_TTL: "600" @@ -82,16 +79,6 @@ backend: VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" - # TORCH_SYMMMEM deliberately NOT set to NVSHMEM here. - # MegaMoE (decode role) allocates its symm buffer with device='cuda' (no - # index, deep_gemm/mega/__init__.py:48) and calls symm_mem.rendezvous. On - # the NVSHMEM backend that reaches a process-global TeamManager singleton - # pinned to the first caller's device (nvshmem_team_manager.hpp:36-44), so a - # TP>1 engine whose workers span devices 0-3 dies with - # "Detected use of TeamManager on multiple devices." - # Measured: fine at TP=1 (DEP16 decode, job 2649641), fails at TP>1 (job - # 2649921 prefill). Leaving it unset selects torch's default CUDA symm-mem - # backend, which has no such singleton. NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -99,83 +86,62 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - NCCL_P2P_LEVEL: NVL + NCCL_P2P_LEVEL: "NVL" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" WITH_NVIDIA_PEERMEM: "0" - # NIXL (venv wheel) bundles its own UCX; NCCL's rdma_sharp_plugin loads - # /opt/hpcx UCX. Both in one process share UCM memory hooks and corrupt the - # heap -- NixlConnector then dies with - # nixl_agent.cpp:470 registerMem: registration failed ... NIXL_ERR_BACKEND - # (and a "corrupted size vs. prev_size" abort at teardown). Disabling the - # external NCCL net plugin leaves exactly one UCX in the process; NCCL falls - # back to its built-in IB/NVLink transports. Verified on theia0263: 4 ranks, - # hpcx libs mapped = 0, 93-region registerMem OK on every rank. NCCL_NET_PLUGIN: "none" - # UCX_MEMTYPE_CACHE=n stops UCX misclassifying a CUDA pointer it did not - # allocate as host memory; REG_WHOLE=n keeps it from registering the whole - # enclosing region instead of the requested range. UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" UCX_RCACHE_MAX_UNRELEASED: "1024" UCX_TCP_AF_PRIO: "inet" - # K3's linear-attention layers hold Mamba conv state. NixlConnector's - # 3-read conv transfer asserts the DS (dim, state_len) layout at worker - # init; without this both roles die with "requires DS conv state layout". - # Must match on prefill and decode, hence the shared anchor. VLLM_SSM_CONV_STATE_LAYOUT: "DS" decode_environment: <<: *kimi_env vllm_config: prefill: - # MultiConnector: Nixl (P->D handshake transfer) + Mooncake (external DRAM KV). - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "moonshotai/Kimi-K3" - enable-prefix-caching: true prefix-match-unit: 128 - load-format: fastsafetensors - kv-cache-dtype: fp8 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 tensor-parallel-size: 8 - # The point of this recipe: DCP reuses the 8 TP ranks and shards the MLA - # KV cache across them instead of replicating it, giving the prefill role - # ~7.5x the KV pool it has under plain TP8. decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" enable-cumem-allocator: true trust-remote-code: true - no-enable-flashinfer-autotune: true max-cudagraph-capture-size: 512 stream-interval: 10 - max_num_batched_tokens: 16384 language-model-only: true attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "moonshotai/Kimi-K3" - enable-prefix-caching: true prefix-match-unit: 128 - load-format: fastsafetensors - kv-cache-dtype: fp8 - # Must equal the prefill TP for the NIXL MLA rank-to-rank mapping (see - # the header); DCP must likewise match at 8. + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 tensor-parallel-size: 8 decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" enable-cumem-allocator: true trust-remote-code: true - no-enable-flashinfer-autotune: true - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 + max-cudagraph-capture-size: 512 stream-interval: 10 - max-num-seqs: 256 language-model-only: true attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true sbatch_directives: mem: "0" @@ -201,5 +167,3 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" - # MODEL, topology, concurrency, duration, offload metadata, and result name - # are inherited from the generated InferenceMAX matrix. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index db0c947ae4..29388bbb67 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -1,18 +1,15 @@ -# Built vLLM image containing the Kimi-K3 DCP, DSpark, and PMU fixes. -# Measured p90-ITL Pareto point: OCI job 489701 (c48), using one DCP8 -# prefill worker, two DCP8 decode workers, DSpark K=4, and synthetic AL 3.36. name: "kimi-k3-vllm-disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic" model: path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -45,11 +42,12 @@ frontend: type: dynamo enable_multiple_frontends: false args: - router-mode: "random" + router-mode: "least-loaded" router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" DYN_TOKENIZER_CACHE_BYTES: "8589934592" + backend: type: vllm connector: null @@ -57,24 +55,23 @@ backend: kv_events_config: prefill: true mooncake_kv_store: + master_extra_args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" store_config: metadata_server: "P2PHANDSHAKE" - # Keep the colleague's measured 150 GB Mooncake segment setting. - global_segment_size: "150GB" + global_segment_size: "160GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false prefill_environment: &kimi_env - # Use the direct DCP a2a/gather kernels rather than the collective - # fallback, matching the B300 arm (kimik3_fp4_b300_vllm_mtp.sh). VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" - # No-op in this recipe: shared-expert sequence-parallel sharding requires - # expert parallelism, which is disabled on both roles. VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" DYN_REQUEST_PLANE: "tcp" ETCD_LEASE_TTL: "600" @@ -82,16 +79,6 @@ backend: VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" - # TORCH_SYMMMEM deliberately NOT set to NVSHMEM here. - # MegaMoE (decode role) allocates its symm buffer with device='cuda' (no - # index, deep_gemm/mega/__init__.py:48) and calls symm_mem.rendezvous. On - # the NVSHMEM backend that reaches a process-global TeamManager singleton - # pinned to the first caller's device (nvshmem_team_manager.hpp:36-44), so a - # TP>1 engine whose workers span devices 0-3 dies with - # "Detected use of TeamManager on multiple devices." - # Measured: fine at TP=1 (DEP16 decode, job 2649641), fails at TP>1 (job - # 2649921 prefill). Leaving it unset selects torch's default CUDA symm-mem - # backend, which has no such singleton. NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -99,83 +86,62 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - NCCL_P2P_LEVEL: NVL + NCCL_P2P_LEVEL: "NVL" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" WITH_NVIDIA_PEERMEM: "0" - # NIXL (venv wheel) bundles its own UCX; NCCL's rdma_sharp_plugin loads - # /opt/hpcx UCX. Both in one process share UCM memory hooks and corrupt the - # heap -- NixlConnector then dies with - # nixl_agent.cpp:470 registerMem: registration failed ... NIXL_ERR_BACKEND - # (and a "corrupted size vs. prev_size" abort at teardown). Disabling the - # external NCCL net plugin leaves exactly one UCX in the process; NCCL falls - # back to its built-in IB/NVLink transports. Verified on theia0263: 4 ranks, - # hpcx libs mapped = 0, 93-region registerMem OK on every rank. NCCL_NET_PLUGIN: "none" - # UCX_MEMTYPE_CACHE=n stops UCX misclassifying a CUDA pointer it did not - # allocate as host memory; REG_WHOLE=n keeps it from registering the whole - # enclosing region instead of the requested range. UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" UCX_RCACHE_MAX_UNRELEASED: "1024" UCX_TCP_AF_PRIO: "inet" - # K3's linear-attention layers hold Mamba conv state. NixlConnector's - # 3-read conv transfer asserts the DS (dim, state_len) layout at worker - # init; without this both roles die with "requires DS conv state layout". - # Must match on prefill and decode, hence the shared anchor. VLLM_SSM_CONV_STATE_LAYOUT: "DS" decode_environment: <<: *kimi_env vllm_config: prefill: - # MultiConnector: Nixl (P->D handshake transfer) + Mooncake (external DRAM KV). - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "moonshotai/Kimi-K3" - enable-prefix-caching: true prefix-match-unit: 128 - load-format: fastsafetensors - kv-cache-dtype: fp8 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 tensor-parallel-size: 8 - # The point of this recipe: DCP reuses the 8 TP ranks and shards the MLA - # KV cache across them instead of replicating it, giving the prefill role - # ~7.5x the KV pool it has under plain TP8. decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" enable-cumem-allocator: true trust-remote-code: true - no-enable-flashinfer-autotune: true max-cudagraph-capture-size: 512 stream-interval: 10 - max_num_batched_tokens: 16384 language-model-only: true attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "moonshotai/Kimi-K3" - enable-prefix-caching: true prefix-match-unit: 128 - load-format: fastsafetensors - kv-cache-dtype: fp8 - # Must equal the prefill TP for the NIXL MLA rank-to-rank mapping (see - # the header); DCP must likewise match at 8. + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 tensor-parallel-size: 8 decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" enable-cumem-allocator: true trust-remote-code: true - no-enable-flashinfer-autotune: true - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 + max-cudagraph-capture-size: 512 stream-interval: 10 - max-num-seqs: 256 language-model-only: true attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true sbatch_directives: mem: "0" @@ -201,5 +167,3 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" - # MODEL, topology, concurrency, duration, offload metadata, and result name - # are inherited from the generated InferenceMAX matrix. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml similarity index 57% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index c24ee7c3bf..39269d445e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -1,18 +1,15 @@ -# Built vLLM image containing the Kimi-K3 DCP, DSpark, and PMU fixes. -# Measured Pareto family: OCI jobs 471040 (c1) and 471041 (c2), using DSpark -# K=7 with synthetic acceptance length 3.84. -name: "kimi-k3-vllm-disagg-gb300-1p1d-dcp8-dcp8-dspark7-mooncake-agentic" +name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic" model: path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -31,9 +28,9 @@ resources: gpu_type: "gb300" gpus_per_node: 4 prefill_nodes: 2 - decode_nodes: 2 + decode_nodes: 6 prefill_workers: 1 - decode_workers: 1 + decode_workers: 3 gpus_per_prefill: 8 gpus_per_decode: 8 @@ -45,11 +42,12 @@ frontend: type: dynamo enable_multiple_frontends: false args: - router-mode: "random" + router-mode: "least-loaded" router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" DYN_TOKENIZER_CACHE_BYTES: "8589934592" + backend: type: vllm connector: null @@ -57,24 +55,23 @@ backend: kv_events_config: prefill: true mooncake_kv_store: + master_extra_args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" store_config: metadata_server: "P2PHANDSHAKE" - # Keep the colleague's measured 150 GB Mooncake segment setting. - global_segment_size: "150GB" + global_segment_size: "160GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false prefill_environment: &kimi_env - # Use the direct DCP a2a/gather kernels rather than the collective - # fallback, matching the B300 arm (kimik3_fp4_b300_vllm_mtp.sh). VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" - # No-op in this recipe: shared-expert sequence-parallel sharding requires - # expert parallelism, which is disabled on both roles. VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" DYN_REQUEST_PLANE: "tcp" ETCD_LEASE_TTL: "600" @@ -82,16 +79,6 @@ backend: VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" - # TORCH_SYMMMEM deliberately NOT set to NVSHMEM here. - # MegaMoE (decode role) allocates its symm buffer with device='cuda' (no - # index, deep_gemm/mega/__init__.py:48) and calls symm_mem.rendezvous. On - # the NVSHMEM backend that reaches a process-global TeamManager singleton - # pinned to the first caller's device (nvshmem_team_manager.hpp:36-44), so a - # TP>1 engine whose workers span devices 0-3 dies with - # "Detected use of TeamManager on multiple devices." - # Measured: fine at TP=1 (DEP16 decode, job 2649641), fails at TP>1 (job - # 2649921 prefill). Leaving it unset selects torch's default CUDA symm-mem - # backend, which has no such singleton. NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -99,83 +86,62 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - NCCL_P2P_LEVEL: NVL + NCCL_P2P_LEVEL: "NVL" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" WITH_NVIDIA_PEERMEM: "0" - # NIXL (venv wheel) bundles its own UCX; NCCL's rdma_sharp_plugin loads - # /opt/hpcx UCX. Both in one process share UCM memory hooks and corrupt the - # heap -- NixlConnector then dies with - # nixl_agent.cpp:470 registerMem: registration failed ... NIXL_ERR_BACKEND - # (and a "corrupted size vs. prev_size" abort at teardown). Disabling the - # external NCCL net plugin leaves exactly one UCX in the process; NCCL falls - # back to its built-in IB/NVLink transports. Verified on theia0263: 4 ranks, - # hpcx libs mapped = 0, 93-region registerMem OK on every rank. NCCL_NET_PLUGIN: "none" - # UCX_MEMTYPE_CACHE=n stops UCX misclassifying a CUDA pointer it did not - # allocate as host memory; REG_WHOLE=n keeps it from registering the whole - # enclosing region instead of the requested range. UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" UCX_RCACHE_MAX_UNRELEASED: "1024" UCX_TCP_AF_PRIO: "inet" - # K3's linear-attention layers hold Mamba conv state. NixlConnector's - # 3-read conv transfer asserts the DS (dim, state_len) layout at worker - # init; without this both roles die with "requires DS conv state layout". - # Must match on prefill and decode, hence the shared anchor. VLLM_SSM_CONV_STATE_LAYOUT: "DS" decode_environment: <<: *kimi_env vllm_config: prefill: - # MultiConnector: Nixl (P->D handshake transfer) + Mooncake (external DRAM KV). - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "moonshotai/Kimi-K3" - enable-prefix-caching: true prefix-match-unit: 128 - load-format: fastsafetensors - kv-cache-dtype: fp8 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 tensor-parallel-size: 8 - # The point of this recipe: DCP reuses the 8 TP ranks and shards the MLA - # KV cache across them instead of replicating it, giving the prefill role - # ~7.5x the KV pool it has under plain TP8. decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" enable-cumem-allocator: true trust-remote-code: true - no-enable-flashinfer-autotune: true max-cudagraph-capture-size: 512 stream-interval: 10 - max_num_batched_tokens: 16384 language-model-only: true attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "moonshotai/Kimi-K3" - enable-prefix-caching: true prefix-match-unit: 128 - load-format: fastsafetensors - kv-cache-dtype: fp8 - # Must equal the prefill TP for the NIXL MLA rank-to-rank mapping (see - # the header); DCP must likewise match at 8. + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 tensor-parallel-size: 8 decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" enable-cumem-allocator: true trust-remote-code: true - no-enable-flashinfer-autotune: true - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 + max-cudagraph-capture-size: 512 stream-interval: 10 - max-num-seqs: 256 language-model-only: true attention-backend: "TOKENSPEED_MLA" attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true sbatch_directives: mem: "0" @@ -201,5 +167,3 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" - # MODEL, topology, concurrency, duration, offload metadata, and result name - # are inherited from the generated InferenceMAX matrix. diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6ab7aee031..0960962ecc 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9580,10 +9580,10 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 16 dp-attn: true -# Kimi-K3 GB300 p90-ITL Pareto points. Each DCP8 worker spans two 4-GPU -# trays: 1P1D uses 16 GPUs and 1P2D uses 24 GPUs. +# Kimi-K3 GB300 points on the measured global p90-ITL Pareto front. Each +# DCP8 worker spans two 4-GPU nodes. kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + image: vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:gb300-nv @@ -9595,14 +9595,14 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: disagg: true scenarios: agentic-coding: - # 0.1664 resolves to 150 GB of host DRAM per 4-GPU server node on + # 0.1775 resolves to 160 GB of host DRAM per 4-GPU server node on # cluster:gb300-nv, matching the recipe's Mooncake segment size. - - dram-utilization: 0.1664 + - dram-utilization: 0.1775 search-space: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [64] + conc-list: [48, 52, 56] prefill: num-worker: 1 tp: 8 @@ -9617,7 +9617,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [48] + conc-list: [16, 32, 48, 64] prefill: num-worker: 1 tp: 8 @@ -9629,11 +9629,25 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } - -# Aggregate p90-ITL Pareto points. One TP8 x DCP8 worker spans two 4-GPU nodes -# and serves both phases; decode num-worker is zero so GPUs are counted once. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [1, 32, 48] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" + decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } +# Aggregate global p90-ITL Pareto points. One TP8 x DCP8 worker spans two +# 4-GPU nodes and serves both phases, so GPUs are counted once. kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + image: vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:gb300-nv @@ -9643,8 +9657,23 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: disagg: false scenarios: agentic-coding: - - dram-utilization: 0.1664 + - dram-utilization: 0.1775 search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [32, 36] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } @@ -9658,15 +9687,13 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: dp-attn: false additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - # Golden AL at DSpark K=7 on the probabilistic/block curve, matching - # the B300 low-latency arm. - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml" -# High-throughput aggregate p90-ITL Pareto points without speculative decoding -# or a max-num-seqs override. +# Aggregate no-spec points measured at concurrency 48 and 70. One TP8 x DCP8 worker +# spans two 4-GPU nodes and serves both phases. kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + image: vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:gb300-nv @@ -9676,11 +9703,11 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: disagg: false scenarios: agentic-coding: - - dram-utilization: 0.1664 + - dram-utilization: 0.1775 search-space: - kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [48, 70, 74] + conc-list: [48, 70] num-nodes: 2 worker: num-worker: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 53a2ba7e75..5e77d0ae83 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6831,3 +6831,13 @@ - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Refresh the Kimi-K3 GB300 AgentX configurations with updated configs." + pr-link: TBD From bcb1b49780833069363fba9f30d8399621dec36c Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:22:16 -0700 Subject: [PATCH 2/5] Add PR number --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5e77d0ae83..ff6f57beec 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6840,4 +6840,4 @@ - agentic-coding description: - "Refresh the Kimi-K3 GB300 AgentX configurations with updated configs." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2811 From 6db7608610f68f7395157daf07f081cb0874ff5d Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:34:14 -0700 Subject: [PATCH 3/5] fix --- ...g-gb300-dcp8-dspark4-mooncake-agentic.yaml | 120 ---------------- ...dcp8-dspark7-maxseq2-mooncake-agentic.yaml | 128 ------------------ configs/nvidia-master.yaml | 48 +------ perf-changelog.yaml | 1 - 4 files changed, 1 insertion(+), 296 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml deleted file mode 100644 index a5ffb6d0c3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml +++ /dev/null @@ -1,120 +0,0 @@ -name: "kimi-k3-vllm-agg-gb300-dcp8-dspark4-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - ETCD_LEASE_TTL: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - MC_SLICE_SIZE: "1048576" - WITH_NVIDIA_PEERMEM: "0" - VLLM_LOG_STATS_INTERVAL: "1" - - vllm_config: - aggregated: - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' - served-model-name: "moonshotai/Kimi-K3" - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - dcp-comm-backend: "a2a" - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - enable-prefix-caching: true - prefix-match-unit: 128 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml deleted file mode 100644 index dafc046abe..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "kimi-k3-vllm-agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - ETCD_LEASE_TTL: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - MC_SLICE_SIZE: "1048576" - WITH_NVIDIA_PEERMEM: "0" - VLLM_LOG_STATS_INTERVAL: "1" - - vllm_config: - aggregated: - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' - served-model-name: "moonshotai/Kimi-K3" - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - dcp-comm-backend: "a2a" - max-num-seqs: 2 - max-num-batched-tokens: 8192 - trust-remote-code: true - max-cudagraph-capture-size: 1024 - stream-interval: 10 - language-model-only: true - moe-backend: "auto" - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' - enable-prefix-caching: true - prefix-match-unit: 128 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0960962ecc..200298e55a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9617,7 +9617,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [16, 32, 48, 64] + conc-list: [32, 48, 64] prefill: num-worker: 1 tp: 8 @@ -9644,52 +9644,6 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } -# Aggregate global p90-ITL Pareto points. One TP8 x DCP8 worker spans two -# 4-GPU nodes and serves both phases, so GPUs are counted once. -kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77 - model: moonshotai/Kimi-K3 - model-prefix: kimik3 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: false - scenarios: - agentic-coding: - - dram-utilization: 0.1775 - search-space: - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [32, 36] - num-nodes: 2 - worker: - num-worker: 1 - tp: 8 - dcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml" - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [1, 4] - num-nodes: 2 - worker: - num-worker: 1 - tp: 8 - dcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml" - # Aggregate no-spec points measured at concurrency 48 and 70. One TP8 x DCP8 worker # spans two 4-GPU nodes and serves both phases. kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ff6f57beec..5f37560eb1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6834,7 +6834,6 @@ - config-keys: - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg - - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg scenario-type: - agentic-coding From bce55d5f2ac8b30e3109824c92facdeb8ed86e89 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:42:57 -0700 Subject: [PATCH 4/5] fixup --- ...dcp8-dspark7-maxseq2-mooncake-agentic.yaml | 128 +++++++++++++ ...3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml | 169 ++++++++++++++++++ perf-changelog.yaml | 1 + 3 files changed, 298 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml new file mode 100644 index 0000000000..dafc046abe --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml @@ -0,0 +1,128 @@ +name: "kimi-k3-vllm-agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + MC_SLICE_SIZE: "1048576" + WITH_NVIDIA_PEERMEM: "0" + VLLM_LOG_STATS_INTERVAL: "1" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + served-model-name: "moonshotai/Kimi-K3" + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + max-num-seqs: 2 + max-num-batched-tokens: 8192 + trust-remote-code: true + max-cudagraph-capture-size: 1024 + stream-interval: 10 + language-model-only: true + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + prefix-match-unit: 128 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml new file mode 100644 index 0000000000..2df5ea74f3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml @@ -0,0 +1,169 @@ +name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 6 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + master_extra_args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + prefill_environment: &kimi_env + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + NCCL_NET_PLUGIN: "none" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + decode_environment: + <<: *kimi_env + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5f37560eb1..ff6f57beec 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6834,6 +6834,7 @@ - config-keys: - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg scenario-type: - agentic-coding From f6984c117827245de7c26ea6496e661dd2e41985 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:47:45 -0700 Subject: [PATCH 5/5] fix --- configs/nvidia-master.yaml | 47 +++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 200298e55a..c3361b057e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9632,7 +9632,22 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [1, 32, 48] + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml" + decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [32, 48] prefill: num-worker: 1 tp: 8 @@ -9644,6 +9659,36 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + +kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.1775 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [1, 4] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml" + # Aggregate no-spec points measured at concurrency 48 and 70. One TP8 x DCP8 worker # spans two 4-GPU nodes and serves both phases. kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: