From 6c9cfa84ab01852ffdab3421e84b3e1ee54aab02 Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 1 Sep 2026 15:59:47 +0530 Subject: [PATCH 01/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- .../agentic/kimik3_fp4_mi355x_mtp.sh | 322 ++++-------------- configs/amd-master.yaml | 6 +- 2 files changed, 61 insertions(+), 267 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index a8f2c42c1c..1ee7ccc1e0 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -1,70 +1,21 @@ #!/usr/bin/env bash set -euo pipefail set -x - -# Agentic trace replay benchmark for Kimi-K3 MXFP4 on MI355X / MI350X (gfx950) -# using vLLM. -# -# The server command is the AMD reference `vllm serve` for this model, i.e. the -# upstream vLLM recipe's amd block (vllm-project/recipes, -# https://recipes.vllm.ai/moonshotai/Kimi-K3) as run in practice: -# -# --trust-remote-code --moe-backend auto --tensor-parallel-size 8 -# --load-format auto --gpu-memory-utilization 0.95 --mm-encoder-tp-mode data -# --max-num-seqs 128 --max-num-batched-tokens 4096 --enable-auto-tool-choice -# --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 -# -# with env VLLM_ROCM_USE_AITER=1 SAFETENSORS_FAST_GPU=1 AITER_SITUV2_A8W4=1 -# AITER_BF16_FP8_MOE_BOUND=0 VLLM_USE_BREAKABLE_CUDAGRAPH=0. -# -# K3 is a 2.8T-parameter natively-multimodal MoE (896 routed experts, 16/token -# plus shared) on Kimi Delta Attention, gated MLA and Attention Residuals, with -# a 1M-token native context. -# -# TP=8 ONLY. The MXFP4 checkpoint is 1.561 TB decimal (1.420 TiB, 96 -# safetensors), ~195 GB/GPU across 8 GPUs of the 288 GB part; TP=4 would need -# ~390 GB/GPU and cannot load. Upstream strategy_min_gpus agrees (single_node_tp -# and multi_node_tep both 8, DEP 16+), which is why there is no DP-attention arm. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE -# -# Perf-search knobs. Each defaults to the reference command's value, so an -# otherwise-unset run reproduces the reference exactly: -# GPU_MEM_UTIL 0.95 (reference) -# MAX_NUM_BATCHED_TOKENS 8192 (default) -# AITER_A8W4 1 (reference; 0 = aiter a16w4 MoE path) -# LANGUAGE_MODEL_ONLY true -# KV_CACHE_DTYPE fp8 (default for every arm; =auto for a bf16 A/B) -# KV_BLOCK_SIZE unset (unset -> vLLM sizes the page; 128 under fp8) -# MAX_MODEL_LEN 1M -# SPEC_DECODE true (this is the _mtp DSpark recipe; =false for a no-spec A/B) -# SPEC_NUM_TOKENS 2 (DSpark draft length; validated by the _mtp config) - source "$(dirname "$0")/../../benchmark_lib.sh" - wait_for_amd_gpu_clean +export AIPERF_EXPERIMENTAL_FAST=0 +export AIPERF_WARMUP_REQUESTS_PER_LANE=1 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi +DP_SIZE=1 +export DP_SIZE +TOTAL_RANKS=$(( TP * DP_SIZE )) -if [ "$TP" -ne 8 ]; then - echo "Error: Kimi-K3 MXFP4 is a 1.56 TB checkpoint and only fits at TP=8 on" >&2 - echo " 288 GB gfx950 parts (~195 GB/GPU). Got TP=$TP." >&2 - exit 1 -fi - -# ROCR/HIP visibility for vLLM 0.14+ if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" fi -# `hf download` creates the target dir if missing and is itself idempotent. The -# 1.56 TB checkpoint is normally pre-staged, so these calls are a no-op there. if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" @@ -75,244 +26,84 @@ else fi rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- resolve_trace_source install_agentic_deps -# ---- Reference env block ---------------------------------------------------- export VLLM_ROCM_AITER_MLA_ASM_PADDING=asm export VLLM_ROCM_USE_AITER=1 -export SAFETENSORS_FAST_GPU=1 +export VLLM_ROCM_USE_AITER_MLA=1 +export VLLM_ROCM_USE_AITER_MOE=1 export VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION="${VLLM_ROCM_QUICK_REDUCE_QUANTIZATION:-NONE}" export AITER_SITUV2_A8W4=1 export AITER_BF16_FP8_MOE_BOUND=0 +export AITER_DISABLE_FMHA_OPUS=1 +export SAFETENSORS_FAST_GPU=1 +export GPU_ARCHS=gfx950 +export HSA_NO_SCRATCH_RECLAIM=1 export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 - -# Workaround for MEC FW <177 RCCL memory reclaim issue (shared with the other -# gfx950 recipes in this tree). -mec_version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') -if [[ "$mec_version" == "" || ${mec_version:-0} -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# 2.8T of weights off a shared/NFS mount takes far longer than the default. -export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-7200}" - -# Long agentic turns against a 1M context: keep the client from timing out -# mid-request while the server is prefill-bound. +export VLLM_K3_KDA_SAFE_STAGES=1 +export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1 +export VLLM_ENGINE_READY_TIMEOUT_S=7200 +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600 export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +export PYTHONNOUSERSITE=1 +export PYTHONHASHSEED=42 -# ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" - SERVER_PID="" -LMCACHE_PID="" cleanup_agentic_services() { local exit_code=$? trap - EXIT INT TERM set +e stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - stop_background_process_tree "$LMCACHE_PID" "LMCache server" exit "$exit_code" } trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM -# ---- KV offload ------------------------------------------------------------- -# TOTAL_CPU_DRAM_GB is the aggregate host-DRAM budget the matrix generator -# derives from dram-utilization and the runner's available-cpu-dram-mib, capped -# at the 3,095,781 MiB (3 TB decimal) agentic limit. Per -# benchmarks/single_node/agentic/README.md it must be consumed as given and -# never replaced with a model-specific constant. -OFFLOAD_ARGS=() - -if agentic_kv_offload_enabled; then -case "${KV_OFFLOAD_BACKEND:-}" in - vllm-simple) - require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TP )) - # Identical prefixes must hash to identical block keys across ranks. - export PYTHONHASHSEED=42 - SIMPLE_LAZY_OFFLOAD="${SIMPLE_LAZY_OFFLOAD:-false}" - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":$CPU_BYTES_PER_RANK,\"lazy_offload\":$SIMPLE_LAZY_OFFLOAD}}" - ) - echo "SimpleCPUOffloadConnector: ${CPU_BYTES_PER_RANK} B/rank x ${TP} ranks, lazy_offload=$SIMPLE_LAZY_OFFLOAD" - ;; - lmcache) - require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - - # Keep the image's tested torch/ROCm stack and install only LMCache's - # missing runtime dependencies, same as the MiniMax-M3 lmcache arm. - LMCACHE_VERSION="0.5.5.dev60+rocm7.2" - LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-rocm" - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-rocm-7-0==14.1.1" \ - "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" - - # LMCache 0.5.5's transfer-channel layer eagerly imports the Mooncake - # backend (mooncake_te_impl.py -> `from mooncake.engine import - # TransferEngine`), whose native .so resolves all of its DT_NEEDED libs at - # import. The vLLM ROCm image ships none of them, so the import sanity - # check below (and the LMCache server) would otherwise fail with - # "ImportError: lib*.so: cannot open shared object file" (first libglog, - # then libjsoncpp, ...). Provision Mooncake's full runtime lib set from the - # distro before importing. apt-get install is idempotent, so run it - # whenever any of the libs is still missing rather than gating on one. - LMCACHE_NATIVE_LIBS=(libglog.so.0 libjsoncpp.so.25 libibverbs.so.1 librdmacm.so.1 libnuma.so.1) - for lib in "${LMCACHE_NATIVE_LIBS[@]}"; do - if ! ldconfig -p | grep -q "$lib"; then - apt-get update - apt-get install -y \ - libgoogle-glog0v5 libjsoncpp25 libibverbs1 librdmacm1 libnuma1 - break - fi - done - python3 -c \ - "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ - >/dev/null - - # One MP server for the node, per the Kimi-K3 recipe - # (docs.lmcache.ai/recipes/kimi_k3.html), with --chunk-size sized for - # THIS stack rather than the recipe's CUDA-path 768: the connector - # requires the chunk to be a multiple of every engine KV group's - # tokens_per_block, and the hybrid KDA/MLA layout here registers - # attention groups at 1536 ("Setting attention block size to 1536", - # run 31644990546) plus a KDA state group at 3072 (run 31645828378), - # so 3072 is the minimum valid chunk. The multi-group layout also - # requires one object group per sliding-window size: - # --separate-object-groups. - LMCACHE_PORT=6555 - LMCACHE_HTTP_PORT=8090 - LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" - - LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" - - LMCACHE_CMD=( - lmcache server - --host 127.0.0.1 - --port "$LMCACHE_PORT" - --http-host 127.0.0.1 - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb 10 - --chunk-size 3072 - --separate-object-groups - --enable-extra-logging - --extra-logging-interval 30 - --max-cpu-workers 8 - --max-gpu-workers 1 - --eviction-policy LRU - --supported-transfer-mode lmcache_driven - --shm-name "" - ) - append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - wait_for_ready \ - --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ - --log "$LMCACHE_LOG" \ - --pid "$LMCACHE_PID" \ - --sleep-interval 1 \ - --timeout 600 - - # 100k-330k-token agentic prefixes make single retrieves large; use the - # same MQ timeout headroom as the MiniMax-M3 arm. - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected vllm-simple or lmcache)" >&2 - exit 1 - ;; -esac +SPEC_ARGS=() +SPEC_ROWS=1 +if [ "$CONC" -le 4 ]; then + SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-8}" + SPEC_ROWS=$(( SPEC_NUM_TOKENS + 1 )) + SPEC_ARGS=(--speculative-config "{\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":4.0}") + MAX_NUM_SEQS=1 + MAX_BATCHED_TOKENS=8192 + DCP_SIZE=1 +else + MAX_NUM_SEQS=$(( CONC + CONC / 4 )) + if [ "$MAX_NUM_SEQS" -gt 80 ]; then MAX_NUM_SEQS=80; fi + MAX_BATCHED_TOKENS=16384 + DCP_SIZE=8 fi +export DCP_SIZE -# ---- LLM server ------------------------------------------------------------ +LADDER=$(( MAX_NUM_SEQS * SPEC_ROWS )) +CUDAGRAPH_CAPTURE_SIZES=$(seq -s, 1 "$LADDER") +COMPILATION_CONFIG_ARGS=(--compilation-config "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"max_cudagraph_capture_size\":$LADDER,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[$CUDAGRAPH_CAPTURE_SIZES]}") -# ---- Parallelism ------------------------------------------------------------ -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) +CP_ARGS=(--attention-backend ROCM_AITER_MLA) +if [ "$DCP_SIZE" -gt 1 ]; then + CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a --cp-kv-cache-interleave-size 1) fi -# ---- Speculative / Util------------------------------------------------------ -case "$CONC" in - # No KV offload; the working set fits in HBM. - 1) - SYNTHETIC_ACCEPT_LEN=3.75 - SPEC_NUM_TOKENS=6 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=16384 - ;; - 2|4|8|10|12|14) - SYNTHETIC_ACCEPT_LEN=3.00 - SPEC_NUM_TOKENS=3 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 - ;; - *) - SPEC_NUM_TOKENS=0 - GPU_MEM_UTIL=0.85 - MAX_NUM_BATCHED_TOKENS=4096 - ;; -esac +GPU_MEM_UTIL=0.9 -SPEC_ARGS=() -if [ "$SPEC_NUM_TOKENS" -gt 0 ]; then -if [ "${EVAL_ONLY:-false}" = "true" ]; then - SPEC_ARGS=( - --speculative-config - "{\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\": \"block\"}" - ) -else - SPEC_ARGS=( - --speculative-config - "{\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" - ) - fi +OFFLOAD_ARGS=() +if agentic_kv_offload_enabled; then + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TOTAL_RANKS )) + OFFLOAD_ARGS=(--kv-transfer-config "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":$CPU_BYTES_PER_RANK,\"lazy_offload\":false}}") fi -# ---- HIP graph ------------------------------------------------------------ -MAX_NUM_SEQS=$((2 * CONC)) -MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * (1 + SPEC_NUM_TOKENS))) -CUDAGRAPH_CAPTURE_SIZES="$(seq -s, 2 "$MAX_CUDAGRAPH_CAPTURE_SIZE")" -COMPILATION_CONFIG_ARGS=(--compilation-config "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"max_cudagraph_capture_size\":$MAX_CUDAGRAPH_CAPTURE_SIZE,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[$CUDAGRAPH_CAPTURE_SIZES]}") - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 -export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS="${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-1200}" - +EP_ARGS=() +if [ "${EP_SIZE:-1}" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel); fi -# ---- DCP ------------------------------------------------------------ -# DCP shards decode KV across the TP ranks, so it must divide TP. -DCP_SIZE="${DCP_SIZE:-8}" -if [ $((TP % DCP_SIZE)) -ne 0 ]; then - echo "Error: TP='$TP' must be divisible by DCP_SIZE='$DCP_SIZE'" >&2 - exit 1 -fi -CP_ARGS=() -ATTN_BE_ARGS=() -if [ "$DCP_SIZE" -gt 1 ]; then - CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) - ATTN_BE_ARGS+=(--attention-backend TRITON_MLA) -fi -export VLLM_USE_DIRECT_DCP_A2A=0 -export VLLM_USE_DIRECT_DCP_Q_GATHER=0 -export VLLM_USE_DIRECT_DCP_KV_GATHER=0 +echo "[cfg] conc=$CONC dcp=$DCP_SIZE gmu=$GPU_MEM_UTIL mns=$MAX_NUM_SEQS ladder=1..$LADDER (mns x $SPEC_ROWS) chunk=$MAX_BATCHED_TOKENS spec=${#SPEC_ARGS[@]} offload=${KV_OFFLOADING:-none}" -{ set +x; } 2>/dev/null VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 @@ -320,34 +111,37 @@ VLLM_CMD=( --trust-remote-code --moe-backend auto --tensor-parallel-size "$TP" - "${EP_ARGS[@]}" --load-format fastsafetensors --gpu-memory-utilization "$GPU_MEM_UTIL" --language-model-only --max-num-seqs "$MAX_NUM_SEQS" + --max-num-batched-tokens "$MAX_BATCHED_TOKENS" + --max-model-len 1048576 + --kv-cache-dtype fp8 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 - --max-model-len 1048576 --enable-prefix-caching - --kv-cache-dtype "fp8" - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --enable-prompt-tokens-details + --no-async-scheduling --attention-config '{"mla_prefill_backend":"ROCM_AITER_FA"}' - "${ATTN_BE_ARGS[@]}" - "${COMPILATION_CONFIG_ARGS[@]}" - "${SPEC_ARGS[@]}" "${OFFLOAD_ARGS[@]}" "${CP_ARGS[@]}" + "${EP_ARGS[@]}" + "${SPEC_ARGS[@]}" + "${COMPILATION_CONFIG_ARGS[@]}" ) + printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" + "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! echo "Server PID: $SERVER_PID" wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [ "${EVAL_ONLY}" = "true" ]; then +if [ "${EVAL_ONLY:-false}" = "true" ]; then run_eval --port "$PORT" else build_replay_cmd "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 62daacca6f..a4e80139ce 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -625,7 +625,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=2" kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds @@ -636,8 +636,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev60+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 16, 32, 48, 52], spec-decoding: mtp} # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From e64ac4dc4991b9c8c4578860ef112f4a8b6e2e1b Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 1 Sep 2026 16:02:39 +0530 Subject: [PATCH 02/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2f58d1b442..af26c097ba 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6730,3 +6730,11 @@ - "Configure model-specific tool and reasoning parsers for the Kimi-K3 and MiniMax-M3 serving paths so opt-in verifier requests produce structured OpenAI tool calls." - "Limit shared AgentX readiness and synthetic-acceptance rewrites to eval-only runs so default throughput recipe rendering remains unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Image : vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2795 From 23cf24ad9f892b19d2dd2638b03c975097b24a5e Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 1 Sep 2026 16:12:01 +0530 Subject: [PATCH 03/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 1ee7ccc1e0..4aa4c6ecb3 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -70,7 +70,12 @@ SPEC_ROWS=1 if [ "$CONC" -le 4 ]; then SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-8}" SPEC_ROWS=$(( SPEC_NUM_TOKENS + 1 )) - SPEC_ARGS=(--speculative-config "{\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":4.0}") + if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_VERIFY="\"rejection_sample_method\":\"standard\"" + else + SPEC_VERIFY="\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":4.0" + fi + SPEC_ARGS=(--speculative-config "{\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",$SPEC_VERIFY}") MAX_NUM_SEQS=1 MAX_BATCHED_TOKENS=8192 DCP_SIZE=1 @@ -102,6 +107,7 @@ fi EP_ARGS=() if [ "${EP_SIZE:-1}" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel); fi +echo "[spec] verify=${SPEC_VERIFY:-none}" echo "[cfg] conc=$CONC dcp=$DCP_SIZE gmu=$GPU_MEM_UTIL mns=$MAX_NUM_SEQS ladder=1..$LADDER (mns x $SPEC_ROWS) chunk=$MAX_BATCHED_TOKENS spec=${#SPEC_ARGS[@]} offload=${KV_OFFLOADING:-none}" VLLM_CMD=( From c197a62a37afea87db2f4deaaebe9469289eb6cc Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 1 Sep 2026 16:21:40 +0530 Subject: [PATCH 04/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 4aa4c6ecb3..6ae4e1f1f1 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -71,7 +71,7 @@ if [ "$CONC" -le 4 ]; then SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-8}" SPEC_ROWS=$(( SPEC_NUM_TOKENS + 1 )) if [ "${EVAL_ONLY:-false}" = "true" ]; then - SPEC_VERIFY="\"rejection_sample_method\":\"standard\"" + SPEC_VERIFY="\"rejection_sample_method\":\"block\"" else SPEC_VERIFY="\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":4.0" fi From 4284eaa8373a17aa7d8ba754bdf87d258f937e7d Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 2 Sep 2026 07:57:02 +0530 Subject: [PATCH 05/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 6 +++--- configs/amd-master.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 6ae4e1f1f1..bf86c700c2 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -70,7 +70,7 @@ SPEC_ROWS=1 if [ "$CONC" -le 4 ]; then SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-8}" SPEC_ROWS=$(( SPEC_NUM_TOKENS + 1 )) - if [ "${EVAL_ONLY:-false}" = "true" ]; then + if [ "${EVAL_ONLY}" = "true" ]; then SPEC_VERIFY="\"rejection_sample_method\":\"block\"" else SPEC_VERIFY="\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":4.0" @@ -147,9 +147,9 @@ echo "Server PID: $SERVER_PID" wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [ "${EVAL_ONLY:-false}" = "true" ]; then +if [ "${EVAL_ONLY}" = "true" ]; then run_eval --port "$PORT" else build_replay_cmd "$RESULT_DIR" run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi +fi \ No newline at end of file diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a60255c054..ad3ff26a0b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -637,7 +637,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: - dram-utilization: 0.60 search-space: - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 16, 32, 48, 52], spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 32, 52], spec-decoding: mtp} # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From 35e59bc08d41b82bcfcbf90b10813b16e3b34d11 Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 2 Sep 2026 09:29:57 +0530 Subject: [PATCH 06/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 2 +- configs/amd-master.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index bf86c700c2..38dfe09ef0 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -152,4 +152,4 @@ if [ "${EVAL_ONLY}" = "true" ]; then else build_replay_cmd "$RESULT_DIR" run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi \ No newline at end of file +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ad3ff26a0b..0f4099d3da 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -637,7 +637,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: - dram-utilization: 0.60 search-space: - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 32, 52], spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [52], spec-decoding: mtp} # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From a2cb4b337c4f93b5b1b590ef275f6e26c4300620 Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 2 Sep 2026 11:28:29 +0530 Subject: [PATCH 07/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 2 +- configs/amd-master.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 38dfe09ef0..16363800e6 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -82,7 +82,7 @@ if [ "$CONC" -le 4 ]; then else MAX_NUM_SEQS=$(( CONC + CONC / 4 )) if [ "$MAX_NUM_SEQS" -gt 80 ]; then MAX_NUM_SEQS=80; fi - MAX_BATCHED_TOKENS=16384 + MAX_BATCHED_TOKENS=8192 DCP_SIZE=8 fi export DCP_SIZE diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0f4099d3da..10254e6be1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -636,8 +636,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [52], spec-decoding: mtp} + # - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: none, kv-offload-backend: { name: vllm-simple }, conc-list: [52], spec-decoding: mtp} # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From 0d96fc85c4f4d8b9442daf5310fd2a15fb151a89 Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 2 Sep 2026 11:30:21 +0530 Subject: [PATCH 08/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 10254e6be1..953e2f7c82 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -637,7 +637,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: - dram-utilization: 0.60 search-space: # - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: none, kv-offload-backend: { name: vllm-simple }, conc-list: [52], spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: none, conc-list: [52], spec-decoding: mtp} # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From b974190ec413de24ae0d56cf33176776dc1ce3fd Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 2 Sep 2026 13:23:25 +0530 Subject: [PATCH 09/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 16363800e6..bfde4aa64a 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -96,7 +96,7 @@ if [ "$DCP_SIZE" -gt 1 ]; then CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a --cp-kv-cache-interleave-size 1) fi -GPU_MEM_UTIL=0.9 +GPU_MEM_UTIL=0.88 OFFLOAD_ARGS=() if agentic_kv_offload_enabled; then From fbe084ed966a96b770e5d5b2d957416ecc692228 Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 2 Sep 2026 15:20:40 +0530 Subject: [PATCH 10/10] [AMD] [AGENTX] Kimi Perf Tuning Signed-off-by: Sirra --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 953e2f7c82..1d172c65e0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -625,7 +625,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=2" kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb + image: vllm/vllm-openai-rocm:nightly-73029d42441321b631779db3475031f5ec26dd6c model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds