perf(agentx): refresh Kimi-K3 MI355X LMCache curve / 刷新 Kimi-K3 MI355X LMCache 曲线

benchmarks/single_node/agentic/kimik3_fp4_mi355x.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export SPEC_DECODING=none
+exec bash "$(dirname "$0")/kimik3_fp4_mi355x_mtp.sh"

benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh

@@ -149,25 +149,18 @@ case "${KV_OFFLOAD_BACKEND:-}" in
       lmcache)
     require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND"
 
-    # Keep the image's tested torch/ROCm stack and install only LMCache's
-    # missing runtime dependencies, same as the MiniMax-M3 lmcache arm.
-    LMCACHE_VERSION="0.5.5.dev60+rocm7.2"
-    LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-rocm"
+    # Keep the stock image's torch/ROCm stack and install the LMCache runtime
+    # dependencies used by the validated reference run.
+    LMCACHE_VERSION="0.5.5rc3+rocm7.2"
+    export KV_OFFLOAD_BACKEND_METADATA="{\"name\":\"lmcache\",\"version\":\"${LMCACHE_VERSION}\"}"
+    LMCACHE_RELEASE="v0.5.5rc3"
+    LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/${LMCACHE_RELEASE}-rocm"
     agentic_pip_install --quiet --no-cache-dir --no-deps \
         "sortedcontainers==2.4.0" \
         "opentelemetry-exporter-prometheus==0.61b0" \
         "cupy-rocm-7-0==14.1.1" \
         "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX"
 
-    # LMCache 0.5.5's transfer-channel layer eagerly imports the Mooncake
-    # backend (mooncake_te_impl.py -> `from mooncake.engine import
-    # TransferEngine`), whose native .so resolves all of its DT_NEEDED libs at
-    # import. The vLLM ROCm image ships none of them, so the import sanity
-    # check below (and the LMCache server) would otherwise fail with
-    # "ImportError: lib*.so: cannot open shared object file" (first libglog,
-    # then libjsoncpp, ...). Provision Mooncake's full runtime lib set from the
-    # distro before importing. apt-get install is idempotent, so run it
-    # whenever any of the libs is still missing rather than gating on one.
     LMCACHE_NATIVE_LIBS=(libglog.so.0 libjsoncpp.so.25 libibverbs.so.1 librdmacm.so.1 libnuma.so.1)
     for lib in "${LMCACHE_NATIVE_LIBS[@]}"; do
         if ! ldconfig -p | grep -q "$lib"; then
@@ -178,17 +171,16 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         fi
     done
     python3 -c \
-        "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
-        >/dev/null
+        "import cupy; import opentelemetry.exporter.prometheus; from lmcache.v1.multiprocess.http_server import run_http_server"
 
     # One MP server for the node, per the Kimi-K3 recipe
     # (docs.lmcache.ai/recipes/kimi_k3.html), with --chunk-size sized for
     # THIS stack rather than the recipe's CUDA-path 768: the connector
     # requires the chunk to be a multiple of every engine KV group's
-    # tokens_per_block, and the hybrid KDA/MLA layout here registers
-    # attention groups at 1536 ("Setting attention block size to 1536",
-    # run 31644990546) plus a KDA state group at 3072 (run 31645828378),
-    # so 3072 is the minimum valid chunk. The multi-group layout also
+    # tokens_per_block. The hybrid KDA/MLA layout registers attention groups
+    # at 1536 tokens and a KDA state group at 3072. Use 12288 for every point
+    # so it is divisible by both group sizes and matches the tested LMCache
+    # scheduler geometry. The multi-group layout also
     # requires one object group per sliding-window size:
     # --separate-object-groups.
     LMCACHE_PORT=6555
@@ -205,7 +197,7 @@ case "${KV_OFFLOAD_BACKEND:-}" in
         --http-port "$LMCACHE_HTTP_PORT"
         --l1-size-gb "$LMCACHE_L1_SIZE_GB"
         --l1-init-size-gb 10
-        --chunk-size 3072
+        --chunk-size 12288
         --separate-object-groups
         --enable-extra-logging
         --extra-logging-interval 30
@@ -248,20 +240,24 @@ if [ "$EP_SIZE" -gt 1 ]; then
 fi
 
 # ---- Speculative / Util------------------------------------------------------
-case "$CONC" in
-    # No KV offload; the working set fits in HBM.
-    1)
+case "${SPEC_DECODING:-mtp}:$CONC" in
+    mtp:1)
         SYNTHETIC_ACCEPT_LEN=3.75
         SPEC_NUM_TOKENS=6
-        GPU_MEM_UTIL=0.9
+        GPU_MEM_UTIL=0.88
         MAX_NUM_BATCHED_TOKENS=16384
         ;;
-    2|4|8|10|12|14)
+    mtp:2|mtp:4|mtp:8|mtp:10|mtp:12|mtp:14)
         SYNTHETIC_ACCEPT_LEN=3.00
         SPEC_NUM_TOKENS=3
-        GPU_MEM_UTIL=0.9
+        GPU_MEM_UTIL=0.88
         MAX_NUM_BATCHED_TOKENS=8192
         ;;
+    none:40|none:44|none:48)
+        SPEC_NUM_TOKENS=0
+        GPU_MEM_UTIL=0.88
+        MAX_NUM_BATCHED_TOKENS=16384
+        ;;
     *)
         SPEC_NUM_TOKENS=0
         GPU_MEM_UTIL=0.85
@@ -285,10 +281,27 @@ else
 fi
 
 # ---- HIP graph ------------------------------------------------------------
-MAX_NUM_SEQS=$((2 * CONC))
-MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * (1 + SPEC_NUM_TOKENS)))
-CUDAGRAPH_CAPTURE_SIZES="$(seq -s, 2 "$MAX_CUDAGRAPH_CAPTURE_SIZE")"
-COMPILATION_CONFIG_ARGS=(--compilation-config "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"max_cudagraph_capture_size\":$MAX_CUDAGRAPH_CAPTURE_SIZE,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[$CUDAGRAPH_CAPTURE_SIZES]}")
+SERVER_STREAM_ARGS=()
+PREFIX_MATCH_ARGS=()
+ATTENTION_CONFIG='{"mla_prefill_backend":"ROCM_AITER_FA"}'
+COMPILATION_CUSTOM_OPS='["+fused_rms_norm_gated"]'
+case "${SPEC_DECODING:-mtp}:$CONC" in
+    none:40|none:44|none:48)
+    MAX_NUM_SEQS=80
+    MAX_CUDAGRAPH_CAPTURE_SIZE=4096
+    CUDAGRAPH_CAPTURE_SIZES="$(seq -s, 1 "$MAX_NUM_SEQS"),128,256,512,1024,2048,4096"
+    SERVER_STREAM_ARGS=(--stream-interval 10)
+    PREFIX_MATCH_ARGS=(--prefix-match-unit 128)
+    ATTENTION_CONFIG='{"mla_prefill_backend":"ROCM_AITER_FA","use_prefill_query_quantization":true}'
+    COMPILATION_CUSTOM_OPS='["+fused_rms_norm_gated","+quant_fp8","+grouped_topk","+sparse_attn_indexer","none"]'
+    ;;
+    *)
+    MAX_NUM_SEQS=$((2 * CONC))
+    MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * (1 + SPEC_NUM_TOKENS)))
+    CUDAGRAPH_CAPTURE_SIZES="$(seq -s, 2 "$MAX_CUDAGRAPH_CAPTURE_SIZE")"
+    ;;
+esac
+COMPILATION_CONFIG_ARGS=(--compilation-config "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"max_cudagraph_capture_size\":$MAX_CUDAGRAPH_CAPTURE_SIZE,\"custom_ops\":$COMPILATION_CUSTOM_OPS,\"cudagraph_capture_sizes\":[$CUDAGRAPH_CAPTURE_SIZES]}")
 
 echo "Starting vllm server..."
 export PYTHONNOUSERSITE=1
@@ -305,8 +318,18 @@ fi
 CP_ARGS=()
 ATTN_BE_ARGS=()
 if [ "$DCP_SIZE" -gt 1 ]; then
-    CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a)
-    ATTN_BE_ARGS+=(--attention-backend TRITON_MLA)
+    CP_KV_CACHE_INTERLEAVE_SIZE=1
+    if [ "${KV_OFFLOAD_BACKEND:-}" = "lmcache" ]; then
+        CP_KV_CACHE_INTERLEAVE_SIZE=1536
+    fi
+    CP_ARGS+=(
+        --decode-context-parallel-size "$DCP_SIZE"
+        --dcp-comm-backend a2a
+        --cp-kv-cache-interleave-size "$CP_KV_CACHE_INTERLEAVE_SIZE"
+    )
+    ATTN_BE_ARGS+=(--attention-backend ROCM_AITER_MLA)
+    export VLLM_ALLOW_DCP_FULL_CUDAGRAPH=1
+    export PREFIX_CACHING_HASH_ALGO=sha256
 fi
 export VLLM_USE_DIRECT_DCP_A2A=0
 export VLLM_USE_DIRECT_DCP_Q_GATHER=0
@@ -329,10 +352,12 @@ VLLM_CMD=(
     --tool-call-parser kimi_k3
     --reasoning-parser kimi_k3
     --max-model-len 1048576
+    "${SERVER_STREAM_ARGS[@]}"
     --enable-prefix-caching
+    "${PREFIX_MATCH_ARGS[@]}"
     --kv-cache-dtype "fp8"
     --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS"
-    --attention-config '{"mla_prefill_backend":"ROCM_AITER_FA"}'
+    --attention-config "$ATTENTION_CONFIG"
     "${ATTN_BE_ARGS[@]}"
     "${COMPILATION_CONFIG_ARGS[@]}"
     "${SPEC_ARGS[@]}"

configs/amd-master.yaml

@@ -625,7 +625,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp:
           - "DECODE_MTP_SIZE=2"
 
 kimik3-fp4-mi355x-vllm-agentic-mtp:
-  image: vllm/vllm-openai-rocm:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45
+  image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb
   model: moonshotai/Kimi-K3
   model-prefix: kimik3
   runner: cluster:mi355x-amds
@@ -636,8 +636,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.60
       search-space:
-      - { tp: 8, kv-offloading: none, conc-list: [1] , spec-decoding: mtp}
-      - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev60+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp }
+      - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc3+rocm7.2" }, conc-list: [1, 8, 14], spec-decoding: mtp }
+      - { tp: 8, dcp-size: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc3+rocm7.2" }, conc-list: [40, 44, 48], spec-decoding: none }
 
 # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark
 # speculative decoding. Acceptance is pinned to the committed golden curve in

perf-changelog.yaml

@@ -6821,3 +6821,13 @@
   description:
     - "Refresh to collect TensorRT-LLM server metrics."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774
+
+- config-keys:
+    - kimik3-fp4-mi355x-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Refresh the Kimi-K3 MI355X AgentX recipe from the merged PR #2787 baseline to vLLM ROCm nightly 7c5dc571 and LMCache 0.5.5rc3+rocm7.2."
+    - "Run C1, C8, and C14 with TP8 DSpark and LMCache DRAM KV offload; run C40, C44, and C48 with TP8/DCP8 and no speculative decoding."
+    - "Use gpu-memory-utilization 0.88 for all six points, a 12288-token LMCache chunk, and a 1536-token DCP KV-cache interleave; C40/C44/C48 retain the 16K batched-token, max-num-seqs 80, and full CUDA-graph profile."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2804