perf(agentx): add LMCache offload arm to Kimi-K3 B300 at conc 56/70

benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh

@@ -99,13 +99,18 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
 
 SERVER_LOG="$RESULT_DIR/server.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
+LMCACHE_LOG="$RESULT_DIR/lmcache_server.log"
 mkdir -p "$RESULT_DIR"
 MOONCAKE_MASTER_PID=""
+LMCACHE_PID=""
 
 cleanup() {
     if [[ -n "$MOONCAKE_MASTER_PID" ]]; then
         kill "$MOONCAKE_MASTER_PID" 2>/dev/null || true
     fi
+    if [[ -n "$LMCACHE_PID" ]]; then
+        stop_background_process_tree "$LMCACHE_PID" "LMCache server"
+    fi
 }
 trap cleanup EXIT
 
@@ -188,8 +193,64 @@ EOF
             '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}'
         )
         ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+        # cp_kv_cache_interleave_size stays at vLLM's default of 1 on this
+        # image, so the connector needs no interleave handling and any DCP-
+        # capable build works. rc2 is the CUDA 13 wheel.
+        LMCACHE_VERSION=0.5.5rc2
+        # LMCache's transfer-channel layer imports mooncake-transfer-engine at
+        # module load; the wheel ships none of its native libs.
+        agentic_pip_install --quiet --no-cache-dir --no-deps \
+            "mooncake-transfer-engine-cuda13==0.3.11.post1"
+        agentic_pip_install --quiet --no-cache-dir --no-deps \
+            "lmcache==$LMCACHE_VERSION"
+        python3 -c "from lmcache.integration.vllm.lmcache_mp_connector import LMCacheMPConnector" >/dev/null
+
+        # Under DCP the chunk must be a multiple of block size x DCP_SIZE.
+        # Kimi-K3's hybrid groups resolve a local block of 1536, so DCP 8
+        # gives 12288.
+        LMCACHE_CHUNK_SIZE=$((1536 * DCP_SIZE))
+        LMCACHE_PORT=$((PORT + 13000))
+        LMCACHE_HTTP_PORT=$((PORT + 14000))
+
+        LMCACHE_CMD=(
+            lmcache server
+            --host 127.0.0.1
+            --port "$LMCACHE_PORT"
+            --http-host 127.0.0.1
+            --http-port "$LMCACHE_HTTP_PORT"
+            --l1-size-gb "$TOTAL_CPU_DRAM_GB"
+            --l1-init-size-gb 10
+            --l1-read-ttl-seconds 3600
+            --chunk-size "$LMCACHE_CHUNK_SIZE"
+            --separate-object-groups
+            --enable-extra-logging
+            --extra-logging-interval 30
+            --max-cpu-workers 8
+            --max-gpu-workers "${LMCACHE_GPU_WORKERS:-8}"
+            --eviction-policy LRU
+            --supported-transfer-mode lmcache_driven
+            --shm-name ""
+        )
+        append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}"
+        "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 &
+        LMCACHE_PID=$!
+        wait_for_ready \
+            --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \
+            --log "$LMCACHE_LOG" \
+            --pid "$LMCACHE_PID" \
+            --sleep-interval 1 \
+            --timeout 600
+
+        # 90k-290k-token agentic prefixes make single retrieves large.
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}"
+        )
+        ;;
     *)
-        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or mooncake)" >&2
+        echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, mooncake, or lmcache)" >&2
         exit 1
         ;;
 esac
@@ -223,6 +284,15 @@ if [ "$NUM_SPEC_TOKENS" -gt 0 ]; then
     SPEC_ARGS=(--speculative-config "$SPEC_CONFIG")
 fi
 
+# The LMCache MP connector shares the KV tensors with its out-of-process server
+# over CUDA IPC, and cuMem-allocated memory has no legacy IPC handle:
+# _share_cuda_() returns cudaErrorInvalidValue and the worker dies during
+# initialize_from_config. No other LMCache recipe in the repo enables cumem.
+CUMEM_ARGS=(--enable-cumem-allocator)
+if [ "${KV_OFFLOAD_BACKEND:-}" = "lmcache" ]; then
+    CUMEM_ARGS=()
+fi
+
 MAX_NUM_SEQS=$((2 * CONC))
 
 # 1 - this is the buffer for what is not sized against the budget: the cudagraph
@@ -275,7 +345,7 @@ VLLM_CMD=(
     --load-format fastsafetensors
     --moe-backend auto
     --no-enable-flashinfer-autotune
-    --enable-cumem-allocator
+    "${CUMEM_ARGS[@]}"
     --enable-prefix-caching
     --prefix-match-unit 128
     --kv-cache-dtype fp8

configs/nvidia-master.yaml

@@ -1505,6 +1505,14 @@ kimik3-fp4-b300-vllm-agentic-dspark:
       # DSpark 3 at conc 16, and not at all above. Keep the concurrencies
       # disjoint across arms so exp-names stay unique.
       - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70] }
+      # LMCache arm at the two points where the offload tier carries the run:
+      # GPU prefix cache hit rate falls to 88.0% at conc 56 and 59.9% at conc
+      # 70, against 90.4-96.3% everywhere below, so most prefix reuse has to
+      # come from the external tier. Both points are no-drafting at
+      # gpu-memory-utilization 0.90. The concurrencies overlap the existing
+      # dram arm on purpose - the backend name is in the exp-name, so each
+      # point still yields a distinct series.
+      - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc2" }, conc-list: [56, 70] }
 
 dsr1-fp8-b200-trt:
   image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14

perf-changelog.yaml

@@ -6831,3 +6831,14 @@
     - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies."
     - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808
+
+- config-keys:
+    - kimik3-fp4-b300-vllm-agentic-dspark
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add an LMCache 0.5.5rc2 DRAM offload arm at concurrency 56 and 70, where the GPU prefix cache hit rate falls to 88.0% and 59.9%."
+    - "Size the LMCache chunk as block size x DCP size (1536 x 8 = 12288), required for chunking under decode context parallel."
+    - "Raise the LMCache L1 read-lock TTL to 3600s: the lock is held from prefetch to retrieve, and at concurrency 70 a queued request can exceed the 300s default, which fails the retrieve."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2802
+  append-only: true