From ff72feb87ce1b4da1595ad96ef944f41d8e77a78 Mon Sep 17 00:00:00 2001 From: Samuel Shen Date: Thu, 3 Sep 2026 07:41:22 +0800 Subject: [PATCH] perf(agentx): add LMCache offload arm to Kimi-K3 B300 at conc 56/70 Adds an LMCache 0.5.5rc2 DRAM offload arm to kimik3-fp4-b300-vllm-agentic-dspark at concurrency 56 and 70. Those are the two points where the offload tier carries the run. In the concurrency sweep from run 33472235399, GPU prefix cache hit rate holds 90.4-96.3% from conc 1 through 48, so nearly all prefix reuse is served out of HBM and the external tier is barely exercised. At conc 56 the pool drops to 21.6M tokens under gpu-memory-utilization 0.90 and GPU hit falls to 88.0%; at conc 70 it falls to 59.9%. Both points are past the drafting cutoff, so they share a topology and memory budget. Under DCP the LMCache chunk must be a multiple of block size x DCP size; Kimi-K3's hybrid KDA/MLA groups resolve a local block of 1536, so DCP 8 gives 12288, matching the validated Kimi-K3 DCP8 e2e run 33463265881. cp_kv_cache_interleave_size stays at vLLM's default of 1: the pinned image defines the option but carries none of the auto-resolution that sets it to the local block size. The cumem allocator is disabled on this arm only. The MP connector shares KV tensors with its out-of-process server over CUDA IPC, and cuMem-allocated memory has no legacy IPC handle, so _share_cuda_() fails with cudaErrorInvalidValue during initialize_from_config. The MP server runs a single GPU transfer worker, matching the validated configuration. --- .../agentic/kimik3_fp4_b300_vllm_mtp.sh | 74 ++++++++++++++++++- configs/nvidia-master.yaml | 8 ++ perf-changelog.yaml | 11 +++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index 86b07e5f2d..38e95fd5ac 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -99,13 +99,18 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 SERVER_LOG="$RESULT_DIR/server.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" MOONCAKE_MASTER_PID="" +LMCACHE_PID="" cleanup() { if [[ -n "$MOONCAKE_MASTER_PID" ]]; then kill "$MOONCAKE_MASTER_PID" 2>/dev/null || true fi + if [[ -n "$LMCACHE_PID" ]]; then + stop_background_process_tree "$LMCACHE_PID" "LMCache server" + fi } trap cleanup EXIT @@ -188,8 +193,64 @@ EOF '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' ) ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # cp_kv_cache_interleave_size stays at vLLM's default of 1 on this + # image, so the connector needs no interleave handling and any DCP- + # capable build works. rc2 is the CUDA 13 wheel. + LMCACHE_VERSION=0.5.5rc2 + # LMCache's transfer-channel layer imports mooncake-transfer-engine at + # module load; the wheel ships none of its native libs. + agentic_pip_install --quiet --no-cache-dir --no-deps \ + "mooncake-transfer-engine-cuda13==0.3.11.post1" + agentic_pip_install --quiet --no-cache-dir --no-deps \ + "lmcache==$LMCACHE_VERSION" + python3 -c "from lmcache.integration.vllm.lmcache_mp_connector import LMCacheMPConnector" >/dev/null + + # Under DCP the chunk must be a multiple of block size x DCP_SIZE. + # Kimi-K3's hybrid groups resolve a local block of 1536, so DCP 8 + # gives 12288. + LMCACHE_CHUNK_SIZE=$((1536 * DCP_SIZE)) + LMCACHE_PORT=$((PORT + 13000)) + LMCACHE_HTTP_PORT=$((PORT + 14000)) + + LMCACHE_CMD=( + lmcache server + --host 127.0.0.1 + --port "$LMCACHE_PORT" + --http-host 127.0.0.1 + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$TOTAL_CPU_DRAM_GB" + --l1-init-size-gb 10 + --l1-read-ttl-seconds 3600 + --chunk-size "$LMCACHE_CHUNK_SIZE" + --separate-object-groups + --enable-extra-logging + --extra-logging-interval 30 + --max-cpu-workers 8 + --max-gpu-workers "${LMCACHE_GPU_WORKERS:-8}" + --eviction-policy LRU + --supported-transfer-mode lmcache_driven + --shm-name "" + ) + append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + wait_for_ready \ + --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ + --log "$LMCACHE_LOG" \ + --pid "$LMCACHE_PID" \ + --sleep-interval 1 \ + --timeout 600 + + # 90k-290k-token agentic prefixes make single retrieves large. + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or mooncake)" >&2 + echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, mooncake, or lmcache)" >&2 exit 1 ;; esac @@ -223,6 +284,15 @@ if [ "$NUM_SPEC_TOKENS" -gt 0 ]; then SPEC_ARGS=(--speculative-config "$SPEC_CONFIG") fi +# The LMCache MP connector shares the KV tensors with its out-of-process server +# over CUDA IPC, and cuMem-allocated memory has no legacy IPC handle: +# _share_cuda_() returns cudaErrorInvalidValue and the worker dies during +# initialize_from_config. No other LMCache recipe in the repo enables cumem. +CUMEM_ARGS=(--enable-cumem-allocator) +if [ "${KV_OFFLOAD_BACKEND:-}" = "lmcache" ]; then + CUMEM_ARGS=() +fi + MAX_NUM_SEQS=$((2 * CONC)) # 1 - this is the buffer for what is not sized against the budget: the cudagraph @@ -275,7 +345,7 @@ VLLM_CMD=( --load-format fastsafetensors --moe-backend auto --no-enable-flashinfer-autotune - --enable-cumem-allocator + "${CUMEM_ARGS[@]}" --enable-prefix-caching --prefix-match-unit 128 --kv-cache-dtype fp8 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6ab7aee031..1ae4b683ec 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1505,6 +1505,14 @@ kimik3-fp4-b300-vllm-agentic-dspark: # DSpark 3 at conc 16, and not at all above. Keep the concurrencies # disjoint across arms so exp-names stay unique. - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70] } + # LMCache arm at the two points where the offload tier carries the run: + # GPU prefix cache hit rate falls to 88.0% at conc 56 and 59.9% at conc + # 70, against 90.4-96.3% everywhere below, so most prefix reuse has to + # come from the external tier. Both points are no-drafting at + # gpu-memory-utilization 0.90. The concurrencies overlap the existing + # dram arm on purpose - the backend name is in the exp-name, so each + # point still yields a distinct series. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc2" }, conc-list: [56, 70] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 53a2ba7e75..9b48e165c9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6831,3 +6831,14 @@ - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add an LMCache 0.5.5rc2 DRAM offload arm at concurrency 56 and 70, where the GPU prefix cache hit rate falls to 88.0% and 59.9%." + - "Size the LMCache chunk as block size x DCP size (1536 x 8 = 12288), required for chunking under decode context parallel." + - "Raise the LMCache L1 read-lock TTL to 3600s: the lock is held from prefetch to retrieve, and at concurrency 70 a queued request can exceed the 300s default, which fails the retrieve." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2802 + append-only: true