diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index 86b07e5f2..38e95fd5a 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -99,13 +99,18 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 SERVER_LOG="$RESULT_DIR/server.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" MOONCAKE_MASTER_PID="" +LMCACHE_PID="" cleanup() { if [[ -n "$MOONCAKE_MASTER_PID" ]]; then kill "$MOONCAKE_MASTER_PID" 2>/dev/null || true fi + if [[ -n "$LMCACHE_PID" ]]; then + stop_background_process_tree "$LMCACHE_PID" "LMCache server" + fi } trap cleanup EXIT @@ -188,8 +193,64 @@ EOF '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' ) ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # cp_kv_cache_interleave_size stays at vLLM's default of 1 on this + # image, so the connector needs no interleave handling and any DCP- + # capable build works. rc2 is the CUDA 13 wheel. + LMCACHE_VERSION=0.5.5rc2 + # LMCache's transfer-channel layer imports mooncake-transfer-engine at + # module load; the wheel ships none of its native libs. + agentic_pip_install --quiet --no-cache-dir --no-deps \ + "mooncake-transfer-engine-cuda13==0.3.11.post1" + agentic_pip_install --quiet --no-cache-dir --no-deps \ + "lmcache==$LMCACHE_VERSION" + python3 -c "from lmcache.integration.vllm.lmcache_mp_connector import LMCacheMPConnector" >/dev/null + + # Under DCP the chunk must be a multiple of block size x DCP_SIZE. + # Kimi-K3's hybrid groups resolve a local block of 1536, so DCP 8 + # gives 12288. + LMCACHE_CHUNK_SIZE=$((1536 * DCP_SIZE)) + LMCACHE_PORT=$((PORT + 13000)) + LMCACHE_HTTP_PORT=$((PORT + 14000)) + + LMCACHE_CMD=( + lmcache server + --host 127.0.0.1 + --port "$LMCACHE_PORT" + --http-host 127.0.0.1 + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$TOTAL_CPU_DRAM_GB" + --l1-init-size-gb 10 + --l1-read-ttl-seconds 3600 + --chunk-size "$LMCACHE_CHUNK_SIZE" + --separate-object-groups + --enable-extra-logging + --extra-logging-interval 30 + --max-cpu-workers 8 + --max-gpu-workers "${LMCACHE_GPU_WORKERS:-8}" + --eviction-policy LRU + --supported-transfer-mode lmcache_driven + --shm-name "" + ) + append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + wait_for_ready \ + --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ + --log "$LMCACHE_LOG" \ + --pid "$LMCACHE_PID" \ + --sleep-interval 1 \ + --timeout 600 + + # 90k-290k-token agentic prefixes make single retrieves large. + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or mooncake)" >&2 + echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty, mooncake, or lmcache)" >&2 exit 1 ;; esac @@ -223,6 +284,15 @@ if [ "$NUM_SPEC_TOKENS" -gt 0 ]; then SPEC_ARGS=(--speculative-config "$SPEC_CONFIG") fi +# The LMCache MP connector shares the KV tensors with its out-of-process server +# over CUDA IPC, and cuMem-allocated memory has no legacy IPC handle: +# _share_cuda_() returns cudaErrorInvalidValue and the worker dies during +# initialize_from_config. No other LMCache recipe in the repo enables cumem. +CUMEM_ARGS=(--enable-cumem-allocator) +if [ "${KV_OFFLOAD_BACKEND:-}" = "lmcache" ]; then + CUMEM_ARGS=() +fi + MAX_NUM_SEQS=$((2 * CONC)) # 1 - this is the buffer for what is not sized against the budget: the cudagraph @@ -275,7 +345,7 @@ VLLM_CMD=( --load-format fastsafetensors --moe-backend auto --no-enable-flashinfer-autotune - --enable-cumem-allocator + "${CUMEM_ARGS[@]}" --enable-prefix-caching --prefix-match-unit 128 --kv-cache-dtype fp8 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6ab7aee03..1ae4b683e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1505,6 +1505,14 @@ kimik3-fp4-b300-vllm-agentic-dspark: # DSpark 3 at conc 16, and not at all above. Keep the concurrencies # disjoint across arms so exp-names stay unique. - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70] } + # LMCache arm at the two points where the offload tier carries the run: + # GPU prefix cache hit rate falls to 88.0% at conc 56 and 59.9% at conc + # 70, against 90.4-96.3% everywhere below, so most prefix reuse has to + # come from the external tier. Both points are no-drafting at + # gpu-memory-utilization 0.90. The concurrencies overlap the existing + # dram arm on purpose - the backend name is in the exp-name, so each + # point still yields a distinct series. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc2" }, conc-list: [56, 70] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 53a2ba7e7..9b48e165c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6831,3 +6831,14 @@ - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add an LMCache 0.5.5rc2 DRAM offload arm at concurrency 56 and 70, where the GPU prefix cache hit rate falls to 88.0% and 59.9%." + - "Size the LMCache chunk as block size x DCP size (1536 x 8 = 12288), required for chunking under decode context parallel." + - "Raise the LMCache L1 read-lock TTL to 3600s: the lock is held from prefetch to retrieve, and at concurrency 70 a queued request can exceed the 300s default, which fails the retrieve." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2802 + append-only: true