diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index c2d1445b2..9be5c651b 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -109,7 +109,7 @@ jobs: ALL_EVALS: ${{ contains(github.event.pull_request.labels.*.name, 'all-evals') }} EVALS_ONLY: ${{ contains(github.event.pull_request.labels.*.name, 'evals-only') }} run: | - pip install pydantic pyyaml + pip install "pydantic>=2" pyyaml CMD=( python3 utils/validate_perf_changelog.py --changelog-file perf-changelog.yaml diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index c1f11f34f..65bd1eebc 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -68,7 +68,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install pytest pydantic pyyaml tabulate + python -m pip install pytest "pydantic>=2" pyyaml tabulate - name: Run validator, reuse, and gating tests run: | diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index a8f2c42c1..f0416e4af 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -35,10 +35,10 @@ set -x # GPU_MEM_UTIL 0.95 (reference) # MAX_NUM_BATCHED_TOKENS 8192 (default) # AITER_A8W4 1 (reference; 0 = aiter a16w4 MoE path) -# LANGUAGE_MODEL_ONLY true +# LANGUAGE_MODEL_ONLY true # KV_CACHE_DTYPE fp8 (default for every arm; =auto for a bf16 A/B) # KV_BLOCK_SIZE unset (unset -> vLLM sizes the page; 128 under fp8) -# MAX_MODEL_LEN 1M +# MAX_MODEL_LEN 1M # SPEC_DECODE true (this is the _mtp DSpark recipe; =false for a no-spec A/B) # SPEC_NUM_TOKENS 2 (DSpark draft length; validated by the _mtp config) @@ -149,10 +149,9 @@ case "${KV_OFFLOAD_BACKEND:-}" in lmcache) require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - # Keep the image's tested torch/ROCm stack and install only LMCache's - # missing runtime dependencies, same as the MiniMax-M3 lmcache arm. - LMCACHE_VERSION="0.5.5.dev60+rocm7.2" + LMCACHE_VERSION=0.5.5.dev89+rocm7.2 LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-rocm" + agentic_pip_install --quiet --no-cache-dir --no-deps \ "sortedcontainers==2.4.0" \ "opentelemetry-exporter-prometheus==0.61b0" \ @@ -197,6 +196,18 @@ case "${KV_OFFLOAD_BACKEND:-}" in LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" + # DCP shards decode KV across the TP ranks, so the LMCache GPU transfer + # pool needs one worker per rank; a non-DCP arm only needs a single worker. + # The DCP KV interleave also needs the larger 12288 chunk; a non-DCP arm + # uses the 3072 minimum (one KDA state group). + if [ "${DCP_SIZE:-1}" -gt 1 ]; then + LMCACHE_MAX_GPU_WORKERS=8 + LMCACHE_CHUNK_SIZE=12288 + else + LMCACHE_MAX_GPU_WORKERS=1 + LMCACHE_CHUNK_SIZE=3072 + fi + LMCACHE_CMD=( lmcache server --host 127.0.0.1 @@ -205,12 +216,12 @@ case "${KV_OFFLOAD_BACKEND:-}" in --http-port "$LMCACHE_HTTP_PORT" --l1-size-gb "$LMCACHE_L1_SIZE_GB" --l1-init-size-gb 10 - --chunk-size 3072 + --chunk-size "$LMCACHE_CHUNK_SIZE" --separate-object-groups --enable-extra-logging --extra-logging-interval 30 --max-cpu-workers 8 - --max-gpu-workers 1 + --max-gpu-workers "$LMCACHE_MAX_GPU_WORKERS" --eviction-policy LRU --supported-transfer-mode lmcache_driven --shm-name "" @@ -256,16 +267,21 @@ case "$CONC" in GPU_MEM_UTIL=0.9 MAX_NUM_BATCHED_TOKENS=16384 ;; - 2|4|8|10|12|14) + 4|8|10|12|14) SYNTHETIC_ACCEPT_LEN=3.00 SPEC_NUM_TOKENS=3 GPU_MEM_UTIL=0.9 MAX_NUM_BATCHED_TOKENS=8192 ;; + 44|48|52) + SPEC_NUM_TOKENS=0 + GPU_MEM_UTIL=0.9 + MAX_NUM_BATCHED_TOKENS=8192 + ;; *) SPEC_NUM_TOKENS=0 - GPU_MEM_UTIL=0.85 - MAX_NUM_BATCHED_TOKENS=4096 + GPU_MEM_UTIL=0.9 + MAX_NUM_BATCHED_TOKENS=8192 ;; esac @@ -306,7 +322,7 @@ CP_ARGS=() ATTN_BE_ARGS=() if [ "$DCP_SIZE" -gt 1 ]; then CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) - ATTN_BE_ARGS+=(--attention-backend TRITON_MLA) + ATTN_BE_ARGS+=(--attention-backend ROCM_AITER_MLA) fi export VLLM_USE_DIRECT_DCP_A2A=0 export VLLM_USE_DIRECT_DCP_Q_GATHER=0 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d380154a..0269ddfa0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -625,7 +625,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=2" kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds @@ -636,8 +636,9 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev60+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev89+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev89+rocm7.2" }, conc-list: [44, 48, 52], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 53a2ba7e7..e02c09384 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6831,3 +6831,13 @@ - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the MI355X K3 vLLM recipe to vLLM ROCm nightly 7c5dc571 and LMCache 0.5.5rc3." + - "Add a TP8/DCP8 spec-mtp DRAM-offload arm at concurrency 44/48/52." + - "Retune the LMCache tier (12288-token chunk, 8 GPU workers) and the no-spec decode profile (gpu-memory-utilization 0.9, 8192 batched tokens); switch the DCP attention backend to ROCM_AITER_MLA." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2810