From d8be7ba658834038af6b5157bd3c112dd9e29088 Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 12:57:31 -0500 Subject: [PATCH 1/7] feat(agentx): dsv4-fp4-mi355x-vllm-agentic-mtp DPattn+EP8 on FlyDSL mega-MoE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase the PR #2381 AgentX MTP recipe onto current main (which carries the AIPerf v1.0.1 bump, 65e0b1ce) and add a DPattn+EP8 arm at concurrency 32. Pure TP8 replicates the MLA KV cache on every GPU, so available KV cache size does not grow with the node. In the AgentX trace-replay scenario, prefix-cache hit rate dominates and is bounded by that pool. DP-attention shards KV per DP rank and EP8 shards the 384 routed experts, freeing HBM for KV. The DEP arm selects --moe-backend flydsl (AITER FlyDSL mega-MoE: fused EP dispatch + both expert GEMMs + combine). AITER resolves launch geometry from mega_moe_tuning_config/flydsl_gfx950_mi355x_IntraNode_ep8.json; DeepSeek-V4-Pro matches its hidden_dim=7168 / topk=6 / local_expert_num=48 fp4 dispatch rules, so the table hits rather than falling back to static geometry defaults. The pure-TP arms are untouched and stay on --moe-backend aiter. 中文：将 PR #2381 的 AgentX MTP 配置变基到当前 main（已包含 AIPerf v1.0.1 升级 65e0b1ce），并新增并发 32 的 DPattn+EP8 配置点。 纯 TP8 会在每张 GPU 上复制一份 MLA KV 缓存，可用 KV 缓存容量无法随节点规模增长。 在 AgentX 轨迹回放场景中，性能主要由前缀缓存(prefix cache)命中率决定，而命中率 受该缓存池大小限制。数据并行注意力(DP attention)按 DP rank 切分 KV，专家并行 EP8 切分 384 个路由专家，从而为 KV 释放出 HBM 空间。 DEP 配置点选用 --moe-backend flydsl（AITER FlyDSL mega-MoE：融合专家并行的 dispatch、两个专家 GEMM 与 combine）。AITER 从 mega_moe_tuning_config/flydsl_gfx950_mi355x_IntraNode_ep8.json 解析 kernel 启动 几何参数；DeepSeek-V4-Pro 与其 hidden_dim=7168 / topk=6 / local_expert_num=48 的 fp4 dispatch 规则匹配，因此可命中调优表，而不会回退到静态默认几何参数。 纯 TP 配置点保持不变，仍使用 --moe-backend aiter。 Co-Authored-By: Claude Opus 5 --- .../agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 541 ++++++++++++++++++ configs/amd-master.yaml | 29 + perf-changelog.yaml | 9 + 3 files changed, 579 insertions(+) create mode 100644 benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh new file mode 100644 index 0000000000..ab8d6ab71c --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -0,0 +1,541 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using vLLM. +# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the +# agentic sweep can probe both interactivity and throughput regimes: +# pure TP (DP_ATTENTION=false, EP_SIZE=1): attention TP-sharded across +# all $TP GPUs in a single engine. Lower TPOT, lower batch. +# TP+EP (DP_ATTENTION=false, EP_SIZE>1): attention TP-sharded, MoE +# experts EP-sharded within the TP group. +# DEP (DP_ATTENTION=true, EP_SIZE>1): per-DP-rank attention with +# experts EP-sharded across DP ranks (per the vLLM blog recipe). +# Highest aggregate throughput at large CONC. +# +# Serving flags follow the validated MI355X recipe from +# https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro?hardware=mi355x +# https://github.com/SemiAnalysisAI/InferenceX/blob/main/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh +# Image is configured in amd-master.yaml. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram requires one of these. +# KV_OFFLOAD_BACKEND=vllm-native. +# KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOAD_BACKEND=lmcache. +# KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# Force the eval framework to lm-eval for this recipe. run_eval derives its +# default as swebench for agentic scenarios (scenario_default=swebench when +# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence +# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so +# setting it here makes the effective framework always lm-eval, never swebench. +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +GPU_COUNT=$TP +if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 + exit 1 +fi +export GPU_COUNT + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi +rocm-smi + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# Nightly ROCm image may be missing runtime deps; ensure they are present. +agentic_pip_install --quiet Pillow fastapi uvicorn + +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 + +# vllm-project/router expands the one HTTP backend into one logical worker per +# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's +# X-Correlation-ID is stable for every turn of a conversation; alias it to the +# router's preferred X-Session-ID header. +USE_VLLM_ROUTER=false +VLLM_BACKEND_PORT="$PORT" +if [ "$DP_ATTENTION" = "true" ]; then + USE_VLLM_ROUTER=true + VLLM_BACKEND_PORT=$((PORT + 1)) + VLLM_ROUTER_VERSION=0.1.14 + VLLM_ROUTER_POLICY=consistent_hash + VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" +fi + +# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. +export VLLM_ENGINE_READY_TIMEOUT_S=3600 + +# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while +# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's +# store mask. 32k matches the trace-replay tuning validated for this workload. +export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 + +# VLLM_PREFIX_CACHE_RETENTION_INTERVAL only applies to sliding-window/Mamba +# models; this vLLM build raises ValueError if it is set for DSv4. + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +ROUTER_PID="" +MOONCAKE_MASTER_PID="" + +OFFLOAD_ARGS=() + +if agentic_kv_offload_enabled; then +case "${KV_OFFLOAD_BACKEND:-}" in + vllm-native) + require_agentic_kv_offload_backend vllm-native + # ---- vLLM native config ---------------------------------------------------------- + unset VLLM_USE_SIMPLE_KV_OFFLOAD + # MI355X nodes have ~2.7 TiB of host DRAM available for offload; + # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for + # worker RSS / page cache / slurm cgroup). + TOTAL_CPU_DRAM_PARTITION_GB="$TOTAL_CPU_DRAM_GB" + # Use vLLM's regular native KV-offload path (OffloadingConnector), + # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to + # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 + # would switch it to SimpleCPUOffloadConnector. We intentionally leave + # that env var UNSET here so the regular OffloadingConnector path is + # used. The shortcut --kv_offloading_backend native + --kv_offloading_size + # form constructs the KVTransferConfig at engine startup + # (vllm/config/vllm.py:662). + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" + ) + + ;; + mooncake) + require_agentic_kv_offload_backend mooncake + # ---- Mooncake config ---------------------------------------------------------- + # Embedded mode contributes one segment per GPU rank to a shared + # distributed store, so pre-divide the aggregate host-memory budget. + PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT)) + + #MOONCAKE_VERSION=0.3.11.post1 + #apt-get update && apt-get install -y libcurl4 libibverbs1 rdma-core librdmacm1 libnuma1 liburing2 + #agentic_pip_install --quiet --no-cache-dir --no-deps \ + # --force-reinstall "mooncake-transfer-engine-non-cuda==$MOONCAKE_VERSION" + + git clone https://github.com/kvcache-ai/Mooncake.git + cd Mooncake + git checkout v0.3.12 + bash dependencies.sh + mkdir build + cd build + cmake .. + make -j + sudo make install # optional, make it ready to be used by vLLM/SGLang + cd .. + cd .. + + python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + + sleep 10 + MOONCAKE_MASTER_PID=$! + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup." >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 + exit 1 + fi + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + ) + + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # ---- Lmcache config ---------------------------------------------------------- + LMCACHE_PID="" + + cleanup_lmcache_server() { + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi + } + + trap cleanup_lmcache_server EXIT + + cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$ROUTER_PID" "vLLM router" + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" + exit "$exit_code" + } + trap cleanup_agentic_services EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 10 + done + + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + } + { set +x; } 2>/dev/null + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + git clone https://github.com/LMCache/LMCache.git + cd LMCache + git checkout v0.5.2 + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation + cd .. + + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + TOTAL_CPU_DRAM_PARTITION_GB="$TOTAL_CPU_DRAM_GB" + # Match the B200 Kimi LMCache setup: keep a 2.5 TB semantic CPU KV + # pool, but let the external MP server own that pool so vLLM does not + # split --kv-offloading-size across TP ranks through the integrated + # LMCache backend. + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector a + # ZMQ-style host string. + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + LMCACHE_L1_SIZE_GB="${TOTAL_CPU_DRAM_PARTITION_GB}" + if [ "$LMCACHE_L1_SIZE_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: LMCACHE_L1_SIZE_GB=$LMCACHE_L1_SIZE_GB exceeds configured capacity $TOTAL_CPU_DRAM_GB" >&2 + exit 1 + fi + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + # LMCache read locks are leases on chunks that lookup has promised + # vLLM can retrieve. The default 300s TTL is too short for this + # long-context agentic queue: TP8/conc32 can spend >300s between + # lookup and retrieve while GPU KV is saturated, which leaves the + # object present in L1 but no longer readable. Keep the 2.5 TB pool + # size unchanged and only extend the lookup-to-retrieve lease. + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-1024}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$TP}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=1200 + LMCACHE_TX_MODE="lmcache_driven" + + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + --supported-transfer-mode "$LMCACHE_TX_MODE" + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + + PREFIX_CACHE_ARGS=(--enable-prefix-caching) + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '${KV_OFFLOAD_BACKEND:-}' (expected: vllm-native, mooncake, lmcache)" >&2 + exit 1 + ;; +esac +fi + +# ---- LLM server config ---------------------------------------------------------- +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") +fi +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi + +# DEP8 (TP8 + DP-attention) is a high-concurrency arm tuned separately from the +# smaller DEP arm: larger prefill token budget and long-prefill chunking so +# decode latency stays bounded at high concurrency. +IS_DEP8=false +if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then + IS_DEP8=true +fi + +MODE_ARGS=() +MNBT=8192 +if [ "$EP_SIZE" -gt 1 ]; then + MODE_ARGS+=(--enable-ep-weight-filter) +fi +if [ "$DP_ATTENTION" = "true" ]; then + MODE_ARGS+=(--prefill-schedule-interval 8) + if [ "$IS_DEP8" = "true" ]; then + MODE_ARGS+=( + --max-num-batched-tokens $((MNBT * 2)) + --long-prefill-token-threshold 16384 + ) + else + MODE_ARGS+=(--max-num-batched-tokens $MNBT) + fi +fi + +# --max-num-seqs is applied PER scheduler. Under DP-attention each of the $TP +# DP ranks runs its own scheduler and the router spreads sessions across them, +# but size the per-rank cap as CONC directly (not CONC/TP) so each rank can +# hold the full session count. In pure-TP there is a single scheduler across +# all GPUs that sees all CONC sessions, so use 2*CONC directly. +if [ "$DP_ATTENTION" = "true" ]; then + MAX_NUM_SEQS=$((CONC)) + if [ "$MAX_NUM_SEQS" -lt 1 ]; then + MAX_NUM_SEQS=1 + fi +else + MAX_NUM_SEQS=$((2 * CONC)) +fi +CONTEXT_LEN=1048576 + +# MTP: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N, +# every uniform decode batch of S seqs verifies S*(1+N) tokens, so capture the +# explicit multiples (1+N), 2*(1+N), ..., MAX_NUM_SEQS*(1+N) -- one graph per +# decode batch of 1..MAX_NUM_SEQS seqs. vLLM rounds configured sizes up to +# multiples of (1+N) and dedups (adjust_cudagraph_sizes_for_spec_decode), so a +# plain 1..MAX_NUM_SEQS list would collapse to coverage of only +# MAX_NUM_SEQS/(1+N) seqs and drop the largest decode batches to eager. +NUM_SPEC_TOKENS=3 +TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) +# Throughput pins synthetic MTP acceptance to the dsv4-pro golden AL (thinking_on, +# num_speculative_tokens=3, golden_al_distribution/dsv4_mtp.yaml). The EVAL_ONLY +# accuracy run uses real target verification instead -- synthetic acceptance +# bypasses verification and corrupts the SWE-bench eval (0.0000 score). +if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" +else + SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 2.49}" +fi + +# These MI355X nodes have a stable ~32 GiB/GPU carveout: only ~256/288 GiB is +# free at init, independent of the KV-offload backend (observed identically on +# both the lmcache and GPU-resident kv-none arms). At +# --gpu-memory-utilization 0.95 vLLM requests 273.6 GiB and every DP worker +# hard-fails ("Free memory ... less than desired GPU memory utilization"). +# But 0.85 (244.8 GiB) leaves only ~23.9 GiB for KV, which is below the +# 24.06 GiB one request at max_model_len=1M needs once the MTP draft layer's +# extra per-token KV is counted -- engine init then dies with +# "available KV cache memory ... larger than ..." on the tighter (eval-only) +# relaunch. 0.86 (247.7 GiB) adds ~2.6 GiB, nearly all to KV (~26.5 GiB), so +# the KV check clears with margin while still keeping ~8 GiB free-mem headroom +# below the ~256 GiB hard-fail ceiling. Additional 0.9 for mooncake headroom +GPU_MEM_UTIL=0.90 + +# Long-context forward passes (~370K tokens with fp8 KV + DRAM offload) can exceed +# vLLM's default 300s worker RPC timeout, killing the engine with +# "RPC call to sample_tokens timed out". Widen it. +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS="${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-1200}" + +echo "Starting vllm server..." +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + +# MoE backend. The pure-TP arms stay on the AITER fused MoE that this recipe +# was validated against. The DEP arms (DP-attention + EP>1) instead select +# AITER's FlyDSL mega-MoE, which fuses the expert-parallel dispatch, the two +# expert GEMMs, and the combine into one pipeline (ROCm/FlyDSL#626). Its +# intranode dispatch/combine kernel is the EP-aware path, so it only applies +# where there is an actual all-to-all to fuse. +# +# Launch geometry comes from AITER's checked-in tuning table, resolved by +# aiter/ops/flydsl/kernels/flydsl_dispatch_combine_intranode_op.py: +# resolve_tuning_config_path() globs mega_moe_tuning_config/ for +# flydsl_*_{kernel_type}_ep{world_size}.json and scores candidates on gfx arch +# then GPU model. On MI355X at EP8 that resolves to +# flydsl_gfx950_mi355x_IntraNode_ep8.json. DSv4-Pro matches its +# hidden_dim=7168 / topk=6 / local_expert_num=48 (384 routed experts / EP8) +# fp4 dispatch rules directly, so the table hits rather than falling back to +# the static 128-block / 4-warp defaults. A shape miss is not fatal: the op +# logs "using static geometry defaults" and still runs. +MOE_BACKEND=aiter +if [ "$DP_ATTENTION" = "true" ] && [ "$EP_SIZE" -gt 1 ]; then + MOE_BACKEND=flydsl +fi + +sleep 180 + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + --trust-remote-code + --async-scheduling + --distributed-executor-backend mp + --kv-cache-dtype fp8 + --block-size 256 + --max-model-len "$CONTEXT_LEN" + "${PARALLEL_ARGS[@]}" + "${MODE_ARGS[@]}" + --gpu-memory-utilization "$GPU_MEM_UTIL" + --moe-backend "$MOE_BACKEND" + --compilation-config '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY"}' + --speculative-config "$SPEC_CONFIG" + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --reasoning-parser deepseek_v4 + --enable-auto-tool-choice + --enable-prefix-caching + --no-disable-hybrid-kv-cache-manager + --max-num-seqs "$MAX_NUM_SEQS" + "${OFFLOAD_ARGS[@]}" +) + +# (srok), not yet + #--attention_config.use_fp4_indexer_cache=True +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_VLLM_ROUTER" = "true" ]; then + echo "Starting native vLLM router on port $PORT for $TP DP ranks..." + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy "$VLLM_ROUTER_POLICY" \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 358da1b337..95d3709523 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1432,6 +1432,35 @@ dsv4-fp4-mi355x-vllm-agentic: - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, conc-list: [32, 40, 48] } +# MTP variant of dsv4-fp4-mi355x-vllm-agentic. Mirrors the base recipe and adds +# spec-decoding: mtp, which routes to +# benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh (MTP, +# num_speculative_tokens=3). +# +# The DEP8 arm (tp 8 + ep 8 + dp-attn) targets the AgentX KV-capacity problem: +# pure TP8 replicates the MLA KV cache on every GPU, so the available KV pool +# -- and therefore the prefix-cache hit rate that dominates this trace-replay +# workload -- does not grow with the node. DP-attention shards KV per DP rank +# and EP8 shards the 384 routed experts, freeing HBM for KV. The launch script +# selects --moe-backend flydsl on this arm (AITER FlyDSL mega-MoE, whose +# intranode dispatch/combine tuning table +# flydsl_gfx950_mi355x_IntraNode_ep8.json covers gfx950 at EP8); the pure-TP +# arms stay on --moe-backend aiter. +dsv4-fp4-mi355x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.60 + search-space: + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } } + # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the # image tag, so bumping sglang is just an image tag bump here. Sweeps diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4dd819f46d..7670bd208d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5460,3 +5460,12 @@ - "Use vllm/vllm-openai:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 with prefix caching enabled, default KV-cache dtype, FlashInfer TRT-LLM attention with FP8 indexer KV, and the EAGLE3 drafter on FLASH_ATTN." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2328 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + description: + - "Add an MTP variant of the DeepSeek-V4-Pro FP4 MI355X vLLM AgentX recipe (new benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh plus an amd-master.yaml entry) on vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20. Adds --speculative-config method=mtp num_speculative_tokens=3; throughput runs pin synthetic acceptance to the dsv4-pro golden AL (synthetic_acceptance_length=2.49) while EVAL_ONLY runs use real target verification, since synthetic acceptance bypasses verification and zeroes the eval score." + - "Carry the shared vllm-agentic fixes: EVAL_FRAMEWORK=lm-eval (never swebench); --max-num-seqs=CONC per DP rank under DP-attention and 2*CONC in pure TP; DEP8 --max-num-batched-tokens 16384 with --long-prefill-token-threshold 16384; --gpu-memory-utilization 0.90; --block-size 256; --max-model-len 1048576; cudagraph FULL_DECODE_ONLY; VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1200 for long-context forward passes." + - "Add a DPattn+EP8 arm (tp 8, ep 8, dp-attn true, kv-offloading none, conc 32) behind the vllm-router. Pure TP8 replicates the MLA KV cache on every GPU, which caps available KV cache size and therefore the prefix-cache hit rate that dominates the AgentX trace-replay scenario; DP-attention shards KV per DP rank and EP8 shards the 384 routed experts, freeing HBM for KV." + - "Select --moe-backend flydsl on the DPattn+EP8 arm only (AITER FlyDSL mega-MoE, which fuses expert-parallel dispatch, both expert GEMMs, and combine). AITER resolves launch geometry from aiter/ops/flydsl/kernels/mega_moe_tuning_config/flydsl_gfx950_mi355x_IntraNode_ep8.json; DeepSeek-V4-Pro matches its hidden_dim=7168 / topk=6 / local_expert_num=48 fp4 dispatch rules, so the table hits instead of falling back to static geometry defaults. The pure-TP arms remain on --moe-backend aiter." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING From 3c6e77c7292798e87c7b8444c3ca63721be982e3 Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 13:05:55 -0500 Subject: [PATCH 2/7] =?UTF-8?q?chore(agentx):=20backfill=20perf-changelog?= =?UTF-8?q?=20pr-link=20to=20#2503=20/=20=E5=9B=9E=E5=A1=AB=20perf-changel?= =?UTF-8?q?og=20=E7=9A=84=20pr-link=20=E4=B8=BA=20#2503?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7670bd208d..d6d3ee50b2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5468,4 +5468,4 @@ - "Carry the shared vllm-agentic fixes: EVAL_FRAMEWORK=lm-eval (never swebench); --max-num-seqs=CONC per DP rank under DP-attention and 2*CONC in pure TP; DEP8 --max-num-batched-tokens 16384 with --long-prefill-token-threshold 16384; --gpu-memory-utilization 0.90; --block-size 256; --max-model-len 1048576; cudagraph FULL_DECODE_ONLY; VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1200 for long-context forward passes." - "Add a DPattn+EP8 arm (tp 8, ep 8, dp-attn true, kv-offloading none, conc 32) behind the vllm-router. Pure TP8 replicates the MLA KV cache on every GPU, which caps available KV cache size and therefore the prefix-cache hit rate that dominates the AgentX trace-replay scenario; DP-attention shards KV per DP rank and EP8 shards the 384 routed experts, freeing HBM for KV." - "Select --moe-backend flydsl on the DPattn+EP8 arm only (AITER FlyDSL mega-MoE, which fuses expert-parallel dispatch, both expert GEMMs, and combine). AITER resolves launch geometry from aiter/ops/flydsl/kernels/mega_moe_tuning_config/flydsl_gfx950_mi355x_IntraNode_ep8.json; DeepSeek-V4-Pro matches its hidden_dim=7168 / topk=6 / local_expert_num=48 fp4 dispatch rules, so the table hits instead of falling back to static geometry defaults. The pure-TP arms remain on --moe-backend aiter." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2503 From 10077e46f89106e8ffd9696d34b4e6c01ae6ebba Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 13:39:25 -0500 Subject: [PATCH 3/7] fix(agentx): drop --moe-backend flydsl, DSv4-Pro MXFP4 rejects it at load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI run 31033203073 shows the DPattn+EP8 arm dying at model load on every worker: ValueError: moe_backend='flydsl' is not supported for MXFP4 MoE. Expected one of ['deep_gemm', 'flashinfer_trtllm', ..., 'aiter', ...] DeepSeek-V4-Pro resolves expert_dtype to 'fp4', so its experts are built by Mxfp4MoEMethod, and map_mxfp4_backend() in vllm/model_executor/layers/fused_moe/oracle/mxfp4.py hard-rejects any backend outside its MXFP4 allowlist. This is a hard failure in FusedMoE.__init__, not a fallback -- it happens before any KV cache is allocated. The rejection is on the quantization path, so it is independent of EP size and of tuning-table coverage: AITER's flydsl_gfx950_mi355x_IntraNode_ep8.json does match DSv4-Pro (hidden_dim=7168 / topk=6 / local_expert_num=48 at EP8), but is never consulted because FlyDSL is unreachable for an MXFP4 MoE in this build. Run all arms on --moe-backend aiter, which is on the allowlist. The DEP8 arm still answers its actual question: whether sharding KV per DP rank and the 384 routed experts across EP8 frees enough HBM to raise the prefix-cache hit rate that dominates AgentX. MOE_BACKEND stays overridable for retesting. 中文：CI 运行 31033203073 显示 DPattn+EP8 配置点在模型加载阶段所有 worker 均崩溃： ValueError: moe_backend='flydsl' is not supported for MXFP4 MoE. DeepSeek-V4-Pro 的 expert_dtype 解析为 'fp4'，其专家层由 Mxfp4MoEMethod 构建， 而 vllm/model_executor/layers/fused_moe/oracle/mxfp4.py 中的 map_mxfp4_backend() 会硬性拒绝其 MXFP4 白名单之外的任何后端。这是 FusedMoE.__init__ 中的致命错误， 而非回退路径——发生在分配任何 KV 缓存之前。 该拒绝发生在量化路径上，因此与 EP 规模及调优表覆盖情况无关：AITER 的 flydsl_gfx950_mi355x_IntraNode_ep8.json 确实与 DSv4-Pro 匹配（EP8 下 hidden_dim=7168 / topk=6 / local_expert_num=48），但在该构建中 FlyDSL 对 MXFP4 MoE 不可达，因此该调优表根本不会被读取。 改为所有配置点均使用白名单内的 --moe-backend aiter。DEP8 配置点仍能回答其真正的 问题：按 DP rank 切分 KV、按 EP8 切分 384 个路由专家，能否释放足够 HBM 以提升 主导 AgentX 性能的前缀缓存命中率。MOE_BACKEND 仍可通过环境变量覆盖以便复测。 Co-Authored-By: Claude Opus 5 --- .../agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 49 +++++++++++-------- configs/amd-master.yaml | 9 ++-- perf-changelog.yaml | 2 +- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index ab8d6ab71c..a637c70ded 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -455,27 +455,36 @@ set -x export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -# MoE backend. The pure-TP arms stay on the AITER fused MoE that this recipe -# was validated against. The DEP arms (DP-attention + EP>1) instead select -# AITER's FlyDSL mega-MoE, which fuses the expert-parallel dispatch, the two -# expert GEMMs, and the combine into one pipeline (ROCm/FlyDSL#626). Its -# intranode dispatch/combine kernel is the EP-aware path, so it only applies -# where there is an actual all-to-all to fuse. +# MoE backend. All arms use the AITER fused MoE this recipe was validated +# against. # -# Launch geometry comes from AITER's checked-in tuning table, resolved by -# aiter/ops/flydsl/kernels/flydsl_dispatch_combine_intranode_op.py: -# resolve_tuning_config_path() globs mega_moe_tuning_config/ for -# flydsl_*_{kernel_type}_ep{world_size}.json and scores candidates on gfx arch -# then GPU model. On MI355X at EP8 that resolves to -# flydsl_gfx950_mi355x_IntraNode_ep8.json. DSv4-Pro matches its -# hidden_dim=7168 / topk=6 / local_expert_num=48 (384 routed experts / EP8) -# fp4 dispatch rules directly, so the table hits rather than falling back to -# the static 128-block / 4-warp defaults. A shape miss is not fatal: the op -# logs "using static geometry defaults" and still runs. -MOE_BACKEND=aiter -if [ "$DP_ATTENTION" = "true" ] && [ "$EP_SIZE" -gt 1 ]; then - MOE_BACKEND=flydsl -fi +# The DEP arms (DP-attention + EP>1) were tried on AITER's FlyDSL mega-MoE +# (--moe-backend flydsl), which fuses the expert-parallel dispatch, both expert +# GEMMs, and the combine into one pipeline (ROCm/FlyDSL#626). That does not work +# for this model: DSv4-Pro resolves expert_dtype to 'fp4', so its experts are +# built by Mxfp4MoEMethod, and vllm/model_executor/layers/fused_moe/oracle/ +# mxfp4.py:map_mxfp4_backend() hard-rejects anything outside its MXFP4 allowlist: +# +# ValueError: moe_backend='flydsl' is not supported for MXFP4 MoE. +# Expected one of ['deep_gemm', 'flashinfer_trtllm', 'flashinfer_trtllm_afp8', +# 'flashinfer_cutlass', 'flashinfer_cutlass_afp8', 'triton', 'triton_unfused', +# 'humming', 'marlin', 'aiter', 'aiter_mxfp4_fp8', 'aiter_mxfp4_mxfp4', 'xpu', +# 'cpu', 'emulation'] +# +# This is a hard failure at model load, not a fallback -- every worker dies in +# FusedMoE.__init__ before any KV cache is allocated. The rejection is on the +# quantization path, so it is independent of EP size and of whether AITER's +# flydsl_gfx950_mi355x_IntraNode_ep8.json tuning table covers the shape (it does: +# hidden_dim=7168 / topk=6 / local_expert_num=48 matches DSv4-Pro at EP8). The +# tuning table is simply never consulted, because FlyDSL is not reachable for an +# MXFP4-quantized MoE in this vLLM build. +# +# So the DEP8 arm runs on --moe-backend aiter, which is on the allowlist. That +# still answers the question this arm exists for: whether sharding KV per DP rank +# and the 384 routed experts across EP8 frees enough HBM to raise the prefix-cache +# hit rate that dominates AgentX. Revisit flydsl once vLLM wires it into the +# MXFP4 backend oracle. Override with MOE_BACKEND to retest without editing this. +MOE_BACKEND="${MOE_BACKEND:-aiter}" sleep 180 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 95d3709523..502d10b266 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1441,11 +1441,10 @@ dsv4-fp4-mi355x-vllm-agentic: # pure TP8 replicates the MLA KV cache on every GPU, so the available KV pool # -- and therefore the prefix-cache hit rate that dominates this trace-replay # workload -- does not grow with the node. DP-attention shards KV per DP rank -# and EP8 shards the 384 routed experts, freeing HBM for KV. The launch script -# selects --moe-backend flydsl on this arm (AITER FlyDSL mega-MoE, whose -# intranode dispatch/combine tuning table -# flydsl_gfx950_mi355x_IntraNode_ep8.json covers gfx950 at EP8); the pure-TP -# arms stay on --moe-backend aiter. +# and EP8 shards the 384 routed experts, freeing HBM for KV. Every arm runs on +# --moe-backend aiter: AITER's FlyDSL mega-MoE was tried on the DEP8 arm and is +# rejected at model load, because DSv4-Pro's experts are MXFP4 and vLLM's MXFP4 +# backend oracle does not list flydsl. See the launch script for details. dsv4-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d6d3ee50b2..5b64647f74 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5467,5 +5467,5 @@ - "Add an MTP variant of the DeepSeek-V4-Pro FP4 MI355X vLLM AgentX recipe (new benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh plus an amd-master.yaml entry) on vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20. Adds --speculative-config method=mtp num_speculative_tokens=3; throughput runs pin synthetic acceptance to the dsv4-pro golden AL (synthetic_acceptance_length=2.49) while EVAL_ONLY runs use real target verification, since synthetic acceptance bypasses verification and zeroes the eval score." - "Carry the shared vllm-agentic fixes: EVAL_FRAMEWORK=lm-eval (never swebench); --max-num-seqs=CONC per DP rank under DP-attention and 2*CONC in pure TP; DEP8 --max-num-batched-tokens 16384 with --long-prefill-token-threshold 16384; --gpu-memory-utilization 0.90; --block-size 256; --max-model-len 1048576; cudagraph FULL_DECODE_ONLY; VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1200 for long-context forward passes." - "Add a DPattn+EP8 arm (tp 8, ep 8, dp-attn true, kv-offloading none, conc 32) behind the vllm-router. Pure TP8 replicates the MLA KV cache on every GPU, which caps available KV cache size and therefore the prefix-cache hit rate that dominates the AgentX trace-replay scenario; DP-attention shards KV per DP rank and EP8 shards the 384 routed experts, freeing HBM for KV." - - "Select --moe-backend flydsl on the DPattn+EP8 arm only (AITER FlyDSL mega-MoE, which fuses expert-parallel dispatch, both expert GEMMs, and combine). AITER resolves launch geometry from aiter/ops/flydsl/kernels/mega_moe_tuning_config/flydsl_gfx950_mi355x_IntraNode_ep8.json; DeepSeek-V4-Pro matches its hidden_dim=7168 / topk=6 / local_expert_num=48 fp4 dispatch rules, so the table hits instead of falling back to static geometry defaults. The pure-TP arms remain on --moe-backend aiter." + - "Run every arm on --moe-backend aiter. AITER FlyDSL mega-MoE (--moe-backend flydsl) was tried on the DPattn+EP8 arm and fails at model load: DeepSeek-V4-Pro resolves expert_dtype to fp4, so its experts are built by Mxfp4MoEMethod, and vLLM map_mxfp4_backend() rejects any backend outside its MXFP4 allowlist with \"moe_backend='flydsl' is not supported for MXFP4 MoE\". The rejection is on the quantization path and is independent of EP size, so aiter/ops/flydsl/kernels/mega_moe_tuning_config/flydsl_gfx950_mi355x_IntraNode_ep8.json is never consulted even though it covers the shape (hidden_dim=7168 / topk=6 / local_expert_num=48 matches DSv4-Pro at EP8). Revisit once vLLM wires FlyDSL into the MXFP4 backend oracle." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2503 From 2fe1f2b0ea072cf57b38eac48bc6fc312368ba00 Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 14:06:34 -0500 Subject: [PATCH 4/7] feat(agentx): capture torch-profiler traces during AgentX replay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add trace capture to the DSv4-Pro MI355X AgentX MTP recipe so the DEP arm (dp-attn + EP8) can be traced under real trace-replay traffic. The shared PROFILE=1 path in benchmark_lib.sh hangs off run_benchmark_serving (--dataset-name random), which the agentic scenario never calls -- AgentX goes through build_replay_cmd + aiperf. profile.yml is likewise fixed-seq-len only (it never sets SCENARIO_SUBDIR). So capture in the recipe instead, driving vLLM's /start_profile and /stop_profile against the live replay. Gated behind AGENTX_TRACE=1 so default sweeps are untouched. The window is short by design: a 3600s replay across 8 DP ranks would produce an unusable full-duration trace. AGENTX_TRACE_DELAY_S (default 1800) must absorb dataset setup (4-14 min on the Weka corpus) plus per-lane warmup so the window lands in steady state; AGENTX_TRACE_WINDOW_S (default 20) bounds the slice. VLLM_TORCH_PROFILER_DIR must be set before the serve command or vLLM does not register the profiler endpoints at all, so it is exported alongside the other server env and pointed at RESULT_DIR/traces -- benchmark-tmpl.yml already uploads results/** as the agentic artifact. Capture is best-effort: it profiles the backend port directly (the router fans out over DP ranks), skips cleanly if the server is unhealthy, and never changes the replay's exit status. 中文：为 DSv4-Pro MI355X AgentX MTP 配置增加 trace 采集，使 DEP 配置点 （dp-attn + EP8）能在真实轨迹回放负载下被采集。 benchmark_lib.sh 中共用的 PROFILE=1 路径挂在 run_benchmark_serving （--dataset-name random）上，而 agentic 场景从不调用它——AgentX 走的是 build_replay_cmd + aiperf。profile.yml 同样只支持 fixed-seq-len（从不设置 SCENARIO_SUBDIR）。因此改为在配置脚本内采集，通过 vLLM 的 /start_profile 与 /stop_profile 端点对运行中的回放进行采样。 采集由 AGENTX_TRACE=1 控制，默认 sweep 不受影响。窗口刻意设短：3600s 回放跨 8 个 DP rank，全程采集会产出无法使用的巨大 trace。AGENTX_TRACE_DELAY_S（默认 1800）需 覆盖数据集准备（Weka 语料通常 4-14 分钟）与各 lane 预热，以确保窗口落在稳态； AGENTX_TRACE_WINDOW_S（默认 20）限定切片长度。 VLLM_TORCH_PROFILER_DIR 必须在 serve 命令之前设置，否则 vLLM 根本不会注册 profiler 端点，因此与其他 server 环境变量一同导出，并指向 RESULT_DIR/traces—— benchmark-tmpl.yml 已将 results/** 作为 agentic 产物上传。 采集为尽力而为：直接对 backend 端口采集（router 会在 DP rank 间分发），服务器不健康 时干净跳过，且不改变回放的退出状态。 Co-Authored-By: Claude Opus 5 --- .github/workflows/benchmark-tmpl.yml | 18 ++++ .github/workflows/e2e-tests.yml | 33 +++++++ .../agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 88 +++++++++++++++++++ 3 files changed, 139 insertions(+) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 6c4fe50fe5..218071710c 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -139,6 +139,21 @@ on: required: false type: boolean default: false + agentx-trace: + description: "Capture a short torch-profiler trace mid-replay (AgentX recipes that implement it)" + required: false + type: boolean + default: false + agentx-trace-delay-s: + description: "Seconds into the AgentX replay before the trace window opens" + required: false + type: string + default: '' + agentx-trace-window-s: + description: "Length of the trace window in seconds" + required: false + type: string + default: '' eval-limit: description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" required: false @@ -186,6 +201,9 @@ env: DURATION: ${{ inputs.duration }} REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }} AIPERF_EXPERIMENTAL_FAST: ${{ inputs.agentx-fast && '1' || '0' }} + AGENTX_TRACE: ${{ inputs.agentx-trace && '1' || '0' }} + AGENTX_TRACE_DELAY_S: ${{ inputs.agentx-trace-delay-s }} + AGENTX_TRACE_WINDOW_S: ${{ inputs.agentx-trace-window-s }} EVAL_LIMIT: ${{ inputs.eval-limit }} SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} AIPERF_FAILED_REQUEST_THRESHOLD: '0.10' diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 9648605fad..1a7c86f56d 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -40,6 +40,21 @@ on: required: false type: boolean default: false + agentx-trace: + description: "Capture a short torch-profiler trace mid-replay (AgentX recipes that implement it)." + required: false + type: boolean + default: false + agentx-trace-delay-s: + description: "Seconds into the AgentX replay before the trace window opens." + required: false + type: string + default: "" + agentx-trace-window-s: + description: "Length of the trace window in seconds." + required: false + type: string + default: "" eval-limit: description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" required: false @@ -120,6 +135,21 @@ on: required: false type: boolean default: false + agentx-trace: + description: "Capture a short torch-profiler trace mid-replay (AgentX recipes that implement it)." + required: false + type: boolean + default: false + agentx-trace-delay-s: + description: "Seconds into the AgentX replay before the trace window opens." + required: false + type: string + default: "" + agentx-trace-window-s: + description: "Length of the trace window in seconds." + required: false + type: string + default: "" eval-limit: description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" required: false @@ -412,6 +442,9 @@ jobs: total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} agentx-fast: ${{ inputs.agentx-fast }} + agentx-trace: ${{ inputs.agentx-trace }} + agentx-trace-delay-s: ${{ inputs.agentx-trace-delay-s }} + agentx-trace-window-s: ${{ inputs.agentx-trace-window-s }} isl: '0' osl: '0' max-model-len: '0' diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index a637c70ded..d66cd4c80c 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -455,6 +455,17 @@ set -x export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +# Torch profiler. vLLM only exposes /start_profile and /stop_profile when +# VLLM_TORCH_PROFILER_DIR is set at server start, so this has to be exported +# before the serve command below. Point it at RESULT_DIR so the traces ride out +# with the agentic_ artifact (benchmark-tmpl.yml uploads +# results/**). See capture_agentx_trace() for the capture window. +if [ "${AGENTX_TRACE:-0}" = "1" ]; then + export VLLM_TORCH_PROFILER_DIR="${VLLM_TORCH_PROFILER_DIR:-$RESULT_DIR/traces}" + mkdir -p "$VLLM_TORCH_PROFILER_DIR" + echo "Torch profiler enabled, traces -> $VLLM_TORCH_PROFILER_DIR" +fi + # MoE backend. All arms use the AITER fused MoE this recipe was validated # against. # @@ -542,9 +553,86 @@ if [ "$USE_VLLM_ROUTER" = "true" ]; then wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" fi +# ---- Torch profiler trace capture (AGENTX_TRACE=1) ------------------------------ +# +# The shared PROFILE=1 path in benchmark_lib.sh hangs off run_benchmark_serving +# (--dataset-name random), which the agentic scenario never calls -- AgentX goes +# through build_replay_cmd + aiperf. So capture here instead, driving vLLM's +# /start_profile and /stop_profile endpoints against the live replay. +# +# The window is deliberately short. This is a 3600s replay across 8 DP ranks; a +# full-duration trace would be enormous and unreadable. We sample a slice once +# the replay has reached steady state, so the trace shows MoE/MLA cost under +# representative AgentX traffic (real prefix-cache hits, real session trees) +# rather than during warmup. +# +# AGENTX_TRACE_DELAY_S is wall-clock from just before aiperf is launched, so it +# has to absorb everything that happens before traffic reaches steady state: +# dataset load/reconstruct (routinely 4-14 min on the Weka corpus, per +# build_replay_cmd) plus AIPERF_WARMUP_REQUESTS_PER_LANE warmup requests per +# lane. The 1800s default leaves margin on both; with agentx-fast (1200s total +# replay) it would land past the end, so pass agentx-trace-delay-s explicitly +# there. Watch for "[TRACE] POST /start_profile" in the log to confirm the +# window actually opened during profiling and not during setup. +# +# Traces land in RESULT_DIR, which benchmark-tmpl.yml uploads wholesale as the +# agentic_ artifact (path: results/**). +capture_agentx_trace() { + local port="$1" + local delay_s="${AGENTX_TRACE_DELAY_S:-1800}" + local window_s="${AGENTX_TRACE_WINDOW_S:-20}" + local url="http://localhost:${port}" + + echo "[TRACE] armed: sleeping ${delay_s}s into the replay, then capturing ${window_s}s" + sleep "$delay_s" + + # Bail out rather than hang the run if the server went away mid-replay. + if ! curl -sf -m 10 "$url/health" > /dev/null 2>&1; then + echo "[TRACE] server not healthy at $url; skipping capture" >&2 + return 0 + fi + + echo "[TRACE] POST /start_profile" + if ! curl -sf -m 60 -X POST "$url/start_profile" > /dev/null 2>&1; then + echo "[TRACE] /start_profile failed; skipping capture" >&2 + return 0 + fi + + sleep "$window_s" + + echo "[TRACE] POST /stop_profile (writes trace; can take minutes to flush)" + # Generous timeout: torch dumps the whole ring buffer on stop. + curl -sf -m 900 -X POST "$url/stop_profile" > /dev/null 2>&1 \ + || echo "[TRACE] /stop_profile failed or timed out; trace may be partial" >&2 + + echo "[TRACE] capture complete; traces in $VLLM_TORCH_PROFILER_DIR" +} + if [ "${EVAL_ONLY}" = "true" ]; then run_eval --port "$PORT" else build_replay_cmd "$RESULT_DIR" + + TRACE_PID="" + if [ "${AGENTX_TRACE:-0}" = "1" ]; then + # Profile against the backend directly, not the router: the router + # fans out over DP ranks, and /start_profile must reach the engine. + capture_agentx_trace "$VLLM_BACKEND_PORT" & + TRACE_PID=$! + echo "Trace capture PID: $TRACE_PID" + fi + + # Let the replay own the exit status; the capture is best-effort. + set +e run_agentic_replay_and_write_outputs "$RESULT_DIR" + REPLAY_RC=$? + set -e + + if [ -n "$TRACE_PID" ]; then + wait "$TRACE_PID" 2>/dev/null || true + fi + + if [ "$REPLAY_RC" -ne 0 ]; then + exit "$REPLAY_RC" + fi fi From 10c68089ec887418f89451fb819ef4695525f26b Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 14:07:40 -0500 Subject: [PATCH 5/7] chore(agentx): add DEP-only config for trace collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both arms of dsv4-fp4-mi355x-vllm-agentic-mtp sit at conc 32, so --conc cannot target the DEP arm alone. Add a separate config file holding only the dp-attn + EP8 entry, so a trace run does not also spend a runner on the TP8 baseline. Not referenced by any sweep: full-sweep dispatches name configs/amd-master.yaml explicitly, so this file is only reachable via an explicit --config-files. 中文：dsv4-fp4-mi355x-vllm-agentic-mtp 的两个配置点并发均为 32，因此 --conc 无法 单独指定 DEP 配置点。新增一个仅包含 dp-attn + EP8 条目的独立 config 文件，使 trace 采集运行不会额外占用一台 runner 去跑 TP8 基线。 该文件不被任何 sweep 引用：full-sweep 派发均显式指定 configs/amd-master.yaml， 因此只能通过显式 --config-files 访问。 Co-Authored-By: Claude Opus 5 --- configs/amd-agentx-trace.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 configs/amd-agentx-trace.yaml diff --git a/configs/amd-agentx-trace.yaml b/configs/amd-agentx-trace.yaml new file mode 100644 index 0000000000..405c549f7b --- /dev/null +++ b/configs/amd-agentx-trace.yaml @@ -0,0 +1,25 @@ +# Trace-collection config. Not part of any sweep -- amd-master.yaml owns the +# published DSv4-Pro MI355X AgentX numbers, and this file exists purely so a +# trace run can target the DEP arm (dp-attn + EP8) on its own without also +# spending a runner on the TP8 baseline. Both arms sit at conc 32 in +# amd-master.yaml, so --conc cannot separate them. +# +# Everything here mirrors dsv4-fp4-mi355x-vllm-agentic-mtp exactly, minus the +# pure-TP search-space entry. Dispatch with agentx-trace: true; the recipe then +# drives vLLM's /start_profile and /stop_profile mid-replay and drops traces in +# results/traces (see benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh). +# +# Keep in sync with the DEP entry in amd-master.yaml if that one changes. +dsv4-fp4-mi355x-vllm-agentic-mtp-trace: + image: vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.60 + search-space: + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } } From 3464ea34da35a8621006b2bc8e201c720618ff39 Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 15:52:46 -0500 Subject: [PATCH 6/7] fix(agentx): configure profiler via --profiler-config, not the removed env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DEP8 trace run reached /start_profile and got HTTP 404: vLLM never registered the profiler endpoints. VLLM_TORCH_PROFILER_DIR no longer exists in this build -- it was dropped from vllm/envs.py and replaced by the structured ProfilerConfig (vllm/config/profiler.py), so setting it only logged "Unknown vLLM environment variable detected" and attach_router() in entrypoints/serve/profile/api_router.py left the routes off, since it gates on profiler_config.profiler being set. Build --profiler-config instead, and bound the capture with max_iterations so the engine self-stops after N profiled steps rather than leaning on the wall-clock window to keep an 8-DP-rank trace readable. Report the HTTP status from /start_profile too -- the 404 was previously swallowed as a bare "failed", which hid the root cause for a whole run. 修复 AgentX trace 采集：改用 --profiler-config，不再用已移除的环境变量 DEP8 trace 运行中 /start_profile 返回 404：vLLM 从未注册 profiler 端点。本 镜像已从 vllm/envs.py 移除 VLLM_TORCH_PROFILER_DIR，改为结构化的 ProfilerConfig，因此设置该环境变量只会打印 "Unknown vLLM environment variable detected"，而 attach_router() 依据 profiler_config.profiler 是否设置来决定 是否挂载路由，于是端点缺失。 改为构造 --profiler-config，并用 max_iterations 限定采集范围，让引擎在 N 个 已采集步后自行停止，而不是依赖墙钟窗口来控制 8 个 DP rank 的 trace 体积。 同时输出 /start_profile 的 HTTP 状态码 —— 之前 404 被吞成笼统的 "failed"， 导致整整一轮运行都没能定位根因。 Co-Authored-By: Claude Opus 5 --- .../agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index d66cd4c80c..8455fa7e92 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -455,15 +455,33 @@ set -x export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -# Torch profiler. vLLM only exposes /start_profile and /stop_profile when -# VLLM_TORCH_PROFILER_DIR is set at server start, so this has to be exported -# before the serve command below. Point it at RESULT_DIR so the traces ride out -# with the agentic_ artifact (benchmark-tmpl.yml uploads +# Torch profiler. vLLM only registers /start_profile and /stop_profile when the +# profiler is configured at server start (entrypoints/serve/profile/api_router.py +# :attach_router gates on profiler_config.profiler being set), so this has to be +# built before the serve command below. +# +# Note this is NOT the old VLLM_TORCH_PROFILER_DIR env var. That variable was +# removed from vllm/envs.py and replaced by the structured --profiler-config on +# the way to this image; setting it only produces "Unknown vLLM environment +# variable detected: VLLM_TORCH_PROFILER_DIR" and the endpoints stay unregistered, +# so /start_profile answers 404. Fields come from vllm/config/profiler.py. +# +# max_iterations makes the engine stop itself after N profiled steps rather than +# relying on the wall-clock window below to be the thing that bounds trace size. +# The window still exists as a backstop, but at 8 DP ranks a purely time-bounded +# capture is what turns a trace unreadable. torch_profiler_dir must be absolute +# (ProfilerConfig validates this) and points into RESULT_DIR so the traces ride +# out with the agentic_ artifact (benchmark-tmpl.yml uploads # results/**). See capture_agentx_trace() for the capture window. +PROFILER_ARGS=() if [ "${AGENTX_TRACE:-0}" = "1" ]; then - export VLLM_TORCH_PROFILER_DIR="${VLLM_TORCH_PROFILER_DIR:-$RESULT_DIR/traces}" - mkdir -p "$VLLM_TORCH_PROFILER_DIR" - echo "Torch profiler enabled, traces -> $VLLM_TORCH_PROFILER_DIR" + AGENTX_TRACE_DIR="${AGENTX_TRACE_DIR:-$RESULT_DIR/traces}" + mkdir -p "$AGENTX_TRACE_DIR" + AGENTX_TRACE_DIR="$(cd "$AGENTX_TRACE_DIR" && pwd)" + PROFILER_ARGS=( + --profiler-config "{\"profiler\":\"torch\",\"torch_profiler_dir\":\"$AGENTX_TRACE_DIR\",\"max_iterations\":${AGENTX_TRACE_MAX_ITERS:-40},\"ignore_frontend\":true,\"torch_profiler_with_stack\":false}" + ) + echo "Torch profiler enabled, traces -> $AGENTX_TRACE_DIR" fi # MoE backend. All arms use the AITER fused MoE this recipe was validated @@ -524,6 +542,7 @@ VLLM_CMD=( --no-disable-hybrid-kv-cache-manager --max-num-seqs "$MAX_NUM_SEQS" "${OFFLOAD_ARGS[@]}" + "${PROFILER_ARGS[@]}" ) # (srok), not yet @@ -566,6 +585,11 @@ fi # representative AgentX traffic (real prefix-cache hits, real session trees) # rather than during warmup. # +# The engine self-stops after max_iterations profiled steps (see PROFILER_ARGS), +# so AGENTX_TRACE_WINDOW_S is a backstop, not the primary bound: we still POST +# /stop_profile to force the flush in case the replay is slow enough that the +# iteration count has not been reached. +# # AGENTX_TRACE_DELAY_S is wall-clock from just before aiperf is launched, so it # has to absorb everything that happens before traffic reaches steady state: # dataset load/reconstruct (routinely 4-14 min on the Weka corpus, per @@ -592,9 +616,19 @@ capture_agentx_trace() { return 0 fi + # Report the status code, not just pass/fail. A 404 here means the profiler + # endpoints were never registered (profiler not configured at server start), + # which is a different bug from the server refusing the request -- and it is + # invisible if the failure is reported as a bare "failed". echo "[TRACE] POST /start_profile" - if ! curl -sf -m 60 -X POST "$url/start_profile" > /dev/null 2>&1; then - echo "[TRACE] /start_profile failed; skipping capture" >&2 + local code + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "$url/start_profile" 2>/dev/null) + if [ "$code" != "200" ]; then + echo "[TRACE] /start_profile returned HTTP $code; skipping capture" >&2 + if [ "$code" = "404" ]; then + echo "[TRACE] 404 means vLLM did not register the profiler endpoints;" >&2 + echo "[TRACE] check that --profiler-config reached the serve command" >&2 + fi return 0 fi @@ -605,7 +639,8 @@ capture_agentx_trace() { curl -sf -m 900 -X POST "$url/stop_profile" > /dev/null 2>&1 \ || echo "[TRACE] /stop_profile failed or timed out; trace may be partial" >&2 - echo "[TRACE] capture complete; traces in $VLLM_TORCH_PROFILER_DIR" + echo "[TRACE] capture complete; traces in ${AGENTX_TRACE_DIR:-$RESULT_DIR/traces}" + ls -la "${AGENTX_TRACE_DIR:-$RESULT_DIR/traces}" 2>/dev/null || true } if [ "${EVAL_ONLY}" = "true" ]; then From 6e9f17c84ba84df2a5fd95b7720a56e9727a4ae1 Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 5 Aug 2026 16:22:01 -0500 Subject: [PATCH 7/7] chore(agentx): add DPattn-only (EP off) config for the DEP comparison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds dsv4-fp4-mi355x-vllm-agentic-mtp-dpa-only: dp-attn true with ep 1, so the launch script emits --tensor-parallel-size 1 --data-parallel-size 8 without --enable-expert-parallel. This is not a degenerate "TP with nothing to shard" configuration. FusedMoEParallelConfig.make() takes the not-use_ep branch, and flatten_tp_across_dp_and_pcp() folds the 8 DP ranks into an effective MoE tp_size of 8, so the experts stay sharded 8 ways -- as tensor-parallel slices of every expert instead of whole experts per rank. Expert weights are not replicated and the KV pool stays per-rank, which makes this a clean EP-vs-MoE-TP comparison against the DEP8 arm at identical attention and KV layout. 新增关闭 EP 的 DPattn-only 配置，用于与 DEP 对比 新增 dsv4-fp4-mi355x-vllm-agentic-mtp-dpa-only：dp-attn 为 true 且 ep 为 1， 启动脚本因此生成 --tensor-parallel-size 1 --data-parallel-size 8，且不带 --enable-expert-parallel。 这并非「TP 无处可切」的退化配置。FusedMoEParallelConfig.make() 走 not-use_ep 分支，flatten_tp_across_dp_and_pcp() 将 8 个 DP rank 折叠为 MoE 的等效 tp_size=8，专家仍按 8 路切分 —— 只是切成每个专家的张量并行分片，而非每 rank 持有整个专家。专家权重不会被复制，KV 池仍按 rank 分片，因此本组与 DEP8 在完全相同的 attention 与 KV 布局下，构成一次干净的 EP 与 MoE-TP 对比。 Co-Authored-By: Claude Opus 5 --- configs/amd-agentx-trace.yaml | 43 ++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/configs/amd-agentx-trace.yaml b/configs/amd-agentx-trace.yaml index 405c549f7b..bbad131056 100644 --- a/configs/amd-agentx-trace.yaml +++ b/configs/amd-agentx-trace.yaml @@ -1,15 +1,42 @@ -# Trace-collection config. Not part of any sweep -- amd-master.yaml owns the -# published DSv4-Pro MI355X AgentX numbers, and this file exists purely so a -# trace run can target the DEP arm (dp-attn + EP8) on its own without also -# spending a runner on the TP8 baseline. Both arms sit at conc 32 in +# Trace-collection / one-off comparison config. Not part of any sweep -- +# amd-master.yaml owns the published DSv4-Pro MI355X AgentX numbers, and this +# file exists so a single arm can be targeted on its own without also spending a +# runner on the arms that are not under test. Both arms sit at conc 32 in # amd-master.yaml, so --conc cannot separate them. # # Everything here mirrors dsv4-fp4-mi355x-vllm-agentic-mtp exactly, minus the -# pure-TP search-space entry. Dispatch with agentx-trace: true; the recipe then -# drives vLLM's /start_profile and /stop_profile mid-replay and drops traces in -# results/traces (see benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh). +# search-space entries that are not under test. Dispatch with agentx-trace: true +# to also drive vLLM's /start_profile and /stop_profile mid-replay and drop +# traces in results/traces (see +# benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh). # -# Keep in sync with the DEP entry in amd-master.yaml if that one changes. +# Keep in sync with the corresponding entries in amd-master.yaml if those change. + +# DP-attention with EP off. The launch script maps dp-attn true to +# --tensor-parallel-size 1 --data-parallel-size 8, and drops +# --enable-expert-parallel when ep is 1. That combination is NOT "TP with +# nothing left to shard": vLLM's FusedMoEParallelConfig.make() takes the +# not-use_ep branch, and flatten_tp_across_dp_and_pcp() folds the 8 DP ranks +# into an effective MoE tp_size of 8, so the experts are still sharded 8 ways -- +# as tensor-parallel slices of every expert rather than whole experts per rank. +# Weights are therefore not replicated, and the KV pool stays per-rank as under +# DEP8. This is the arm that isolates EP vs MoE-TP at identical attention and KV +# layout. +dsv4-fp4-mi355x-vllm-agentic-mtp-dpa-only: + image: vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.60 + search-space: + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } } + +# DP-attention + EP8. Already measured in run 31037991308. dsv4-fp4-mi355x-vllm-agentic-mtp-trace: image: vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20 model: deepseek-ai/DeepSeek-V4-Pro