diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh new file mode 100755 index 0000000000..be705c0c21 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# MiniMax-M3 NVFP4 B200 AgentX with EAGLE3-GQA and synthetic acceptance. +# DRAM KV offload uses vLLM's SimpleCPUOffloadConnector in lazy mode. +# +# Port of the validated agentic/minimaxm3_fp4_b300_mtp.sh. Spec-decode only, +# per the AgentX policy that agentic recipes are run and published with +# speculative decoding enabled rather than as an STP/MTP A/B (MODELS.md: +# MiniMax-M3 agentic non-EAGLE3 is deprecated after 2026-08-03, and the 8k1k +# scenario that carried the B200 MiniMax-M3 curves was removed in #2493 -- +# without this recipe MiniMax-M3 has no active B200 config at all). +# +# The only B200 deltas are the two blocks marked "B200:" below -- the +# checkpoint-resolution guard and the draft staging path. Every engine flag is +# the B300 script unchanged so the two SKU curves stay directly comparable. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" +NUM_SPEC_TOKENS=3 +# Golden AL for the GQA draft head: golden_al_distribution/minimaxm3_eagle3_gqa.yaml +# minimax-m3.thinking_on[3]. The non-GQA curve (minimaxm3_eagle3.yaml) reads 2.83 +# at the same level -- that head is not what this script runs. The curve is a +# property of the target/draft pair, not of the SKU it was measured on, so B200 +# pins the same 2.78 the B300 sibling does. +SYNTHETIC_ACCEPT_LEN=2.78 + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# A non-empty directory is NOT a staged checkpoint: an aborted pull leaves +# config.json and friends behind with no weights, and an `ls -A` guard accepts +# it, so the cell skips the download and dies later inside the loader. Check +# that every shard the index names is actually present -- and accept the +# single-file layout the EAGLE3 draft head ships in, which has no index. +checkpoint_is_complete() { + local dir="$1" + [[ -d "$dir" && -f "$dir/config.json" ]] || return 1 + CKPT_DIR="$dir" python3 - <<'PYEOF' +import glob, json, os, sys + +d = os.environ["CKPT_DIR"] +index = os.path.join(d, "model.safetensors.index.json") +if os.path.isfile(index): + with open(index) as fh: + shards = sorted(set(json.load(fh)["weight_map"].values())) + missing = [s for s in shards if not os.path.isfile(os.path.join(d, s))] + if missing: + print( + f"{len(missing)}/{len(shards)} shards missing, e.g. {missing[:3]}", + file=sys.stderr, + ) + sys.exit(1) +elif not glob.glob(os.path.join(d, "*.safetensors")): + print("no shard index and no .safetensors present", file=sys.stderr) + sys.exit(1) +PYEOF +} + +# B200: runners/launch_b200-dgxc.sh resolves the checkpoint to a cluster-local +# path (/scratch/fsw/models/MiniMax-M3-NVFP4) and then rewrites MODEL to that +# path before handing off to this script, so `hf download "$MODEL"` cannot work +# on this runner the way it does on b300-nv, where MODEL stays the HF repo id. +# Keep the repo id separate for the case where the checkpoint is not staged. +HF_MODEL_ID="${HF_MODEL_ID:-nvidia/MiniMax-M3-NVFP4}" + +if [[ -n "${MODEL_PATH:-}" ]]; then + if ! checkpoint_is_complete "$MODEL_PATH"; then + # Every concurrency of this sweep runs as its own allocation against + # the same shared path, so serialize: one cell pulls the ~250 GB + # checkpoint and the rest wait on it rather than racing as writers. + # `hf download` resumes into a partially-populated --local-dir. + mkdir -p "$MODEL_PATH" + MODEL_DOWNLOAD_LOCK="${MODEL_PATH%/}.download.lock" + echo "Checkpoint at $MODEL_PATH is incomplete; acquiring $MODEL_DOWNLOAD_LOCK" + exec 9>"$MODEL_DOWNLOAD_LOCK" + flock -w "${MODEL_DOWNLOAD_LOCK_TIMEOUT:-21600}" 9 || { + echo "Error: timed out waiting for another cell to stage $MODEL_PATH" >&2 + exit 1 + } + if checkpoint_is_complete "$MODEL_PATH"; then + echo "Another cell staged $MODEL_PATH while we waited" + else + hf download "$HF_MODEL_ID" --local-dir "$MODEL_PATH" + fi + flock -u 9 + exec 9>&- + checkpoint_is_complete "$MODEL_PATH" || { + echo "Error: $MODEL_PATH is still incomplete after hf download $HF_MODEL_ID." >&2 + exit 1 + } + fi +else + hf download "$HF_MODEL_ID" + export MODEL_PATH="$HF_MODEL_ID" +fi + +# B200: the B300 sibling stages the draft under /data/models, which does not +# exist on b200-dgxc. The launcher bind-mounts only $MODEL_PATH itself +# (--container-mounts=...,$MODEL_PATH:$MODEL_PATH,...), so its parent exists +# solely inside the container overlay -- writable, but per-job. Stage the draft +# there rather than inside $MODEL_PATH, which is the shared read-mostly +# checkpoint directory and must not be polluted. The EAGLE3-GQA head is small +# next to the target, so the per-job pull is cheap; this is what the +# now-deprecated 8k1k B200 MiniMax-M3 MTP recipe did as well. +DRAFT_MODEL_PATH="${DRAFT_MODEL_PATH:-$(dirname "${MODEL_PATH%/}")/${DRAFT_MODEL##*/}}" +if ! checkpoint_is_complete "$DRAFT_MODEL_PATH"; then + hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" + checkpoint_is_complete "$DRAFT_MODEL_PATH" || { + echo "Error: $DRAFT_MODEL_PATH is incomplete after hf download $DRAFT_MODEL." >&2 + exit 1 + } +fi + +nvidia-smi +resolve_trace_source +install_agentic_deps + +OFFLOAD_ARGS=() +if require_agentic_kv_offload_backend vllm-simple; then + CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) + export VLLM_USE_SIMPLE_KV_OFFLOAD=1 + OFFLOAD_CONFIG=$(printf \ + '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":%d,"lazy_offload":true}}' \ + "$CPU_OFFLOAD_BYTES") + OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG") +fi + +export PYTHONNOUSERSITE=1 +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_FLOAT32_MATMUL_PRECISION=high +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + +# Same 0.9 the B300 sibling and the deprecated 8k1k B200 MiniMax-M3 NVFP4 MTP +# recipe ran. Exposed as an override because B200's 180 GB leaves little beyond +# the ~250 GB checkpoint: TP2 could not host the 1M-context KV for even one +# request at this value and was dropped from the search space, so TP4 is the +# smallest topology this script is expected to serve. +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +if [ "${EVAL_ONLY:-}" = "true" ]; then + SPEC_CONFIG=$(printf \ + '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN"}' \ + "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS") +else + SPEC_CONFIG=$(printf \ + '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":%.2f}' \ + "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS" "$SYNTHETIC_ACCEPT_LEN") +fi + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --block-size 128 + --language-model-only + --enable-prefix-caching + --no-enable-flashinfer-autotune + --reasoning-parser minimax_m3 + --tool-call-parser minimax_m3 + --enable-auto-tool-choice + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' + --attention-config '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + --kv-cache-dtype fp8 + --max-cudagraph-capture-size 512 + --max-num-batched-tokens 16384 + --stream-interval 20 + --trust-remote-code + --speculative-config "$SPEC_CONFIG" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" +set -x + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5ac5166aa1..0e344c330c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7045,6 +7045,52 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode # throughput. Dynamo KV routing and AIPerf conversation-aware routing remain # enabled by framework=dynamo-vllm. + +# B200 sibling of minimaxm3-fp4-b300-vllm-agentic-mtp: same image, same +# EAGLE3-GQA draft at 3 speculative tokens, same golden AL 2.78, and the same +# SimpleCPU offload arm, so the two SKU curves are directly comparable. This is +# MiniMax-M3's only active B200 configuration -- the 8k1k scenario that carried +# the previous B200 curves was removed in #2493. +# +# TP4-only. The B300 sibling also runs a TP2 arm, but TP2 does not fit on B200: +# the ~250 GB NVFP4 checkpoint leaves a 31.72 GiB KV pool at +# --gpu-memory-utilization 0.9, and one request at the model's 1,048,576-token +# max_model_len needs 37.62 GiB, so vLLM raises in _check_enough_kv_cache_memory +# and the engine never reaches serving (run 31141913741 -- both TP2 cells, c1 +# and c2, died identically at init; every TP4 cell passed). This is structural, +# not a concurrency cliff: the estimated max model length at TP2 is 883,840, +# below the corpus context, so no conc-list would have made the arm run. +# +# Concurrency on the surviving arms still differs from B300, whose TP4 pool is +# ~787 GB against B200's ~398 GB: +# TP4 B300 runs 1-20; B200 samples 5-15 densely because the cliff lands +# inside that range at roughly half the KV, and retains 20 past it. +# TP4 + SimpleCPU picks up at 20 rather than B300's 30, since the host tier +# has to start absorbing the working set one step earlier, and stops at +# 40 rather than following B300 to 75. Past 40 the host tier is +# absorbing a working set that never fit in B200's ~398 GB TP4 pool to +# begin with, so those points cost four engine starts each to trace a +# tail that is offload bandwidth, not the SKU. +# +# dram-utilization 0.683 is the B300 value verbatim, and it resolves to the same +# 1,024 GB engine-level CPU KV budget at TP4: b200-dgxc and b300-nv both report +# more installed DRAM than MAX_AGENTIC_AVAILABLE_CPU_DRAM_MIB, so both clamp to +# the same 3 TB AgentX ceiling before the proportional-GPU rule is applied. +# GPU-resident points receive a zero budget. +minimaxm3-fp4-b200-vllm-agentic-mtp: + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.683 + search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5, 8, 10, 12, 15, 20] } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 30, 40] } dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8: image: vllm/vllm-openai:v0.23.0 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f8e593b9dd..b0ef6a1839 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5601,3 +5601,20 @@ description: - "Kimi K2.5 NVFP4 B300 vLLM: nightly image, TP/DEP/TEP sweep, TP8 conc-1 only, DEP gmu 0.85" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2513 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the MiniMax-M3 NVFP4 B200 AgentX (agentic-coding) recipe: nvidia/MiniMax-M3-NVFP4 on single-node vLLM with EAGLE3-GQA speculative decoding, routed to benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B, because synthetic acceptance already makes spec-decode results comparable across submissions. This is MiniMax-M3's only active B200 configuration: the Single-turn 8k1k scenario that carried the previous B200 MiniMax-M3 curves (minimaxm3-fp4-b200-vllm, minimaxm3-fp4-b200-vllm-mtp) was deprecated and archived under configs/deprecated/ in #2493." + - "Speculative config is the Inferact/MiniMax-M3-EAGLE3-GQA draft head at three speculative tokens with the drafter pinned to FLASH_ATTN, matching the B300 sibling minimaxm3-fp4-b300-vllm-agentic-mtp. Throughput runs pin vLLM synthetic rejection sampling to the committed golden AL: rejection_sample_method=synthetic with synthetic_acceptance_length 2.78, the thinking_on K=3 entry of golden_al_distribution/minimaxm3_eagle3_gqa.yaml. EVAL_ONLY runs drop synthetic acceptance and keep real target verification, since synthetic acceptance bypasses verification and zeroes the eval score." + - "Image vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 -- the tag the B300 AgentX MTP sibling runs, and the same tag the deprecated minimaxm3-fp4-b200-vllm 8k1k entry was bumped to in #2468, so it is already proven on this model, precision and SKU." + - "Engine flags are the B300 script verbatim so the two SKU curves stay comparable: FlashInfer TRT-LLM attention with FP8 indexer KV, --kv-cache-dtype fp8, --block-size 128 (mandatory for the MSA sparse/index cache), --language-model-only, --enable-prefix-caching, --max-cudagraph-capture-size 512, --max-num-batched-tokens 16384, --stream-interval 20, the minimax_m3 reasoning and tool-call parsers, and thinking_mode enabled. DRAM offload arms use vLLM's SimpleCPUOffloadConnector in lazy mode. --gpu-memory-utilization stays 0.9 (the value both the B300 sibling and the deprecated 8k1k B200 NVFP4 MTP recipe ran) and is now overridable via GPU_MEMORY_UTILIZATION for bring-up debugging." + - "First B200 delta: the checkpoint-resolution guard. runners/launch_b200-dgxc.sh resolves the checkpoint to /scratch/fsw/models/MiniMax-M3-NVFP4 and then rewrites MODEL to that path, so `hf download \"$MODEL\"` cannot work there the way it does on b300-nv, where MODEL stays the HF repo id. The script keeps the repo id in HF_MODEL_ID, verifies the checkpoint by walking model.safetensors.index.json rather than trusting an ls -A emptiness test, and serializes any download behind a flock so the concurrencies of one sweep do not race as writers on the shared path." + - "Second B200 delta: draft staging. The B300 sibling stages the EAGLE3-GQA head under /data/models, which does not exist on b200-dgxc; the launcher bind-mounts only $MODEL_PATH itself, so the draft is staged in the container-local parent directory instead of polluting the shared checkpoint directory. Same approach the deprecated 8k1k B200 MiniMax-M3 MTP recipe used." + - "Search space is TP4-only: TP4 GPU-resident conc [1, 2, 5, 8, 10, 12, 15, 20] and TP4 SimpleCPU conc [20, 30, 40]. The 250 GB NVFP4 checkpoint leaves roughly 398 GB of KV at TP4 on B200's 180 GB HBM3e against 787 GB on B300, so the host tier picks up one step earlier than the B300 sibling. Same B200/B300 asymmetry the qwen3.5 NVFP4 AgentX MTP pair already publishes." + - "The B300 sibling's TP2 arm is dropped rather than ported: TP2 does not fit on B200. With the checkpoint resident, the KV pool is 31.72 GiB at --gpu-memory-utilization 0.9 while a single request at the model's 1,048,576-token max_model_len needs 37.62 GiB, so vLLM raises in _check_enough_kv_cache_memory and the engine never reaches serving. Observed on run 31141913741, where both TP2 cells (c1 and c2) died identically at engine init while every TP4 cell passed. The failure is structural, not a concurrency cliff -- vLLM estimates the max model length at TP2 as 883,840, below the agentic corpus context -- so no conc-list would have made the arm run." + - "The whole recipe stops at concurrency 40. The B300 sibling's SimpleCPU arm runs to 75, but on B200 everything past 40 is the host tier absorbing a working set that never fit in the ~398 GB TP4 pool to begin with, so that tail traces offload bandwidth rather than the SKU and costs one engine start per point to do it. Cutting it drops the recipe from 18 configs to 13." + - "dram-utilization 0.683 is carried over unchanged from the B300 entry and resolves to the same 1,024 GB engine-level CPU KV budget at TP4: b200-dgxc and b300-nv both report more installed DRAM than MAX_AGENTIC_AVAILABLE_CPU_DRAM_MIB, so both clamp to the same 3 TB AgentX ceiling before the proportional-GPU rule applies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2511