diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 8b9b38f2e..ba2d810c3 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -55,19 +55,19 @@ require_agentic_kv_offload_backend() { fi return 1 ;; - dram) + dram|nvme|dram+nvme) if [[ "${KV_OFFLOAD_BACKEND:-}" != "$expected_backend" ]]; then - echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=dram, got '${KV_OFFLOAD_BACKEND:-}'" >&2 + echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=$KV_OFFLOADING, got '${KV_OFFLOAD_BACKEND:-}'" >&2 exit 1 fi - if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then - echo "Error: DRAM KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2 + if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: $KV_OFFLOADING KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2 exit 1 fi return 0 ;; *) - echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2 + echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2 exit 1 ;; esac @@ -108,18 +108,18 @@ if [[ "$_benchmark_caller" == */agentic/* || exit 1 fi ;; - dram) + dram|nvme|dram+nvme) if [[ -z "${KV_OFFLOAD_BACKEND:-}" || "${KV_OFFLOAD_BACKEND:-}" == "none" ]]; then - echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=dram" >&2 + echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=$KV_OFFLOADING" >&2 exit 1 fi - if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then - echo "Error: DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2 + if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: $KV_OFFLOADING KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2 exit 1 fi ;; *) - echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2 + echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2 exit 1 ;; esac diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh index 9e5f8fc8d..c095106ab 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh @@ -2,7 +2,7 @@ set -eo pipefail set -x -# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional Mooncake DRAM KV offload. +# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional DRAM or NVMe KV offload. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -97,13 +97,37 @@ EOF --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' ) +elif [ "$KV_OFFLOADING" = "nvme" ]; then + require_agentic_kv_offload_backend vllm-simple + : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}" + NVME_OFFLOAD_TOTAL_BYTES=8000000000000 + NVME_OFFLOAD_PER_RANK_BYTES=$((NVME_OFFLOAD_TOTAL_BYTES / TP)) + # vLLM appends .rank_ to give each TP rank its own file. + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$NVME_OFFLOAD_PER_RANK_BYTES,\"disk_buffer_slots\":4,\"lazy_offload\":false}}" + ) +elif [ "$KV_OFFLOADING" = "dram+nvme" ]; then + require_agentic_kv_offload_backend vllm-native + : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}" + TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824)) + PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB)) + if (( PER_RANK_GIB <= 0 )); then + echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and DRAM+NVMe KV offload" >&2 + exit 1 + fi + CPU_OFFLOAD_TOTAL_BYTES=$((PER_RANK_GIB * TP * 1073741824)) + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":$CPU_OFFLOAD_TOTAL_BYTES,\"eviction_policy\":\"lru\",\"secondary_tiers\":[{\"type\":\"fs\",\"root_dir\":\"$NVME_OFFLOAD_DIR\",\"n_read_threads\":32,\"n_write_threads\":16,\"locality\":\"LOCAL\"}]}}" + ) else echo "Error: unsupported KV_OFFLOADING='$KV_OFFLOADING'" >&2 exit 1 fi export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:prompt_tokens_cached_by_source" NUM_SPEC_TOKENS=3 TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0d5bd0c7d..4e30ee930 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7499,7 +7499,7 @@ qwen3.5-fp4-b200-trt-mtp: - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] } minimaxm3-fp8-h100-vllm-agentic-mtp: - image: vllm/vllm-openai:v0.27.1 + image: ttl.sh/cquil11-vllm-tier-progress-b9e9d720-amd64-20260902:24h model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: cluster:h100-dgxc @@ -7508,11 +7508,12 @@ minimaxm3-fp8-h100-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - # The fast sweep places the resident HBM cliff between c5 and c6. + # Narrow validation sweep for physical cache-source attribution. - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8] } + - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [30] } + - { tp: 8, spec-decoding: mtp, kv-offloading: [dram, nvme], kv-offload-backend: { name: vllm-native }, conc-list: [20] } minimaxm3-fp8-h200-vllm-agentic-mtp: image: vllm/vllm-openai:v0.27.1 diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 6db5d351e..6aa3e2073 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -210,6 +210,11 @@ For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`, Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2266-L2268)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. +The H100 MiniMax-M3 NVMe and DRAM+NVMe AgentX launcher allows 420 minutes for Slurm: +measured warmup alone took over four hours before the one-hour profile. +Other H100 single-node cases retain the 300-minute default, and an explicit +`SALLOC_TIME_LIMIT` overrides either value. The canonical workload is unchanged. + ## 8. Preserve trace and run provenance AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L2023-L2102), [replay semantics](../benchmarks/benchmark_lib.sh#L2104-L2270)). diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 6e8c8e03c..26c117ca7 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -210,6 +210,11 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ Fast 结果只能作为 bring-up 证据,绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效;只能用于 smoke 诊断([源码](../benchmarks/benchmark_lib.sh#L2266-L2268))。Fast 运行健康后,必须对完全相同的 candidate 进行 canonical 运行,才能宣称 benchmark 成功。 +H100 MiniMax-M3 NVMe 和 DRAM+NVMe AgentX 启动器的 Slurm 时限为 420 分钟: +实测 warmup 本身超过四小时,之后还需运行一小时的 profiling。 +其他 H100 单节点场景仍默认使用 300 分钟;显式设置 `SALLOC_TIME_LIMIT` +可覆盖任一默认值。canonical 工作负载保持不变。 + ## 8. 保留 trace 与运行 provenance AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量,但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时,才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定,否则所选 trace corpus 依赖模型 family;resolver 会同时记录 loader 与 Hugging Face dataset([trace 解析](../benchmarks/benchmark_lib.sh#L2023-L2102)、[replay 语义](../benchmarks/benchmark_lib.sh#L2104-L2270))。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4eb0a2caa..6d9f3aab1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6821,3 +6821,35 @@ description: - "Refresh to collect TensorRT-LLM server metrics." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 + +- config-keys: + - minimaxm3-fp8-h100-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Run a narrow H100 MiniMax-M3 AgentX validation on the patched vLLM image that exports the bounded cached-token sources device, cpu, disk, p2p, and external and includes vLLM PR #53087's bounded fallback for stalled tier-primary writes." + - "Validate one Mooncake DRAM point at TP8 concurrency 8 and one NVMe-only point at TP8 concurrency 30 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity." + - "Validate one declarative kv-offloading [dram, nvme] point at TP8 concurrency 20, mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier." + - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container." + - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796 + +- config-keys: + - minimaxm3-fp8-h100-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Allow seven hours for the native DRAM+NVMe point after its canonical warmup exceeded four hours; retain the 300-minute H100 default for other cases." + - "Pin the AgentX client fix that excludes automatic warmup baselines from profiling server metrics and require the cached-token-source metric in JSON/CSV artifacts." + - "Keep exactly Mooncake DRAM c8, Simple NVMe c30, and native DRAM+NVMe c20, with unchanged canonical warmup, one-hour profiling, images, and full evals." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796 + +- config-keys: + - minimaxm3-fp8-h100-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use the Python-only token-source image b9e9d720 on official vLLM nightly 7c5dc571, retaining the HIT_PENDING fix and backporting upstream PR #45406 so an unschedulable queue head cannot strand completed async KV loads." + - "Allow seven hours for Simple NVMe as well as native DRAM+NVMe; the NVMe warmup took four hours and its previous five-hour allocation ended before profiling completed." + - "Preserve exactly Mooncake DRAM c8, Simple NVMe c30 with 8 TB aggregate capacity, and native DRAM+NVMe c20, with full evals and unchanged canonical warmup and one-hour profiling." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 50c6a8ad1..0d4b2965c 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -298,13 +298,40 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" - salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --time=180 --no-shell --job-name="$RUNNER_NAME" + # These NVMe AgentX points can need >4 hours of warmup before a 1-hour profile. + if [[ "${MODEL_PREFIX:-}" == "minimaxm3" && "${SCENARIO_TYPE:-}" == "agentic-coding" && ( "${KV_OFFLOADING:-}" == "nvme" || "${KV_OFFLOADING:-}" == "dram+nvme" ) ]]; then + SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-420}" + fi + SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-300}" + salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ + --gres="gpu:$GPU_COUNT" --exclusive --time="$SALLOC_TIME_LIMIT" \ + --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) if [[ -z "$JOB_ID" ]]; then echo "ERROR: failed to resolve H100 Slurm allocation" >&2 exit 1 fi - trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT + cleanup_allocation() { + local rc=$? + trap - EXIT INT TERM + scancel "$JOB_ID" 2>/dev/null || true + exit "$rc" + } + trap cleanup_allocation EXIT INT TERM + + NVME_CONTAINER_MOUNT="" + if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "dram+nvme" ]]; then + NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)" + NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID" + srun --jobid="$JOB_ID" bash -c " + set -e + test -w '$NVME_HOST_ROOT' + mkdir -m 700 '$NVME_HOST_DIR' + findmnt -T '$NVME_HOST_DIR' + " + NVME_CONTAINER_MOUNT=",$NVME_HOST_DIR:/kv-offload" + export NVME_OFFLOAD_DIR=/kv-offload + fi # Check the shared cache before opening its lock. A valid squash file is # immutable, so readers do not need to touch a lock owned by another user. @@ -327,12 +354,10 @@ else srun --jobid=$JOB_ID \ --container-image=$SQUASH_FILE \ - --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ + --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache$NVME_CONTAINER_MOUNT \ --no-container-mount-home \ --container-workdir=/workspace/ \ --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \ bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_h100${SPEC_SUFFIX}.sh - scancel $JOB_ID - fi diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index aed48695d..7e6646117 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -26,6 +26,35 @@ def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str ) +def test_h100_nvme_agentx_time_limit_preserves_defaults_and_override() -> None: + launcher = (REPO_ROOT / "runners" / "launch_h100-dgxc-slurm.sh").read_text() + start = launcher.index(" # These NVMe AgentX points") + stop = launcher.index(" salloc ", start) + configure = launcher[start:stop] + cases = [ + ("minimaxm3", "agentic-coding", "dram+nvme", "", "420"), + ("minimaxm3", "agentic-coding", "dram+nvme", "480", "480"), + ("minimaxm3", "agentic-coding", "dram", "", "300"), + ("minimaxm3", "agentic-coding", "nvme", "", "420"), + ("minimaxm3", "agentic-coding", "nvme", "480", "480"), + ("other", "agentic-coding", "dram+nvme", "", "300"), + ("minimaxm3", "fixed-sequence", "dram+nvme", "", "300"), + ] + for model, scenario, offload, override, expected in cases: + result = run_bash( + 'MODEL_PREFIX="$1"; SCENARIO_TYPE="$2"; KV_OFFLOADING="$3"; ' + 'SALLOC_TIME_LIMIT="$4";\n' + + configure + + '\nprintf "%s" "$SALLOC_TIME_LIMIT"', + model, + scenario, + offload, + override, + ) + assert result.returncode == 0, result.stderr + assert result.stdout == expected + + def test_copy_agentic_results_stages_only_matching_points(tmp_path: Path) -> None: source = tmp_path / "source" workspace = tmp_path / "workspace" diff --git a/utils/aiperf b/utils/aiperf index 754356e9a..dcbd94265 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 754356e9a39acc6cc6afb242d123bb57c3fb6f75 +Subproject commit dcbd942654ef87b357c7722837855dbccedc2353 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index ecd5bd7ba..7ed29fdec 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -399,7 +399,7 @@ def agentic_dram_offload_gb( budgeted separately if it ever gains its own pool). """ kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none") - if kv_offloading != "dram": + if kv_offloading != "dram" and kv_offloading != ["dram", "nvme"]: return 0 available_mib = min( @@ -433,13 +433,21 @@ def agentic_dram_offload_gb( def agentic_kv_offload_suffix( - kv_offloading: str, + kv_offloading: str | list[str], kv_offload_backend: dict | None, ) -> str: """Return a compact exp-name suffix for agentic KV offload settings.""" if kv_offloading == "none": return "kvnone" - return f"kv{kv_offloading}-{kv_offload_backend['name']}" + mode = "+".join(kv_offloading) if isinstance(kv_offloading, list) else kv_offloading + return f"kv{mode}-{kv_offload_backend['name']}" + + +def agentic_kv_offload_runtime_value(kv_offloading: str | list[str]) -> str: + """Convert declarative tier lists into the workflow's string input.""" + if isinstance(kv_offloading, list): + return "+".join(kv_offloading) + return kv_offloading def multinode_agentic_exp_name( @@ -1148,7 +1156,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.PREFILL.value: prefill, Fields.DECODE.value: decode, Fields.CONC.value: conc_batch, - Fields.KV_OFFLOADING.value: kv_offloading, + Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading), Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, Fields.EXP_NAME.value: multinode_agentic_exp_name( @@ -1185,7 +1193,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False, Fields.SPEC_DECODING.value: spec_decoding, Fields.CONC.value: conc, - Fields.KV_OFFLOADING.value: kv_offloading, + Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading), Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, Fields.EXP_NAME.value: ( @@ -1447,7 +1455,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.PREFILL.value: prefill, Fields.DECODE.value: decode, Fields.CONC.value: conc_batch, - Fields.KV_OFFLOADING.value: kv_offloading, + Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading), Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, Fields.EXP_NAME.value: multinode_agentic_exp_name( @@ -1483,7 +1491,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False, Fields.SPEC_DECODING.value: spec_decoding, Fields.CONC.value: conc, - Fields.KV_OFFLOADING.value: kv_offloading, + Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading), Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, Fields.EXP_NAME.value: ( diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index ef305ab23..0e65793f1 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2429,6 +2429,56 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): } assert all(entry["duration"] == 3600 for entry in result) + def test_multi_tier_agentic_uses_dram_budget_and_distinct_name( + self, sample_runner_config + ): + config = { + "dsv4-b300-agentic": { + "image": "vllm/vllm-openai:v0.23.0", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", + "precision": "fp4", + "framework": "vllm", + "runner": "cluster:b300-nv", + "multinode": False, + "scenarios": { + "agentic-coding": [{ + "dram-utilization": 0.80, + "search-space": [ + { + "tp": 8, + "kv-offloading": "nvme", + "kv-offload-backend": {"name": "vllm-simple"}, + "conc-list": [7], + }, + { + "tp": 8, + "kv-offloading": ["dram", "nvme"], + "kv-offload-backend": {"name": "vllm-native"}, + "conc-list": [7], + }, + ], + }], + }, + }, + } + args = argparse.Namespace( + config_keys=["dsv4-b300-agentic"], + seq_lens=None, + conc=None, + scenario_type=["agentic-coding"], + runner_node_filter=None, + ) + + result = generate_test_config_sweep(args, config, sample_runner_config) + + assert [entry["kv-offloading"] for entry in result] == ["nvme", "dram+nvme"] + assert [entry["total-cpu-dram-gb"] for entry in result] == [0, 2399] + assert [entry["exp-name"] for entry in result] == [ + "dsv4_tp8_conc7_kvnvme-vllm-simple", + "dsv4_tp8_conc7_kvdram+nvme-vllm-native", + ] + def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_config): config = { "dsv4-b300-agentic": { diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index c57c93ae8..cf021ca22 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -504,6 +504,49 @@ def test_dram_kv_offload_requires_dram_utilization(self): }], }) + def test_multi_tier_kv_offload_requires_dram_utilization(self): + with pytest.raises(Exception, match="dram-utilization"): + AgenticCodingConfig(**{ + "search-space": [{ + "tp": 8, + "kv-offloading": ["dram", "nvme"], + "kv-offload-backend": {"name": "vllm-native"}, + "conc-list": [7, 8], + }], + }) + + def test_multi_tier_kv_offload_accepts_dram_capacity_config(self): + config = AgenticCodingConfig(**{ + "dram-utilization": 0.99, + "search-space": [{ + "tp": 8, + "kv-offloading": ["dram", "nvme"], + "kv-offload-backend": {"name": "vllm-native"}, + "conc-list": [7, 8], + }], + }) + + assert config.search_space[0].kv_offloading == ["dram", "nvme"] + assert config.dram_utilization == 0.99 + + @pytest.mark.parametrize("kv_offloading", ["nvme", ["dram", "nvme"]]) + def test_nvme_kv_offload_rejects_multinode_agentic_entries( + self, kv_offloading + ): + with pytest.raises(Exception, match="only for single-node agentic"): + AgenticCodingSearchSpaceEntry(**{ + "worker": { + "tp": 8, + "pp": 1, + "ep": 1, + "dp-attn": False, + }, + "num-nodes": 2, + "kv-offloading": kv_offloading, + "kv-offload-backend": {"name": "vllm-native"}, + "conc-list": [7], + }) + def test_agentic_search_space_rejects_total_cpu_dram_gb(self): with pytest.raises(Exception, match="total-cpu-dram-gb"): AgenticCodingSearchSpaceEntry(**{ diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 7014e5429..f10509bdf 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -14,6 +14,11 @@ CLUSTER_LABEL_PREFIX = "cluster:" DEFAULT_AGENTIC_DURATION_SECONDS = 3600 +KVOffloadingTier = Literal["dram", "nvme"] +KVOffloadingConfig = Union[ + Literal["none", "dram", "nvme"], + List[KVOffloadingTier], +] """ The below class defines the field names expected to be present in the JSON entries @@ -328,7 +333,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel): default="none", alias=Fields.SPEC_DECODING.value ) conc: int - kv_offloading: Literal["none", "dram"] = Field( + kv_offloading: Literal["none", "dram", "nvme", "dram+nvme"] = Field( alias=Fields.KV_OFFLOADING.value ) kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field( @@ -531,7 +536,13 @@ def _validate_kv_offload_fields(self): f"{Fields.KV_OFFLOADING.value}" ) return self - if self.kv_offloading == "none": + if isinstance(self.kv_offloading, list): + if self.kv_offloading != ["dram", "nvme"]: + raise ValueError( + f"The only supported multi-tier {Fields.KV_OFFLOADING.value} " + "configuration is ['dram', 'nvme']" + ) + elif self.kv_offloading == "none": if backend is not None: raise ValueError( f"{Fields.KV_OFFLOAD_BACKEND.value} can only be set when " @@ -662,7 +673,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel): decode: Optional[WorkerConfig] = None num_nodes: Optional[int] = Field( default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True) - kv_offloading: Optional[Literal["none", "dram"]] = Field( + kv_offloading: Optional[KVOffloadingConfig] = Field( default=None, alias=Fields.KV_OFFLOADING.value ) kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field( @@ -710,6 +721,11 @@ def validate_topology_fields(self): ) _validate_tp_context_topology(self) if has_aggregate_worker or has_complete_multinode: + if self.kv_offloading in ("nvme", ["dram", "nvme"]): + raise ValueError( + f"{Fields.KV_OFFLOADING.value}={self.kv_offloading!r} is " + "currently supported only for single-node agentic entries" + ) explicitly_single_node_fields = { "pp", "dcp_size", @@ -744,7 +760,7 @@ class AgenticCodingConfig(BaseModel): @model_validator(mode='after') def validate_dram_offload_capacity(self): for entry in self.search_space: - if entry.kv_offloading != "dram": + if entry.kv_offloading != "dram" and entry.kv_offloading != ["dram", "nvme"]: continue if self.dram_utilization is None: raise ValueError(