From 5d793511cd3ae5602a7a1553bd4a96f14f5b9fe1 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 00:22:03 -0500
Subject: [PATCH 01/23] feat(agentx): add H100 NVMe KV offload sweep

---
 benchmarks/benchmark_lib.sh                   | 16 ++++++-------
 .../agentic/minimaxm3_fp8_h100_mtp.sh         | 11 ++++++++-
 configs/nvidia-master.yaml                    |  7 +++---
 runners/launch_h100-dgxc-slurm.sh             | 23 +++++++++++++++++--
 utils/matrix_logic/validation.py              |  4 ++--
 5 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh
index 4819f50599..fee5a0be2a 100644
--- a/benchmarks/benchmark_lib.sh
+++ b/benchmarks/benchmark_lib.sh
@@ -49,19 +49,19 @@ require_agentic_kv_offload_backend() {
             fi
             return 1
             ;;
-        dram)
+        dram|nvme)
             if [[ "${KV_OFFLOAD_BACKEND:-}" != "$expected_backend" ]]; then
-                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=dram, got '${KV_OFFLOAD_BACKEND:-}'" >&2
+                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=$KV_OFFLOADING, got '${KV_OFFLOAD_BACKEND:-}'" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+            if [[ "$KV_OFFLOADING" == "dram" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
                 echo "Error: DRAM KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             return 0
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme)" >&2
             exit 1
             ;;
     esac
@@ -86,18 +86,18 @@ if [[ "$_benchmark_caller" == */agentic/* ||
                 exit 1
             fi
             ;;
-        dram)
+        dram|nvme)
             if [[ -z "${KV_OFFLOAD_BACKEND:-}" || "${KV_OFFLOAD_BACKEND:-}" == "none" ]]; then
-                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=dram" >&2
+                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=$KV_OFFLOADING" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+            if [[ "$KV_OFFLOADING" == "dram" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
                 echo "Error: DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme)" >&2
             exit 1
             ;;
     esac
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index 9e5f8fc8d6..38e9da901d 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -2,7 +2,7 @@
 set -eo pipefail
 set -x
 
-# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional Mooncake DRAM KV offload.
+# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional DRAM or NVMe KV offload.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -97,6 +97,15 @@ EOF
         --kv-transfer-config
         '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
     )
+elif [ "$KV_OFFLOADING" = "nvme" ]; then
+    require_agentic_kv_offload_backend vllm-simple
+    : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
+    NVME_OFFLOAD_TOTAL_BYTES=8000000000000
+    NVME_OFFLOAD_PER_RANK_BYTES=$((NVME_OFFLOAD_TOTAL_BYTES / TP))
+    OFFLOAD_ARGS=(
+        --kv-transfer-config
+        "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$NVME_OFFLOAD_PER_RANK_BYTES,\"disk_buffer_slots\":4,\"lazy_offload\":false}}"
+    )
 else
     echo "Error: unsupported KV_OFFLOADING='$KV_OFFLOADING'" >&2
     exit 1
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 438a3e9da0..52dbf12cde 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7341,7 +7341,7 @@ qwen3.5-fp4-b200-trt-mtp:
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
 
 minimaxm3-fp8-h100-vllm-agentic-mtp:
-  image: vllm/vllm-openai:v0.27.1
+  image: ttl.sh/cquil11-vllm-tier-dfb25dbef4-20260828:24h
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: cluster:h100-dgxc
@@ -7353,8 +7353,9 @@ minimaxm3-fp8-h100-vllm-agentic-mtp:
     # The fast sweep places the resident HBM cliff between c5 and c6.
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] }
-      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
 
 minimaxm3-fp8-h200-vllm-agentic-mtp:
   image: vllm/vllm-openai:v0.27.1
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 1334c95542..139f06fae2 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -295,7 +295,26 @@ else
         echo "ERROR: failed to resolve H100 Slurm allocation" >&2
         exit 1
     fi
-    trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
+    NVME_HOST_DIR=""
+    cleanup_allocation() {
+        local rc=$?
+        trap - EXIT INT TERM
+        if [[ -n "$NVME_HOST_DIR" ]]; then
+            srun --jobid="$JOB_ID" bash -c "rm -rf -- '$NVME_HOST_DIR'" 2>/dev/null || true
+        fi
+        scancel "$JOB_ID" 2>/dev/null || true
+        exit "$rc"
+    }
+    trap cleanup_allocation EXIT INT TERM
+
+    NVME_CONTAINER_MOUNT=""
+    if [[ "${KV_OFFLOADING:-none}" == "nvme" ]]; then
+        NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
+        NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
+        srun --jobid="$JOB_ID" bash -c "set -e; test -w '$NVME_HOST_ROOT'; mkdir -m 700 '$NVME_HOST_DIR'; findmnt -T '$NVME_HOST_DIR'"
+        NVME_CONTAINER_MOUNT=",$NVME_HOST_DIR:/kv-offload"
+        export NVME_OFFLOAD_DIR=/kv-offload
+    fi
 
     # flock-serialize the enroot import so concurrent sweep jobs on the same
     # shared NFS path don't race each other into 'File already exists' (race
@@ -317,7 +336,7 @@ else
 
     srun --jobid=$JOB_ID \
         --container-image=$SQUASH_FILE \
-        --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \
+        --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache$NVME_CONTAINER_MOUNT \
         --no-container-mount-home \
         --container-workdir=/workspace/ \
         --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \
diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
index 436ea97b39..cbb353c1ec 100644
--- a/utils/matrix_logic/validation.py
+++ b/utils/matrix_logic/validation.py
@@ -318,7 +318,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
         default="none", alias=Fields.SPEC_DECODING.value
     )
     conc: int
-    kv_offloading: Literal["none", "dram"] = Field(
+    kv_offloading: Literal["none", "dram", "nvme"] = Field(
         alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -644,7 +644,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel):
     decode: Optional[WorkerConfig] = None
     num_nodes: Optional[int] = Field(
         default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True)
-    kv_offloading: Optional[Literal["none", "dram"]] = Field(
+    kv_offloading: Optional[Literal["none", "dram", "nvme"]] = Field(
         default=None, alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(

From 44d1af31714804dfb215696d11ac642a9f65ef1b Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 00:31:33 -0500
Subject: [PATCH 02/23] feat(agentx): add tiered DRAM and NVMe sweep

---
 benchmarks/benchmark_lib.sh                      | 16 ++++++++--------
 .../agentic/minimaxm3_fp8_h100_mtp.sh            | 14 ++++++++++++++
 configs/nvidia-master.yaml                       |  1 +
 runners/launch_h100-dgxc-slurm.sh                |  2 +-
 utils/matrix_logic/generate_sweep_configs.py     |  2 +-
 utils/matrix_logic/validation.py                 |  6 +++---
 6 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh
index fee5a0be2a..75d6001aea 100644
--- a/benchmarks/benchmark_lib.sh
+++ b/benchmarks/benchmark_lib.sh
@@ -49,19 +49,19 @@ require_agentic_kv_offload_backend() {
             fi
             return 1
             ;;
-        dram|nvme)
+        dram|nvme|tiered)
             if [[ "${KV_OFFLOAD_BACKEND:-}" != "$expected_backend" ]]; then
                 echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=$KV_OFFLOADING, got '${KV_OFFLOAD_BACKEND:-}'" >&2
                 exit 1
             fi
-            if [[ "$KV_OFFLOADING" == "dram" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
-                echo "Error: DRAM KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+                echo "Error: $KV_OFFLOADING KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             return 0
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, tiered)" >&2
             exit 1
             ;;
     esac
@@ -86,18 +86,18 @@ if [[ "$_benchmark_caller" == */agentic/* ||
                 exit 1
             fi
             ;;
-        dram|nvme)
+        dram|nvme|tiered)
             if [[ -z "${KV_OFFLOAD_BACKEND:-}" || "${KV_OFFLOAD_BACKEND:-}" == "none" ]]; then
                 echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=$KV_OFFLOADING" >&2
                 exit 1
             fi
-            if [[ "$KV_OFFLOADING" == "dram" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
-                echo "Error: DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+                echo "Error: $KV_OFFLOADING KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, tiered)" >&2
             exit 1
             ;;
     esac
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index 38e9da901d..fd2c9b66f4 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -106,6 +106,20 @@ elif [ "$KV_OFFLOADING" = "nvme" ]; then
         --kv-transfer-config
         "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$NVME_OFFLOAD_PER_RANK_BYTES,\"disk_buffer_slots\":4,\"lazy_offload\":false}}"
     )
+elif [ "$KV_OFFLOADING" = "tiered" ]; then
+    require_agentic_kv_offload_backend vllm-native
+    : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
+    TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824))
+    PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB))
+    if (( PER_RANK_GIB <= 0 )); then
+        echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and tiered KV offload" >&2
+        exit 1
+    fi
+    CPU_OFFLOAD_TOTAL_BYTES=$((PER_RANK_GIB * TP * 1073741824))
+    OFFLOAD_ARGS=(
+        --kv-transfer-config
+        "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":$CPU_OFFLOAD_TOTAL_BYTES,\"eviction_policy\":\"lru\",\"secondary_tiers\":[{\"type\":\"fs\",\"root_dir\":\"$NVME_OFFLOAD_DIR\",\"n_read_threads\":32,\"n_write_threads\":16,\"locality\":\"LOCAL\"}]}}"
+    )
 else
     echo "Error: unsupported KV_OFFLOADING='$KV_OFFLOADING'" >&2
     exit 1
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 52dbf12cde..647de4b5ef 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7356,6 +7356,7 @@ minimaxm3-fp8-h100-vllm-agentic-mtp:
       - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8] }
       - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10] }
       - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: tiered, kv-offload-backend: { name: vllm-native }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
 
 minimaxm3-fp8-h200-vllm-agentic-mtp:
   image: vllm/vllm-openai:v0.27.1
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 139f06fae2..99f263da41 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -308,7 +308,7 @@ else
     trap cleanup_allocation EXIT INT TERM
 
     NVME_CONTAINER_MOUNT=""
-    if [[ "${KV_OFFLOADING:-none}" == "nvme" ]]; then
+    if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "tiered" ]]; then
         NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
         NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
         srun --jobid="$JOB_ID" bash -c "set -e; test -w '$NVME_HOST_ROOT'; mkdir -m 700 '$NVME_HOST_DIR'; findmnt -T '$NVME_HOST_DIR'"
diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py
index e3872c3d77..59a9152cf3 100644
--- a/utils/matrix_logic/generate_sweep_configs.py
+++ b/utils/matrix_logic/generate_sweep_configs.py
@@ -322,7 +322,7 @@ def agentic_dram_offload_gb(
       budgeted separately if it ever gains its own pool).
     """
     kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none")
-    if kv_offloading != "dram":
+    if kv_offloading not in {"dram", "tiered"}:
         return 0
 
     available_mib = min(
diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
index cbb353c1ec..d7078842a6 100644
--- a/utils/matrix_logic/validation.py
+++ b/utils/matrix_logic/validation.py
@@ -318,7 +318,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
         default="none", alias=Fields.SPEC_DECODING.value
     )
     conc: int
-    kv_offloading: Literal["none", "dram", "nvme"] = Field(
+    kv_offloading: Literal["none", "dram", "nvme", "tiered"] = Field(
         alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -644,7 +644,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel):
     decode: Optional[WorkerConfig] = None
     num_nodes: Optional[int] = Field(
         default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True)
-    kv_offloading: Optional[Literal["none", "dram", "nvme"]] = Field(
+    kv_offloading: Optional[Literal["none", "dram", "nvme", "tiered"]] = Field(
         default=None, alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -726,7 +726,7 @@ class AgenticCodingConfig(BaseModel):
     @model_validator(mode='after')
     def validate_dram_offload_capacity(self):
         for entry in self.search_space:
-            if entry.kv_offloading != "dram":
+            if entry.kv_offloading not in {"dram", "tiered"}:
                 continue
             if self.dram_utilization is None:
                 raise ValueError(

From 923f60dba5c79f3b8dbf7801adc08884ac8541cf Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 00:33:19 -0500
Subject: [PATCH 03/23] test(agentx): cover tiered KV offload matrices

---
 .../test_generate_sweep_configs.py            | 50 +++++++++++++++++++
 utils/matrix_logic/test_validation.py         | 25 ++++++++++
 2 files changed, 75 insertions(+)

diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py
index cbde1afc4a..95cc6978a0 100644
--- a/utils/matrix_logic/test_generate_sweep_configs.py
+++ b/utils/matrix_logic/test_generate_sweep_configs.py
@@ -2385,6 +2385,56 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config):
         }
         assert all(entry["duration"] == 3600 for entry in result)
 
+    def test_tiered_agentic_uses_dram_budget_and_distinct_name(
+        self, sample_runner_config
+    ):
+        config = {
+            "dsv4-b300-agentic": {
+                "image": "vllm/vllm-openai:v0.23.0",
+                "model": "deepseek-ai/DeepSeek-V4-Pro",
+                "model-prefix": "dsv4",
+                "precision": "fp4",
+                "framework": "vllm",
+                "runner": "cluster:b300-nv",
+                "multinode": False,
+                "scenarios": {
+                    "agentic-coding": [{
+                        "dram-utilization": 0.80,
+                        "search-space": [
+                            {
+                                "tp": 8,
+                                "kv-offloading": "nvme",
+                                "kv-offload-backend": {"name": "vllm-simple"},
+                                "conc-list": [7],
+                            },
+                            {
+                                "tp": 8,
+                                "kv-offloading": "tiered",
+                                "kv-offload-backend": {"name": "vllm-native"},
+                                "conc-list": [7],
+                            },
+                        ],
+                    }],
+                },
+            },
+        }
+        args = argparse.Namespace(
+            config_keys=["dsv4-b300-agentic"],
+            seq_lens=None,
+            conc=None,
+            scenario_type=["agentic-coding"],
+            runner_node_filter=None,
+        )
+
+        result = generate_test_config_sweep(args, config, sample_runner_config)
+
+        assert [entry["kv-offloading"] for entry in result] == ["nvme", "tiered"]
+        assert [entry["total-cpu-dram-gb"] for entry in result] == [0, 2399]
+        assert [entry["exp-name"] for entry in result] == [
+            "dsv4_tp8_conc7_kvnvme-vllm-simple",
+            "dsv4_tp8_conc7_kvtiered-vllm-native",
+        ]
+
     def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_config):
         config = {
             "dsv4-b300-agentic": {
diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py
index f20c36b3de..d585806c7e 100644
--- a/utils/matrix_logic/test_validation.py
+++ b/utils/matrix_logic/test_validation.py
@@ -530,6 +530,31 @@ def test_dram_kv_offload_requires_dram_utilization(self):
                 }],
             })
 
+    def test_tiered_kv_offload_requires_dram_utilization(self):
+        with pytest.raises(Exception, match="dram-utilization"):
+            AgenticCodingConfig(**{
+                "search-space": [{
+                    "tp": 8,
+                    "kv-offloading": "tiered",
+                    "kv-offload-backend": {"name": "vllm-native"},
+                    "conc-list": [7, 8],
+                }],
+            })
+
+    def test_tiered_kv_offload_accepts_dram_capacity_config(self):
+        config = AgenticCodingConfig(**{
+            "dram-utilization": 0.99,
+            "search-space": [{
+                "tp": 8,
+                "kv-offloading": "tiered",
+                "kv-offload-backend": {"name": "vllm-native"},
+                "conc-list": [7, 8],
+            }],
+        })
+
+        assert config.search_space[0].kv_offloading == "tiered"
+        assert config.dram_utilization == 0.99
+
     def test_agentic_search_space_rejects_total_cpu_dram_gb(self):
         with pytest.raises(Exception, match="total-cpu-dram-gb"):
             AgenticCodingSearchSpaceEntry(**{

From 3ebc7337995b0c345ab4b5ddba0093fe550c36f5 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 00:39:17 -0500
Subject: [PATCH 04/23] refactor(agentx): represent tiered offload as tier list

---
 benchmarks/benchmark_lib.sh                   |  8 +++----
 .../agentic/minimaxm3_fp8_h100_mtp.sh         |  4 ++--
 configs/nvidia-master.yaml                    |  2 +-
 runners/launch_h100-dgxc-slurm.sh             |  2 +-
 utils/matrix_logic/generate_sweep_configs.py  | 22 +++++++++++++------
 .../test_generate_sweep_configs.py            |  8 +++----
 utils/matrix_logic/test_validation.py         | 10 ++++-----
 utils/matrix_logic/validation.py              | 19 ++++++++++++----
 8 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh
index 75d6001aea..e84caa0b33 100644
--- a/benchmarks/benchmark_lib.sh
+++ b/benchmarks/benchmark_lib.sh
@@ -49,7 +49,7 @@ require_agentic_kv_offload_backend() {
             fi
             return 1
             ;;
-        dram|nvme|tiered)
+        dram|nvme|dram+nvme)
             if [[ "${KV_OFFLOAD_BACKEND:-}" != "$expected_backend" ]]; then
                 echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=$KV_OFFLOADING, got '${KV_OFFLOAD_BACKEND:-}'" >&2
                 exit 1
@@ -61,7 +61,7 @@ require_agentic_kv_offload_backend() {
             return 0
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, tiered)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2
             exit 1
             ;;
     esac
@@ -86,7 +86,7 @@ if [[ "$_benchmark_caller" == */agentic/* ||
                 exit 1
             fi
             ;;
-        dram|nvme|tiered)
+        dram|nvme|dram+nvme)
             if [[ -z "${KV_OFFLOAD_BACKEND:-}" || "${KV_OFFLOAD_BACKEND:-}" == "none" ]]; then
                 echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=$KV_OFFLOADING" >&2
                 exit 1
@@ -97,7 +97,7 @@ if [[ "$_benchmark_caller" == */agentic/* ||
             fi
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, tiered)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2
             exit 1
             ;;
     esac
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index fd2c9b66f4..fd9d9d2163 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -106,13 +106,13 @@ elif [ "$KV_OFFLOADING" = "nvme" ]; then
         --kv-transfer-config
         "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$NVME_OFFLOAD_PER_RANK_BYTES,\"disk_buffer_slots\":4,\"lazy_offload\":false}}"
     )
-elif [ "$KV_OFFLOADING" = "tiered" ]; then
+elif [ "$KV_OFFLOADING" = "dram+nvme" ]; then
     require_agentic_kv_offload_backend vllm-native
     : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
     TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824))
     PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB))
     if (( PER_RANK_GIB <= 0 )); then
-        echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and tiered KV offload" >&2
+        echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and DRAM+NVMe KV offload" >&2
         exit 1
     fi
     CPU_OFFLOAD_TOTAL_BYTES=$((PER_RANK_GIB * TP * 1073741824))
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 647de4b5ef..626ecba08f 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7356,7 +7356,7 @@ minimaxm3-fp8-h100-vllm-agentic-mtp:
       - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8] }
       - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10] }
       - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
-      - { tp: 8, spec-decoding: mtp, kv-offloading: tiered, kv-offload-backend: { name: vllm-native }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: [dram, nvme], kv-offload-backend: { name: vllm-native }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
 
 minimaxm3-fp8-h200-vllm-agentic-mtp:
   image: vllm/vllm-openai:v0.27.1
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 99f263da41..f5829a3b90 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -308,7 +308,7 @@ else
     trap cleanup_allocation EXIT INT TERM
 
     NVME_CONTAINER_MOUNT=""
-    if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "tiered" ]]; then
+    if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "dram+nvme" ]]; then
         NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
         NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
         srun --jobid="$JOB_ID" bash -c "set -e; test -w '$NVME_HOST_ROOT'; mkdir -m 700 '$NVME_HOST_DIR'; findmnt -T '$NVME_HOST_DIR'"
diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py
index 59a9152cf3..48ba2d96d2 100644
--- a/utils/matrix_logic/generate_sweep_configs.py
+++ b/utils/matrix_logic/generate_sweep_configs.py
@@ -322,7 +322,7 @@ def agentic_dram_offload_gb(
       budgeted separately if it ever gains its own pool).
     """
     kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none")
-    if kv_offloading not in {"dram", "tiered"}:
+    if kv_offloading != "dram" and kv_offloading != ["dram", "nvme"]:
         return 0
 
     available_mib = min(
@@ -356,13 +356,21 @@ def agentic_dram_offload_gb(
 
 
 def agentic_kv_offload_suffix(
-    kv_offloading: str,
+    kv_offloading: str | list[str],
     kv_offload_backend: dict | None,
 ) -> str:
     """Return a compact exp-name suffix for agentic KV offload settings."""
     if kv_offloading == "none":
         return "kvnone"
-    return f"kv{kv_offloading}-{kv_offload_backend['name']}"
+    mode = "+".join(kv_offloading) if isinstance(kv_offloading, list) else kv_offloading
+    return f"kv{mode}-{kv_offload_backend['name']}"
+
+
+def agentic_kv_offload_runtime_value(kv_offloading: str | list[str]) -> str:
+    """Convert declarative tier lists into the workflow's string input."""
+    if isinstance(kv_offloading, list):
+        return "+".join(kv_offloading)
+    return kv_offloading
 
 
 def multinode_agentic_exp_name(
@@ -1029,7 +1037,7 @@ def generate_full_sweep(args, all_config_data, runner_data):
                                 Fields.PREFILL.value: prefill,
                                 Fields.DECODE.value: decode,
                                 Fields.CONC.value: conc_batch,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: multinode_agentic_exp_name(
@@ -1066,7 +1074,7 @@ def generate_full_sweep(args, all_config_data, runner_data):
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
                                 Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (
@@ -1328,7 +1336,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                                 Fields.PREFILL.value: prefill,
                                 Fields.DECODE.value: decode,
                                 Fields.CONC.value: conc_batch,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: multinode_agentic_exp_name(
@@ -1364,7 +1372,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
                                 Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (
diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py
index 95cc6978a0..6cf79d1b89 100644
--- a/utils/matrix_logic/test_generate_sweep_configs.py
+++ b/utils/matrix_logic/test_generate_sweep_configs.py
@@ -2385,7 +2385,7 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config):
         }
         assert all(entry["duration"] == 3600 for entry in result)
 
-    def test_tiered_agentic_uses_dram_budget_and_distinct_name(
+    def test_multi_tier_agentic_uses_dram_budget_and_distinct_name(
         self, sample_runner_config
     ):
         config = {
@@ -2409,7 +2409,7 @@ def test_tiered_agentic_uses_dram_budget_and_distinct_name(
                             },
                             {
                                 "tp": 8,
-                                "kv-offloading": "tiered",
+                                "kv-offloading": ["dram", "nvme"],
                                 "kv-offload-backend": {"name": "vllm-native"},
                                 "conc-list": [7],
                             },
@@ -2428,11 +2428,11 @@ def test_tiered_agentic_uses_dram_budget_and_distinct_name(
 
         result = generate_test_config_sweep(args, config, sample_runner_config)
 
-        assert [entry["kv-offloading"] for entry in result] == ["nvme", "tiered"]
+        assert [entry["kv-offloading"] for entry in result] == ["nvme", "dram+nvme"]
         assert [entry["total-cpu-dram-gb"] for entry in result] == [0, 2399]
         assert [entry["exp-name"] for entry in result] == [
             "dsv4_tp8_conc7_kvnvme-vllm-simple",
-            "dsv4_tp8_conc7_kvtiered-vllm-native",
+            "dsv4_tp8_conc7_kvdram+nvme-vllm-native",
         ]
 
     def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_config):
diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py
index d585806c7e..671d91d753 100644
--- a/utils/matrix_logic/test_validation.py
+++ b/utils/matrix_logic/test_validation.py
@@ -530,29 +530,29 @@ def test_dram_kv_offload_requires_dram_utilization(self):
                 }],
             })
 
-    def test_tiered_kv_offload_requires_dram_utilization(self):
+    def test_multi_tier_kv_offload_requires_dram_utilization(self):
         with pytest.raises(Exception, match="dram-utilization"):
             AgenticCodingConfig(**{
                 "search-space": [{
                     "tp": 8,
-                    "kv-offloading": "tiered",
+                    "kv-offloading": ["dram", "nvme"],
                     "kv-offload-backend": {"name": "vllm-native"},
                     "conc-list": [7, 8],
                 }],
             })
 
-    def test_tiered_kv_offload_accepts_dram_capacity_config(self):
+    def test_multi_tier_kv_offload_accepts_dram_capacity_config(self):
         config = AgenticCodingConfig(**{
             "dram-utilization": 0.99,
             "search-space": [{
                 "tp": 8,
-                "kv-offloading": "tiered",
+                "kv-offloading": ["dram", "nvme"],
                 "kv-offload-backend": {"name": "vllm-native"},
                 "conc-list": [7, 8],
             }],
         })
 
-        assert config.search_space[0].kv_offloading == "tiered"
+        assert config.search_space[0].kv_offloading == ["dram", "nvme"]
         assert config.dram_utilization == 0.99
 
     def test_agentic_search_space_rejects_total_cpu_dram_gb(self):
diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
index d7078842a6..7e0c4b7a13 100644
--- a/utils/matrix_logic/validation.py
+++ b/utils/matrix_logic/validation.py
@@ -14,6 +14,11 @@
 
 CLUSTER_LABEL_PREFIX = "cluster:"
 DEFAULT_AGENTIC_DURATION_SECONDS = 3600
+KVOffloadingTier = Literal["dram", "nvme"]
+KVOffloadingConfig = Union[
+    Literal["none", "dram", "nvme"],
+    List[KVOffloadingTier],
+]
 
 """
     The below class defines the field names expected to be present in the JSON entries
@@ -318,7 +323,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
         default="none", alias=Fields.SPEC_DECODING.value
     )
     conc: int
-    kv_offloading: Literal["none", "dram", "nvme", "tiered"] = Field(
+    kv_offloading: Literal["none", "dram", "nvme", "dram+nvme"] = Field(
         alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -513,7 +518,13 @@ def _validate_kv_offload_fields(self):
                 f"{Fields.KV_OFFLOADING.value}"
             )
         return self
-    if self.kv_offloading == "none":
+    if isinstance(self.kv_offloading, list):
+        if self.kv_offloading != ["dram", "nvme"]:
+            raise ValueError(
+                f"The only supported multi-tier {Fields.KV_OFFLOADING.value} "
+                "configuration is ['dram', 'nvme']"
+            )
+    elif self.kv_offloading == "none":
         if backend is not None:
             raise ValueError(
                 f"{Fields.KV_OFFLOAD_BACKEND.value} can only be set when "
@@ -644,7 +655,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel):
     decode: Optional[WorkerConfig] = None
     num_nodes: Optional[int] = Field(
         default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True)
-    kv_offloading: Optional[Literal["none", "dram", "nvme", "tiered"]] = Field(
+    kv_offloading: Optional[KVOffloadingConfig] = Field(
         default=None, alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -726,7 +737,7 @@ class AgenticCodingConfig(BaseModel):
     @model_validator(mode='after')
     def validate_dram_offload_capacity(self):
         for entry in self.search_space:
-            if entry.kv_offloading not in {"dram", "tiered"}:
+            if entry.kv_offloading != "dram" and entry.kv_offloading != ["dram", "nvme"]:
                 continue
             if self.dram_utilization is None:
                 raise ValueError(

From 0e41fd17b3f66d4d54ccc76d1e48c7d5a726328e Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 00:47:34 -0500
Subject: [PATCH 05/23] perf: schedule H100 MiniMax-M3 offload sweep

---
 perf-changelog.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index ab92e7921c..e535c534b7 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6576,3 +6576,15 @@
     - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})."
     - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730
+
+- config-keys:
+    - minimaxm3-fp8-h100-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Run the official H100 MiniMax-M3 AgentX sweep on the patched vLLM image that exports cached prompt tokens by physical source tier."
+    - "Retain the no-offload TP8 points at concurrency 1-8 and the Mooncake DRAM points at concurrency 5-10; add NVMe-only points at concurrency 7-14 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity."
+    - "Add the same concurrency 7-14 range with declarative kv-offloading [dram, nvme], mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
+    - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container and remove it during allocation cleanup."
+    - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2775

From ef3c36e63b4ebaf50d3dc83df3b2e1512f518cec Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 03:58:33 -0500
Subject: [PATCH 06/23] fix(agentx): extend H100 allocation time

---
 runners/launch_h100-dgxc-slurm.sh | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index f5829a3b90..6bfaacd112 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -289,7 +289,25 @@ else
 
     export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
-    salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --time=180 --no-shell --job-name="$RUNNER_NAME"
+    if [[ -z "${SALLOC_TIME_LIMIT:-}" ]]; then
+        if [[ "${SCENARIO_TYPE:-}" == "agentic-coding" ]]; then
+            # AgentX includes large snapshot-prefill warmup before its timed
+            # profile. MiniMax-M3 can exceed three hours end-to-end even when
+            # the measured profile itself is only one hour.
+            SALLOC_TIME_LIMIT=300
+        else
+            SALLOC_TIME_LIMIT=180
+        fi
+    fi
+
+    salloc \
+        --partition="$SLURM_PARTITION" \
+        --account="$SLURM_ACCOUNT" \
+        --gres="gpu:$GPU_COUNT" \
+        --exclusive \
+        --time="$SALLOC_TIME_LIMIT" \
+        --no-shell \
+        --job-name="$RUNNER_NAME"
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
     if [[ -z "$JOB_ID" ]]; then
         echo "ERROR: failed to resolve H100 Slurm allocation" >&2

From 366d271491d227a75dca81c1ca8b8f0036f5fab2 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 05:03:47 -0500
Subject: [PATCH 07/23] fix(h100): clean NVMe before releasing allocation

---
 runners/launch_h100-dgxc-slurm.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 6bfaacd112..47cd22e34c 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -360,6 +360,4 @@ else
         --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \
         bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_h100${SPEC_SUFFIX}.sh
 
-    scancel $JOB_ID
-
 fi

From 0545bda3633ac0356d541bf085399e0e70c8d14d Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 05:06:05 -0500
Subject: [PATCH 08/23] fix(agentx): reject multinode NVMe offload

---
 utils/matrix_logic/test_validation.py | 18 ++++++++++++++++++
 utils/matrix_logic/validation.py      |  5 +++++
 2 files changed, 23 insertions(+)

diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py
index 671d91d753..be34d6f393 100644
--- a/utils/matrix_logic/test_validation.py
+++ b/utils/matrix_logic/test_validation.py
@@ -555,6 +555,24 @@ def test_multi_tier_kv_offload_accepts_dram_capacity_config(self):
         assert config.search_space[0].kv_offloading == ["dram", "nvme"]
         assert config.dram_utilization == 0.99
 
+    @pytest.mark.parametrize("kv_offloading", ["nvme", ["dram", "nvme"]])
+    def test_nvme_kv_offload_rejects_multinode_agentic_entries(
+        self, kv_offloading
+    ):
+        with pytest.raises(Exception, match="only for single-node agentic"):
+            AgenticCodingSearchSpaceEntry(**{
+                "worker": {
+                    "tp": 8,
+                    "pp": 1,
+                    "ep": 1,
+                    "dp-attn": False,
+                },
+                "num-nodes": 2,
+                "kv-offloading": kv_offloading,
+                "kv-offload-backend": {"name": "vllm-native"},
+                "conc-list": [7],
+            })
+
     def test_agentic_search_space_rejects_total_cpu_dram_gb(self):
         with pytest.raises(Exception, match="total-cpu-dram-gb"):
             AgenticCodingSearchSpaceEntry(**{
diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
index 7e0c4b7a13..201c69c738 100644
--- a/utils/matrix_logic/validation.py
+++ b/utils/matrix_logic/validation.py
@@ -703,6 +703,11 @@ def validate_topology_fields(self):
                 )
             _validate_tp_context_topology(self)
         if has_aggregate_worker or has_complete_multinode:
+            if self.kv_offloading in ("nvme", ["dram", "nvme"]):
+                raise ValueError(
+                    f"{Fields.KV_OFFLOADING.value}={self.kv_offloading!r} is "
+                    "currently supported only for single-node agentic entries"
+                )
             explicitly_single_node_fields = {
                 "pp",
                 "dcp_size",

From 8de9e47c717a705a1cfbb47aeceda8c5e0b2cfa1 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 05:27:00 -0500
Subject: [PATCH 09/23] fix(h100): exclude node with broken NVLink P2P

---
 runners/launch_h100-dgxc-slurm.sh | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 47cd22e34c..19f69de3c9 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -300,14 +300,25 @@ else
         fi
     fi
 
-    salloc \
-        --partition="$SLURM_PARTITION" \
-        --account="$SLURM_ACCOUNT" \
-        --gres="gpu:$GPU_COUNT" \
-        --exclusive \
-        --time="$SALLOC_TIME_LIMIT" \
-        --no-shell \
+    # hpc-gpu-1-17 consistently fails NCCL initialization because P2P is
+    # disabled between NVLink-connected GPUs 5 and 0. Keep this H100-specific
+    # default separate from SALLOC_EXCLUDE, which currently contains B300 node
+    # names in the shared benchmark workflow. The override makes the node
+    # usable again without a code change after the hardware is repaired.
+    SALLOC_EXCLUDE_H100="${SALLOC_EXCLUDE_H100:-hpc-gpu-1-17}"
+    SALLOC_ARGS=(
+        --partition="$SLURM_PARTITION"
+        --account="$SLURM_ACCOUNT"
+        --gres="gpu:$GPU_COUNT"
+        --exclusive
+        --time="$SALLOC_TIME_LIMIT"
+        --no-shell
         --job-name="$RUNNER_NAME"
+    )
+    if [[ -n "$SALLOC_EXCLUDE_H100" ]]; then
+        SALLOC_ARGS+=(--exclude="$SALLOC_EXCLUDE_H100")
+    fi
+    salloc "${SALLOC_ARGS[@]}"
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
     if [[ -z "$JOB_ID" ]]; then
         echo "ERROR: failed to resolve H100 Slurm allocation" >&2

From ae673c5dbc3578a3cabfabdb762a5b369a2f929f Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 07:03:47 -0500
Subject: [PATCH 10/23] fix(h100): clean stale vLLM offload memory

---
 runners/launch_h100-dgxc-slurm.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 19f69de3c9..fee5e7c026 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -325,17 +325,29 @@ else
         exit 1
     fi
     NVME_HOST_DIR=""
+    cleanup_offload_shm() {
+        # Native vLLM tiering backs its DRAM tier with a shared-memory file.
+        # Abruptly cancelled jobs can leave a nearly 1 TB file behind, which
+        # makes the next exclusive job on the node hang under memory pressure.
+        srun --jobid="$JOB_ID" bash -c \
+            'find /dev/shm -maxdepth 1 -type f -user "$(id -u)" -name "vllm_offload_*.mmap" -delete'
+    }
     cleanup_allocation() {
         local rc=$?
         trap - EXIT INT TERM
         if [[ -n "$NVME_HOST_DIR" ]]; then
             srun --jobid="$JOB_ID" bash -c "rm -rf -- '$NVME_HOST_DIR'" 2>/dev/null || true
         fi
+        cleanup_offload_shm 2>/dev/null || true
         scancel "$JOB_ID" 2>/dev/null || true
         exit "$rc"
     }
     trap cleanup_allocation EXIT INT TERM
 
+    # The allocation is exclusive, so any user-owned vLLM offload mmap left
+    # on this node is stale from an earlier job and is safe to remove.
+    cleanup_offload_shm
+
     NVME_CONTAINER_MOUNT=""
     if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "dram+nvme" ]]; then
         NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"

From d83dd9d5b6d2ec16e1845a0095133061932c32db Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 07:06:36 -0500
Subject: [PATCH 11/23] fix(h100): purge stale NVMe offload directories

---
 runners/launch_h100-dgxc-slurm.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index fee5e7c026..236c7c2a2d 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -352,7 +352,16 @@ else
     if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "dram+nvme" ]]; then
         NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
         NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
-        srun --jobid="$JOB_ID" bash -c "set -e; test -w '$NVME_HOST_ROOT'; mkdir -m 700 '$NVME_HOST_DIR'; findmnt -T '$NVME_HOST_DIR'"
+        NVME_OWNER_UID="$(id -u)"
+        srun --jobid="$JOB_ID" bash -c "
+            set -e
+            test -w '$NVME_HOST_ROOT'
+            find '$NVME_HOST_ROOT' -mindepth 1 -maxdepth 1 -type d \
+                -uid '$NVME_OWNER_UID' -name 'inferencex-kv-*' \
+                -exec rm -rf -- {} +
+            mkdir -m 700 '$NVME_HOST_DIR'
+            findmnt -T '$NVME_HOST_DIR'
+        "
         NVME_CONTAINER_MOUNT=",$NVME_HOST_DIR:/kv-offload"
         export NVME_OFFLOAD_DIR=/kv-offload
     fi

From f436e23047191623f462548e4b5c0024685a7fd5 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 12:25:49 -0500
Subject: [PATCH 12/23] fix(h100): bound offload cleanup steps

---
 docs/recovery-results-procedures.md    | 13 +++++++++++++
 docs/recovery-results-procedures_zh.md | 13 +++++++++++++
 runners/launch_h100-dgxc-slurm.sh      | 21 ++++++++++++++++++---
 3 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/docs/recovery-results-procedures.md b/docs/recovery-results-procedures.md
index 019843a806..e7ca87ced0 100644
--- a/docs/recovery-results-procedures.md
+++ b/docs/recovery-results-procedures.md
@@ -266,6 +266,19 @@ After pushing, never rebase, locally squash, amend, or force-push this carrier c
 
 Canonical source: [complete failed-ingest recovery command](https://github.com/SemiAnalysisAI/InferenceX/blob/0c28706b33d4a796b82f6f9c3594c19c46365575/.claude/commands/recover-failed-ingest.md).
 
+## H100 offload cleanup stalls
+
+An H100 AgentX benchmark can finish and write all result files while the GitHub job remains in `Launch job script`. Check the launcher process tree on the login node. A long-running child matching either of these commands means teardown, rather than the benchmark, is stalled:
+
+```text
+srun --jobid=<id> bash -c find /dev/shm ... vllm_offload_*.mmap ... -delete
+srun --jobid=<id> bash -c rm -rf -- /mnt/numa0/.../inferencex-kv-<id>
+```
+
+Preserve and verify the result files before intervening. The H100 launcher bounds each pre-run and post-run offload cleanup step with `H100_OFFLOAD_CLEANUP_TIMEOUT_S` (120 seconds by default), followed by a 15-second forced-termination window. A post-run timeout is best-effort: the launcher cancels the allocation and preserves the benchmark exit status so artifact uploads can proceed. A pre-run timeout fails closed because stale shared memory or NVMe state can invalidate the next benchmark.
+
+For an already-running job created before this guard existed, terminate only the stuck login-node `srun` client after confirming the benchmark outputs are complete. The launcher's cleanup trap then continues to `scancel` the allocation. Do not cancel the GitHub job or delete its workspace; doing so can discard uploadable evidence.
+
 ## AMD root-owned workspace prevention and recovery
 
 ### Prevent recurrence
diff --git a/docs/recovery-results-procedures_zh.md b/docs/recovery-results-procedures_zh.md
index 8c73f2a604..9aeddd26dc 100644
--- a/docs/recovery-results-procedures_zh.md
+++ b/docs/recovery-results-procedures_zh.md
@@ -266,6 +266,19 @@ git diff --check origin/main...HEAD
 
 权威来源：[完整失败摄取恢复命令](https://github.com/SemiAnalysisAI/InferenceX/blob/0c28706b33d4a796b82f6f9c3594c19c46365575/.claude/commands/recover-failed-ingest.md)。
 
+## H100 offload 清理停滞
+
+H100 AgentX 基准可能已经完成并写出所有结果文件，但 GitHub 任务仍停留在 `Launch job script`。请在登录节点检查 launcher 进程树。若以下任一命令的子进程长时间运行，说明停滞发生在 teardown，而不是基准本身：
+
+```text
+srun --jobid=<id> bash -c find /dev/shm ... vllm_offload_*.mmap ... -delete
+srun --jobid=<id> bash -c rm -rf -- /mnt/numa0/.../inferencex-kv-<id>
+```
+
+干预前必须保留并验证结果文件。H100 launcher 使用 `H100_OFFLOAD_CLEANUP_TIMEOUT_S` 限制每个运行前和运行后的 offload 清理步骤；默认超时为 120 秒，随后保留 15 秒强制终止窗口。运行后超时按 best-effort 处理：launcher 会取消 allocation，并保留基准退出状态，使 artifact 上传可以继续。运行前超时则会 fail closed，因为残留的共享内存或 NVMe 状态可能使下一次基准结果无效。
+
+对于在该保护逻辑加入前已经启动的任务，只有在确认基准输出完整后，才能终止登录节点上卡住的 `srun` client。随后 launcher 的 cleanup trap 会继续执行 `scancel` 来释放 allocation。不要取消 GitHub 任务，也不要删除其工作区，否则可能丢失仍可上传的证据。
+
 ## AMD root-owned 工作区的预防与恢复
 
 ### 防止复发
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 236c7c2a2d..034b912880 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -325,18 +325,33 @@ else
         exit 1
     fi
     NVME_HOST_DIR=""
+    H100_OFFLOAD_CLEANUP_TIMEOUT_S="${H100_OFFLOAD_CLEANUP_TIMEOUT_S:-120}"
+    run_bounded_cleanup_step() {
+        local description="$1"
+        shift
+        local rc
+        if timeout --kill-after=15s "${H100_OFFLOAD_CLEANUP_TIMEOUT_S}s" \
+            srun --jobid="$JOB_ID" "$@"; then
+            return 0
+        else
+            rc=$?
+            echo "WARNING: H100 cleanup step '$description' failed or timed out after ${H100_OFFLOAD_CLEANUP_TIMEOUT_S}s (rc=$rc)" >&2
+            return "$rc"
+        fi
+    }
     cleanup_offload_shm() {
         # Native vLLM tiering backs its DRAM tier with a shared-memory file.
         # Abruptly cancelled jobs can leave a nearly 1 TB file behind, which
         # makes the next exclusive job on the node hang under memory pressure.
-        srun --jobid="$JOB_ID" bash -c \
+        run_bounded_cleanup_step "vLLM offload shared memory" bash -c \
             'find /dev/shm -maxdepth 1 -type f -user "$(id -u)" -name "vllm_offload_*.mmap" -delete'
     }
     cleanup_allocation() {
         local rc=$?
         trap - EXIT INT TERM
         if [[ -n "$NVME_HOST_DIR" ]]; then
-            srun --jobid="$JOB_ID" bash -c "rm -rf -- '$NVME_HOST_DIR'" 2>/dev/null || true
+            run_bounded_cleanup_step "NVMe offload directory $NVME_HOST_DIR" \
+                bash -c "rm -rf -- '$NVME_HOST_DIR'" 2>/dev/null || true
         fi
         cleanup_offload_shm 2>/dev/null || true
         scancel "$JOB_ID" 2>/dev/null || true
@@ -353,7 +368,7 @@ else
         NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
         NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
         NVME_OWNER_UID="$(id -u)"
-        srun --jobid="$JOB_ID" bash -c "
+        run_bounded_cleanup_step "stale NVMe offload directories" bash -c "
             set -e
             test -w '$NVME_HOST_ROOT'
             find '$NVME_HOST_ROOT' -mindepth 1 -maxdepth 1 -type d \

From e631356ec99afacf658e7621e0a642b1253bcc7b Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 16:30:19 -0500
Subject: [PATCH 13/23] simplify H100 offload lifecycle

---
 .../agentic/minimaxm3_fp8_h100_mtp.sh         |  4 ++
 runners/launch_h100-dgxc-slurm.sh             | 58 +------------------
 2 files changed, 6 insertions(+), 56 deletions(-)

diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index fd9d9d2163..9b30833da5 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -47,6 +47,9 @@ cleanup_services() {
     set +e
     stop_background_process_tree "$SERVER_PID" "vLLM server" 60
     stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" 30
+    if [[ -n "${NVME_OFFLOAD_DIR:-}" ]]; then
+        find "$NVME_OFFLOAD_DIR" -mindepth 1 -delete
+    fi
     exit "$exit_code"
 }
 trap cleanup_services EXIT
@@ -102,6 +105,7 @@ elif [ "$KV_OFFLOADING" = "nvme" ]; then
     : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
     NVME_OFFLOAD_TOTAL_BYTES=8000000000000
     NVME_OFFLOAD_PER_RANK_BYTES=$((NVME_OFFLOAD_TOTAL_BYTES / TP))
+    # vLLM appends .rank_<CUDA device index> to give each TP rank its own file.
     OFFLOAD_ARGS=(
         --kv-transfer-config
         "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$NVME_OFFLOAD_PER_RANK_BYTES,\"disk_buffer_slots\":4,\"lazy_offload\":false}}"
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 034b912880..eb33303f05 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -289,23 +289,7 @@ else
 
     export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
-    if [[ -z "${SALLOC_TIME_LIMIT:-}" ]]; then
-        if [[ "${SCENARIO_TYPE:-}" == "agentic-coding" ]]; then
-            # AgentX includes large snapshot-prefill warmup before its timed
-            # profile. MiniMax-M3 can exceed three hours end-to-end even when
-            # the measured profile itself is only one hour.
-            SALLOC_TIME_LIMIT=300
-        else
-            SALLOC_TIME_LIMIT=180
-        fi
-    fi
-
-    # hpc-gpu-1-17 consistently fails NCCL initialization because P2P is
-    # disabled between NVLink-connected GPUs 5 and 0. Keep this H100-specific
-    # default separate from SALLOC_EXCLUDE, which currently contains B300 node
-    # names in the shared benchmark workflow. The override makes the node
-    # usable again without a code change after the hardware is repaired.
-    SALLOC_EXCLUDE_H100="${SALLOC_EXCLUDE_H100:-hpc-gpu-1-17}"
+    SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-300}"
     SALLOC_ARGS=(
         --partition="$SLURM_PARTITION"
         --account="$SLURM_ACCOUNT"
@@ -315,65 +299,27 @@ else
         --no-shell
         --job-name="$RUNNER_NAME"
     )
-    if [[ -n "$SALLOC_EXCLUDE_H100" ]]; then
-        SALLOC_ARGS+=(--exclude="$SALLOC_EXCLUDE_H100")
-    fi
     salloc "${SALLOC_ARGS[@]}"
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
     if [[ -z "$JOB_ID" ]]; then
         echo "ERROR: failed to resolve H100 Slurm allocation" >&2
         exit 1
     fi
-    NVME_HOST_DIR=""
-    H100_OFFLOAD_CLEANUP_TIMEOUT_S="${H100_OFFLOAD_CLEANUP_TIMEOUT_S:-120}"
-    run_bounded_cleanup_step() {
-        local description="$1"
-        shift
-        local rc
-        if timeout --kill-after=15s "${H100_OFFLOAD_CLEANUP_TIMEOUT_S}s" \
-            srun --jobid="$JOB_ID" "$@"; then
-            return 0
-        else
-            rc=$?
-            echo "WARNING: H100 cleanup step '$description' failed or timed out after ${H100_OFFLOAD_CLEANUP_TIMEOUT_S}s (rc=$rc)" >&2
-            return "$rc"
-        fi
-    }
-    cleanup_offload_shm() {
-        # Native vLLM tiering backs its DRAM tier with a shared-memory file.
-        # Abruptly cancelled jobs can leave a nearly 1 TB file behind, which
-        # makes the next exclusive job on the node hang under memory pressure.
-        run_bounded_cleanup_step "vLLM offload shared memory" bash -c \
-            'find /dev/shm -maxdepth 1 -type f -user "$(id -u)" -name "vllm_offload_*.mmap" -delete'
-    }
     cleanup_allocation() {
         local rc=$?
         trap - EXIT INT TERM
-        if [[ -n "$NVME_HOST_DIR" ]]; then
-            run_bounded_cleanup_step "NVMe offload directory $NVME_HOST_DIR" \
-                bash -c "rm -rf -- '$NVME_HOST_DIR'" 2>/dev/null || true
-        fi
-        cleanup_offload_shm 2>/dev/null || true
         scancel "$JOB_ID" 2>/dev/null || true
         exit "$rc"
     }
     trap cleanup_allocation EXIT INT TERM
 
-    # The allocation is exclusive, so any user-owned vLLM offload mmap left
-    # on this node is stale from an earlier job and is safe to remove.
-    cleanup_offload_shm
-
     NVME_CONTAINER_MOUNT=""
     if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "dram+nvme" ]]; then
         NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
         NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
-        NVME_OWNER_UID="$(id -u)"
-        run_bounded_cleanup_step "stale NVMe offload directories" bash -c "
+        srun --jobid="$JOB_ID" bash -c "
             set -e
             test -w '$NVME_HOST_ROOT'
-            find '$NVME_HOST_ROOT' -mindepth 1 -maxdepth 1 -type d \
-                -uid '$NVME_OWNER_UID' -name 'inferencex-kv-*' \
-                -exec rm -rf -- {} +
             mkdir -m 700 '$NVME_HOST_DIR'
             findmnt -T '$NVME_HOST_DIR'
         "

From fc9433ae18d8580dcd60a36cd7ef5a29a5569b8d Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 16:39:40 -0500
Subject: [PATCH 14/23] test updated cache-source image

---
 configs/nvidia-master.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 626ecba08f..151500b7d1 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7341,7 +7341,7 @@ qwen3.5-fp4-b200-trt-mtp:
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
 
 minimaxm3-fp8-h100-vllm-agentic-mtp:
-  image: ttl.sh/cquil11-vllm-tier-dfb25dbef4-20260828:24h
+  image: ttl.sh/cquil11-vllm-tier-3565dfefe1-20260828:24h
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: cluster:h100-dgxc

From 1a0793b4ec74c97678ffd5c389f8a34ea28a9a2c Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 16:47:13 -0500
Subject: [PATCH 15/23] bound in-container NVMe cleanup

---
 benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index 9b30833da5..533601cae9 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -48,7 +48,7 @@ cleanup_services() {
     stop_background_process_tree "$SERVER_PID" "vLLM server" 60
     stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" 30
     if [[ -n "${NVME_OFFLOAD_DIR:-}" ]]; then
-        find "$NVME_OFFLOAD_DIR" -mindepth 1 -delete
+        timeout --kill-after=5s 120s find "$NVME_OFFLOAD_DIR" -mindepth 1 -delete
     fi
     exit "$exit_code"
 }

From 3cb3641cf90ba11fcea4bbb15361a7b0a28700f1 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 18:34:44 -0500
Subject: [PATCH 16/23] simplify H100 offload launcher

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
---
 docs/recovery-results-procedures.md    | 13 -------------
 docs/recovery-results-procedures_zh.md | 13 -------------
 runners/launch_h100-dgxc-slurm.sh      | 13 +++----------
 3 files changed, 3 insertions(+), 36 deletions(-)

diff --git a/docs/recovery-results-procedures.md b/docs/recovery-results-procedures.md
index e7ca87ced0..019843a806 100644
--- a/docs/recovery-results-procedures.md
+++ b/docs/recovery-results-procedures.md
@@ -266,19 +266,6 @@ After pushing, never rebase, locally squash, amend, or force-push this carrier c
 
 Canonical source: [complete failed-ingest recovery command](https://github.com/SemiAnalysisAI/InferenceX/blob/0c28706b33d4a796b82f6f9c3594c19c46365575/.claude/commands/recover-failed-ingest.md).
 
-## H100 offload cleanup stalls
-
-An H100 AgentX benchmark can finish and write all result files while the GitHub job remains in `Launch job script`. Check the launcher process tree on the login node. A long-running child matching either of these commands means teardown, rather than the benchmark, is stalled:
-
-```text
-srun --jobid=<id> bash -c find /dev/shm ... vllm_offload_*.mmap ... -delete
-srun --jobid=<id> bash -c rm -rf -- /mnt/numa0/.../inferencex-kv-<id>
-```
-
-Preserve and verify the result files before intervening. The H100 launcher bounds each pre-run and post-run offload cleanup step with `H100_OFFLOAD_CLEANUP_TIMEOUT_S` (120 seconds by default), followed by a 15-second forced-termination window. A post-run timeout is best-effort: the launcher cancels the allocation and preserves the benchmark exit status so artifact uploads can proceed. A pre-run timeout fails closed because stale shared memory or NVMe state can invalidate the next benchmark.
-
-For an already-running job created before this guard existed, terminate only the stuck login-node `srun` client after confirming the benchmark outputs are complete. The launcher's cleanup trap then continues to `scancel` the allocation. Do not cancel the GitHub job or delete its workspace; doing so can discard uploadable evidence.
-
 ## AMD root-owned workspace prevention and recovery
 
 ### Prevent recurrence
diff --git a/docs/recovery-results-procedures_zh.md b/docs/recovery-results-procedures_zh.md
index 9aeddd26dc..8c73f2a604 100644
--- a/docs/recovery-results-procedures_zh.md
+++ b/docs/recovery-results-procedures_zh.md
@@ -266,19 +266,6 @@ git diff --check origin/main...HEAD
 
 权威来源：[完整失败摄取恢复命令](https://github.com/SemiAnalysisAI/InferenceX/blob/0c28706b33d4a796b82f6f9c3594c19c46365575/.claude/commands/recover-failed-ingest.md)。
 
-## H100 offload 清理停滞
-
-H100 AgentX 基准可能已经完成并写出所有结果文件，但 GitHub 任务仍停留在 `Launch job script`。请在登录节点检查 launcher 进程树。若以下任一命令的子进程长时间运行，说明停滞发生在 teardown，而不是基准本身：
-
-```text
-srun --jobid=<id> bash -c find /dev/shm ... vllm_offload_*.mmap ... -delete
-srun --jobid=<id> bash -c rm -rf -- /mnt/numa0/.../inferencex-kv-<id>
-```
-
-干预前必须保留并验证结果文件。H100 launcher 使用 `H100_OFFLOAD_CLEANUP_TIMEOUT_S` 限制每个运行前和运行后的 offload 清理步骤；默认超时为 120 秒，随后保留 15 秒强制终止窗口。运行后超时按 best-effort 处理：launcher 会取消 allocation，并保留基准退出状态，使 artifact 上传可以继续。运行前超时则会 fail closed，因为残留的共享内存或 NVMe 状态可能使下一次基准结果无效。
-
-对于在该保护逻辑加入前已经启动的任务，只有在确认基准输出完整后，才能终止登录节点上卡住的 `srun` client。随后 launcher 的 cleanup trap 会继续执行 `scancel` 来释放 allocation。不要取消 GitHub 任务，也不要删除其工作区，否则可能丢失仍可上传的证据。
-
 ## AMD root-owned 工作区的预防与恢复
 
 ### 防止复发
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index eb33303f05..a8f86e411f 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -290,16 +290,9 @@ else
     export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
     SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-300}"
-    SALLOC_ARGS=(
-        --partition="$SLURM_PARTITION"
-        --account="$SLURM_ACCOUNT"
-        --gres="gpu:$GPU_COUNT"
-        --exclusive
-        --time="$SALLOC_TIME_LIMIT"
-        --no-shell
-        --job-name="$RUNNER_NAME"
-    )
-    salloc "${SALLOC_ARGS[@]}"
+    salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \
+        --gres="gpu:$GPU_COUNT" --exclusive --time="$SALLOC_TIME_LIMIT" \
+        --no-shell --job-name="$RUNNER_NAME"
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
     if [[ -z "$JOB_ID" ]]; then
         echo "ERROR: failed to resolve H100 Slurm allocation" >&2

From 893cd1f6f38c1ec501730a5f5c19d669db5fe413 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Fri, 28 Aug 2026 18:34:44 -0500
Subject: [PATCH 17/23] test tiering timeout image

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
---
 configs/nvidia-master.yaml | 2 +-
 perf-changelog.yaml        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index a2f7c207e6..552c38ec69 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7341,7 +7341,7 @@ qwen3.5-fp4-b200-trt-mtp:
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
 
 minimaxm3-fp8-h100-vllm-agentic-mtp:
-  image: ttl.sh/cquil11-vllm-tier-3565dfefe1-20260828:24h
+  image: ttl.sh/cquil11-vllm-tier-3565dfefe1-pr53087-20260828:24h
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: cluster:h100-dgxc
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 3dd872dd45..f8245ebbdf 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6582,7 +6582,7 @@
   scenario-type:
     - agentic-coding
   description:
-    - "Run the official H100 MiniMax-M3 AgentX sweep on the patched vLLM image that exports cached prompt tokens by physical source tier."
+    - "Run the official H100 MiniMax-M3 AgentX sweep on the patched vLLM image that exports cached prompt tokens by physical source tier and includes vLLM PR #53087's bounded fallback for stalled tier-primary writes."
     - "Retain the no-offload TP8 points at concurrency 1-8 and the Mooncake DRAM points at concurrency 5-10; add NVMe-only points at concurrency 7-14 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity."
     - "Add the same concurrency 7-14 range with declarative kv-offloading [dram, nvme], mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
     - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container and clear its contents from the benchmark's existing service-shutdown trap."

From 8a00c0ad544e388e9e25f5930436d4d4e473d1c0 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Mon, 31 Aug 2026 14:31:59 -0500
Subject: [PATCH 18/23] clean stale vLLM offload mmap files

---
 benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index 533601cae9..e2ddbcd53e 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -39,6 +39,12 @@ SERVER_LOG="$RESULT_DIR/server.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
 mkdir -p "$RESULT_DIR"
 
+# Native vLLM tiering uses host /dev/shm files that survive a cancelled
+# container. This job owns the node exclusively, so remove only our stale
+# tiering files before reserving DRAM for either offload backend.
+find /dev/shm -maxdepth 1 -type f -user "$(id -u)" \
+    -name 'vllm_offload_*.mmap' -print -delete
+
 SERVER_PID=""
 MOONCAKE_MASTER_PID=""
 cleanup_services() {
@@ -47,6 +53,8 @@ cleanup_services() {
     set +e
     stop_background_process_tree "$SERVER_PID" "vLLM server" 60
     stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" 30
+    find /dev/shm -maxdepth 1 -type f -user "$(id -u)" \
+        -name 'vllm_offload_*.mmap' -print -delete
     if [[ -n "${NVME_OFFLOAD_DIR:-}" ]]; then
         timeout --kill-after=5s 120s find "$NVME_OFFLOAD_DIR" -mindepth 1 -delete
     fi

From a4c2a6cc1b4bb52ae1887973123441b77464468b Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Mon, 31 Aug 2026 14:34:57 -0500
Subject: [PATCH 19/23] document stale offload cleanup

---
 perf-changelog.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index f8245ebbdf..34e3ecaf0e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6586,6 +6586,7 @@
     - "Retain the no-offload TP8 points at concurrency 1-8 and the Mooncake DRAM points at concurrency 5-10; add NVMe-only points at concurrency 7-14 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity."
     - "Add the same concurrency 7-14 range with declarative kv-offloading [dram, nvme], mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
     - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container and clear its contents from the benchmark's existing service-shutdown trap."
+    - "Remove same-user stale native-tier mmap files at startup and after shutdown on the exclusive H100 node so an interrupted run cannot starve later DRAM-offload jobs."
     - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2775
 

From cb0fec25d774e88abeae788e5d83f123f4238ef5 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Tue, 1 Sep 2026 00:39:23 -0500
Subject: [PATCH 20/23] Narrow H100 cache-source validation sweep

---
 .../single_node/agentic/minimaxm3_fp8_h100_mtp.sh     | 11 -----------
 configs/nvidia-master.yaml                            | 11 +++++------
 perf-changelog.yaml                                   |  9 ++++-----
 3 files changed, 9 insertions(+), 22 deletions(-)

diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index e2ddbcd53e..04a25bf10c 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -39,12 +39,6 @@ SERVER_LOG="$RESULT_DIR/server.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
 mkdir -p "$RESULT_DIR"
 
-# Native vLLM tiering uses host /dev/shm files that survive a cancelled
-# container. This job owns the node exclusively, so remove only our stale
-# tiering files before reserving DRAM for either offload backend.
-find /dev/shm -maxdepth 1 -type f -user "$(id -u)" \
-    -name 'vllm_offload_*.mmap' -print -delete
-
 SERVER_PID=""
 MOONCAKE_MASTER_PID=""
 cleanup_services() {
@@ -53,11 +47,6 @@ cleanup_services() {
     set +e
     stop_background_process_tree "$SERVER_PID" "vLLM server" 60
     stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" 30
-    find /dev/shm -maxdepth 1 -type f -user "$(id -u)" \
-        -name 'vllm_offload_*.mmap' -print -delete
-    if [[ -n "${NVME_OFFLOAD_DIR:-}" ]]; then
-        timeout --kill-after=5s 120s find "$NVME_OFFLOAD_DIR" -mindepth 1 -delete
-    fi
     exit "$exit_code"
 }
 trap cleanup_services EXIT
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 552c38ec69..8fcb7765a1 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7341,7 +7341,7 @@ qwen3.5-fp4-b200-trt-mtp:
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
 
 minimaxm3-fp8-h100-vllm-agentic-mtp:
-  image: ttl.sh/cquil11-vllm-tier-3565dfefe1-pr53087-20260828:24h
+  image: ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: cluster:h100-dgxc
@@ -7350,13 +7350,12 @@ minimaxm3-fp8-h100-vllm-agentic-mtp:
   multinode: false
   scenarios:
     agentic-coding:
-    # The fast sweep places the resident HBM cliff between c5 and c6.
+    # Narrow validation sweep for physical cache-source attribution.
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8] }
-      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10] }
-      - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
-      - { tp: 8, spec-decoding: mtp, kv-offloading: [dram, nvme], kv-offload-backend: { name: vllm-native }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [30] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: [dram, nvme], kv-offload-backend: { name: vllm-native }, conc-list: [20] }
 
 minimaxm3-fp8-h200-vllm-agentic-mtp:
   image: vllm/vllm-openai:v0.27.1
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 38edde1d1d..b4c76c64ec 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6582,11 +6582,10 @@
   scenario-type:
     - agentic-coding
   description:
-    - "Run the official H100 MiniMax-M3 AgentX sweep on the patched vLLM image that exports cached prompt tokens by physical source tier and includes vLLM PR #53087's bounded fallback for stalled tier-primary writes."
-    - "Retain the no-offload TP8 points at concurrency 1-8 and the Mooncake DRAM points at concurrency 5-10; add NVMe-only points at concurrency 7-14 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity."
-    - "Add the same concurrency 7-14 range with declarative kv-offloading [dram, nvme], mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
-    - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container and clear its contents from the benchmark's existing service-shutdown trap."
-    - "Remove same-user stale native-tier mmap files at startup and after shutdown on the exclusive H100 node so an interrupted run cannot starve later DRAM-offload jobs."
+    - "Run a narrow H100 MiniMax-M3 AgentX validation on the patched vLLM image that exports the bounded cached-token sources device, cpu, disk, p2p, and external and includes vLLM PR #53087's bounded fallback for stalled tier-primary writes."
+    - "Validate one Mooncake DRAM point at TP8 concurrency 8 and one NVMe-only point at TP8 concurrency 30 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity."
+    - "Validate one declarative kv-offloading [dram, nvme] point at TP8 concurrency 20, mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
+    - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container."
     - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2775
 

From 90dc0488f4e5ea23dcc01938fc1f8ad307bf8767 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Tue, 1 Sep 2026 10:07:34 -0500
Subject: [PATCH 21/23] Link reduced cache-source validation PR

---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index b4c76c64ec..b832b88bd4 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6587,7 +6587,7 @@
     - "Validate one declarative kv-offloading [dram, nvme] point at TP8 concurrency 20, mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
     - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container."
     - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters."
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2775
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796
 
 - config-keys:
     - dsv4-fp4-mi355x-atom-agentic-mtp

From c23f2c4507267ebcffc18835280e72d393715515 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Tue, 1 Sep 2026 15:34:36 -0500
Subject: [PATCH 22/23] fix(agentx): allow tiered warmup to finish and isolate
 profiling metrics

---
 .../agentic/minimaxm3_fp8_h100_mtp.sh         |  2 +-
 docs/eval-agentx-procedures.md                |  5 ++++
 docs/eval-agentx-procedures_zh.md             |  5 ++++
 perf-changelog.yaml                           | 10 +++++++
 runners/launch_h100-dgxc-slurm.sh             |  4 +++
 runners/test_slurm_utils.py                   | 28 +++++++++++++++++++
 utils/aiperf                                  |  2 +-
 7 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
index 04a25bf10c..c095106ab3 100755
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh
@@ -127,7 +127,7 @@ else
 fi
 
 export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics"
-export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:"
+export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:prompt_tokens_cached_by_source"
 
 NUM_SPEC_TOKENS=3
 TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS))
diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md
index 6db5d351e8..c465a8ef17 100644
--- a/docs/eval-agentx-procedures.md
+++ b/docs/eval-agentx-procedures.md
@@ -210,6 +210,11 @@ For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`,
 
 Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2266-L2268)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success.
 
+The H100 MiniMax-M3 DRAM+NVMe AgentX launcher allows 420 minutes for Slurm:
+the measured warmup alone took over four hours before the one-hour profile.
+Other H100 single-node cases retain the 300-minute default, and an explicit
+`SALLOC_TIME_LIMIT` overrides either value. The canonical workload is unchanged.
+
 ## 8. Preserve trace and run provenance
 
 AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L2023-L2102), [replay semantics](../benchmarks/benchmark_lib.sh#L2104-L2270)).
diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md
index 6e8c8e03c8..0e9f364a13 100644
--- a/docs/eval-agentx-procedures_zh.md
+++ b/docs/eval-agentx-procedures_zh.md
@@ -210,6 +210,11 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \
 
 Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2266-L2268)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。
 
+H100 MiniMax-M3 DRAM+NVMe AgentX 启动器的 Slurm 时限为 420 分钟：
+实测 warmup 本身超过四小时，之后还需运行一小时的 profiling。
+其他 H100 单节点场景仍默认使用 300 分钟；显式设置 `SALLOC_TIME_LIMIT`
+可覆盖任一默认值。canonical 工作负载保持不变。
+
 ## 8. 保留 trace 与运行 provenance
 
 AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L2023-L2102)、[replay 语义](../benchmarks/benchmark_lib.sh#L2104-L2270)）。
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 656ed47e43..37f46bba85 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6753,3 +6753,13 @@
     - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container."
     - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796
+
+- config-keys:
+    - minimaxm3-fp8-h100-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Allow seven hours for the native DRAM+NVMe point after its canonical warmup exceeded four hours; retain the 300-minute H100 default for other cases."
+    - "Pin the AgentX client fix that excludes automatic warmup baselines from profiling server metrics and require the cached-token-source metric in JSON/CSV artifacts."
+    - "Keep exactly Mooncake DRAM c8, Simple NVMe c30, and native DRAM+NVMe c20, with unchanged canonical warmup, one-hour profiling, images, and full evals."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 6cd0323988..1420069ece 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -298,6 +298,10 @@ else
 
     export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
+    # This tiered AgentX point needs >4 hours of warmup before its 1-hour profile.
+    if [[ "${MODEL_PREFIX:-}" == "minimaxm3" && "${SCENARIO_TYPE:-}" == "agentic-coding" && "${KV_OFFLOADING:-}" == "dram+nvme" ]]; then
+        SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-420}"
+    fi
     SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-300}"
     salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \
         --gres="gpu:$GPU_COUNT" --exclusive --time="$SALLOC_TIME_LIMIT" \
diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py
index 162acab380..de34b9f714 100644
--- a/runners/test_slurm_utils.py
+++ b/runners/test_slurm_utils.py
@@ -24,6 +24,34 @@ def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str
     )
 
 
+def test_h100_tiered_agentx_time_limit_preserves_defaults_and_override() -> None:
+    launcher = (REPO_ROOT / "runners" / "launch_h100-dgxc-slurm.sh").read_text()
+    start = launcher.index("    # This tiered AgentX point")
+    stop = launcher.index("    salloc ", start)
+    configure = launcher[start:stop]
+    cases = [
+        ("minimaxm3", "agentic-coding", "dram+nvme", "", "420"),
+        ("minimaxm3", "agentic-coding", "dram+nvme", "480", "480"),
+        ("minimaxm3", "agentic-coding", "dram", "", "300"),
+        ("minimaxm3", "agentic-coding", "nvme", "", "300"),
+        ("other", "agentic-coding", "dram+nvme", "", "300"),
+        ("minimaxm3", "fixed-sequence", "dram+nvme", "", "300"),
+    ]
+    for model, scenario, offload, override, expected in cases:
+        result = run_bash(
+            'MODEL_PREFIX="$1"; SCENARIO_TYPE="$2"; KV_OFFLOADING="$3"; '
+            'SALLOC_TIME_LIMIT="$4";\n'
+            + configure
+            + '\nprintf "%s" "$SALLOC_TIME_LIMIT"',
+            model,
+            scenario,
+            offload,
+            override,
+        )
+        assert result.returncode == 0, result.stderr
+        assert result.stdout == expected
+
+
 def test_copy_agentic_results_stages_only_matching_points(tmp_path: Path) -> None:
     source = tmp_path / "source"
     workspace = tmp_path / "workspace"
diff --git a/utils/aiperf b/utils/aiperf
index 754356e9a3..dcbd942654 160000
--- a/utils/aiperf
+++ b/utils/aiperf
@@ -1 +1 @@
-Subproject commit 754356e9a39acc6cc6afb242d123bb57c3fb6f75
+Subproject commit dcbd942654ef87b357c7722837855dbccedc2353

From 992df5d5a7925f680938aa21f4fc2b4cfbe6604c Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 2 Sep 2026 10:58:47 -0500
Subject: [PATCH 23/23] fix(agentx): recover offload scheduler progress and
 budget NVMe warmup

---
 configs/nvidia-master.yaml        |  2 +-
 docs/eval-agentx-procedures.md    |  4 ++--
 docs/eval-agentx-procedures_zh.md |  2 +-
 perf-changelog.yaml               | 10 ++++++++++
 runners/launch_h100-dgxc-slurm.sh |  4 ++--
 runners/test_slurm_utils.py       |  7 ++++---
 6 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 82ea328ffe..4e30ee9303 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7499,7 +7499,7 @@ qwen3.5-fp4-b200-trt-mtp:
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
 
 minimaxm3-fp8-h100-vllm-agentic-mtp:
-  image: ttl.sh/cquil11-vllm-tier-d67b417bca-pr53087-20260901:24h
+  image: ttl.sh/cquil11-vllm-tier-progress-b9e9d720-amd64-20260902:24h
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: cluster:h100-dgxc
diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md
index c465a8ef17..6aa3e20730 100644
--- a/docs/eval-agentx-procedures.md
+++ b/docs/eval-agentx-procedures.md
@@ -210,8 +210,8 @@ For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`,
 
 Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2266-L2268)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success.
 
-The H100 MiniMax-M3 DRAM+NVMe AgentX launcher allows 420 minutes for Slurm:
-the measured warmup alone took over four hours before the one-hour profile.
+The H100 MiniMax-M3 NVMe and DRAM+NVMe AgentX launcher allows 420 minutes for Slurm:
+measured warmup alone took over four hours before the one-hour profile.
 Other H100 single-node cases retain the 300-minute default, and an explicit
 `SALLOC_TIME_LIMIT` overrides either value. The canonical workload is unchanged.
 
diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md
index 0e9f364a13..26c117ca74 100644
--- a/docs/eval-agentx-procedures_zh.md
+++ b/docs/eval-agentx-procedures_zh.md
@@ -210,7 +210,7 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \
 
 Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2266-L2268)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。
 
-H100 MiniMax-M3 DRAM+NVMe AgentX 启动器的 Slurm 时限为 420 分钟：
+H100 MiniMax-M3 NVMe 和 DRAM+NVMe AgentX 启动器的 Slurm 时限为 420 分钟：
 实测 warmup 本身超过四小时，之后还需运行一小时的 profiling。
 其他 H100 单节点场景仍默认使用 300 分钟；显式设置 `SALLOC_TIME_LIMIT`
 可覆盖任一默认值。canonical 工作负载保持不变。
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 48260e4150..6d9f3aab10 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6843,3 +6843,13 @@
     - "Pin the AgentX client fix that excludes automatic warmup baselines from profiling server metrics and require the cached-token-source metric in JSON/CSV artifacts."
     - "Keep exactly Mooncake DRAM c8, Simple NVMe c30, and native DRAM+NVMe c20, with unchanged canonical warmup, one-hour profiling, images, and full evals."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796
+
+- config-keys:
+    - minimaxm3-fp8-h100-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Use the Python-only token-source image b9e9d720 on official vLLM nightly 7c5dc571, retaining the HIT_PENDING fix and backporting upstream PR #45406 so an unschedulable queue head cannot strand completed async KV loads."
+    - "Allow seven hours for Simple NVMe as well as native DRAM+NVMe; the NVMe warmup took four hours and its previous five-hour allocation ended before profiling completed."
+    - "Preserve exactly Mooncake DRAM c8, Simple NVMe c30 with 8 TB aggregate capacity, and native DRAM+NVMe c20, with full evals and unchanged canonical warmup and one-hour profiling."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2796
diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh
index 1420069ece..0d4b2965cb 100644
--- a/runners/launch_h100-dgxc-slurm.sh
+++ b/runners/launch_h100-dgxc-slurm.sh
@@ -298,8 +298,8 @@ else
 
     export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
-    # This tiered AgentX point needs >4 hours of warmup before its 1-hour profile.
-    if [[ "${MODEL_PREFIX:-}" == "minimaxm3" && "${SCENARIO_TYPE:-}" == "agentic-coding" && "${KV_OFFLOADING:-}" == "dram+nvme" ]]; then
+    # These NVMe AgentX points can need >4 hours of warmup before a 1-hour profile.
+    if [[ "${MODEL_PREFIX:-}" == "minimaxm3" && "${SCENARIO_TYPE:-}" == "agentic-coding" && ( "${KV_OFFLOADING:-}" == "nvme" || "${KV_OFFLOADING:-}" == "dram+nvme" ) ]]; then
         SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-420}"
     fi
     SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-300}"
diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py
index 4a7f1c9467..7e6646117c 100644
--- a/runners/test_slurm_utils.py
+++ b/runners/test_slurm_utils.py
@@ -26,16 +26,17 @@ def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str
     )
 
 
-def test_h100_tiered_agentx_time_limit_preserves_defaults_and_override() -> None:
+def test_h100_nvme_agentx_time_limit_preserves_defaults_and_override() -> None:
     launcher = (REPO_ROOT / "runners" / "launch_h100-dgxc-slurm.sh").read_text()
-    start = launcher.index("    # This tiered AgentX point")
+    start = launcher.index("    # These NVMe AgentX points")
     stop = launcher.index("    salloc ", start)
     configure = launcher[start:stop]
     cases = [
         ("minimaxm3", "agentic-coding", "dram+nvme", "", "420"),
         ("minimaxm3", "agentic-coding", "dram+nvme", "480", "480"),
         ("minimaxm3", "agentic-coding", "dram", "", "300"),
-        ("minimaxm3", "agentic-coding", "nvme", "", "300"),
+        ("minimaxm3", "agentic-coding", "nvme", "", "420"),
+        ("minimaxm3", "agentic-coding", "nvme", "480", "480"),
         ("other", "agentic-coding", "dram+nvme", "", "300"),
         ("minimaxm3", "fixed-sequence", "dram+nvme", "", "300"),
     ]