From 9c88dd9c399da4c1c34c51830a61c973b6935fa9 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:08:07 -0400
Subject: [PATCH 1/8] qwen3.8next-fp8-h200-sglang-agentic-mtp: day-zero
 Qwen3.8-Flash-Next AgentX on H200
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the Qwen3.8-Flash-Next AgentX recipe on H200, served by SGLang with native
NEXTN MTP. H200 is Hopper, so this arm serves the FP8 checkpoint rather than
the NVFP4 one the Blackwell arms use: NVFP4 needs SM100 tensor cores. TP8 keeps
the 172.8 GiB checkpoint near 22 GiB per rank, and the attention backend stays
flashinfer for sm_90. Throughput runs pin an interim acceptance length of 3.24
at three speculative tokens; eval-only runs keep real target verification.

新增 H200 上的 Qwen3.8-Flash-Next AgentX 配方，由 SGLang 以原生 NEXTN MTP 提供
服务。H200 属 Hopper 架构，无 SM100 张量核心，无法运行 NVFP4，故本分支使用 FP8
权重而非 Blackwell 分支所用的 NVFP4。TP8 下每卡权重约 22 GiB，注意力后端保持
sm_90 适用的 flashinfer。吞吐运行按 3 个投机 token 锁定临时接受长度 3.24，仅评测
运行仍使用真实目标验证。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../qwen3.8next_fp8_h200_sglang_mtp.sh        | 194 ++++++++++++++++++
 configs/nvidia-master.yaml                    |  18 ++
 perf-changelog.yaml                           |   9 +
 3 files changed, 221 insertions(+)
 create mode 100755 benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
new file mode 100755
index 0000000000..f8d76a54d3
--- /dev/null
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# Agentic trace replay benchmark for Qwen3.8-Flash-Next FP8 on H200 using
+# SGLang with MTP speculative decoding. Day-zero recipe; SGLang is the
+# plan-of-record engine for this model (MODELS.md), and it is spec-decode only,
+# per the AgentX policy that new agentic arms ship with speculative decoding
+# enabled rather than as an STP/MTP A/B.
+#
+# H200 is Hopper, so this arm is FP8 (Qwen/Qwen3.8-Flash-Next-FP8, 172.8 GiB at
+# TP8) rather than the NVFP4 checkpoint the Blackwell arms use: NVFP4 needs
+# SM100 tensor cores. Attention backend stays flashinfer (sm_90).
+#
+# Structure follows the proven H100 MTP AgentX replay path
+# (HiCache host-DRAM offload, the multi_tokenizer cached_tokens_details patch,
+# aiperf-driven trace replay). H200's 141 GB HBM3e is roomier than H100's 80 GB,
+# so --mem-fraction-static is 0.8 rather than 0.75, matching
+# fixed_seq_len/qwen3.5_fp8_h200_mtp.sh. Attention backend stays flashinfer
+# (sm_90); the trtllm_mha path is Blackwell-only.
+#
+# Speculative decoding mirrors fixed_seq_len/qwen3.5_fp8_h100_mtp.sh:
+# SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, 3 steps, eagle-topk
+# 1 and 4 draft tokens, i.e. 3 speculative tokens per verification step.
+#
+# Throughput runs pin acceptance to the committed golden AL through SGLang's
+# simulated-acceptance path; the EVAL_ONLY accuracy run leaves it off and keeps
+# real verification. See the SGLANG_SIMULATE_ACC_* block.
+#
+# Required env vars:
+#   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
+#
+# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE
+
+SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+fi
+
+# `hf download` creates the target dir if missing and is itself idempotent.
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# Either way, MODEL_PATH is what the server is launched with.
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+nvidia-smi
+
+# ---- Resolve traces and install deps ----------------------------------------
+# Keep the 256k-capped with-subagents corpus the H100 Qwen3.5 AgentX recipe
+# uses (470 traces, max in+out <= 256k). The unfiltered corpus has requests up
+# to ~1M proxy tokens that the server would reject; H200's extra HBM raises the
+# context ceiling but not past 256k for this model at TP8.
+export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k
+
+resolve_trace_source
+install_agentic_deps
+
+# ---- Server config ----------------------------------------------------------
+SERVER_LOG="$RESULT_DIR/server.log"
+mkdir -p "$RESULT_DIR"
+
+CACHE_ARGS=()
+if require_agentic_kv_offload_backend hicache; then
+    # HiCache extends RadixAttention, so do not pass --disable-radix-cache.
+    # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank.
+    REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}"
+    if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
+        echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
+        exit 1
+    fi
+    TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB"
+    HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}"
+    HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
+    MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT))
+    HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
+    if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
+        echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2
+        exit 1
+    fi
+    if [ "$HICACHE_SIZE_GB" -lt 1 ]; then
+        echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2
+        exit 1
+    fi
+    echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}"
+    CACHE_ARGS=(
+        --page-size 64
+        --enable-hierarchical-cache
+        --hicache-size "$HICACHE_SIZE_GB"
+        --hicache-io-backend kernel
+        --hicache-mem-layout page_first
+        --hicache-write-policy "$HICACHE_WRITE_POLICY"
+    )
+fi
+
+echo "Starting SGLang server..."
+export PYTHONNOUSERSITE=1
+export SGLANG_ENABLE_SPEC_V2=1
+
+# 3 speculative tokens per step (num-steps 3, eagle-topk 1, 4 draft tokens),
+# the same MTP shape as the fixed-seq-len Qwen3.5 recipes.
+SPEC_ARGS=(
+    --speculative-algorithm EAGLE
+    --speculative-num-steps 3
+    --speculative-eagle-topk 1
+    --speculative-num-draft-tokens 4
+)
+
+# AgentX pins acceptance to the committed golden AL so submissions are compared
+# on system performance at a fixed acceptance target rather than on draft-head
+# quality (golden_al_distribution/README.md). 3.39 is the Qwen3.5 MTP curve at
+# num_speculative_tokens=3, thinking_on (golden_al_distribution/qwen3.5_mtp.yaml)
+# -- the same value the GB300 Qwen3.5 AgentX srt-slurm recipes pin.
+# SGLANG_SIMULATE_ACC_TOKEN_MODE landed in SGLang v0.5.16, which is why this
+# recipe pins that image rather than the non-MTP agentic sibling's v0.5.12.
+#
+# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens
+# regardless of the target logits, so generated text is wrong and the eval would
+# score ~0.
+if [ "${EVAL_ONLY:-false}" != "true" ]; then
+    # Qwen3.8-Flash-Next acceptance length, SPEED-Bench coding.
+    # --speculative-num-steps 3 with 4 draft tokens is 3 speculative
+    # tokens per verification step, i.e. the MTP=3 cell -> AL 3.24.
+    # Measured thinking=off in speedbench-al run 33031708148; the
+    # thinking=on collection is still in flight, so this is an
+    # interim value and is not yet a committed golden_al_distribution
+    # curve. Refresh once qwen3.8next_mtp.yaml lands.
+    export SGLANG_SIMULATE_ACC_LEN=3.24
+    export SGLANG_SIMULATE_ACC_METHOD=match-expected
+    export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
+fi
+
+SGLANG_MULTI_TOKENIZER=/sgl-workspace/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py
+if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/p' "$SGLANG_MULTI_TOKENIZER" \
+    | grep -q 'cached_tokens_details=_extract_field_by_index'; then
+    sed -i '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/ {
+        /cached_tokens=_extract_field_by_index(output, "cached_tokens", i),/a\
+            cached_tokens_details=_extract_field_by_index(\
+                output, "cached_tokens_details", i\
+            ),
+    }' "$SGLANG_MULTI_TOKENIZER"
+fi
+
+{ set +x; } 2>/dev/null
+SGLANG_CMD=(
+    python3 -m sglang.launch_server
+    --model-path="$MODEL_PATH" --served-model-name="$MODEL"
+    --host=0.0.0.0
+    --port="$PORT"
+    --trust-remote-code
+    --tensor-parallel-size="$TP"
+    --data-parallel-size=1
+    --expert-parallel-size="$EP_SIZE"
+    --quantization fp8
+    --kv-cache-dtype fp8_e4m3
+    --mamba-ssm-dtype bfloat16
+    --attention-backend flashinfer
+    --enable-flashinfer-allreduce-fusion
+    # --cuda-graph-max-bs "$CONC"
+    # --max-running-requests "$CONC"
+    # --max-prefill-tokens 8192
+    # --chunked-prefill-size 8192
+    --mem-fraction-static 0.8
+    --stream-interval 50
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
+    --tokenizer-worker-num 6
+    --tokenizer-path "$MODEL"
+    --enable-metrics
+    "${SPEC_ARGS[@]}"
+    "${CACHE_ARGS[@]}"
+)
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+echo "Server PID: $SERVER_PID"
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "${EVAL_ONLY}" = "true" ]; then
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+fi
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 4c1036915d..0c44d5ff20 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7184,6 +7184,24 @@ qwen3.5-fp8-h200-sglang-agentic-mtp:
       - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16] }
       - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] }
 
+
+# Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP.
+# Day-zero recipe. H200 is Hopper, so this arm is FP8 rather than the NVFP4
+# checkpoint the Blackwell arms use: NVFP4 needs SM100 tensor cores. TP8 keeps
+# the 172.8 GiB checkpoint at ~22 GiB per rank, leaving HBM for long traces.
+qwen3.8next-fp8-h200-sglang-agentic-mtp:
+  image: lmsysorg/sglang:qwen38flashnext
+  model: Qwen/Qwen3.8-Flash-Next-FP8
+  model-prefix: qwen3.8next
+  runner: cluster:h200-dgxc
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.8
+      search-space:
+      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] }
 # H200 AgentX MTP frontier with DRAM HiCache. This is intentionally an MTP-only
 # submission; the model's non-speculative AgentX arm is not included.
 qwen3.5-fp8-h200-sglang-agentic-hicache-mtp:
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index de9533314a..cc92c4738c 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6490,3 +6490,12 @@
     - "Required lanes promote exporter startup timeouts, launch failures, and endpoint resolution failures to blocking validation failures, so a recipe cannot publish a result whose power collection never started."
     - "Route only enabled recipes through the immutable producer fork and preserve non-power launcher revisions."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2688
+- config-keys:
+    - qwen3.8next-fp8-h200-sglang-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add the day-zero Qwen3.8-Flash-Next FP8 AgentX recipe on H200 with SGLang native NEXTN MTP at TP8 and concurrency 1/4/8/12/16."
+    - "Use the FP8 checkpoint rather than NVFP4 because Hopper has no SM100 tensor cores, and keep the flashinfer attention backend for sm_90."
+    - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
+  pr-link: TBD

From e3a111c89180ea1f25947c3b252a9aafbbae8c9a Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:08:29 -0400
Subject: [PATCH 2/8] =?UTF-8?q?Fill=20perf-changelog=20pr-link=20for=20#27?=
 =?UTF-8?q?53=20/=20=E8=A1=A5=E5=85=A8=20#2753=20=E7=9A=84=20perf-changelo?=
 =?UTF-8?q?g=20pr-link?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index cc92c4738c..8e496bdd68 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6498,4 +6498,4 @@
     - "Add the day-zero Qwen3.8-Flash-Next FP8 AgentX recipe on H200 with SGLang native NEXTN MTP at TP8 and concurrency 1/4/8/12/16."
     - "Use the FP8 checkpoint rather than NVFP4 because Hopper has no SM100 tensor cores, and keep the flashinfer attention backend for sm_90."
     - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
-  pr-link: TBD
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2753

From 65a1a76197485cae211196bdc7ef5d7805eb4fe9 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:30:27 -0400
Subject: [PATCH 3/8] Use the cookbook's verified TP4/EP4 command and the
 committed golden AL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The recipe was copied from the Qwen3.5 FP8 H200 sibling and inherited TP8/EP1
plus that model's EAGLE speculative algorithm. The SGLang cookbook playground's
verified low-latency single-node command for Qwen3.8-Flash-Next on H200 is
--tp 4 --ep 4, sharding the 512-expert MoE with expert parallelism rather than
sharding attention eight ways, with memory fraction 0.85, chunked prefill 8192,
the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM
and NEXTN rather than EAGLE. NVFP4 is greyed out for Hopper in the cookbook,
confirming FP8 for this arm. Acceptance length moves from the interim 3.24 to
the committed golden thinking_on value of 2.32.

本配方原样照搬自 Qwen3.5 FP8 H200 同类配方，带入了 TP8/EP1 与该模型的 EAGLE
投机算法。SGLang cookbook playground 给出的 Qwen3.8-Flash-Next H200 低时延单节点
验证命令为 --tp 4 --ep 4，即以专家并行切分 512 专家 MoE，而非把注意力切成八份；
并使用显存占用 0.85、chunked prefill 8192、flashinfer 线性注意力预填充与解码后端、
bfloat16 Mamba SSM，投机算法为 NEXTN 而非 EAGLE。cookbook 中 Hopper 的 NVFP4
为不可选，印证本分支应使用 FP8。接受长度由临时值 3.24 改为已提交黄金曲线的
thinking_on 取值 2.32。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../qwen3.8next_fp8_h200_sglang_mtp.sh        | 54 ++++++++++---------
 configs/nvidia-master.yaml                    |  9 ++--
 perf-changelog.yaml                           |  3 +-
 3 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
index f8d76a54d3..1ff1623c02 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
@@ -109,7 +109,7 @@ export SGLANG_ENABLE_SPEC_V2=1
 # 3 speculative tokens per step (num-steps 3, eagle-topk 1, 4 draft tokens),
 # the same MTP shape as the fixed-seq-len Qwen3.5 recipes.
 SPEC_ARGS=(
-    --speculative-algorithm EAGLE
+    --speculative-algorithm NEXTN
     --speculative-num-steps 3
     --speculative-eagle-topk 1
     --speculative-num-draft-tokens 4
@@ -127,14 +127,12 @@ SPEC_ARGS=(
 # regardless of the target logits, so generated text is wrong and the eval would
 # score ~0.
 if [ "${EVAL_ONLY:-false}" != "true" ]; then
-    # Qwen3.8-Flash-Next acceptance length, SPEED-Bench coding.
-    # --speculative-num-steps 3 with 4 draft tokens is 3 speculative
-    # tokens per verification step, i.e. the MTP=3 cell -> AL 3.24.
-    # Measured thinking=off in speedbench-al run 33031708148; the
-    # thinking=on collection is still in flight, so this is an
-    # interim value and is not yet a committed golden_al_distribution
-    # curve. Refresh once qwen3.8next_mtp.yaml lands.
-    export SGLANG_SIMULATE_ACC_LEN=3.24
+    # golden_al_distribution/qwen3.8next_mtp.yaml:
+    # qwen3.8-flash-next-fp8.thinking_on[3] = 2.32.
+    # --speculative-num-steps 3 with 4 draft tokens is 3 speculative tokens
+    # per verification step, i.e. the MTP=3 cell. AgentX replays run with
+    # thinking on, so the thinking_on row is the right one.
+    export SGLANG_SIMULATE_ACC_LEN=2.32
     export SGLANG_SIMULATE_ACC_METHOD=match-expected
     export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
 fi
@@ -153,29 +151,35 @@ fi
 { set +x; } 2>/dev/null
 SGLANG_CMD=(
     python3 -m sglang.launch_server
-    --model-path="$MODEL_PATH" --served-model-name="$MODEL"
-    --host=0.0.0.0
-    --port="$PORT"
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
     --trust-remote-code
-    --tensor-parallel-size="$TP"
-    --data-parallel-size=1
-    --expert-parallel-size="$EP_SIZE"
-    --quantization fp8
-    --kv-cache-dtype fp8_e4m3
+    # Verified flags from the SGLang cookbook playground for this model on
+    # H200 / FP8 / low latency / single node. NVFP4 is greyed out for Hopper,
+    # so FP8 is the whole verified surface here. TP4 with EP4, not TP8: the
+    # cookbook shards the 512-expert MoE across four ranks with expert
+    # parallelism rather than sharding attention eight ways.
+    --tp-size "$TP"
+    --ep-size "$EP_SIZE"
+    --dp-size 1
+    --mem-fraction-static 0.85
+    --chunked-prefill-size 8192
+    --linear-attn-prefill-backend flashinfer
+    --linear-attn-decode-backend flashinfer
     --mamba-ssm-dtype bfloat16
-    --attention-backend flashinfer
-    --enable-flashinfer-allreduce-fusion
-    # --cuda-graph-max-bs "$CONC"
-    # --max-running-requests "$CONC"
-    # --max-prefill-tokens 8192
-    # --chunked-prefill-size 8192
-    --mem-fraction-static 0.8
+    "${SPEC_ARGS[@]}"
+    --reasoning-parser auto
+    # NEXTN silently resets --max-running-requests to 48 when it is unset, so
+    # this must stay explicit and sized to the AgentX concurrency.
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
     --stream-interval 50
     --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
     --tokenizer-worker-num 6
     --tokenizer-path "$MODEL"
     --enable-metrics
-    "${SPEC_ARGS[@]}"
     "${CACHE_ARGS[@]}"
 )
 printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 0c44d5ff20..6a2f7e504d 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7186,9 +7186,10 @@ qwen3.5-fp8-h200-sglang-agentic-mtp:
 
 
 # Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP.
-# Day-zero recipe. H200 is Hopper, so this arm is FP8 rather than the NVFP4
-# checkpoint the Blackwell arms use: NVFP4 needs SM100 tensor cores. TP8 keeps
-# the 172.8 GiB checkpoint at ~22 GiB per rank, leaving HBM for long traces.
+# Day-zero recipe. H200 is Hopper, so this arm is FP8: NVFP4 is greyed out for
+# this part in the SGLang cookbook. TP4 with EP4 per the cookbook's verified
+# low-latency single-node command, which shards the 512-expert MoE with expert
+# parallelism rather than sharding attention eight ways.
 qwen3.8next-fp8-h200-sglang-agentic-mtp:
   image: lmsysorg/sglang:qwen38flashnext
   model: Qwen/Qwen3.8-Flash-Next-FP8
@@ -7201,7 +7202,7 @@ qwen3.8next-fp8-h200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.8
       search-space:
-      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] }
+      - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] }
 # H200 AgentX MTP frontier with DRAM HiCache. This is intentionally an MTP-only
 # submission; the model's non-speculative AgentX arm is not included.
 qwen3.5-fp8-h200-sglang-agentic-hicache-mtp:
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 8e496bdd68..ec27122d0f 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6497,5 +6497,6 @@
   description:
     - "Add the day-zero Qwen3.8-Flash-Next FP8 AgentX recipe on H200 with SGLang native NEXTN MTP at TP8 and concurrency 1/4/8/12/16."
     - "Use the FP8 checkpoint rather than NVFP4 because Hopper has no SM100 tensor cores, and keep the flashinfer attention backend for sm_90."
-    - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
+    - "Correct the serve flags to the SGLang cookbook's verified low-latency single-node command for this model: TP4 with EP4 rather than TP8, memory fraction 0.85, chunked prefill 8192, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and the NEXTN speculative algorithm."
+    - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2753

From 1fa09be6ba89f51688ff7da002f4f00ff8c70df4 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:56:31 -0400
Subject: [PATCH 4/8] Define MAX_RUNNING_REQUESTS and CUDA_GRAPH_MAX_BS on the
 H200 arm
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The H200 run died at server launch with

  qwen3.8next_fp8_h200_sglang_mtp.sh: line 184: MAX_RUNNING_REQUESTS: unbound
  variable

The Qwen3.5 H200 template this was derived from leaves --max-running-requests
and --cuda-graph-max-bs commented out, so unlike the B200, B300 and MI355X
templates it never defines either variable. The rewrite to the cookbook command
passes both flags, and the script runs under set -u.

Define them the way the other three arms do: two times the AgentX concurrency
for the scheduler, and graph capture at the concurrency capped to 64. Keeping
--max-running-requests explicit also matters here because NEXTN silently resets
it to 48 when it is unset. Audited the serve command of all four arms for other
variables that are neither assigned in the script nor supplied by the matrix:
none.

H200 运行在启动服务时失败：MAX_RUNNING_REQUESTS 未定义。本脚本所参照的 Qwen3.5
H200 模板把 --max-running-requests 与 --cuda-graph-max-bs 注释掉了，因此不同于
B200、B300 与 MI355X 模板，它从未定义这两个变量；而改写为 cookbook 命令后这两个
参数都会传入，且脚本运行在 set -u 下。

现按其余三个分支的做法定义：调度器取 AgentX 并发的两倍，图捕获取并发值并以 64
封顶。此处保持 --max-running-requests 显式设置同样重要，因为 NEXTN 在其未设置时
会静默重置为 48。已审计四个分支服务命令中所有既未在脚本内赋值、也非由矩阵提供的
变量：无遗漏。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../agentic/qwen3.8next_fp8_h200_sglang_mtp.sh        | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
index 1ff1623c02..f90b038bb1 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
@@ -149,6 +149,17 @@ if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_va
 fi
 
 { set +x; } 2>/dev/null
+# AgentX concurrency counts live session trees rather than individual HTTP
+# requests. Leave room for subagent fan-out, and do not spend HBM capturing
+# graphs above the batch sizes that stay useful for this long-context workload.
+# The Qwen3.5 H200 template left both flags commented out, so neither variable
+# existed; NEXTN silently caps --max-running-requests at 48 when it is unset.
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS="$CONC"
+if [ "$CUDA_GRAPH_MAX_BS" -gt 64 ]; then
+    CUDA_GRAPH_MAX_BS=64
+fi
+
 SGLANG_CMD=(
     python3 -m sglang.launch_server
     --model-path "$MODEL_PATH"

From f1da353965f46c40465e51a49be402414644a414 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 00:08:22 -0400
Subject: [PATCH 5/8] Use a float32 Mamba SSM state so the NEXTN verify kernel
 accepts it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The H200 cells got all the way to CUDA graph capture and died there:

  AssertionError: initial_state must be float32, got torch.bfloat16
  flashinfer/gdn_decode.py:761 in gated_delta_rule_mtp
  via gdn_backend.py target_verify, during "Capture target verify CUDA graph"

With NEXTN on, the GDN linear-attention backend routes verification through
flashinfer's gated_delta_rule_mtp, and that kernel requires the SSM state in
float32. The cookbook command pairs --mamba-ssm-dtype bfloat16 with NEXTN, but
this flashinfer build rejects the combination. Of the two halves, the state
dtype is the one that can move: the linear-attention backends are what the
cookbook verifies, and dropping speculative decoding would defeat the recipe.

H200 各单元一直跑到 CUDA graph 捕获阶段才失败：flashinfer 的
gated_delta_rule_mtp 断言 initial_state 必须为 float32，而实际为 bfloat16。启用
NEXTN 后，GDN 线性注意力后端的验证路径会走到该内核，而它要求 SSM 状态为
float32。cookbook 命令把 --mamba-ssm-dtype bfloat16 与 NEXTN 并列，但当前
flashinfer 构建不接受该组合。两者之中可调整的是状态精度：线性注意力后端正是
cookbook 所验证的部分，而去掉投机解码则会使该配方失去意义。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../agentic/qwen3.8next_fp8_h200_sglang_mtp.sh         | 10 +++++++++-
 perf-changelog.yaml                                    |  1 +
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
index f90b038bb1..5199d539cc 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh
@@ -179,7 +179,15 @@ SGLANG_CMD=(
     --chunked-prefill-size 8192
     --linear-attn-prefill-backend flashinfer
     --linear-attn-decode-backend flashinfer
-    --mamba-ssm-dtype bfloat16
+    # float32, not the cookbook's bfloat16. With NEXTN enabled the GDN linear
+    # attention backend routes verification through flashinfer's
+    # gated_delta_rule_mtp, which asserts initial_state.dtype == torch.float32
+    # and aborts CUDA graph capture on a bf16 SSM state:
+    #   AssertionError: initial_state must be float32, got torch.bfloat16
+    #   flashinfer/gdn_decode.py:761, via gdn_backend.py target_verify
+    # The cookbook command pairs bfloat16 with NEXTN, but this flashinfer build
+    # rejects that combination, and the state dtype is the half that can move.
+    --mamba-ssm-dtype float32
     "${SPEC_ARGS[@]}"
     --reasoning-parser auto
     # NEXTN silently resets --max-running-requests to 48 when it is unset, so
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index ec27122d0f..d9aa5e7fbd 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6499,4 +6499,5 @@
     - "Use the FP8 checkpoint rather than NVFP4 because Hopper has no SM100 tensor cores, and keep the flashinfer attention backend for sm_90."
     - "Correct the serve flags to the SGLang cookbook's verified low-latency single-node command for this model: TP4 with EP4 rather than TP8, memory fraction 0.85, chunked prefill 8192, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and the NEXTN speculative algorithm."
     - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification."
+    - "Use a float32 Mamba SSM state: flashinfer's gated_delta_rule_mtp verify kernel asserts float32 and aborts CUDA graph capture on the bfloat16 state the cookbook command specifies."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2753

From d08ce6c8b8fcfb5467af5087a3fb59c1cec0b7bc Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 01:56:14 -0400
Subject: [PATCH 6/8] Separate the appended changelog entry from history with a
 blank line
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

merge_with_reuse.sh refused PR #2753 with

  ERROR: new changelog entries must be separated from history by one empty
  line and appended at the end

validate_perf_changelog.py requires the appended suffix to start with
"\n- config-keys:" unless the base file already ends in a blank line. Main's
perf-changelog.yaml now ends "...pull/2688\n", so the entry needed one blank
line ahead of it and did not have one. Historical bytes were already exact;
this only inserts the separator.

merge_with_reuse.sh 因变更记录格式校验拒绝了 PR #2753：追加条目必须与历史之间空一
行。validate_perf_changelog.py 要求追加部分以 "\n- config-keys:" 开头（除非基础
文件本身以空行结尾），而当前 main 的 perf-changelog.yaml 以 "...pull/2688\n" 结尾，
因此需要在条目前补一个空行。历史字节本就完全一致，此处仅插入该分隔空行。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 perf-changelog.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index d9aa5e7fbd..f25a76fb08 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6490,6 +6490,7 @@
     - "Required lanes promote exporter startup timeouts, launch failures, and endpoint resolution failures to blocking validation failures, so a recipe cannot publish a result whose power collection never started."
     - "Route only enabled recipes through the immutable producer fork and preserve non-power launcher revisions."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2688
+
 - config-keys:
     - qwen3.8next-fp8-h200-sglang-agentic-mtp
   scenario-type:

From c1e509dc8879be2a9c363571ad9667f03207a609 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 01:56:30 -0400
Subject: [PATCH 7/8] chore: refresh PR #2753 for sweep reuse [skip-sweep]


From 2eccb1a36a4df6afc994e27d77e7b67a038e0aa1 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 01:57:11 -0400
Subject: [PATCH 8/8] chore: refresh PR #2753 for sweep reuse [skip-sweep]