diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp8_mi355x_sglang.sh b/benchmarks/single_node/agentic/qwen3.8next_fp8_mi355x_sglang.sh new file mode 100755 index 0000000000..6954bc6006 --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.8next_fp8_mi355x_sglang.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# AgentX trace replay for Qwen3.8-Flash-Next FP8 on MI355X with SGLang. +# Day-zero recipe; SGLang is the plan-of-record engine for this model +# (MODELS.md). +# +# Two deliberate differences from the NVIDIA arms: +# +# * No speculative decoding. The MI355X SGLang path for this model does not +# drive MTP yet, so this arm runs the target model alone and carries no +# synthetic-acceptance pin. Add MTP and the golden AL once ROCm supports +# it. Until then this arm is not directly comparable to the spec-decode +# NVIDIA arms on the published frontier. +# +# * FP8, not FP4. NVFP4 is greyed out for MI355X in the SGLang cookbook and +# there is no AMD FP4 checkpoint for this model. +# +# KNOWN BLOCKER, upstream, not in this recipe: on the current +# lmsysorg/sglang-rocm:qwen38flashnext image the server aborts while loading +# weights with +# AssertionError: Expected 1.0, got 0.00019931793212890625 in skipped +# model.layers.1.ple.ple_embedding.ngram_embedding.weight_scale +# qwen4_exp.py:2039 in load_weights +# The model registers no parameter for the PLE ngram embedding, so load_weights +# takes the "skipped" branch, which assumes any orphaned _scale must be a no-op +# and asserts it is 1.0. The checkpoint really does quantize that embedding: +# its 128 shard_N.weight tensors are F8_E4M3 with a single BF16 weight_scale of +# ~1.99e-4, and modules_to_not_convert lists ple.conv1d, ple.key_proj and +# ple.value_proj but not the ngram embedding. +# +# Suppressing the assert is NOT a fix: the shards would load as raw FP8 with +# the scale never applied, leaving that embedding wrong by ~5000x with no error +# reported. The CUDA image carries a different SGLang build and loads the same +# checkpoint, so this is specific to the ROCm build. It clears when that image +# implements the quantized PLE ngram embedding; the recipe below is otherwise +# the cookbook's verified balanced command and needs no further change. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars \ + MODEL TP CONC EP_SIZE KV_OFFLOADING \ + TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +rocm-smi || true +amd-smi || true + +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k +resolve_trace_source +install_agentic_deps + +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "SGLang server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +CACHE_ARGS=() +if require_agentic_kv_offload_backend hicache; then + HICACHE_RATIO="${HICACHE_RATIO:-1.5}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + echo "HiCache CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT, dram_budget=${TOTAL_CPU_DRAM_GB} GB, tp=$TP" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-ratio "$HICACHE_RATIO" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) +fi + +PARALLEL_ARGS=( + --tp "$TP" + --dp 1 + --ep-size "$EP_SIZE" +) + +TOKENIZER_ARGS=() +if [ "$TP" -ge 4 ]; then + TOKENIZER_ARGS=(--tokenizer-worker-num 6) +fi + +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS="$CONC" +[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + +export PYTHONNOUSERSITE=1 +export SGLANG_USE_AITER=1 +export SGLANG_USE_AITER_UNIFIED_ATTN=1 +export AITER_FLYDSL_FORCE=1 +export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export SGLANG_TIMEOUT_KEEP_ALIVE=1800 + + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + # Verified flags from the SGLang cookbook playground for this model on + # MI355X / FP8 / balanced / single node. Low-latency and high-throughput + # are not offered for this part, and NVFP4 is greyed out, so balanced FP8 + # at TP8 is the whole of the verified AMD surface today. + --tp-size "$TP" + --attention-backend aiter + --page-size 32 + --kv-cache-dtype auto + --chunked-prefill-size 16384 + --watchdog-timeout 1200 + --mem-fraction-static 0.9 + --model-loader-extra-config '{"enable_multithread_load": true}' + --max-running-requests "$MAX_RUNNING_REQUESTS" + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --stream-interval 50 + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" + "${TOKENIZER_ARGS[@]}" + --tokenizer-path "$MODEL" + --enable-metrics + --enable-cache-report + "${CACHE_ARGS[@]}" +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY:-false}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index acf6efef1a..285d7e5c46 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -377,6 +377,28 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 48, 56, 64] } - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32] } + +# Qwen3.8-Flash-Next FP8 AgentX on MI355X via SGLang, without speculative +# decoding: the MI355X SGLang path for this model does not drive MTP yet, so +# this arm runs the target model alone and carries no synthetic-acceptance pin. +# FP8 at TP8 per the cookbook's verified balanced single-node command; NVFP4 +# is greyed out for MI355X and there is no AMD FP4 checkpoint for this model. +# Blocked upstream: the current ROCm image asserts a 1.0 weight_scale on the +# PLE ngram embedding that this checkpoint genuinely quantizes to F8_E4M3. +# See the script header; suppressing the assert would corrupt the embedding. +qwen3.8next-fp8-mi355x-sglang-agentic: + image: lmsysorg/sglang-rocm:qwen38flashnext + model: Qwen/Qwen3.8-Flash-Next-FP8 + model-prefix: qwen3.8next + runner: cluster:mi355x-amds + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.8 + search-space: + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 model: amd/Qwen3.5-397B-A17B-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index de9533314a..602722d541 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6490,3 +6490,14 @@ - "Required lanes promote exporter startup timeouts, launch failures, and endpoint resolution failures to blocking validation failures, so a recipe cannot publish a result whose power collection never started." - "Route only enabled recipes through the immutable producer fork and preserve non-power launcher revisions." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2688 +- config-keys: + - qwen3.8next-fp8-mi355x-sglang-agentic + scenario-type: + - agentic-coding + description: + - "Add the day-zero Qwen3.8-Flash-Next FP8 AgentX recipe on MI355X with SGLang at TP8 and concurrency 1/4/8/12/16." + - "Run without speculative decoding because the MI355X SGLang path for this model does not drive MTP yet, so this arm carries no synthetic-acceptance pin." + - "Serve the FP8 checkpoint because no AMD FP4 checkpoint exists for this model yet and NVFP4 is unavailable for MI355X." + - "Correct the serve flags to the SGLang cookbook's verified balanced single-node command for this model: TP8 rather than TP4, page size 32, automatic KV cache dtype, chunked prefill 16384, memory fraction 0.9, and multithreaded model loading." + - "Mount the repository away from the container path this image keeps its python packages under, so the bind mount stops hiding the SGLang and aiter trees the image puts on PYTHONPATH, and point the shared library's workspace variable at the new mount." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2754 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 49c3af0cf5..1b92ff1f2f 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -313,12 +313,32 @@ else BENCHMARK_SCRIPT="$SCRIPT_FALLBACK" fi + # Some bring-up images ship their python packages under /workspace and put + # them on PYTHONPATH, e.g. the qwen38flashnext ROCm image sets + # PYTHONPATH=/workspace/sglang-qwen-next/python:/workspace/aiter-pr4882 + # and has no pip-installed sglang at all. Bind mounting the repo over + # /workspace masks those trees, and the server dies with + # "No module named 'sglang'". Mount the repo elsewhere for such images and + # move RESULT_DIR with it; the host side of the mount is unchanged, so the + # workflow's artifact staging still finds everything under GITHUB_WORKSPACE. + WS_MOUNT_DIR="/workspace" + if [[ "$IMAGE" == *qwen38flashnext* ]]; then + WS_MOUNT_DIR="/inferencex" + export RESULT_DIR="${RESULT_DIR/#\/workspace//inferencex}" + # benchmark_lib.sh resolves AGENTIC_DIR, AIPERF_DIR and every `cd` into + # the repo through this variable, which defaults to /workspace. Move it + # with the mount or install_agentic_deps looks for the requirements + # file at /workspace/utils/... inside the image and fails. + export INFMAX_CONTAINER_WORKSPACE="$WS_MOUNT_DIR" + echo "Image ships packages under /workspace; mounting the repo at $WS_MOUNT_DIR (RESULT_DIR=$RESULT_DIR)" + fi + srun --jobid=$JOB_ID \ --container-image=$SQUASH_FILE \ - --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ + --container-mounts=$GITHUB_WORKSPACE:$WS_MOUNT_DIR/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ $SLRUM_HOME_MOUNT \ --container-writable \ - --container-workdir=/workspace/ \ + --container-workdir=$WS_MOUNT_DIR/ \ --container-remap-root \ --no-container-entrypoint --export=ALL,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \ bash "$BENCHMARK_SCRIPT"