From 0f07dfeadbddd70722f3bcb5015bc21b8351e2f3 Mon Sep 17 00:00:00 2001 From: Ruodi Lu Date: Mon, 3 Aug 2026 05:55:44 +0000 Subject: [PATCH 1/2] [None][test] Raise perf-core TIMEOUT for large-checkpoint cases on slower platforms Five llm_perf_core cases fail with pytest-timeout rather than a real error on the weekly QA run. In each case the harness reports only "Failed: Timeout (>3600.0s) from pytest-timeout" with no exception, and the platforms hitting it are the slowest / smallest-memory members of their condition (RTX 6000 Server, H20-3e), so the wall time is dominated by checkpoint load or by a long low-concurrency decode rather than by a defect. Give those five cases an explicit TIMEOUT so the run reflects real perf signal instead of a truncated one: * llama_v3.3_70b_instruct_fp8 20000,2000 reqs:1000 120 -> 180 min * llama_v3.1_nemotron_ultra_253b bf16 128,128 60 -> 180 min * qwen3.5_397b_a17b_fp8 128,128 ep:8 60 -> 120 min * llama_v3.3_70b_instruct_fp4 8000,1000 reqs:3000 60 -> 120 min * qwen3_235b_a22b_fp4 1000,2000 con:8 60 -> 120 min No cases are added or removed and no other case is affected. Note this only removes the timeout noise; for the two largest checkpoints (474G nemotron_ultra_253b, 380G qwen3.5_397b_a17b_fp8) the load time itself is long because of host-memory checkpoint prefetch, tracked separately in https://nvbugs/6272421 and https://nvbugs/6272397. Signed-off-by: Ruodi Lu --- .../test_lists/qa/llm_perf_core.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index ea30931f8b0a..3ad4e1acd585 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -310,7 +310,9 @@ llm_perf_core: tests: #llama_v3.3_70b_instruct_fp8 #pytorch backend - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:4096-maxnt:20000-kv_frac:0.85-input_output_len:20000,2000-reqs:1000-tp:8-gpus:8] TIMEOUT(120) + # 20K ISL x 1000 reqs does not finish in 120min on the slowest platform in this + # condition (RTX 6000 Server); bumped to 180min. + - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:4096-maxnt:20000-kv_frac:0.85-input_output_len:20000,2000-reqs:1000-tp:8-gpus:8] TIMEOUT(180) #minimax_m2.5 (FP8 216G, 8-GPU) - use EP=8 + attention DP; TP=8 would split intermediate_size 1536/8=192 (not divisible by 128 block size) - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-gpus:8] - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-gpus:8] @@ -328,7 +330,9 @@ llm_perf_core: - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-con:250-tp:8-gpus:8] #max_throughput #qwen3.5_397b_a17b_fp8 (MoE FP8 380G, 8-GPU ep=8) - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] + # 128,128 is a trivial workload but the 380G checkpoint load alone exceeds the + # 60min default on the slower platforms in this condition. + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:2000,500-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:8000,1000-ep:8-tp:8-gpus:8] @@ -364,7 +368,9 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput #llama_v3.1_nemotron_ultra_253b (nemotron-nas BF16 474G, 8-GPU) - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:128,128-tp:8-gpus:8] + # Largest checkpoint in this list; the 474G load alone blows the 60min default + # on H20-3e. + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:128,128-tp:8-gpus:8] TIMEOUT(180) - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:8-gpus:8] @@ -384,8 +390,11 @@ llm_perf_core: #llama_v3.3_70b - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-bfloat16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-input_output_len:128,128-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-maxbs:1024-maxnt:4096-kv_frac:0.85-input_output_len:8000,1000-reqs:3000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:8-ep:8-tp:8-gpus:8] + # 3000 reqs x 9K tokens does not finish within the 60min default on RTX 6000 Server. + - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-maxbs:1024-maxnt:4096-kv_frac:0.85-input_output_len:8000,1000-reqs:3000-tp:8-gpus:8] TIMEOUT(120) + # con:8 keeps the GPUs far from saturation, so wall time is dominated by the + # 2000-token decode of each request; does not fit the 60min default here. + - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:8-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:512-ep:8-tp:8-gpus:8] # deepseek_r1_0528 - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,2000-reqs:10-ep:4-tp:8-gpus:8] TIMEOUT(120) From 272e405074dbcbe9f5df662b5b293c89b7ac8dcb Mon Sep 17 00:00:00 2001 From: Ruodi Lu Date: Mon, 3 Aug 2026 06:13:33 +0000 Subject: [PATCH 2/2] [None][test] Drop 645G DeepSeek FP8 perf cases from 96G-per-GPU parts deepseek_r1_0528_fp8 and deepseek_v3.2_fp8 are ~645G checkpoints run at ep:8 tp:8, i.e. ~81G/GPU of weights. On a 96G part (NVML reports ~95.0 GiB usable) that leaves only ~14G per GPU for KV cache, activations and comm buffers, so these four cases cannot produce a meaningful perf number there and fail every weekly run. Condition 9 already gates on gpu_memory (gt: 90000 MiB, which is how 80G H100 is excluded), but a 96G part clears that bar. Split the four cases into a sibling condition with gt: 120000 MiB, which keeps H20-3e (~143.8K MiB), B200 (~183.4K MiB) and B300 (~294.9K MiB) and excludes the 96G part (~97.9K MiB). Effective per-platform case counts, before -> after: H100-SC 99 -> 99 H20-3e 117 -> 117 B200 177 -> 177 B300 177 -> 177 RTX 6000 Server 96 -> 92 (the four cases above) L40S 22 -> 22 Only the 96G part loses coverage; no case is deleted outright, so the larger-memory platforms keep tracking https://nvbugs/6272397. Signed-off-by: Ruodi Lu --- .../test_lists/qa/llm_perf_core.yml | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index 3ad4e1acd585..b1b807b237b0 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -12,6 +12,7 @@ llm_perf_core: # 7: B200, B300 test cases # 8: H100, H20, B200, B300, RTX6000-Server test cases # 9: H20, B200, B300, RTX6000-Server test cases +# 9b: H20, B200, B300 test cases (>120G per GPU; excludes 96G RTX6000-Server) # 10: RTX-6000 Server test cases # =============================================================================== @@ -353,12 +354,6 @@ llm_perf_core: gpu_memory: gt: 90000 tests: - # deepseek_r1_0528 - - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:8000,1000-reqs:20000-ep:8-tp:8-gpus:8] TIMEOUT(120) - - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:1000,2000-reqs:3000-ep:8-tp:8-gpus:8] TIMEOUT(100) - #deepseek_v3.2_fp8 (FP8 645G, 8-GPU ep=8) - - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:8000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput #qwen3.5_397b_a17b_fp4 (MoE FP4 234G, 8-GPU ep=8) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:500,2000-ep:8-tp:8-gpus:8] @@ -378,6 +373,28 @@ llm_perf_core: - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:8-gpus:8] #max_throughput + +# 9b: H20, B200, B300 test cases - same as 9 but excludes 96G-per-GPU parts. +# These checkpoints are ~645G, i.e. ~81G/GPU of weights at tp:8, which leaves +# only ~14G per GPU for KV cache, activations and comm buffers on a 96G part. +- condition: + ranges: + system_gpu_count: + gte: 8 + compute_capability: + gte: 9.0 + lte: 12.0 + gpu_memory: + gt: 120000 + tests: + # deepseek_r1_0528 + - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:8000,1000-reqs:20000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:1000,2000-reqs:3000-ep:8-tp:8-gpus:8] TIMEOUT(100) + #deepseek_v3.2_fp8 (FP8 645G, 8-GPU ep=8) + - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] + - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:8000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + + # 10: RTX-6000 Server test cases - condition: ranges: