From 31d88fbd63729da53d1f9ee51ebdfd6c3ebc0c83 Mon Sep 17 00:00:00 2001 From: Pranav Shrestha <254760092+pranav-nvidia@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:24:10 -0700 Subject: [PATCH 1/3] [https://nvbugs/6476233][fix] Cap max_seq_len on H200 DeepSeek-V3.2 blockscale test KV cache size estimation allocates a temporary cache sized from max_seq_len, so the model's native 163842 costs 12.1 GiB on top of ~100 GiB of weights. That left disable_skip_indexer peaking at 138.89 of 139.80 GiB, which OOMs in the autotuner warmup prefill once an earlier param in the same session has run. Capping at 8192 drops the temporary cache to 0.71 GiB and the peak to 134.72 GiB. Validated on 8xH200 in CI order (baseline, latency_default, disable_skip_indexer): 3 passed, all scores at or above reference. Signed-off-by: Pranav Shrestha <254760092+pranav-nvidia@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 6 +++++- tests/integration/test_lists/waives.txt | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index a65e2c70ef3e..572ce41149cc 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -3390,6 +3390,7 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, attention_dp, cuda_graph, overlap_scheduler, max_batch_size, moe_backend, disable_skip_indexer, enable_heuristic_topk, use_cute_dsl_topk): + extra_llm_args = {} if get_sm_version() == 100 or get_sm_version() == 103: moe_backend = "DEEPGEMM" if moe_backend == "_DEFAULT" else moe_backend moe_config = MoeConfig(backend=moe_backend, max_num_tokens=16384) @@ -3402,6 +3403,8 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, pytest.skip("Not supported MoE backend!") moe_config = MoeConfig() kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.7) + # Cap max_seq_len based on the evaluation task (NVBug 6476233). + extra_llm_args["max_seq_len"] = 8192 pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, @@ -3442,7 +3445,8 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, **pytorch_config, enable_attention_dp=attention_dp, speculative_config=mtp_config, - sparse_attention_config=dsa_config) as llm: + sparse_attention_config=dsa_config, + **extra_llm_args) as llm: # GPQA Diamond takes too long to run, we enable it only for fp8kv. if fp8kv: diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 32ca8df0c60d..9402d7d4ecdb 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -189,8 +189,6 @@ full:DGX_B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Pro::test_gsm8k_fu full:DGX_B200/disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] SKIP (https://nvbugs/6594241) full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-gemma4_26b_a4b_nvfp4_blackwell-gemma4_26b_a4b_nvfp4_tp1_1k1k] SKIP (https://nvbugs/6571410) full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b_spec_decode-llama8b_spec_bs1_128_128] SKIP (https://nvbugs/6571408) -full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/6476233) -full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6476233) full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True] SKIP (https://nvbugs/6525893) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt SKIP (https://nvbugs/6276923) From 0f5cc15306a083e3a8786bc6d661d1a4464b2a52 Mon Sep 17 00:00:00 2001 From: Pranav Shrestha <254760092+pranav-nvidia@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:31:20 -0700 Subject: [PATCH 2/3] [https://nvbugs/6476233][fix] Size the max_seq_len cap by evaluation task The fp8kv param evaluates GPQA Diamond (4096 in + 32768 out) rather than MMLU/GSM8K, so a flat 8192 would truncate its generation budget. Unreachable while fp8kv skips on pre-Blackwell, but the else branch also covers SM >= 100 outside 100/103, where that skip does not apply. Signed-off-by: Pranav Shrestha <254760092+pranav-nvidia@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 572ce41149cc..fc3f59fde1d9 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -3404,7 +3404,7 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, moe_config = MoeConfig() kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.7) # Cap max_seq_len based on the evaluation task (NVBug 6476233). - extra_llm_args["max_seq_len"] = 8192 + extra_llm_args["max_seq_len"] = 40960 if fp8kv else 8192 pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, From dd76b8b73cfdf0ae32a58b6a840cb8593d487caa Mon Sep 17 00:00:00 2001 From: Pranav Shrestha <254760092+pranav-nvidia@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:09:05 -0700 Subject: [PATCH 3/3] [https://nvbugs/6476233][fix] Derive the max_seq_len cap from the task classes Read MAX_INPUT_LEN + MAX_OUTPUT_LEN off the AccuracyTask classes the param actually evaluates and pad up to an 8 KiB boundary, so the cap tracks the tasks instead of restating their budgets as literals. Values are unchanged: 40960 for the GPQA Diamond (fp8kv) path, 8192 for MMLU + GSM8K. Signed-off-by: Pranav Shrestha <254760092+pranav-nvidia@users.noreply.github.com> --- .../integration/defs/accuracy/test_llm_api_pytorch.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index fc3f59fde1d9..dcf1ff8a436e 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -37,6 +37,7 @@ SADecodingConfig, SamplingParams, SchedulerConfig, SkipSoftmaxAttentionConfig, SAEnhancerConfig, TorchCompileConfig) # isort: on +from tensorrt_llm.math_utils import pad_up from tensorrt_llm.quantization import QuantAlgo from ..conftest import (check_device_contain, get_device_count, @@ -3403,8 +3404,15 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, pytest.skip("Not supported MoE backend!") moe_config = MoeConfig() kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.7) + # Cap max_seq_len based on the evaluation task (NVBug 6476233). - extra_llm_args["max_seq_len"] = 40960 if fp8kv else 8192 + MAX_SEQ_LEN_GRANULARITY = 8 * 1024 # Padding boundary for max_seq_len + + task_classes = (GPQADiamond, ) if fp8kv else (MMLU, GSM8K) + max_task_seq_len = max(task.MAX_INPUT_LEN + task.MAX_OUTPUT_LEN + for task in task_classes) + extra_llm_args["max_seq_len"] = pad_up(max_task_seq_len, + MAX_SEQ_LEN_GRANULARITY) pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler,