From 983de685477965eaca27c6807fef7a0148953373 Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Sun, 26 Apr 2026 00:36:42 -0700 Subject: [PATCH 1/2] [https://nvbugs/6105768][fix] Fix disaggregated cancel stress test OOM on L40S The test_disaggregated_cancel_large_context_requests test fails with OOM on L40S (44.4 GiB) because DeepSeek-V3-Lite bf16 requires ~37 GiB per disaggregated worker, and two workers sharing a single GPU need ~74 GiB. Add runtime GPU memory detection to fall back to TinyLlama with a smaller config on single-GPU systems with <80 GiB memory. This preserves the test's cancellation stress-test coverage while fitting within L40S memory constraints. On H100 or multi-GPU systems, the original DeepSeek-V3-Lite bf16 model is still used. Also adds a TinyLlama-compatible disagg config with conservative memory fractions to prevent KV cache allocation races on shared GPUs, and removes the test waiver from waives.txt. Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- ...isagg_config_cancel_stress_test_small.yaml | 39 +++++++++++++++++++ .../defs/disaggregated/test_disaggregated.py | 35 ++++++++++++++--- 2 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_cancel_stress_test_small.yaml diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_cancel_stress_test_small.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_cancel_stress_test_small.yaml new file mode 100644 index 000000000000..3bf22772483a --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_cancel_stress_test_small.yaml @@ -0,0 +1,39 @@ +hostname: localhost +model: TinyLlama/TinyLlama-1.1B-Chat-v1.0 +backend: pytorch +enable_autotuner: false +context_servers: + disable_overlap_scheduler: true + num_instances: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + max_num_tokens: 2048 + max_seq_len: 2048 + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: true + enable_partial_reuse: true + free_gpu_memory_fraction: 0.2 + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 2048 + cuda_graph_config: + enable_padding: true + max_batch_size: 1 +generation_servers: + num_instances: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + max_num_tokens: 2048 + max_seq_len: 2048 + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: true + enable_partial_reuse: true + free_gpu_memory_fraction: 0.3 + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 2048 + cuda_graph_config: + enable_padding: true + max_batch_size: 64 diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 6b9bb8469e1b..dd5b2ebde8da 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -418,6 +418,8 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml", "cancel_stress_test": f"{test_configs_root}/disagg_config_cancel_stress_test.yaml", + "cancel_stress_test_small": + f"{test_configs_root}/disagg_config_cancel_stress_test_small.yaml", "qwen3_8b": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_qwen3_8b.yaml", "mamba_conc_greater_than_mbs": @@ -3988,7 +3990,8 @@ def run_disaggregated_cancel_test(example_dir, requests_per_burst=64, server_start_timeout=1200, model_path=None, - cwd=None): + cwd=None, + prompt_len_range=(2000, 8000)): """Run disaggregated test with request cancellation stress test.""" cleanup_output_files() run_env = env.copy() @@ -4014,7 +4017,8 @@ def run_disaggregated_cancel_test(example_dir, # Run the cancel stress test run_cancel_stress_test(server_url, num_bursts=num_bursts, - requests_per_burst=requests_per_burst) + requests_per_burst=requests_per_burst, + prompt_len_range=prompt_len_range) # Create a temporary client config with the correct dynamic port client_config = config.copy() @@ -4057,16 +4061,35 @@ def test_disaggregated_cancel_large_context_requests(disaggregated_test_root, This test sends bursts of requests with large contexts and cancels them during prefill to stress test resource cleanup. + + DeepSeek-V3-Lite bf16 (~37 GiB) requires two disagg workers on separate + GPUs. On single-GPU systems with <80 GiB, fall back to TinyLlama to avoid + OOM while still exercising the cancellation code path. """ - setup_model_symlink(llm_venv, deepseek_v3_model_root, - "DeepSeek-V3-Lite/bf16") + import torch + gpu_mem_gib = torch.cuda.get_device_properties(0).total_memory / (1024**3) + num_gpus = torch.cuda.device_count() + + if gpu_mem_gib < 80 and num_gpus < 2: + model_path = os.path.join(llm_models_root(), "llama-models-v2", + "TinyLlama-1.1B-Chat-v1.0") + setup_model_symlink(llm_venv, model_path, + "TinyLlama/TinyLlama-1.1B-Chat-v1.0") + test_desc = "cancel_stress_test_small" + prompt_len_range = (200, 800) + else: + model_path = deepseek_v3_model_root + setup_model_symlink(llm_venv, model_path, "DeepSeek-V3-Lite/bf16") + test_desc = "cancel_stress_test" + prompt_len_range = (2000, 8000) run_disaggregated_cancel_test(disaggregated_example_root, - "cancel_stress_test", + test_desc, env=llm_venv._new_env, num_bursts=5, requests_per_burst=32, - model_path=deepseek_v3_model_root, + model_path=model_path, + prompt_len_range=prompt_len_range, cwd=llm_venv.get_working_directory()) From c4cd9e87b7a03a12bb85cd2d33541b94545e46b9 Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:36:08 -0700 Subject: [PATCH 2/2] [nvbugs/6105768][chore] Remove stale waiver after fix Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 052014687eb0..0655b913cb0a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -38,7 +38,6 @@ accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_ cpp/test_multi_gpu.py::test_cache_transceiver[8proc-mooncake_kvcache-90] SKIP (https://nvbugs/5838199) cpp/test_multi_gpu.py::test_cache_transceiver[8proc-nixl_kvcache-90] SKIP (https://nvbugs/5838199) cpp/test_multi_gpu.py::test_cache_transceiver[8proc-ucx_kvcache-90] SKIP (https://nvbugs/5838199) -disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_cache_aware_balance[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_conditional[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_gen_only[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322)