From 16fcf030310344c57107d97ee39f60188b010a67 Mon Sep 17 00:00:00 2001 From: Allison Lim Date: Tue, 18 Aug 2026 14:40:06 -0700 Subject: [PATCH 1/2] [test] unwaive Nemotron Nano FP8 CUDA graph test on DGX B200 Signed-off-by: Allison Lim --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index dd9297f4f941..d79b7992810e 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -173,7 +173,6 @@ full:B300/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_servi full:B300/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6475623) full:B300/unittest/_torch/attention/test_attention_backends.py::test_attention_backend[qwen2_0_5b_gqa_hd64-ctx-bf16-HND-p32-v1] SKIP (https://nvbugs/6610548) full:DGX_B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Pro::test_gsm8k_full_accuracy SKIP (https://nvbugs/6571418) -full:DGX_B200/accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph] SKIP (https://nvbugs/6631019) full:DGX_B200/disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] SKIP (https://nvbugs/6594241) full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-gemma4_26b_a4b_nvfp4_blackwell-gemma4_26b_a4b_nvfp4_tp1_1k1k] SKIP (https://nvbugs/6571410) full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b_spec_decode-llama8b_spec_bs1_128_128] SKIP (https://nvbugs/6571408) From 81149dc5de7a347aee7ed7b3abea6edc97f5c41a Mon Sep 17 00:00:00 2001 From: Allison Lim Date: Tue, 18 Aug 2026 16:00:51 -0700 Subject: [PATCH 2/2] [test] reset compile mode in reused MPI workers Signed-off-by: Allison Lim --- tests/test_common/grouped_test_utils.py | 18 +++++++++--------- tests/unittest/llmapi/test_session_reuse.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/test_common/grouped_test_utils.py b/tests/test_common/grouped_test_utils.py index aa77aba7ed1a..174c6a59887b 100644 --- a/tests/test_common/grouped_test_utils.py +++ b/tests/test_common/grouped_test_utils.py @@ -59,17 +59,17 @@ def submit_sync_per_worker(mpi_session, fn, timeout: float = 60.0) -> list: def reset_worker_torch_compile_state() -> None: - """Reset per-worker torch.compile / Dynamo state (runs inside each worker). + """Reset per-worker torch.compile state (runs inside each worker). - Dynamo's recompile counter is process-global and per-code-object. When - worker processes are reused across LLMs (shared ``MpiPoolSession``), each - ``torch_compile`` case recompiles the same ``model.forward`` code object - under new guards; the count accumulates and eventually trips - ``recompile_limit`` (16), which is a HARD failure under ``fullgraph=True`` - (``FailOnRecompileLimitHit``) and aborts the whole MPI job. Resetting - between cases makes each LLM start from a clean compile cache, like a fresh - process. Run on every worker via ``submit_sync_per_worker``. + Both Dynamo's recompile counter and TensorRT-LLM's compile-mode flag are + process-global. When worker processes are reused across LLMs (shared + ``MpiPoolSession``), each case must start with a clean compile cache and + compile mode disabled, like a fresh process. Run on every worker via + ``submit_sync_per_worker``. """ import torch + from tensorrt_llm._torch.utils import set_torch_compiling + torch._dynamo.reset() + set_torch_compiling(False) diff --git a/tests/unittest/llmapi/test_session_reuse.py b/tests/unittest/llmapi/test_session_reuse.py index ef693c291c10..e20b670d2576 100644 --- a/tests/unittest/llmapi/test_session_reuse.py +++ b/tests/unittest/llmapi/test_session_reuse.py @@ -83,6 +83,22 @@ def test_reuse_hands_back_same_pool(reuse_cache): assert len(reuse_cache.prefetch.restocks) == 1 # reuse does not create a shadow +def test_worker_reset_clears_tensorrt_llm_compile_mode(monkeypatch): + import torch + from test_common.grouped_test_utils import reset_worker_torch_compile_state + + from tensorrt_llm._torch.utils import is_torch_compiling, set_torch_compiling + + dynamo_resets = [] + monkeypatch.setattr(torch._dynamo, "reset", lambda: dynamo_resets.append(True)) + set_torch_compiling(True) + + reset_worker_torch_compile_state() + + assert dynamo_resets == [True] + assert not is_torch_compiling() + + def test_cached_handover_reaps_in_flight_retires(reuse_cache): # Two same-size pools released back-to-back (concurrent LLMs in one # test): the duplicate is retired in a BACKGROUND thread while holding