From 7e1ee241582ff97d1e465a378d211259d331f6dd Mon Sep 17 00:00:00 2001 From: William Zhang <133824995+2ez4bz@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:16:45 -0700 Subject: [PATCH 1/2] [https://nvbugs/6336747][ci] Waive intermittently hanging E2E tests Signed-off-by: William Zhang <133824995+2ez4bz@users.noreply.github.com> --- tests/integration/defs/accuracy/test_epd_disagg_multimodal.py | 2 ++ tests/integration/test_lists/waives.txt | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tests/integration/defs/accuracy/test_epd_disagg_multimodal.py b/tests/integration/defs/accuracy/test_epd_disagg_multimodal.py index 20659f9b7ca9..f32d0098b668 100644 --- a/tests/integration/defs/accuracy/test_epd_disagg_multimodal.py +++ b/tests/integration/defs/accuracy/test_epd_disagg_multimodal.py @@ -282,6 +282,8 @@ def _run_videomme(self, llm, variant: EPDVariant) -> None: ), ], ) + # `torch.compile` uses a thread pool to compile and it's used in audio pre-processing. + @pytest.mark.threadleak(enabled=False) def test_disaggregated_videomme(self, variant: EPDVariant) -> None: """Run VideoMME shard through a model-specific llmapi E/PD config.""" with self._launch_epd(variant) as llm: diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 54ee277060a7..381214b8fefd 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -6,6 +6,8 @@ accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram SKIP (https://nvbugs/6245651) accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] SKIP (https://nvbugs/6402054) accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) +accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] SKIP (https://nvbugs/6336747) +accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct] SKIP (https://nvbugs/6336747) accuracy/test_llm_api.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] SKIP (https://nvbugs/5346443) accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma3n_e2b_it SKIP (https://nvbugs/6417265) accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8 SKIP (https://nvbugs/6396422) @@ -85,6 +87,8 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency] SKIP (https:/ accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3] SKIP (https://nvbugs/6367805) accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=3] SKIP (https://nvbugs/6367805) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) +accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph] (https://nvbugs/6336747) +accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[nvfp4] SKIP (https://nvbugs/6336747) accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3] SKIP (https://nvbugs/6367805) cpp/test_e2e.py::test_benchmarks[bart-90] SKIP (https://nvbugs/5550689) cpp/test_e2e.py::test_benchmarks[gpt-80] SKIP (https://nvbugs/5550689) From 91447f49786227d2d18e958c5f46aa2588d4c9a8 Mon Sep 17 00:00:00 2001 From: William Zhang <133824995+2ez4bz@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:44:00 -0700 Subject: [PATCH 2/2] fix waive entry Signed-off-by: William Zhang <133824995+2ez4bz@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index a6768d20a1d3..6c05c3b03f88 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -91,7 +91,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency] SKIP (https:/ accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3] SKIP (https://nvbugs/6367805) accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=3] SKIP (https://nvbugs/6367805) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) -accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph] (https://nvbugs/6336747) +accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph] SKIP (https://nvbugs/6336747) accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[nvfp4] SKIP (https://nvbugs/6336747) accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3] SKIP (https://nvbugs/6367805) cpp/test_e2e.py::test_benchmarks[bart-90] SKIP (https://nvbugs/5550689)