From 83cb292bb2790c97d24afee5bedda0bb4b5b952f Mon Sep 17 00:00:00 2001 From: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> Date: Sat, 30 May 2026 05:14:05 -0700 Subject: [PATCH 1/5] [nvbugs/6221055][fix] Add GB300 rows to agg_unit_mem_df.csv The GB300-4_GPUs-PyTorch-Post-Merge-1 stage was timing out at the 4h Slurm budget because test_unittests_v2 fell back to num_workers=1 on GB300 nodes. The fallback was triggered by a missing lookup row in tests/integration/defs/agg_unit_mem_df.csv: pynvml reports the GB300 device name as 'NVIDIA B300 SXM6 AC' but the CSV only had entries for B200 / H100 / B200-bring-up. With xdist disabled, the 5,011-case unittest/_torch/thop/parallel suite alone needs ~3h and the stage cannot finish in time. Add 'NVIDIA B300 SXM6 AC' rows for the unittest/_torch cases scheduled on GB300 stages (l0_gb300.yml, l0_gb300_multi_gpus.yml), mirroring the parallel_factor values already used for B200/H100. test_unittests_v2 clamps num_workers to 8, so 16 for thop/parallel matches the other Blackwell stages. Signed-off-by: tensorrt-cicd <90828364+tensorrt-cicd@users.noreply.github.com> --- tests/integration/defs/agg_unit_mem_df.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/defs/agg_unit_mem_df.csv b/tests/integration/defs/agg_unit_mem_df.csv index 637c57aca632..8da8b8929d77 100644 --- a/tests/integration/defs/agg_unit_mem_df.csv +++ b/tests/integration/defs/agg_unit_mem_df.csv @@ -138,3 +138,7 @@ unittest/_torch/misc,NVIDIA H100,4, unittest/_torch/thop/parallel,NVIDIA H100,16, unittest/_torch/thop/parallel_hw_agnostic,NVIDIA H100,16, unittest/_torch/flashinfer/test_trtllm_flashinfer_symbol_collision.py,NVIDIA H100 PCIe,1, +unittest/_torch/attention,NVIDIA B300 SXM6 AC,4, +unittest/_torch/thop/parallel,NVIDIA B300 SXM6 AC,16, +unittest/_torch/thop/serial,NVIDIA B300 SXM6 AC,4, +unittest/_torch/executor,NVIDIA B300 SXM6 AC,4, From f8b9ca9568e5a1031dc93d956c592f34142b9bb6 Mon Sep 17 00:00:00 2001 From: QI JUN <22017000+QiJune@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:52:27 +0800 Subject: [PATCH 2/5] unwaive thop/parallel Signed-off-by: QI JUN <22017000+QiJune@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 008aeef59266..54bae2a7999d 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -260,7 +260,6 @@ full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_d full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) full:GB300/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) -full:GB300/unittest/_torch/thop/parallel SKIP (https://nvbugs/6422443) full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) full:H100/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6313072) From c89e1037b6dce14d67f4bbd13b388289c14901a6 Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:58:54 +0800 Subject: [PATCH 3/5] [CI] Print GPU name for unittest parallel config Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/defs/test_unittests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index da4e71922ad0..a535044e69af 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -84,13 +84,19 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): # This dataframe is not manually edited. Infra team will regularly generate this dataframe based on test execution results. # If you need to override this policy, please use postprocess code as below. - agg_unit_mem_df = pd.read_csv( - f'{test_root}/integration/defs/agg_unit_mem_df.csv') + agg_unit_mem_path = f'{test_root}/integration/defs/agg_unit_mem_df.csv' + print(f'Loading unittest parallel config from: {agg_unit_mem_path}') + agg_unit_mem_df = pd.read_csv(agg_unit_mem_path) gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0) gpu_name = pynvml.nvmlDeviceGetName(gpu_handle) if isinstance(gpu_name, bytes): gpu_name = gpu_name.decode() + print(f'GPU name from NVML (index 0): {gpu_name!r}') + print(f'GPU names available in parallel config: ' + f'{sorted(agg_unit_mem_df["gpu"].dropna().unique().tolist())}') agg_unit_mem_df = agg_unit_mem_df[agg_unit_mem_df['gpu'] == gpu_name] + print(f'Matched parallel config rows for GPU {gpu_name!r}: ' + f'{len(agg_unit_mem_df)}') print(agg_unit_mem_df) parallel_dict = {} @@ -101,6 +107,7 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): print(parallel_dict) cur_key = (gpu_name, case) + print(f'Parallel config lookup key: {cur_key!r}') if cur_key in parallel_dict: num_workers = parallel_dict[cur_key] num_workers = min(num_workers, 8) From 8636cf7e62d3fa6621f234441b078226e0c59f0f Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:19:38 +0800 Subject: [PATCH 4/5] [CI] Add GB300 unittest parallel config Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/defs/agg_unit_mem_df.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/defs/agg_unit_mem_df.csv b/tests/integration/defs/agg_unit_mem_df.csv index 32a14beaf364..ca2814c2a55a 100644 --- a/tests/integration/defs/agg_unit_mem_df.csv +++ b/tests/integration/defs/agg_unit_mem_df.csv @@ -142,3 +142,7 @@ unittest/_torch/attention,NVIDIA B300 SXM6 AC,4, unittest/_torch/thop/parallel,NVIDIA B300 SXM6 AC,16, unittest/_torch/thop/serial,NVIDIA B300 SXM6 AC,4, unittest/_torch/executor,NVIDIA B300 SXM6 AC,4, +unittest/_torch/attention,NVIDIA GB300,4, +unittest/_torch/thop/parallel,NVIDIA GB300,16, +unittest/_torch/thop/serial,NVIDIA GB300,4, +unittest/_torch/executor,NVIDIA GB300,4, From ebf9a261fd229c00d6a59326d456c0f29e10030b Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:23:35 +0800 Subject: [PATCH 5/5] [CI] Remove unused B300 unittest parallel config Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/defs/agg_unit_mem_df.csv | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/defs/agg_unit_mem_df.csv b/tests/integration/defs/agg_unit_mem_df.csv index ca2814c2a55a..12c220501c5f 100644 --- a/tests/integration/defs/agg_unit_mem_df.csv +++ b/tests/integration/defs/agg_unit_mem_df.csv @@ -138,10 +138,6 @@ unittest/_torch/misc,NVIDIA H100,4, unittest/_torch/thop/parallel,NVIDIA H100,16, unittest/_torch/thop/parallel_hw_agnostic,NVIDIA H100,16, unittest/_torch/flashinfer/test_trtllm_flashinfer_symbol_collision.py,NVIDIA H100 PCIe,1, -unittest/_torch/attention,NVIDIA B300 SXM6 AC,4, -unittest/_torch/thop/parallel,NVIDIA B300 SXM6 AC,16, -unittest/_torch/thop/serial,NVIDIA B300 SXM6 AC,4, -unittest/_torch/executor,NVIDIA B300 SXM6 AC,4, unittest/_torch/attention,NVIDIA GB300,4, unittest/_torch/thop/parallel,NVIDIA GB300,16, unittest/_torch/thop/serial,NVIDIA GB300,4,