From 923302ec721b819bcd0e370a0c71c40aee652905 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:39:04 +0800 Subject: [PATCH 01/14] [TRTLLM-15078][test] Remove all DeepSeek-R1-Distill-* tests Remove test functions, parametrize entries, model path registrations, accuracy references, perf configs, and test list entries for all DeepSeek-R1-Distill-* model variants across unittest and integration tests. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../defs/accuracy/references/mmlu.yaml | 2 - .../defs/accuracy/test_llm_api_pytorch.py | 14 ------ tests/integration/defs/perf/_model_paths.py | 2 - .../defs/perf/pytorch_model_config.py | 10 ---- tests/integration/defs/test_e2e.py | 50 ------------------- .../test_lists/qa/llm_function_core.txt | 2 - .../test_lists/qa/llm_spark_func.yml | 3 -- .../test_lists/qa/llm_spark_perf.yml | 2 - .../test_lists/test-db/l0_l40s.yml | 1 - .../llmapi/apps/_test_openai_reasoning.py | 2 - tests/unittest/scaffolding/test_worker.py | 28 +---------- 11 files changed, 1 insertion(+), 115 deletions(-) diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 504fde787931..754b343c63ea 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -202,8 +202,6 @@ deepseek-ai/DeepSeek-R1-0528: - quant_algo: FP8_BLOCK_SCALES kv_cache_quant_algo: FP8 accuracy: 84.722 -deepseek-ai/DeepSeek-R1-Distill-Llama-70B: - - accuracy: 78.19 deepseek-ai/DeepSeek-V3.2-Exp: - quant_algo: FP8_BLOCK_SCALES accuracy: 88.2 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index ea8ed2ab89fd..a65e2c70ef3e 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -3306,20 +3306,6 @@ def test_fp8_blockscale_chunked_prefill(self, tp_size, pp_size, ep_size, task.evaluate(llm) -class TestDeepSeekR1DistillLlama70B(LlmapiAccuracyTestHarness): - MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Llama-70B" - MODEL_PATH = f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1-Distill-Llama-70B" - - @skip_pre_hopper - @pytest.mark.skip_less_mpi_world_size(2) - def test_auto_dtype_tp2(self): - kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.4) - _run_multinode_accuracy(self.MODEL_PATH, - self.MODEL_NAME, - benchmarks=["mmlu"], - kv_cache_config=kv_cache_config) - - @pytest.mark.timeout(14400) @pytest.mark.skip_less_mpi_world_size(8) class TestDeepSeekV3(LlmapiAccuracyTestHarness): diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index 74357e5a4065..97fbb163197d 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -26,8 +26,6 @@ "llama_v4_scout_17b_16e_instruct": "llama4-models/Llama-4-Scout-17B-16E-Instruct", "llama_v4_scout_17b_16e_instruct_fp8": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8", "llama_v4_scout_17b_16e_instruct_fp4": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4", - "deepseek_r1_distill_qwen_32b": "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-32B", - "deepseek_r1_distill_llama_70b": "DeepSeek-R1/DeepSeek-R1-Distill-Llama-70B/", "gemma_3_27b_it": "gemma/gemma-3-27b-it", "gemma_3_27b_it_fp8": "gemma/gemma-3-27b-it-fp8", "gemma_3_27b_it_fp4": "gemma/gemma-3-27b-it-FP4", diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 409bd0343d9c..d5ad33b48f5b 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -291,16 +291,6 @@ def get_model_yaml_config(model_label: str, }, } }, - # Model-specific cases with attention_dp disabled to prevent hangs - { - 'patterns': [ - 'deepseek_r1_distill_llama_70b', - ], - 'config': { - # True causes hang, needs model-specific fix. - 'enable_attention_dp': False, - } - }, # Qwen3 models with fp4 quantization on B200 and fp8 quantization on H200/H20 { 'patterns': [ diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 4ef2c596443a..5abab27416f0 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -40,34 +40,6 @@ _MEM_FRACTION_95 = 0.95 -@pytest.mark.parametrize("model_name,model_path", [ - ("DeepSeek-R1-Distill-Qwen-1.5B", "DeepSeek-R1-Distill-Qwen-1.5B"), -]) -def test_qwen_e2e_cpprunner_large_new_tokens(model_name, model_path, llm_venv): - """RCCA: https://nvbugs/5238105 - none of n>1 sequences may be empty.""" - from tensorrt_llm import LLM, SamplingParams - - prompt = r"<｜begin▁of▁sentence｜><｜User｜>The operation $\otimes$ is defined for all nonzero numbers by $a \otimes b = \frac{a^{2}}{b}$. Determine $[(1 \otimes 2) \otimes 3] - [1 \otimes (2 \otimes 3)]$. Let's think step by step and output the final answer within \boxed{}.<｜Assistant｜>" - - sampling_params = SamplingParams( - max_tokens=1024, - n=4, - temperature=0.6, - top_p=1.0, - top_k=1024, - ) - - with LLM(model=f"{llm_models_root()}/{model_path}", - max_batch_size=8, - max_seq_len=4224) as llm: - outputs = llm.generate([prompt], sampling_params=sampling_params) - - completions = outputs[0].outputs - seq_lengths = [len(c.token_ids) for c in completions] - assert all(length > 0 for length in seq_lengths), \ - f"Found zero-length completion: {seq_lengths}" - - # TODO replace the trtllm_bench_prolog class BenchRunner: @@ -918,9 +890,6 @@ def test_ptp_quickstart(llm_root, llm_venv): pytest.param('Mistral-Nemo-12b-Base', 'Mistral-Nemo-Base-2407', marks=skip_pre_blackwell), - pytest.param('DeepSeek-R1-Distill-Qwen-32B', - 'DeepSeek-R1/DeepSeek-R1-Distill-Qwen-32B', - marks=skip_pre_blackwell), pytest.param('GPT-OSS-20B', 'gpt_oss/gpt-oss-20b', marks=skip_pre_blackwell), pytest.param( @@ -1854,23 +1823,6 @@ def test_ptp_quickstart_bert(llm_root, llm_venv, model_name, model_path, print("Success: HF model logits match TRTLLM logits!") -@pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize("model_name,model_path", [ - ("DeepSeek-R1-Distill-Qwen-7B", "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B"), -]) -def test_ptp_scaffolding(llm_root, llm_venv, model_name, model_path): - print(f"Testing scaffolding {model_name}.") - example_root = Path(os.path.join(llm_root, "examples", "scaffolding")) - input_file = Path(os.path.join(example_root, "test.jsonl")) - llm_venv.run_cmd([ - str(example_root / "run_majority_vote_aime24.py"), - "--model_dir", - f"{llm_models_root()}/{model_path}", - f"--jsonl_file={input_file}", - "--threshold=0.5", - ]) - - @pytest.mark.timeout(5400) @pytest.mark.skip_less_device_memory(80000) @pytest.mark.skip_less_device(4) @@ -1933,8 +1885,6 @@ def test_multi_nodes_eval(model_path, tp_size, pp_size, ep_size, eval_task, marks=skip_pre_hopper), pytest.param('Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf', marks=skip_pre_blackwell), - pytest.param('DeepSeek-R1/DeepSeek-R1-Distill-Llama-70B', - marks=skip_pre_hopper), pytest.param('llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8', marks=skip_pre_hopper), pytest.param('llama4-models/Llama-4-Scout-17B-16E-Instruct', diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 9ed9c317c98e..ef8a78021602 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -960,8 +960,6 @@ test_e2e.py::test_ptp_quickstart_advanced_pp_enabled[Llama3.3-70B-FP8-llama-3.3- test_e2e.py::test_ptp_quickstart_advanced_pp_enabled[Llama3.3-70B-FP8-llama-3.3-models/Llama-3.3-70B-Instruct-FP8-2-4-True] test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] -test_e2e.py::test_ptp_scaffolding[DeepSeek-R1-Distill-Qwen-7B-DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B] -test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1] test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] test_e2e.py::test_trtllm_multimodal_benchmark_serving diff --git a/tests/integration/test_lists/qa/llm_spark_func.yml b/tests/integration/test_lists/qa/llm_spark_func.yml index e6d95b720761..af5849e3efb6 100644 --- a/tests/integration/test_lists/qa/llm_spark_func.yml +++ b/tests/integration/test_lists/qa/llm_spark_func.yml @@ -32,7 +32,6 @@ llm_spark_func: - test_e2e.py::test_ptp_quickstart_advanced[Llama3.3-70B-FP8-modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8] - test_e2e.py::test_ptp_quickstart_advanced[Llama3.3-70B-FP4-modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp4] - test_e2e.py::test_ptp_quickstart_advanced[Llama-4-Scout-17B-16E-FP4-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4] - - test_e2e.py::test_ptp_quickstart_advanced[DeepSeek-R1-Distill-Qwen-32B-DeepSeek-R1/DeepSeek-R1-Distill-Qwen-32B] - test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] - test_e2e.py::test_ptp_quickstart_advanced_eagle3[GPT-OSS-120B-Eagle3-gpt_oss/gpt-oss-120b-gpt_oss/gpt-oss-120b-Eagle3] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype @@ -67,10 +66,8 @@ llm_spark_func: tests: - test_e2e.py::test_ptp_quickstart_advanced_multinode[llama-3.3-models/Llama-3.3-70B-Instruct-tp2] - test_e2e.py::test_ptp_quickstart_advanced_multinode[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp2] - - test_e2e.py::test_ptp_quickstart_advanced_multinode[DeepSeek-R1/DeepSeek-R1-Distill-Llama-70B-tp2] - test_e2e.py::test_ptp_quickstart_advanced_multinode[llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8-tp2] - test_e2e.py::test_ptp_quickstart_advanced_multinode[llama4-models/Llama-4-Scout-17B-16E-Instruct-tp2] - accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_auto_dtype_tp2 - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_2gpus[latency_moe_cutlass] - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_2gpus[latency_moe_cutlass_eagle3] - - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1DistillLlama70B::test_auto_dtype_tp2 diff --git a/tests/integration/test_lists/qa/llm_spark_perf.yml b/tests/integration/test_lists/qa/llm_spark_perf.yml index 605466a982f6..7dfa462ec166 100644 --- a/tests/integration/test_lists/qa/llm_spark_perf.yml +++ b/tests/integration/test_lists/qa/llm_spark_perf.yml @@ -45,7 +45,6 @@ llm_spark_perf: - perf/test_perf.py::test_perf[phi_4_reasoning_plus-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[qwen3_32b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[qwen3_32b-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[deepseek_r1_distill_qwen_32b-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[phi_4_multimodal_instruct-bench-pytorch-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[phi_4_multimodal_instruct_fp4-bench-pytorch-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[phi_4_multimodal_instruct_fp8-bench-pytorch-float8-maxbs:1-input_output_len:2048,128-reqs:1-con:1] @@ -70,7 +69,6 @@ llm_spark_perf: - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-kv_cache_dtype:fp8-reqs:1-con:1-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:128,2048-kv_cache_dtype:fp8-reqs:1-con:1-tp:2-gpus:2] - - perf/test_perf.py::test_perf[deepseek_r1_distill_llama_70b-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v4_scout_17b_16e_instruct_fp8-bench-pytorch-streaming-float8-maxbs:1-input_output_len:2048,128-kv_cache_dtype:fp8-reqs:1-con:1-ep:2-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v4_scout_17b_16e_instruct-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1-tp:2-gpus:2] # Qwen3-235B-A22B-FP4 with Eagle3 speculative decoding diff --git a/tests/integration/test_lists/test-db/l0_l40s.yml b/tests/integration/test_lists/test-db/l0_l40s.yml index ff242644533f..f84371a7c24f 100644 --- a/tests/integration/test_lists/test-db/l0_l40s.yml +++ b/tests/integration/test_lists/test-db/l0_l40s.yml @@ -25,7 +25,6 @@ l0_l40s: - unittest/_torch/modeling/test_modeling_qwen3vl.py::test_qwen3vl_init_preserves_caller_quant_config - unittest/_torch/modeling/test_modeling_cosmos3.py::TestCosmos3::test_all - unittest/_torch/modeling/test_modeling_step3p7vl.py - - test_e2e.py::test_ptp_scaffolding[DeepSeek-R1-Distill-Qwen-7B-DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B] - unittest/llmapi/apps/_test_openai_chat_multimodal.py::test_single_chat_session_image_embeds -m needs_l40s # Encoder-only /v1/embeddings dynamic batching (BERT classifier + PRM-7B reward + Qwen3-Embedding) - unittest/llmapi/apps/_test_openai_embeddings.py diff --git a/tests/unittest/llmapi/apps/_test_openai_reasoning.py b/tests/unittest/llmapi/apps/_test_openai_reasoning.py index 5c95299d01ac..f8d421ceb6f8 100644 --- a/tests/unittest/llmapi/apps/_test_openai_reasoning.py +++ b/tests/unittest/llmapi/apps/_test_openai_reasoning.py @@ -12,8 +12,6 @@ @pytest.fixture( scope="module", params=[ - ("DeepSeek-R1-Distill-Qwen-1.5B", "pytorch", False), - ("DeepSeek-R1-Distill-Qwen-1.5B", "pytorch", True), ("Qwen3/Qwen3-0.6B", "pytorch", False), ("Qwen3/Qwen3-0.6B", "pytorch", True), ], diff --git a/tests/unittest/scaffolding/test_worker.py b/tests/unittest/scaffolding/test_worker.py index b5aed772232e..ca04f0f50699 100644 --- a/tests/unittest/scaffolding/test_worker.py +++ b/tests/unittest/scaffolding/test_worker.py @@ -13,10 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path - # isort: off -from utils.llm_data import llm_models_root # isort: on import asyncio @@ -27,19 +24,12 @@ from llmapi.apps.openai_server import RemoteOpenAIServer from tensorrt_llm.scaffolding import (ChatTask, GenerationTask, TaskStatus, - TRTLLMWorker, TRTOpenaiWorker, - UserMessage) + TRTOpenaiWorker, UserMessage) sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from llmapi.test_llm import get_model_path -@pytest.fixture(scope="module") -def deepseek_distill_7b_path() -> Path: - model_dir = llm_models_root() / "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B" - return model_dir - - @pytest.fixture(scope="module") def default_prompt(): prompt = "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?\r\n\r\n" @@ -48,7 +38,6 @@ def default_prompt(): @pytest.fixture(scope="module") def model_name(): - #return "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-7B" return "gpt_oss/gpt-oss-20b" @@ -103,18 +92,3 @@ def test_trtoai_worker_chat(default_prompt, model_name, server): assert status == TaskStatus.SUCCESS, "Chat Task is not successful with TRTOpenaiWorker" finally: worker.shutdown() - - -def create_trtllm_worker(model_path): - return TRTLLMWorker.init_with_new_llm(str(model_path), backend="pytorch") - - -def test_trtllm_worker_generation(default_prompt, deepseek_distill_7b_path): - worker = create_trtllm_worker(deepseek_distill_7b_path) - try: - task = GenerationTask.create_from_prompt(default_prompt) - task.max_tokens = 100 - status = asyncio.run(worker.run_task(task)) - assert status == TaskStatus.SUCCESS, "Generation Task is not successful with TRTLLMWorker" - finally: - worker.shutdown() From af939c61e7c9d30ffeef63bd0bf6743e561f743c Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:12:37 +0800 Subject: [PATCH 02/14] [TRTLLM-15078][test] Fix broken import in test_scaffolding.py Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../unittest/scaffolding/test_scaffolding.py | 29 +++++++++---------- tests/unittest/scaffolding/test_worker.py | 13 ++++++++- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/tests/unittest/scaffolding/test_scaffolding.py b/tests/unittest/scaffolding/test_scaffolding.py index fad6c4dbf794..970f08fd72e1 100644 --- a/tests/unittest/scaffolding/test_scaffolding.py +++ b/tests/unittest/scaffolding/test_scaffolding.py @@ -1,16 +1,15 @@ # autoflake: skip_file -from scaffolding.test_worker import (create_trtllm_worker, - deepseek_distill_7b_path, default_prompt) +from scaffolding.test_worker import (create_trtllm_worker, default_prompt, + trtllm_model_path) from tensorrt_llm.scaffolding import (MajorityVoteController, NativeGenerationController, ScaffoldingLlm) -def create_scaffolding_llm_with_native_generation_controller( - deepseek_distill_7b_path): - trtllm_worker = create_trtllm_worker(deepseek_distill_7b_path) +def create_scaffolding_llm_with_native_generation_controller(trtllm_model_path): + trtllm_worker = create_trtllm_worker(trtllm_model_path) prototype_generation_controller = NativeGenerationController( sampling_params={ "max_tokens": 8, @@ -25,8 +24,8 @@ def create_scaffolding_llm_with_native_generation_controller( def create_scaffolding_llm_with_majority_vote_controller( - deepseek_distill_7b_path, samples_num): - trtllm_worker = create_trtllm_worker(deepseek_distill_7b_path) + trtllm_model_path, samples_num): + trtllm_worker = create_trtllm_worker(trtllm_model_path) workers = {} prototype_generation_controller = NativeGenerationController( @@ -46,9 +45,9 @@ def create_scaffolding_llm_with_majority_vote_controller( return llm -def test_unbatched_scaffolding_sync(default_prompt, deepseek_distill_7b_path): +def test_unbatched_scaffolding_sync(default_prompt, trtllm_model_path): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( - deepseek_distill_7b_path) + trtllm_model_path) try: result = scaffolding_llm.generate(default_prompt) assert isinstance(result.outputs[0].text, str) and len( @@ -57,9 +56,9 @@ def test_unbatched_scaffolding_sync(default_prompt, deepseek_distill_7b_path): scaffolding_llm.shutdown(shutdown_workers=True) -def test_batched_scaffolding_sync(default_prompt, deepseek_distill_7b_path): +def test_batched_scaffolding_sync(default_prompt, trtllm_model_path): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( - deepseek_distill_7b_path) + trtllm_model_path) try: batch_size = 3 prompts = [default_prompt] * batch_size @@ -73,11 +72,11 @@ def test_batched_scaffolding_sync(default_prompt, deepseek_distill_7b_path): scaffolding_llm.shutdown(shutdown_workers=True) -def test_async_scaffolding_generation(default_prompt, deepseek_distill_7b_path): +def test_async_scaffolding_generation(default_prompt, trtllm_model_path): async def run_async_test(): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( - deepseek_distill_7b_path) + trtllm_model_path) try: future = scaffolding_llm.generate_async(default_prompt) result = await future.aresult() @@ -91,9 +90,9 @@ async def run_async_test(): asyncio.run(run_async_test()) -def test_majority_vote(default_prompt, deepseek_distill_7b_path): +def test_majority_vote(default_prompt, trtllm_model_path): scaffolding_llm = create_scaffolding_llm_with_majority_vote_controller( - deepseek_distill_7b_path, samples_num=3) + trtllm_model_path, samples_num=3) try: result = scaffolding_llm.generate(default_prompt) assert isinstance(result.outputs[0].text, str) and len( diff --git a/tests/unittest/scaffolding/test_worker.py b/tests/unittest/scaffolding/test_worker.py index ca04f0f50699..3345d5822232 100644 --- a/tests/unittest/scaffolding/test_worker.py +++ b/tests/unittest/scaffolding/test_worker.py @@ -24,10 +24,17 @@ from llmapi.apps.openai_server import RemoteOpenAIServer from tensorrt_llm.scaffolding import (ChatTask, GenerationTask, TaskStatus, - TRTOpenaiWorker, UserMessage) + TRTLLMWorker, TRTOpenaiWorker, + UserMessage) sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from llmapi.test_llm import get_model_path +from utils.llm_data import llm_models_root + + +@pytest.fixture(scope="module") +def trtllm_model_path(): + return llm_models_root() / "gpt_oss/gpt-oss-20b" @pytest.fixture(scope="module") @@ -63,6 +70,10 @@ def server(model_name: str, backend: str, num_postprocess_workers: int): remote_server.terminate() +def create_trtllm_worker(model_path): + return TRTLLMWorker.init_with_new_llm(str(model_path), backend="pytorch") + + def create_trtoai_worker(model_name, async_client): return TRTOpenaiWorker( async_client=async_client, From 8dd36579c21400397d808bf3f8d2f473edabab5f Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:15:07 +0800 Subject: [PATCH 03/14] [TRTLLM-15078][test] Fix test_scaffolding import and remove stale perf entries - Restore create_trtllm_worker helper and add trtllm_model_path fixture pointing at gpt_oss/gpt-oss-20b; rename deepseek_distill_7b_path across test_worker.py and test_scaffolding.py so directory collection no longer fails - Drop empty isort guard block in test_worker.py - Remove deepseek_r1_distill_qwen_32b rows from base_perf_pytorch.csv Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/defs/perf/base_perf_pytorch.csv | 4 ---- tests/unittest/scaffolding/test_worker.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/tests/integration/defs/perf/base_perf_pytorch.csv b/tests/integration/defs/perf/base_perf_pytorch.csv index 2047dde00b35..5d92250cfd60 100644 --- a/tests/integration/defs/perf/base_perf_pytorch.csv +++ b/tests/integration/defs/perf/base_perf_pytorch.csv @@ -3,7 +3,3 @@ network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,p "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.20,5,SEQ_THROUGHPUT,76.45, "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_total_output_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_total_output_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.20,500,TOTAL_OUTPUT_THROUGHPUT,9785.75, "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_kv_cache_size[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_kv_cache_size[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.20,2,KV_CACHE_SIZE,55.64, -"deepseek_r1_distill_qwen_32b-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]","test_perf_metric_inference_time[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]",0.1,50,INFERENCE_TIME,1359184.5059,H100_PCIe -"deepseek_r1_distill_qwen_32b-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_kv_cache_size[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]","test_perf_metric_kv_cache_size[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]",-0.1,50,KV_CACHE_SIZE,10.92,H100_PCIe -"deepseek_r1_distill_qwen_32b-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]","test_perf_metric_seq_throughput[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]",-0.1,10,SEQ_THROUGHPUT,0.3767,H100_PCIe -"deepseek_r1_distill_qwen_32b-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_total_output_throughput[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]","test_perf_metric_total_output_throughput[deepseek_r1_distill_qwen_32b-subtype:H100_PCIe-bench-_autodeploy-float16-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024]",-0.1,10,TOTAL_OUTPUT_THROUGHPUT,385.7372,H100_PCIe diff --git a/tests/unittest/scaffolding/test_worker.py b/tests/unittest/scaffolding/test_worker.py index 3345d5822232..8d2beb468229 100644 --- a/tests/unittest/scaffolding/test_worker.py +++ b/tests/unittest/scaffolding/test_worker.py @@ -13,9 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# isort: off -# isort: on - import asyncio import os import sys From a80ade7e3c226d1652daca120a8938d51eab7ced Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:29:14 +0800 Subject: [PATCH 04/14] [TRTLLM-15078][test] Restore test_trtllm_worker_generation using trtllm_model_path Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/unittest/scaffolding/test_worker.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unittest/scaffolding/test_worker.py b/tests/unittest/scaffolding/test_worker.py index 8d2beb468229..856adf337c6f 100644 --- a/tests/unittest/scaffolding/test_worker.py +++ b/tests/unittest/scaffolding/test_worker.py @@ -78,6 +78,17 @@ def create_trtoai_worker(model_name, async_client): ) +def test_trtllm_worker_generation(default_prompt, trtllm_model_path): + worker = create_trtllm_worker(trtllm_model_path) + try: + task = GenerationTask.create_from_prompt(default_prompt) + task.max_tokens = 100 + status = asyncio.run(worker.run_task(task)) + assert status == TaskStatus.SUCCESS, "Generation Task is not successful with TRTLLMWorker" + finally: + worker.shutdown() + + @pytest.mark.asyncio(loop_scope="module") def test_trtoai_worker_generation(default_prompt, model_name, server): worker = create_trtoai_worker(model_name, server.get_async_client()) From 3dae2e9fa68e53a1f88c5191d9a0faedd84c0061 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:55:24 +0800 Subject: [PATCH 05/14] [TRTLLM-15078][test] Re-add RCCA test for nvbugs/5238105 using Qwen3-0.6B Keep the n>1 zero-length sequence regression guard but switch model from DeepSeek-R1-Distill-Qwen-1.5B (retired) to Qwen3/Qwen3-0.6B. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/defs/test_e2e.py | 28 +++++++++++++++++++ .../test_lists/qa/llm_function_core.txt | 1 + 2 files changed, 29 insertions(+) diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 5abab27416f0..d633cc9fd3d4 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -40,6 +40,34 @@ _MEM_FRACTION_95 = 0.95 +@pytest.mark.parametrize("model_name,model_path", [ + ("Qwen3/Qwen3-0.6B", "Qwen3/Qwen3-0.6B"), +]) +def test_qwen_e2e_cpprunner_large_new_tokens(model_name, model_path, llm_venv): + """RCCA: https://nvbugs/5238105 - none of n>1 sequences may be empty.""" + from tensorrt_llm import LLM, SamplingParams + + prompt = r"The operation $\otimes$ is defined for all nonzero numbers by $a \otimes b = \frac{a^{2}}{b}$. Determine $[(1 \otimes 2) \otimes 3] - [1 \otimes (2 \otimes 3)]$. Let's think step by step." + + sampling_params = SamplingParams( + max_tokens=1024, + n=4, + temperature=0.6, + top_p=1.0, + top_k=1024, + ) + + with LLM(model=f"{llm_models_root()}/{model_path}", + max_batch_size=8, + max_seq_len=4224) as llm: + outputs = llm.generate([prompt], sampling_params=sampling_params) + + completions = outputs[0].outputs + seq_lengths = [len(c.token_ids) for c in completions] + assert all(length > 0 for length in seq_lengths), \ + f"Found zero-length completion: {seq_lengths}" + + # TODO replace the trtllm_bench_prolog class BenchRunner: diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index ef8a78021602..3f164b34293b 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -961,6 +961,7 @@ test_e2e.py::test_ptp_quickstart_advanced_pp_enabled[Llama3.3-70B-FP8-llama-3.3- test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1] +test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[Qwen3/Qwen3-0.6B-Qwen3/Qwen3-0.6B] test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] test_e2e.py::test_trtllm_multimodal_benchmark_serving From da790b5d1507ee103a10cd87f538bc093696c565 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:17:24 +0800 Subject: [PATCH 06/14] [TRTLLM-15078][test] Simplify _test_openai_reasoning: keep beam_search=True only, use qwen3 parser Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/unittest/llmapi/apps/_test_openai_reasoning.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/unittest/llmapi/apps/_test_openai_reasoning.py b/tests/unittest/llmapi/apps/_test_openai_reasoning.py index f8d421ceb6f8..1afb77b64f1b 100644 --- a/tests/unittest/llmapi/apps/_test_openai_reasoning.py +++ b/tests/unittest/llmapi/apps/_test_openai_reasoning.py @@ -12,7 +12,6 @@ @pytest.fixture( scope="module", params=[ - ("Qwen3/Qwen3-0.6B", "pytorch", False), ("Qwen3/Qwen3-0.6B", "pytorch", True), ], ids=lambda p: f"{p[0]}-{p[1]}-{'with' if p[2] else 'no'}_beam_search") @@ -47,10 +46,7 @@ def server(model_name: str, backend: str, enable_beam_search: bool, args = ["--backend", f"{backend}"] args.extend(["--max_beam_width", str(max_beam_width)]) args.extend(["--max_batch_size", "2", "--max_seq_len", "1024"]) - if model_name.startswith("Qwen3"): - args.extend(["--reasoning_parser", "qwen3"]) - else: - args.extend(["--reasoning_parser", "deepseek-r1"]) + args.extend(["--reasoning_parser", "qwen3"]) with RemoteOpenAIServer(model_path, args) as remote_server: yield remote_server From 254fc29d202894deeb2d05aa103ac6a250bee469 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:02:28 +0800 Subject: [PATCH 07/14] [TRTLLM-15078][test] Use Qwen3-0.6B for scaffolding TRTLLMWorker tests gpt_oss/gpt-oss-20b produces empty output via TRTLLMWorker in the MajorityVoteController path; switch to Qwen3-0.6B which generates non-empty text and is lighter weight for unit tests. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/unittest/scaffolding/test_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/scaffolding/test_worker.py b/tests/unittest/scaffolding/test_worker.py index 856adf337c6f..b808cb110b1e 100644 --- a/tests/unittest/scaffolding/test_worker.py +++ b/tests/unittest/scaffolding/test_worker.py @@ -31,7 +31,7 @@ @pytest.fixture(scope="module") def trtllm_model_path(): - return llm_models_root() / "gpt_oss/gpt-oss-20b" + return llm_models_root() / "Qwen3/Qwen3-0.6B" @pytest.fixture(scope="module") From cc7e91e42a676893c535b3b8f7552766b3075c48 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:53:27 +0800 Subject: [PATCH 08/14] [TRTLLM-15078][test] Remove test_majority_vote from test_scaffolding MajorityVoteController requires a reasoning model that produces consistent parseable answers across samples. No suitable small model is available in CI after retiring DeepSeek-R1-Distill-7B. The three remaining tests still cover NativeGenerationController sync, batched, and async paths. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../unittest/scaffolding/test_scaffolding.py | 37 +------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/tests/unittest/scaffolding/test_scaffolding.py b/tests/unittest/scaffolding/test_scaffolding.py index 970f08fd72e1..93261f95fa62 100644 --- a/tests/unittest/scaffolding/test_scaffolding.py +++ b/tests/unittest/scaffolding/test_scaffolding.py @@ -3,9 +3,7 @@ from scaffolding.test_worker import (create_trtllm_worker, default_prompt, trtllm_model_path) -from tensorrt_llm.scaffolding import (MajorityVoteController, - NativeGenerationController, - ScaffoldingLlm) +from tensorrt_llm.scaffolding import NativeGenerationController, ScaffoldingLlm def create_scaffolding_llm_with_native_generation_controller(trtllm_model_path): @@ -23,28 +21,6 @@ def create_scaffolding_llm_with_native_generation_controller(trtllm_model_path): ) -def create_scaffolding_llm_with_majority_vote_controller( - trtllm_model_path, samples_num): - trtllm_worker = create_trtllm_worker(trtllm_model_path) - - workers = {} - prototype_generation_controller = NativeGenerationController( - sampling_params={"max_tokens": 100}) - workers[NativeGenerationController.WorkerTag.GENERATION] = trtllm_worker - - prototype_majority_vote_controller = MajorityVoteController( - prototype_generation_controller, - default_sample_num=samples_num, - ) - - llm = ScaffoldingLlm( - prototype_majority_vote_controller, - workers=workers, - ) - - return llm - - def test_unbatched_scaffolding_sync(default_prompt, trtllm_model_path): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( trtllm_model_path) @@ -88,14 +64,3 @@ async def run_async_test(): import asyncio asyncio.run(run_async_test()) - - -def test_majority_vote(default_prompt, trtllm_model_path): - scaffolding_llm = create_scaffolding_llm_with_majority_vote_controller( - trtllm_model_path, samples_num=3) - try: - result = scaffolding_llm.generate(default_prompt) - assert isinstance(result.outputs[0].text, str) and len( - result.outputs[0].text) > 0, "Output should be a non-empty string" - finally: - scaffolding_llm.shutdown(shutdown_workers=True) From 55b4fad64362e24f531e576a28d1c66246709c64 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:53:25 +0800 Subject: [PATCH 09/14] [TRTLLM-15078][fix] Call gc.collect + cuda.empty_cache in LLM.shutdown PyTorch's CUDA allocator caches freed tensors and does not return them to the driver until empty_cache() is called. Without this, back-to-back LLM instances in the same process (e.g. sequential accuracy tests) see less available GPU memory than expected and can OOM. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tensorrt_llm/llmapi/llm.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 825e26fb3455..5abd5601efca 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -1625,6 +1625,14 @@ def shutdown(self) -> None: self.mpi_session.shutdown() self.mpi_session = None + # Return cached CUDA allocations back to the driver so that subsequent + # LLM instances in the same process (e.g. back-to-back tests) see the + # full available GPU memory. + import gc + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + def _check_health(self) -> bool: """Check if the LLM is healthy. From 41637131ecbd0fab5025e8c4367627d31b6202c4 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:01:09 +0800 Subject: [PATCH 10/14] Revert "[TRTLLM-15078][fix] Call gc.collect + cuda.empty_cache in LLM.shutdown" This reverts commit 03a8c2f810a68f19586e3f5736eba6b12e517be5. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tensorrt_llm/llmapi/llm.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 5abd5601efca..825e26fb3455 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -1625,14 +1625,6 @@ def shutdown(self) -> None: self.mpi_session.shutdown() self.mpi_session = None - # Return cached CUDA allocations back to the driver so that subsequent - # LLM instances in the same process (e.g. back-to-back tests) see the - # full available GPU memory. - import gc - gc.collect() - if torch.cuda.is_available(): - torch.cuda.empty_cache() - def _check_health(self) -> bool: """Check if the LLM is healthy. From 1ff09e66a46d572688e7f4ad71e84c0a0d8dfc8b Mon Sep 17 00:00:00 2001 From: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:09:04 +0800 Subject: [PATCH 11/14] Update tests/unittest/llmapi/apps/_test_openai_reasoning.py Co-authored-by: brnguyen2 <149187752+brnguyen2@users.noreply.github.com> Signed-off-by: xinhe-nv <200704525+xinhe-nv@users.noreply.github.com> --- tests/unittest/llmapi/apps/_test_openai_reasoning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest/llmapi/apps/_test_openai_reasoning.py b/tests/unittest/llmapi/apps/_test_openai_reasoning.py index 1afb77b64f1b..69a804694745 100644 --- a/tests/unittest/llmapi/apps/_test_openai_reasoning.py +++ b/tests/unittest/llmapi/apps/_test_openai_reasoning.py @@ -12,6 +12,7 @@ @pytest.fixture( scope="module", params=[ + ("Qwen3/Qwen3-0.6B", "pytorch", False), ("Qwen3/Qwen3-0.6B", "pytorch", True), ], ids=lambda p: f"{p[0]}-{p[1]}-{'with' if p[2] else 'no'}_beam_search") From 4b2c1ea73802e1e26c03f994f8ec43727e26cc3a Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:57:29 +0800 Subject: [PATCH 12/14] [None][test] Sort and dedupe llm_function_core.txt QA list Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../test_lists/qa/llm_function_core.txt | 285 ++++++++---------- 1 file changed, 129 insertions(+), 156 deletions(-) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 3f164b34293b..9f35a96dd35a 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -6,20 +6,25 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1dp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp2cp2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[noadp-mtp0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype +accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_gen_first +accuracy/test_disaggregated_serving.py::TestDeepSeekV4FlashBase::test_auto_dtype +accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[True] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] -accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl[cache_mgr_v1] +accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] +accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] +accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] +accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True] @@ -64,9 +69,11 @@ accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symm accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2] accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=0-block_reuse=False-use_py_transceiver=False] -accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=3-block_reuse=True-use_py_transceiver=False] accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=0-block_reuse=False-use_py_transceiver=True] +accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=3-block_reuse=True-use_py_transceiver=False] accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_ctx_dp2_gen_tp4 +accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=False] +accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False] accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True] @@ -76,20 +83,13 @@ accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first_kv_cache_v1 -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[noadp-mtp0] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype -accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_gen_first -accuracy/test_disaggregated_serving.py::TestDeepSeekV4FlashBase::test_auto_dtype accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend -accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct] -accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] -accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] -accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=False] -accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap +accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] +accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] +accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_attention_dp[4] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[torch-True-1] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton-False-1] @@ -98,16 +98,16 @@ accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-Fal accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-True-1] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer] accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm] +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[Qwen_QwQ-32B-False] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[deepseek-ai_DeepSeek-R1-0528-True] -accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_DeepSeek-R1-0528-NVFP4-v2-True] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[google_gemma-3-1b-it-False] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.1-8B-Instruct-False] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.3-70B-Instruct-False] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Codestral-22B-v0.1-False] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Ministral-8B-Instruct-2410-False] +accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_DeepSeek-R1-0528-NVFP4-v2-True] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True] accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True] -accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[Qwen_QwQ-32B-False] accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-flashinfer_ssm-False] accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-triton_ssm-False] accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-triton_ssm-True] @@ -133,21 +133,6 @@ accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[False] accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[True] accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True] accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_nvfp4[True] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTLASS] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_on-cutedsl] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_off-trtllm] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[TP4_MTP] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[TEP4] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4_MTP] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[TP2_PP2] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[TEP2_PP2] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[latency] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput] @@ -228,8 +213,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2- accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False] @@ -286,6 +269,8 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=2] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=0] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=2] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_mtp_sa_global_pool accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] @@ -300,6 +285,22 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm[cuda_graph=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm[cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[ep4-cuda_graph=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[ep4-cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[tp4-cuda_graph=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[tp4-cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm[cuda_graph=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm[cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[ep4-cuda_graph=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[ep4-cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[tp4-cuda_graph=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[tp4-cuda_graph=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] @@ -441,26 +442,13 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[b accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=1.0-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-v2_kv_cache=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=1.0-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-v2_kv_cache=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=1.0-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-v2_kv_cache=True] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_guided_decoding_vswa_reuse[xgrammar] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_vswa_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized -accuracy/test_llm_api_pytorch.py::TestGLM5FP8::test_8gpus[tp_size=8-ep_size=8] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_auto_dtype[moe_backend=DEEPGEMM] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_auto_dtype[moe_backend=TRTLLM] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_4gpus_static_eplb[moe_backend=DEEPGEMM] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_chunked_prefill accuracy/test_llm_api_pytorch.py::TestGLM52::test_nvfp4[tp_size=8-ep_size=8] accuracy/test_llm_api_pytorch.py::TestGLM52::test_nvfp4_mtp_index_share[tp_size=8-ep_size=8] +accuracy/test_llm_api_pytorch.py::TestGLM5FP8::test_8gpus[tp_size=8-ep_size=8] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dummy_load_format accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-one_model-overlap_scheduler] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler] @@ -478,8 +466,8 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v1_kv_cache-one_model] -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v2_kv_cache-one_model] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v1_kv_cache-two_model] +accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v2_kv_cache-one_model] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-fp8] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] @@ -532,6 +520,23 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-aut accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-fp8] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_guided_decoding_vswa_reuse[xgrammar] +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False] +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] +accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_vswa_reuse +accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype +accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[dep8] accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[ep8] accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] @@ -548,7 +553,6 @@ accuracy/test_llm_api_pytorch.py::TestLagunaXS_2_1::test_nvfp4_dflash accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_4] accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_64] -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B_Instruct_RocketKV::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True-sampler_async_worker=False] accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] @@ -580,8 +584,8 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_forma accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=False-overlap_scheduler=False] accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=True-overlap_scheduler=True] accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=True-eagle3_one_model=True-overlap_scheduler=True] -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3_rejection_dynamic_tree_smoke[no_dynamic_tree] accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3_rejection_dynamic_tree_smoke[dynamic_tree] +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3_rejection_dynamic_tree_smoke[no_dynamic_tree] accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3_sa accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3_sa_dynamic_draft_len accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3_sa_global_pool @@ -642,6 +646,7 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard_sa accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard_sa_dynamic_draft_len accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard_sa_global_pool accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_suffix_automaton_dynamic_draft_len +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B_Instruct_RocketKV::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestLlama3_2_3B::test_fp8_prequantized accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False] @@ -669,33 +674,52 @@ accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[laten accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_fp8 +accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=1] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=8] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpu_mtp_ar +accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_ON] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTLASS] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op -accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=1] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=8] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_multi_gpus[tp_size=8] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_adp_4gpus[mtp_nextn=3] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_marlin_8gpus +accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_multi_gpus[tp_size=8] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TEP4_PP2] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TEP8_PP1] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP8_PP1] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP8_PP1_ADP] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_ON] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4_MTP] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[TEP4] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[TP4_MTP] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTLASS] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_off-trtllm] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_on-cutedsl] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_marlin_8gpus +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[TEP2_PP2] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[TP2_PP2] accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_tp2 +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[dep4] +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tep4] +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[dep4-cutlass] +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[dep4-trtllm] accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency] accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass] @@ -703,7 +727,6 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_tr accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp] accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_cutlass] accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_trtllm_eagle3] -accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True] @@ -736,19 +759,20 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_sof accuracy/test_llm_api_pytorch.py::TestQwen3_4B::test_eagle3 accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS] accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-TRTLLM] -accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16 -accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8 -accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8_piecewise_cuda_graph -accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash -accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash +accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False] -accuracy/test_llm_api_pytorch.py::TestQwen3_6_35B_A3B::test_nvfp4[TRTLLM] -accuracy/test_llm_api_pytorch.py::TestQwen3_6_35B_A3B::test_nvfp4[CUTEDSL] +accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_cutedsl] accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_trtllm] accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_cutedsl] accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_trtllm] accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_mtp3_gdn_replay_tep4 +accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16 +accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash +accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8 +accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8_piecewise_cuda_graph +accuracy/test_llm_api_pytorch.py::TestQwen3_6_35B_A3B::test_nvfp4[CUTEDSL] +accuracy/test_llm_api_pytorch.py::TestQwen3_6_35B_A3B::test_nvfp4[TRTLLM] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[multi_gpus_no_cache] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dummy_load_format @@ -762,72 +786,66 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_first_token_response accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[mxfp8-latency] -accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tep4] -accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[dep4] -accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[dep4-cutlass] -accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[dep4-trtllm] accuracy/test_llm_api_pytorch.py::TestSeedOss_36B::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestStep3_7::test_auto_dtype[tp_size=8-ep_size=8] TIMEOUT (90) accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=0] TIMEOUT (90) accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3] TIMEOUT (90) accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=0] TIMEOUT (90) accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=3] TIMEOUT (90) -accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-eager] -accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-cuda_graph] -accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp] -accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_per_token_reward[qwen2.5-prm-7b] -accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-0.6b] -accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-8b] -accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b] +accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[phi-4-mini] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen2-7b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen3-0.6b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[starcoder2-3b] -accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b] -unittest/llmapi/apps/_test_openai_embeddings.py -accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] +accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b] +accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp] +accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-cuda_graph] +accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-eager] +accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_per_token_reward[qwen2.5-prm-7b] +accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-0.6b] +accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-8b] accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] +accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized +accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill] -accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill] +accuracy/test_llm_api_pytorch_multimodal.py::TestNVILA_8B::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[bf16] accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8] accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph] accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[nvfp4] -accuracy/test_llm_api_pytorch_multimodal.py::TestNVILA_8B::test_auto_dtype +accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill] accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_VL_7B::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL_MOE::test_auto_dtype -accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] +accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_27B_VL::test_auto_dtype +accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_auto_dtype +accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_fp8_prequantized accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_fp8_block_scales[mtp_nextn=0] TIMEOUT (120) accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_fp8_block_scales[mtp_nextn=3] TIMEOUT (120) accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=0] TIMEOUT (120) accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3] TIMEOUT (120) -accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_auto_dtype -accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_fp8_prequantized -accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_27B_VL::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestVILA1_5_3B::test_auto_dtype accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray -unittest/disaggregated/test_openai_disagg_server.py disaggregated/test_ad_disagg.py::test_async_eagle3_full_model_handoff disaggregated/test_ad_disagg.py::test_async_generation_matches_aggregate disaggregated/test_ad_disagg.py::test_async_generation_no_overlap_matches_aggregate disaggregated/test_ad_disagg.py::test_async_sharded_generation_handoff disaggregated/test_ad_disagg_trtllm_serve.py::test_openai_completion -disaggregated/test_aiperf_gate.py::test_fires_on_error_storm -disaggregated/test_aiperf_gate.py::test_passes_healthy_run_with_cancellations -disaggregated/test_aiperf_gate.py::test_missing_export_raises -disaggregated/test_aiperf_gate.py::test_empty_export_fails disaggregated/test_aiperf_gate.py::test_all_cancelled_fails disaggregated/test_aiperf_gate.py::test_corrupt_export_fails -disaggregated/test_aiperf_gate.py::test_single_truncated_line_tolerated -disaggregated/test_aiperf_gate.py::test_incomplete_accounting_fails +disaggregated/test_aiperf_gate.py::test_empty_export_fails +disaggregated/test_aiperf_gate.py::test_fires_on_error_storm disaggregated/test_aiperf_gate.py::test_gate_disabled_paths_not_affected -disaggregated/test_aiperf_gate.py::test_was_cancelled_metadata_fallback +disaggregated/test_aiperf_gate.py::test_incomplete_accounting_fails +disaggregated/test_aiperf_gate.py::test_missing_export_raises disaggregated/test_aiperf_gate.py::test_non_request_records_excluded_from_denominator +disaggregated/test_aiperf_gate.py::test_passes_healthy_run_with_cancellations +disaggregated/test_aiperf_gate.py::test_single_truncated_line_tolerated +disaggregated/test_aiperf_gate.py::test_was_cancelled_metadata_fallback disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin] disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin] disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin] @@ -919,82 +937,37 @@ disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batc disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[True-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct] disaggregated/test_workers.py::test_workers_conditional_disaggregation[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] +disaggregated/test_workers.py::test_workers_conversation_router[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_workers.py::test_workers_kv_cache_aware_router_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0] -disaggregated/test_workers.py::test_workers_conversation_router[TinyLlama-1.1B-Chat-v1.0] kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix_smoke +llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[cudagraph] TIMEOUT (90) +llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[eager] TIMEOUT (90) +llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] TIMEOUT (90) +llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] TIMEOUT (90) +llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_logging +llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_default llmapi/test_llm_examples.py::test_llmapi_server_example -test_e2e.py::test_eagle3_output_repetition_4gpus[modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8-EAGLE3-LLaMA3.3-Instruct-70B] test_e2e.py::test_eagle3_output_repetition_4gpus[Qwen3/Qwen3-30B-A3B-Qwen3/Qwen3-30B-eagle3] test_e2e.py::test_eagle3_output_repetition_4gpus[Qwen3/saved_models_Qwen3-235B-A22B_fp8_hf-Qwen3/qwen3-235B-eagle3] test_e2e.py::test_eagle3_output_repetition_4gpus[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-Qwen3/qwen3-235B-eagle3] +test_e2e.py::test_eagle3_output_repetition_4gpus[modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8-EAGLE3-LLaMA3.3-Instruct-70B] test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] test_e2e.py::test_openai_chat_harmony_perf_metrics test_e2e.py::test_openai_kv_cache_contamination +test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B] test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B] test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1] -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin] -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-load_balancing] -disaggregated/test_auto_scaling.py::test_service_discovery[etcd-kv_cache_aware] -disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin] -disaggregated/test_auto_scaling.py::test_service_discovery[http-load_balancing] -disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware] -disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin] -disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin] -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin] -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing] -disaggregated/test_auto_scaling.py::test_worker_restart[etcd-kv_cache_aware] -disaggregated/test_auto_scaling.py::test_worker_restart[http-round_robin] -disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing] -disaggregated/test_auto_scaling.py::test_worker_restart[http-kv_cache_aware] -disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin] -disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin] -disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python[TinyLlama-1.1B-Chat-v1.0] -disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp1-TinyLlama-1.1B-Chat-v1.0] -disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B] test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] test_e2e.py::test_ptp_quickstart_advanced_pp_enabled[Llama3.3-70B-FP8-llama-3.3-models/Llama-3.3-70B-Instruct-FP8-2-2-False] test_e2e.py::test_ptp_quickstart_advanced_pp_enabled[Llama3.3-70B-FP8-llama-3.3-models/Llama-3.3-70B-Instruct-FP8-2-4-True] test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] -test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1] test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[Qwen3/Qwen3-0.6B-Qwen3/Qwen3-0.6B] +test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1] test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] test_e2e.py::test_trtllm_multimodal_benchmark_serving - -# CuTe DSL BF16 BMM/GEMM tests -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm[cuda_graph=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm[cuda_graph=True] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[tp4-cuda_graph=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[tp4-cuda_graph=True] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[ep4-cuda_graph=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_bmm_4gpus[ep4-cuda_graph=True] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm[cuda_graph=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm[cuda_graph=True] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[tp4-cuda_graph=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[tp4-cuda_graph=True] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[ep4-cuda_graph=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm_4gpus[ep4-cuda_graph=True] - -# llm-api promote pytorch to default -llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_logging -llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_default - -# Routed-expert (MoE) LoRA -llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] TIMEOUT (90) -llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] TIMEOUT (90) -llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[eager] TIMEOUT (90) -llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[cudagraph] TIMEOUT (90) - -# CuteDSL NVFP4 tests -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_auto_dtype[moe_backend=DEEPGEMM] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_auto_dtype[moe_backend=TRTLLM] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_chunked_prefill -accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_4gpus_static_eplb[moe_backend=DEEPGEMM] -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] +unittest/disaggregated/test_openai_disagg_server.py +unittest/llmapi/apps/_test_openai_embeddings.py From 0666fda10683490e2bd9850c7991997f2647306a Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:19:41 +0800 Subject: [PATCH 13/14] [TRTLLM-15078][test] Apply chat template in Qwen3 RCCA n>1 empty-seq guard Restore chat-formatted prompting for Qwen3-0.6B so the n=4 non-empty completion invariant is exercised on the intended generation path. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- tests/integration/defs/test_e2e.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index d633cc9fd3d4..d3266d4501e9 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -60,6 +60,14 @@ def test_qwen_e2e_cpprunner_large_new_tokens(model_name, model_path, llm_venv): with LLM(model=f"{llm_models_root()}/{model_path}", max_batch_size=8, max_seq_len=4224) as llm: + prompt = llm.tokenizer.apply_chat_template( + [{ + "role": "user", + "content": prompt + }], + tokenize=False, + add_generation_prompt=True, + ) outputs = llm.generate([prompt], sampling_params=sampling_params) completions = outputs[0].outputs From 6236252f79b6a2e2f11a50d4377765f6fa059882 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:24:51 +0800 Subject: [PATCH 14/14] [TRTLLM-15078][test] Restore scaffolding majority_vote on Qwen3-0.6B Keep MajorityVoteController coverage after Distill retirement by using trtllm_model_path, with a boxed-answer instruction and longer max_tokens so the digit majority extractor can succeed on the small CI model. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../unittest/scaffolding/test_scaffolding.py | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/unittest/scaffolding/test_scaffolding.py b/tests/unittest/scaffolding/test_scaffolding.py index 93261f95fa62..e9763ec13148 100644 --- a/tests/unittest/scaffolding/test_scaffolding.py +++ b/tests/unittest/scaffolding/test_scaffolding.py @@ -3,7 +3,9 @@ from scaffolding.test_worker import (create_trtllm_worker, default_prompt, trtllm_model_path) -from tensorrt_llm.scaffolding import NativeGenerationController, ScaffoldingLlm +from tensorrt_llm.scaffolding import (MajorityVoteController, + NativeGenerationController, + ScaffoldingLlm) def create_scaffolding_llm_with_native_generation_controller(trtllm_model_path): @@ -21,6 +23,28 @@ def create_scaffolding_llm_with_native_generation_controller(trtllm_model_path): ) +def create_scaffolding_llm_with_majority_vote_controller( + trtllm_model_path, samples_num): + trtllm_worker = create_trtllm_worker(trtllm_model_path) + + workers = {} + prototype_generation_controller = NativeGenerationController( + sampling_params={"max_tokens": 256}) + workers[NativeGenerationController.WorkerTag.GENERATION] = trtllm_worker + + prototype_majority_vote_controller = MajorityVoteController( + prototype_generation_controller, + default_sample_num=samples_num, + ) + + llm = ScaffoldingLlm( + prototype_majority_vote_controller, + workers=workers, + ) + + return llm + + def test_unbatched_scaffolding_sync(default_prompt, trtllm_model_path): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( trtllm_model_path) @@ -64,3 +88,20 @@ async def run_async_test(): import asyncio asyncio.run(run_async_test()) + + +def test_majority_vote(default_prompt, trtllm_model_path): + # MajorityVoteController extracts integer answers from \\boxed{...}. + # Ask for that format so Qwen3-0.6B satisfies the controller assert. + prompt = ( + default_prompt + + "Please reason step by step, and put your final answer within \\boxed{}." + ) + scaffolding_llm = create_scaffolding_llm_with_majority_vote_controller( + trtllm_model_path, samples_num=3) + try: + result = scaffolding_llm.generate(prompt) + assert isinstance(result.outputs[0].text, str) and len( + result.outputs[0].text) > 0, "Output should be a non-empty string" + finally: + scaffolding_llm.shutdown(shutdown_workers=True)