diff --git a/examples/auto_deploy/model_registry/models.yaml b/examples/auto_deploy/model_registry/models.yaml index 78a4ae2afa0b..f1e925a6d978 100644 --- a/examples/auto_deploy/model_registry/models.yaml +++ b/examples/auto_deploy/model_registry/models.yaml @@ -83,12 +83,6 @@ models: # - name: google/gemma-1.1-7b-it # config_id: default_ws_2 # yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: google/gemma-2-2b-it - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: google/gemma-2-9b-it - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 # - name: google/codegemma-7b-it # config_id: default_ws_2 @@ -229,9 +223,6 @@ models: - name: Qwen/QwQ-32B config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: google/gemma-2-27b-it - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # OOM during AutoDeploy run. # - name: perplexity-ai/r1-1776-distill-llama-70b # config_id: default_ws_4 diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index 672f44ea42db..c15fab97de1e 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -1,37 +1,3 @@ -google/gemma-3-1b-it: - - accuracy: 19.0 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 20.699 -google/gemma-3-27b-it: - - accuracy: 28.90 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 27.90 -gpt2: - - accuracy: 18.408 - - quant_algo: W8A16 - accuracy: 18.408 - - quant_algo: W4A16 - accuracy: 15.251 - - kv_cache_quant_algo: INT8 - accuracy: 18.208 - - quant_algo: W8A8_SQ_PER_TENSOR_PLUGIN - accuracy: 18.377 - - quant_algo: W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN - accuracy: 18.408 - - extra_acc_spec: beam_width=4 - accuracy: 24.933 - - extra_acc_spec: beam_width=256 - num_samples: 32 - accuracy: 29.422 - - extra_acc_spec: beam_width=8;beam_width_array=[2,3,4,5] - num_samples: 32 - accuracy: 26.994 -gpt2-medium: - - accuracy: 22.730 - - quant_algo: FP8 - accuracy: 22.249 gpt-next: - accuracy: 25.516 state-spaces/mamba-130m-hf: @@ -136,15 +102,6 @@ google/gemma-7b: accuracy: 19.357 - quant_algo: W4A16_AWQ accuracy: 20.283 -google/gemma-2-9b-it: - - accuracy: 29.519 - - quant_algo: W8A16 - accuracy: 29.679 - - quant_algo: W4A16 - accuracy: 29.536 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 26.670 nvidia/Nemotron-Mini-4B-Instruct: - quant_algo: FP8 accuracy: 25.247 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index 5f7b439ec890..fa6c03539295 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -316,16 +316,6 @@ nvidia/Nemotron-MOE: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 63.268 -google/gemma-3-1b-it: - - accuracy: 25.52 # score getting from lm-eval with HF implementation - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 23.96 -google/gemma-3-27b-it: - - accuracy: 91.66 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 90.66 google/gemma-4-26B-A4B-it: - accuracy: 90.83 google/gemma-3n-E2B-it: diff --git a/tests/integration/defs/accuracy/references/json_mode_eval.yaml b/tests/integration/defs/accuracy/references/json_mode_eval.yaml index 6b43fa4e39c6..3fda403fb2c8 100644 --- a/tests/integration/defs/accuracy/references/json_mode_eval.yaml +++ b/tests/integration/defs/accuracy/references/json_mode_eval.yaml @@ -10,10 +10,6 @@ deepseek-ai/DeepSeek-V3-Lite: - accuracy: 77.00 - spec_dec_algo: MTP accuracy: 77.00 -google/gemma-3-1b-it: - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 61.00 GPT-OSS/120B-MXFP4: - quant_algo: W4A16_MXFP4 spec_dec_algo: Eagle diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 1143669dbc57..251dc86e5e97 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -35,18 +35,6 @@ mistralai/Mistral-Small-3.1-24B-Instruct-2503: - quant_algo: FP8 kv_cache_quant_algo: FP8 accuracy: 81.1 -google/gemma-2-9b-it: - - accuracy: 73.05 -google/gemma-3-1b-it: - - accuracy: 39.0 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 37.5 -google/gemma-3-27b-it: - - accuracy: 77.80 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 76.80 google/gemma-4-26B-A4B-it: - accuracy: 71.296 google/gemma-3n-E2B-it: diff --git a/tests/integration/defs/accuracy/references/mmmu.yaml b/tests/integration/defs/accuracy/references/mmmu.yaml index df6f2115e39d..7fc2c909710e 100644 --- a/tests/integration/defs/accuracy/references/mmmu.yaml +++ b/tests/integration/defs/accuracy/references/mmmu.yaml @@ -1,20 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -google/gemma-3-27b-it: - - accuracy: 52.0 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 50.0 - # Blackwell FP8 cubins for Gemma3 multimodal route through a different - # FlashInfer path than Hopper, producing ~5pt lower MMMU. Use a separate - # Blackwell-calibrated reference instead of relaxing the Hopper one. - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - extra_acc_spec: sm100_fp8 - accuracy: 46.0 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - accuracy: 48.0 google/gemma-4-26B-A4B-it: - accuracy: 56.667 # B200 PyTorch backend baseline for nvidia/Gemma-4-26B-A4B-NVFP4. @@ -25,14 +10,6 @@ google/gemma-4-26B-A4B-it: kv_cache_quant_algo: FP8 spec_dec_algo: MTP accuracy: 54.0 -google/gemma-3-12b-it: - - accuracy: 50.44 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 49.0 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - accuracy: 50.11 LGAI-EXAONE/EXAONE-4.5-33B: # Empirically achieved ~46.5 across H20/B200/B300/GB200/GB300 since the # model was added (PR #12873). The original 51.22 was committed without diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index cda65eec2cbb..39af5bf0aa96 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -30,40 +30,6 @@ allow_module_level=True) -class TestGpt2(CliFlowAccuracyTestHarness): - MODEL_NAME = "gpt2" - MODEL_PATH = f"{llm_models_root()}/gpt2" - EXAMPLE_FOLDER = "models/core/gpt" - - def test_auto_dtype(self): - # float16 - self.run(dtype='auto') - - @skip_post_blackwell - @pytest.mark.parametrize("precision", ["int8", "int4"]) - def test_weight_only(self, precision: str): - quant_algo = QuantAlgo.W8A16 if precision == "int8" else QuantAlgo.W4A16 - self.run(quant_algo=quant_algo) - - def test_beam_search(self): - self.run(extra_acc_spec="beam_width=4", - extra_build_args=["--max_beam_width=4"], - extra_summarize_args=["--num_beams=4", "--length_penalty=2.0"]) - - -class TestGpt2Medium(CliFlowAccuracyTestHarness): - MODEL_NAME = "gpt2-medium" - MODEL_PATH = f"{llm_models_root()}/gpt2-medium" - EXAMPLE_FOLDER = "models/core/gpt" - - def test_auto_dtype(self): - self.run(dtype='auto') - - @skip_pre_ada - def test_fp8(self): - self.run(quant_algo=QuantAlgo.FP8) - - class TestStarcoder2_15B(CliFlowAccuracyTestHarness): MODEL_NAME = "bigcode/starcoder2-15b" MODEL_PATH = f"{llm_models_root()}/starcoder2-model" @@ -315,29 +281,3 @@ def test_weight_only(self, precision: str): @skip_pre_ada def test_fp8(self): self.run(quant_algo=QuantAlgo.FP8, kv_cache_quant_algo=QuantAlgo.FP8) - - -@pytest.mark.skip_less_device_memory(40000) -class TestGemma2_9BIt(CliFlowAccuracyTestHarness): - MODEL_NAME = "google/gemma-2-9b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-2-9b-it" - EXAMPLE_FOLDER = "models/core/gemma" - - @skip_post_blackwell - def test_auto_dtype(self): - self.run(tasks=[CnnDailymail(self.MODEL_NAME), - MMLU(self.MODEL_NAME)], - dtype='auto', - extra_convert_args=["--ckpt-type=hf"]) - - @skip_post_blackwell - @pytest.mark.parametrize("precision", ["int8", "int4"]) - def test_weight_only(self, precision: str): - quant_algo = QuantAlgo.W8A16 if precision == "int8" else QuantAlgo.W4A16 - self.run(quant_algo=quant_algo, extra_convert_args=["--ckpt-type=hf"]) - - @skip_pre_hopper - def test_fp8(self): - self.run(quant_algo=QuantAlgo.FP8, - kv_cache_quant_algo=QuantAlgo.FP8, - extra_convert_args=["--device_map=sequential"]) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 78432c45d5ce..03bce01d822b 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1384,157 +1384,6 @@ def test_gen_first(self, enable_attention_dp, mtp_nextn): run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) -@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) -class TestGemma3_1BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-1b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-1b-it/" - - # Literal NIXL bypasses the harness's legacy UCX fallback. Omitting the - # runtime then exercises Gemma's automatic preference for Python V2. - - @pytest.mark.skip_less_device(2) - @pytest.mark.parametrize("block_reuse", [False, True]) - @skip_pre_hopper - def test_auto_dtype(self, block_reuse): - - ctx_server_config = { - "disable_overlap_scheduler": True, - "cuda_graph_config": None, - "cache_transceiver_config": { - "backend": "NIXL", - "max_tokens_in_buffer": 4096 - } - } - gen_server_config = { - "disable_overlap_scheduler": False, - "cuda_graph_config": None, - "cache_transceiver_config": { - "backend": "NIXL", - "max_tokens_in_buffer": 4096 - } - } - ctx_server_config["kv_cache_config"] = { - "max_attention_window": [512, 512, 512, 512, 512, 32768], - "enable_block_reuse": block_reuse, - "enable_partial_reuse": block_reuse, - } - gen_server_config["kv_cache_config"] = { - "max_attention_window": [512, 512, 512, 512, 512, 32768], - "enable_block_reuse": block_reuse, - "enable_partial_reuse": block_reuse, - } - disaggregated_server_config = { - "hostname": "localhost", - "backend": "pytorch", - "context_servers": { - "num_instances": 1 - }, - "generation_servers": { - "num_instances": 1 - } - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"]) - - @pytest.mark.skip_less_device(2) - @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], - ids=["cache_mgr_v1", "cache_mgr_v2"]) - @skip_pre_hopper - def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): - """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" - ctx_server_config = { - "disable_overlap_scheduler": True, - "cuda_graph_config": None, - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": use_kv_cache_manager_v2 - }, - "cache_transceiver_config": { - "backend": "NIXL", - "transceiver_runtime": "PYTHON" - } - } - gen_server_config = { - "disable_overlap_scheduler": True, - "cuda_graph_config": None, - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": use_kv_cache_manager_v2 - }, - "cache_transceiver_config": { - "backend": "NIXL", - "transceiver_runtime": "PYTHON" - } - } - disaggregated_server_config = { - "hostname": "localhost", - "port": 8000, - "backend": "pytorch", - "context_servers": { - "num_instances": 1, - "urls": ["localhost:8001"] - }, - "generation_servers": { - "num_instances": 1, - "urls": ["localhost:8002"] - } - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"]) - - @skip_pre_hopper - @pytest.mark.skip_less_device(2) - @parametrize_with_ids("enable_block_reuse", [True]) - @parametrize_with_ids("use_kv_cache_manager_v2", [True, False]) - def test_pipelined_kv_transfer_nixl_python_accuracy( - self, enable_block_reuse: bool, use_kv_cache_manager_v2: bool): - """Test pipelined Python transfer with Gemma 3 VSWA.""" - kv_cache_config = { - "use_kv_cache_manager_v2": use_kv_cache_manager_v2, - "enable_block_reuse": enable_block_reuse, - "enable_partial_reuse": enable_block_reuse, - "max_attention_window": [512, 512, 512, 512, 512, 32768], - } - cache_transceiver_config = { - "backend": "NIXL", - "transceiver_runtime": "PYTHON", - "max_tokens_in_buffer": 4096, - "enable_pipelined_transfer": True, - } - ctx_server_config = { - "max_num_tokens": 256, - "cuda_graph_config": None, - "kv_cache_config": dict(kv_cache_config), - "cache_transceiver_config": dict(cache_transceiver_config), - "enable_chunked_prefill": True, - } - gen_server_config = { - "cuda_graph_config": None, - "kv_cache_config": dict(kv_cache_config), - "cache_transceiver_config": dict(cache_transceiver_config), - "enable_chunked_prefill": True, - } - disaggregated_server_config = { - "hostname": "localhost", - "backend": "pytorch", - "schedule_style": "generation_first", - "context_servers": { - "num_instances": 1, - }, - "generation_servers": { - "num_instances": 1, - }, - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) - - @skip_pre_blackwell @pytest.mark.skip_less_device_memory(80000) class TestGPTOSS(LlmapiAccuracyTestHarness): diff --git a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py index 1f2394634ea4..6d166da2c38e 100644 --- a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py +++ b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @@ -1438,8 +1438,6 @@ class TestModelRegistryAccuracy(LlmapiAccuracyTestHarness): pytest.param("nvidia/Llama-3.1-8B-Instruct-NVFP4", {}, [MMLU, GSM8K], marks=skip_pre_blackwell, id="nvidia_Llama-3.1-8B-Instruct-NVFP4"), - pytest.param("google/gemma-3-1b-it", {}, [MMLU, GSM8K], - id="google_gemma-3-1b-it"), pytest.param("mistralai/Ministral-8B-Instruct-2410", {}, [MMLU, GSM8K], id="mistralai_Ministral-8B-Instruct-2410"), pytest.param( diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 99689df243fe..fa9a8904568f 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1222,359 +1222,6 @@ def test_fp8(self): pytest.skip("FP8 pre-quantized Ministral-8B model not available") -@skip_post_blackwell -@skip_pre_hopper -class TestGemma3_27BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-27b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-27b-it/" - - def test_auto_dtype(self): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - free_gpu_memory_fraction=0.5, - ) - # We use FlashInfer as the attention backend for Gemma3 VLM to support custom mask for images. - # So, testing with it here. - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - attn_backend="FLASHINFER", - cuda_graph_config=None, - max_batch_size=128, - max_seq_len=4096) as llm: - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - - def test_fp8_prequantized(self): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig(enable_block_reuse=False, - enable_partial_reuse=False, - dtype="fp8") - # Note: This has only the LLM part quantized. Vision part is in bfloat16. - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-27b-it-fp8/" - with LLM(prequantized_model_path, - kv_cache_config=kv_cache_config, - attn_backend="FLASHINFER", - cuda_graph_config=None, - max_seq_len=4096) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - -@skip_pre_hopper -class TestGemma3_1BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-1b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-1b-it/" - - # NOTE: Disable block reuse for SWA window model. - kv_cache_config = KvCacheConfig(enable_block_reuse=True) - - def test_auto_dtype(self): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - ) - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - @parametrize_with_ids("torch_compile", [False, True]) - def test_fp8_prequantized(self, torch_compile): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig(enable_block_reuse=False, - enable_partial_reuse=False, - dtype="fp8") - torch_compile_config = _get_default_torch_compile_config(torch_compile) - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-1b-it-fp8/" - with LLM(prequantized_model_path, - kv_cache_config=kv_cache_config, - torch_compile_config=torch_compile_config) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_fp8_vswa_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-1b-it-fp8/" - with LLM(prequantized_model_path, - kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - @pytest.mark.parametrize("backend", ["xgrammar"]) - def test_fp8_guided_decoding_vswa_reuse(self, backend: str, mocker): - mocker.patch.dict(os.environ, {"TRTLLM_XGUIDANCE_LENIENT": "1"}) - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-1b-it-fp8/" - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - cuda_graph_config = CudaGraphConfig(enable_padding=True) - llm = LLM(prequantized_model_path, - guided_decoding_backend=backend, - kv_cache_config=kv_cache_config, - cuda_graph_config=cuda_graph_config) - with llm: - task = JsonModeEval(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_without_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_without_reuse_low_memory_available(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - free_gpu_memory_fraction=0.1, - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - disable_overlap_scheduler=True) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_disable_overlap_scheduler(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - disable_overlap_scheduler=True) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_partial_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - enable_partial_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - free_gpu_memory_fraction=0.1, - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - enable_partial_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - free_gpu_memory_fraction=0.1, - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_kv_cache_stats(self): - """Mirror of test_auto_dtype_vswa_reuse that collects per-iteration stats. - - Collects per-iteration KV cache statistics and writes them to a JSON - file for offline visualization with - ``scripts/visualize_kv_cache_stats.py``. - """ - import json - import time - from pathlib import Path - - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - iteration_stats_interval=1, - ) - - all_stats = [] - - def drain_stats(llm, phase_label): - """Drain the stats queue and tag each entry. - - Tags each entry with wall-clock time and a human-readable phase - label. - """ - stats = llm.get_stats(timeout=2) - ts = time.time() - for entry in stats: - entry["_collectedAt"] = ts - entry["_phase"] = phase_label - all_stats.extend(stats) - - with LLM( - self.MODEL_PATH, - kv_cache_config=kv_cache_config, - max_stats_len=-1, - enable_iter_perf_stats=True, - ) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - drain_stats(llm, "GSM8K") - - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - drain_stats(llm, "MMLU") - - # Write collected stats to JSON - out_dir = Path( - os.environ.get( - "KV_CACHE_STATS_OUTPUT_DIR", - "kv_cache_stats_output", - )) - out_dir.mkdir(parents=True, exist_ok=True) - timestamp = time.strftime("%Y%m%d_%H%M%S") - out_path = out_dir / f"kv_cache_stats_{timestamp}.json" - - payload = { - "model": self.MODEL_NAME, - "kv_cache_config": { - "enable_block_reuse": - kv_cache_config.enable_block_reuse, - "max_attention_window": - kv_cache_config.max_attention_window, - "iteration_stats_interval": - kv_cache_config.iteration_stats_interval, - }, - "num_entries": len(all_stats), - "stats": all_stats, - } - out_path.write_text(json.dumps(payload, indent=2)) - print(f"\n[kv_cache_stats] Wrote {len(all_stats)} entries to " - f"{out_path}") - - def test_auto_dtype_vswa_chunked_prefill_without_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - # chunked prefill case or more features - extra_llm_config = dict( - enable_chunked_prefill=True, - max_num_tokens=1024, - ) - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - **extra_llm_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_chunked_prefill_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - # chunked prefill case or more features - extra_llm_config = dict( - enable_chunked_prefill=True, - max_num_tokens=1024, - ) - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - **extra_llm_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - # This class has extensively parameterized test methods, which yield totally 200 test cases. # This is because this model requires high test coverage over the feature combinations. # Normally we should not parameterize test methods so extensively -- just test on the typical/important feature combinations. diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py index 07602993da95..c75cffdb842e 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py @@ -240,7 +240,6 @@ def test_qwen3_text_embedding_matches_huggingface(self, model_name, model_path): # # One representative per distinct TRT-LLM architecture class: # LlamaForCausalLM — TinyLlama (also covers Mistral, which aliases LlamaModel) -# Gemma3ForCausalLM — Gemma-3-1B (sliding window + global alternation) # Qwen3ForCausalLM — Qwen3-0.6B (QKNorm) DECODER_MODELS = [ # -- LlamaForCausalLM (covers Llama + Mistral family) -- @@ -249,12 +248,6 @@ def test_qwen3_text_embedding_matches_huggingface(self, model_name, model_path): f"{llm_models_root()}/llama-models-v2/TinyLlama-1.1B-Chat-v1.0", id="tinyllama-1.1b", ), - # -- Gemma3ForCausalLM -- - pytest.param( - "google/gemma-3-1b-it", - f"{llm_models_root()}/gemma/gemma-3-1b-it/", - id="gemma-3-1b", - ), # -- Qwen3ForCausalLM -- pytest.param( "Qwen/Qwen3-0.6B", @@ -274,8 +267,7 @@ class TestDecoderEncode(LlmapiAccuracyTestHarness): ] # Top-K size used for the argmax-in-top-K containment / overlap checks. - # Chosen to be robust to near-tie argmax flips under FP16/BF16 rounding - # on very large vocabularies (Gemma-3 has 262K tokens). + # This is robust to near-tie argmax flips under FP16/BF16 rounding. TOPK = 5 TOPK_MIN_OVERLAP = 3 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 1dd10b4c4ee8..e984117f758c 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -32,7 +32,6 @@ from tensorrt_llm.quantization import QuantAlgo from ..conftest import ( - get_sm_version, llm_models_root, skip_post_blackwell_ultra, skip_pre_blackwell, @@ -120,113 +119,6 @@ def test_auto_dtype(self, enable_chunked_prefill, max_num_tokens): ) -@skip_pre_hopper -class TestGemma3_27BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-27b-it" - # Note: This has only the LLM part quantized. Vision part is in bfloat16. - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-27b-it-fp8/" - MAX_NUM_TOKENS = 12800 - - sampling_params = SamplingParams( - max_tokens=MAX_NUM_TOKENS, truncate_prompt_tokens=MMMU.MAX_INPUT_LEN, stop="" - ) - - # Gemma3 VLM needs KV cache reuse disabled for custom mask support. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - free_gpu_memory_fraction=0.4, - dtype="fp8", - ) - - def _make_llm(self, model_path: str): - # Gemma3 VLM needs FlashInfer attention backend for custom mask support. - return LLM( - model_path, - max_batch_size=16, - max_num_tokens=self.MAX_NUM_TOKENS, - max_seq_len=8704, # 8192 + 512. - kv_cache_config=self.kv_cache_config, - attn_backend="FLASHINFER", - enable_chunked_prefill=False, - ) - - def test_fp8_prequantized(self): - # Blackwell FP8 numerics differ from Hopper at the cubin level - # (~5pt drop on MMMU). Route to a Blackwell-calibrated reference - # rather than relaxing the Hopper one. - extra_acc_spec = "sm100_fp8" if get_sm_version() >= 100 else None - with self._make_llm(self.MODEL_PATH) as llm: - task = MMMU(self.MODEL_NAME) - task.evaluate( - llm, - extra_acc_spec=extra_acc_spec, - sampling_params=self.sampling_params, - ) - - @skip_pre_blackwell - def test_nvfp4_prequantized(self): - model_path = f"{llm_models_root()}/gemma/gemma-3-27b-it-FP4" - with self._make_llm(model_path) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - -@skip_pre_hopper -class TestGemma3_12BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-12b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-12b-it" - MAX_NUM_TOKENS = 12800 - - sampling_params = SamplingParams( - max_tokens=MAX_NUM_TOKENS, truncate_prompt_tokens=MMMU.MAX_INPUT_LEN, stop="" - ) - - # Gemma3 VLM needs KV cache reuse disabled for custom mask support. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - free_gpu_memory_fraction=0.6, - ) - - kv_cache_config_fp8 = kv_cache_config.model_copy(update={"dtype": "fp8"}) - - def _make_llm(self, model_path: str, kv_cache_config: KvCacheConfig = None): - # Gemma3 VLM needs FlashInfer attention backend for custom mask support. - if kv_cache_config is None: - kv_cache_config = self.kv_cache_config - return LLM( - model_path, - max_batch_size=16, - max_num_tokens=self.MAX_NUM_TOKENS, - max_seq_len=8704, # 8192 + 512. - kv_cache_config=kv_cache_config, - attn_backend="FLASHINFER", - enable_chunked_prefill=False, - ) - - def test_auto_dtype(self): - with self._make_llm(self.MODEL_PATH) as llm: - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - def test_fp8_prequantized(self): - model_path = f"{llm_models_root()}/gemma/gemma-3-12b-it-fp8" - with self._make_llm(model_path, self.kv_cache_config_fp8) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - @skip_pre_blackwell - def test_nvfp4_prequantized(self): - model_path = f"{llm_models_root()}/gemma/gemma-3-12b-it-fp4" - with self._make_llm(model_path, self.kv_cache_config_fp8) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - @pytest.mark.skip_device_not_contain(["B200", "GB10"]) class TestGemma4_26B_A4B(LlmapiAccuracyTestHarness): MODEL_NAME = "google/gemma-4-26B-A4B-it" diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index d9f7fc882ff4..12650ea507f3 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -568,20 +568,6 @@ def llm_replit_code_v1_5_3b_model_root(): raise RuntimeError("replit-code-v1_5-3b must be cached") -@pytest.fixture(scope="module") -@cached_in_llm_models_root("gpt2", True) -def llm_gpt2_model_root(): - "Get gpt2 model root" - raise RuntimeError("gpt2 must be cached") - - -@pytest.fixture(scope="module") -@cached_in_llm_models_root("gpt2-medium", True) -def llm_gpt2_medium_model_root(): - "Get gpt2 medium model root" - raise RuntimeError("gpt2-medium must be cached") - - @pytest.fixture(scope="module") @cached_in_llm_models_root("GPT-2B-001_bf16_tp1.nemo", True) def llm_gpt2_next_model_root(): diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index 4308d916b19c..206ec0d9f2a0 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -19,13 +19,6 @@ "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct", "llama_v3.1_8b_instruct_fp8": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8", "llama_v3.1_8b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4", - "gemma_3_27b_it": "gemma/gemma-3-27b-it", - "gemma_3_27b_it_fp8": "gemma/gemma-3-27b-it-fp8", - "gemma_3_27b_it_fp4": "gemma/gemma-3-27b-it-FP4", - "gemma_3_12b_it": "gemma/gemma-3-12b-it", - "gemma_3_12b_it_fp8": "gemma/gemma-3-12b-it-fp8", - "gemma_3_12b_it_fp4": "gemma/gemma-3-12b-it-fp4", - "gemma_3_1b_it": "gemma/gemma-3-1b-it", "gemma_4_26b_a4b_nvfp4": "gemma/nvidia-Gemma-4-26B-A4B-NVFP4", "gemma_4_31b_it_nvfp4": "gemma/nvidia-Gemma-4-31B-IT-NVFP4", "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", diff --git a/tests/integration/defs/triton_server/conftest.py b/tests/integration/defs/triton_server/conftest.py index 86fb46412bce..59188db72eec 100644 --- a/tests/integration/defs/triton_server/conftest.py +++ b/tests/integration/defs/triton_server/conftest.py @@ -295,18 +295,6 @@ def llama_v2_tokenizer_model_root(): return llama_v2_tokenizer_model_root -@pytest.fixture(scope="session") -def gpt_tokenizer_model_root(llm_backend_venv): - models_root = llm_models_root() - assert models_root, "Did you set LLM_MODELS_ROOT?" - gpt_tokenizer_model_root = os.path.join(models_root, "gpt2") - - assert os.path.exists( - gpt_tokenizer_model_root - ), f"{gpt_tokenizer_model_root} does not exist under NFS LLM_MODELS_ROOT dir" - return gpt_tokenizer_model_root - - @pytest.fixture(scope="session") def gptj_tokenizer_model_root(llm_backend_venv): models_root = llm_models_root() @@ -319,18 +307,6 @@ def gptj_tokenizer_model_root(llm_backend_venv): return gptj_tokenizer_model_root -@pytest.fixture(scope="session") -def gpt2_medium_tokenizer_model_root(): - models_root = llm_models_root() - assert models_root, "Did you set LLM_MODELS_ROOT?" - gpt_tokenizer_model_root = os.path.join(models_root, "gpt2-medium") - - assert os.path.exists( - gpt_tokenizer_model_root - ), f"{gpt_tokenizer_model_root} does not exist under NFS LLM_MODELS_ROOT dir" - return gpt_tokenizer_model_root - - @pytest.fixture(scope="session") def gpt_next_ptuning_model_root(): models_root = llm_models_root() diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index b3d743f6766f..7d9d9177d534 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -19,10 +19,6 @@ accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[True] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] @@ -463,23 +459,6 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-aut accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-fp8] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_guided_decoding_vswa_reuse[xgrammar] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_vswa_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[dep8] accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[ep8] accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] @@ -723,7 +702,6 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8_Flash_Next::test_fp8_1gpu_mtp3_trt accuracy/test_llm_api_pytorch.py::TestQwen3_8_Flash_Next::test_nvfp4_1gpu_mtp3_cutedsl_ple_offload accuracy/test_llm_api_pytorch.py::TestSeedOss_36B::test_auto_dtype accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b] -accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen3-0.6b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b] accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp] @@ -733,7 +711,6 @@ accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embe accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-8b] accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] -accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4 accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (120) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] diff --git a/tests/integration/test_lists/qa/llm_spark_func.yml b/tests/integration/test_lists/qa/llm_spark_func.yml index a59385112d68..0db4b20e4ece 100644 --- a/tests/integration/test_lists/qa/llm_spark_func.yml +++ b/tests/integration/test_lists/qa/llm_spark_func.yml @@ -31,11 +31,6 @@ llm_spark_func: - test_e2e.py::test_ptp_quickstart_advanced[Qwen3.6-35B-A3B-nvfp4-Qwen3.6-35B-A3B-NVFP4] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_nvfp4_prequantized - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_auto_dtype - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_fp8_prequantized - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_nvfp4_prequantized - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4_no_mtp - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4 - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] diff --git a/tests/integration/test_lists/qa/llm_spark_perf.yml b/tests/integration/test_lists/qa/llm_spark_perf.yml index 966db8a62ef9..7cbb2ef70e70 100644 --- a/tests/integration/test_lists/qa/llm_spark_perf.yml +++ b/tests/integration/test_lists/qa/llm_spark_perf.yml @@ -38,12 +38,6 @@ llm_spark_perf: - perf/test_perf.py::test_perf[qwen3_30b_a3b-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[qwen3_32b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[qwen3_32b-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_27b_it-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_27b_it_fp8-bench-pytorch-streaming-float8-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_27b_it_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_12b_it-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_12b_it_fp8-bench-pytorch-streaming-float8-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_12b_it_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] # Qwen3.6-35B-A3B NVFP4: baseline and the same checkpoint with MTP drafting - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,256-reqs:1-con:1] diff --git a/tests/integration/test_lists/test-db/l0_a100.yml b/tests/integration/test_lists/test-db/l0_a100.yml index 6a8a81066bd1..2b0c7fd7359c 100644 --- a/tests/integration/test_lists/test-db/l0_a100.yml +++ b/tests/integration/test_lists/test-db/l0_a100.yml @@ -33,7 +33,6 @@ l0_a100: - accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-cuda_graph] - accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp] - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b] - - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b] - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen3-0.6b] - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b] # Moved from l0_a10.yml: Llama-7B + KV cache + spec resources OOM on A10's 22 GB. diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 63755e48bd6e..1f8b32e2b389 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -17,9 +17,6 @@ l0_dgx_b200: tests: - unittest/_torch/misc/test_autotuner.py::test_autotuner_distributed_strategy - accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-TRTLLM] - # ------------- Disaggregated Serving: Pipelined KV Transfer (multi-GPU) --------------- - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_pipelined_kv_transfer_nixl_python_accuracy[use_kv_cache_manager_v2=False-enable_block_reuse=True] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_pipelined_kv_transfer_nixl_python_accuracy[use_kv_cache_manager_v2=True-enable_block_reuse=True] # ------------- KV Cache V2 Scheduler IT (multi-GPU) --------------- - kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_draft_tokens - kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_chunked_draft_tokens diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 7c671ca75747..89a28d10e33e 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -38,10 +38,6 @@ l0_dgx_h100: - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[no_block_reuse] diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index dc72fe9b1e3b..260219f26bbc 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -127,17 +127,6 @@ l0_h100: - unittest/scaffolding - unittest/usage/test_transport.py - unittest/usage/test_e2e_capture.py - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler - accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap] - accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=TRTLLM] TIMEOUT (90) @@ -166,7 +155,6 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized - accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8] - accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency] @@ -261,7 +249,6 @@ l0_h100: backend: pytorch orchestrator: ray tests: - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency] - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False] @@ -439,10 +426,6 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=vanilla-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=none-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized - - accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d87fcd2ce625..99653c1d593e 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -157,7 +157,6 @@ full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_ full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash SKIP (https://nvbugs/6316985) full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) -full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6479708) full:GB200/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:GB300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2] SKIP (https://nvbugs/6295740) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6432818) @@ -176,7 +175,6 @@ full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16 SKIP (htt full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_8_2_4T_A95B::test_fp8_tp16_mtp3_trtllm SKIP (https://nvbugs/6694922) full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) -full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6479708) full:GB300/disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6581064) full:GB300/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:GB300/test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] SKIP (https://nvbugs/6732110) @@ -342,7 +340,6 @@ unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfe unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) -unittest/llmapi/test_llm.py::test_generate_with_detokenization_stop_words_streaming[/scratch.trt_llm_data/llm-models/gemma/gemma-3-1b-it] SKIP (https://nvbugs/6566772) unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp2[False-False-True] SKIP (https://nvbugs/6432826) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504) diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 1132748a33c0..000000000000 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -metadata: - model_name: gemma_3_1b_it - precision: bf16 - model_dir_name: gemma-3-1b-it - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '256' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - print_iter_log: true - max_batch_size: 256 - max_num_tokens: 512 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - kv_cache_config: - use_kv_cache_manager_v2: false - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: auto - max_attention_window: [512, 512, 512, 512, 512, 32768] - cache_transceiver_config: - max_tokens_in_buffer: 4096 - backend: NIXL - transceiver_runtime: PYTHON - disable_overlap_scheduler: false - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 32 - max_num_tokens: 8192 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: null - kv_cache_config: - use_kv_cache_manager_v2: false - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: auto - max_attention_window: [512, 512, 512, 512, 512, 32768] - cache_transceiver_config: - max_tokens_in_buffer: 4096 - backend: NIXL - transceiver_runtime: PYTHON - disable_overlap_scheduler: false diff --git a/tests/test_common/llm_data.py b/tests/test_common/llm_data.py index eb07cdad1d87..51f4555cb225 100644 --- a/tests/test_common/llm_data.py +++ b/tests/test_common/llm_data.py @@ -53,7 +53,6 @@ "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": "NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", # AutoDeploy accuracy tests - overlapping with model registry - "google/gemma-3-1b-it": "gemma/gemma-3-1b-it", "google/gemma-3n-E2B-it": "gemma/gemma-3n-E2B-it", "google/gemma-4-E2B-it": "gemma/gemma-4-E2B-it", "nvidia/Qwen3.5-397B-A17B-NVFP4": "Qwen3.5-397B-A17B-NVFP4", diff --git a/tests/unittest/llmapi/test_llm.py b/tests/unittest/llmapi/test_llm.py index fcea6626a701..823bd0c9b4ff 100644 --- a/tests/unittest/llmapi/test_llm.py +++ b/tests/unittest/llmapi/test_llm.py @@ -504,91 +504,6 @@ def test_generate_with_stop_words(): stop_reasons=["I J"]) -@force_ampere -@pytest.mark.part0 -@pytest.mark.parametrize("model_path", [ - get_model_path('gemma/gemma-3-1b-it'), -]) -def test_generate_with_detokenization_stop_words(model_path): - llm = LLM( - model=model_path, - kv_cache_config=global_kvcache_config, - ) - - # Format the prompt using chat template - messages = [{ - "role": "user", - "content": "Say exactly: Hello there! How can I help" - }] - - formatted_prompt = llm.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True) - - detokenization_prompts = [formatted_prompt] - - # Test case 1: Stop word "How" should be detected after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello there!"], - sampling_params=SamplingParams(stop="How", max_tokens=10), - finish_reasons=['stop'], - stop_reasons=["How"]) - - # Test case 2: Stop word "there" should be detected after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello"], - sampling_params=SamplingParams(stop="there", - max_tokens=10), - finish_reasons=['stop'], - stop_reasons=["there"]) - - # Test case 3: Stop word that should not be found after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello there! How can I help"], - sampling_params=SamplingParams(stop="XYZ", max_tokens=10), - finish_reasons=['length'], - stop_reasons=[None]) - - # Test case 4: Multiple stop words, one should be found after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello"], - sampling_params=SamplingParams(stop=["XYZ", "there"], - max_tokens=10), - finish_reasons=['stop'], - stop_reasons=["there"]) - - -@force_ampere -@pytest.mark.part0 -@pytest.mark.parametrize("model_path", [ - get_model_path('gemma/gemma-3-1b-it'), -]) -def test_generate_with_detokenization_stop_words_streaming(model_path): - llm = LLM( - model=model_path, - kv_cache_config=global_kvcache_config, - ) - - # Format the prompt using chat template - messages = [{ - "role": "user", - "content": "Say exactly: Hello there! How can I help" - }] - - formatted_prompt = llm.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True) - - sampling_params = SamplingParams(stop="How", max_tokens=10) - - for output in llm.generate_async(formatted_prompt, - sampling_params=sampling_params, - streaming=True): - if output.outputs[0].finish_reason == 'stop': - assert output.outputs[0].stop_reason == "How" - break - elif output.outputs[0].finish_reason == 'length': - assert False, f"Expected to find stop word 'How' but reached max_tokens. Generated: {output.outputs[0].text}" - - @force_ampere @pytest.mark.part0 def test_generate_with_bad_words():