From 029161f7f6e1b7613611a466cd5f48d240940e8a Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:59:57 +0800 Subject: [PATCH 1/3] test: prune Gemma 3 checkpoint tests Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../accuracy/references/cnn_dailymail.yaml | 10 - .../defs/accuracy/references/gsm8k.yaml | 10 - .../accuracy/references/json_mode_eval.yaml | 4 - .../defs/accuracy/references/mmlu.yaml | 10 - .../defs/accuracy/references/mmmu.yaml | 23 - .../accuracy/test_disaggregated_serving.py | 151 ------ .../test_kv_pool_rebalance_accuracy.py | 171 ------- .../defs/accuracy/test_llm_api_autodeploy.py | 2 - .../defs/accuracy/test_llm_api_pytorch.py | 353 ------------- .../accuracy/test_llm_api_pytorch_encode.py | 10 +- .../test_llm_api_pytorch_multimodal.py | 108 ---- .../visual_gen/test_visual_gen_ltx2.py | 466 ------------------ .../README_test_visual_gen_perf_sanity.md | 3 +- tests/integration/defs/perf/_model_paths.py | 7 - .../defs/perf/test_visual_gen_perf_sanity.py | 4 - .../test_lists/qa/llm_function_core.txt | 23 - .../test_lists/qa/llm_spark_func.yml | 5 - .../test_lists/qa/llm_spark_perf.yml | 6 - .../test_lists/test-db/l0_a100.yml | 1 - .../test_lists/test-db/l0_b200.yml | 7 - .../l0_b200_visual_gen_perf_sanity.yml | 3 - .../test_lists/test-db/l0_dgx_h100.yml | 4 - .../test_lists/test-db/l0_h100.yml | 19 - tests/integration/test_lists/waives.txt | 8 +- ...ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml | 103 ---- tests/test_common/llm_data.py | 1 - .../_torch/visual_gen/test_cache_dit.py | 126 +---- .../_torch/visual_gen/test_ltx2_pipeline.py | 46 +- tests/unittest/llmapi/test_llm.py | 85 ---- tests/unittest/llmapi/test_llm_pytorch.py | 143 +----- 30 files changed, 20 insertions(+), 1892 deletions(-) delete mode 100644 tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py delete mode 100644 tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py delete mode 100644 tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index 672f44ea42db..e576a74c4364 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -1,13 +1,3 @@ -google/gemma-3-1b-it: - - accuracy: 19.0 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 20.699 -google/gemma-3-27b-it: - - accuracy: 28.90 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 27.90 gpt2: - accuracy: 18.408 - quant_algo: W8A16 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index 5f7b439ec890..fa6c03539295 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -316,16 +316,6 @@ nvidia/Nemotron-MOE: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 63.268 -google/gemma-3-1b-it: - - accuracy: 25.52 # score getting from lm-eval with HF implementation - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 23.96 -google/gemma-3-27b-it: - - accuracy: 91.66 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 90.66 google/gemma-4-26B-A4B-it: - accuracy: 90.83 google/gemma-3n-E2B-it: diff --git a/tests/integration/defs/accuracy/references/json_mode_eval.yaml b/tests/integration/defs/accuracy/references/json_mode_eval.yaml index 6b43fa4e39c6..3fda403fb2c8 100644 --- a/tests/integration/defs/accuracy/references/json_mode_eval.yaml +++ b/tests/integration/defs/accuracy/references/json_mode_eval.yaml @@ -10,10 +10,6 @@ deepseek-ai/DeepSeek-V3-Lite: - accuracy: 77.00 - spec_dec_algo: MTP accuracy: 77.00 -google/gemma-3-1b-it: - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 61.00 GPT-OSS/120B-MXFP4: - quant_algo: W4A16_MXFP4 spec_dec_algo: Eagle diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 1143669dbc57..d682a80068ab 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -37,16 +37,6 @@ mistralai/Mistral-Small-3.1-24B-Instruct-2503: accuracy: 81.1 google/gemma-2-9b-it: - accuracy: 73.05 -google/gemma-3-1b-it: - - accuracy: 39.0 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 37.5 -google/gemma-3-27b-it: - - accuracy: 77.80 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 76.80 google/gemma-4-26B-A4B-it: - accuracy: 71.296 google/gemma-3n-E2B-it: diff --git a/tests/integration/defs/accuracy/references/mmmu.yaml b/tests/integration/defs/accuracy/references/mmmu.yaml index df6f2115e39d..7fc2c909710e 100644 --- a/tests/integration/defs/accuracy/references/mmmu.yaml +++ b/tests/integration/defs/accuracy/references/mmmu.yaml @@ -1,20 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -google/gemma-3-27b-it: - - accuracy: 52.0 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 50.0 - # Blackwell FP8 cubins for Gemma3 multimodal route through a different - # FlashInfer path than Hopper, producing ~5pt lower MMMU. Use a separate - # Blackwell-calibrated reference instead of relaxing the Hopper one. - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - extra_acc_spec: sm100_fp8 - accuracy: 46.0 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - accuracy: 48.0 google/gemma-4-26B-A4B-it: - accuracy: 56.667 # B200 PyTorch backend baseline for nvidia/Gemma-4-26B-A4B-NVFP4. @@ -25,14 +10,6 @@ google/gemma-4-26B-A4B-it: kv_cache_quant_algo: FP8 spec_dec_algo: MTP accuracy: 54.0 -google/gemma-3-12b-it: - - accuracy: 50.44 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 49.0 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - accuracy: 50.11 LGAI-EXAONE/EXAONE-4.5-33B: # Empirically achieved ~46.5 across H20/B200/B300/GB200/GB300 since the # model was added (PR #12873). The original 51.22 was committed without diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 78432c45d5ce..03bce01d822b 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1384,157 +1384,6 @@ def test_gen_first(self, enable_attention_dp, mtp_nextn): run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) -@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) -class TestGemma3_1BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-1b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-1b-it/" - - # Literal NIXL bypasses the harness's legacy UCX fallback. Omitting the - # runtime then exercises Gemma's automatic preference for Python V2. - - @pytest.mark.skip_less_device(2) - @pytest.mark.parametrize("block_reuse", [False, True]) - @skip_pre_hopper - def test_auto_dtype(self, block_reuse): - - ctx_server_config = { - "disable_overlap_scheduler": True, - "cuda_graph_config": None, - "cache_transceiver_config": { - "backend": "NIXL", - "max_tokens_in_buffer": 4096 - } - } - gen_server_config = { - "disable_overlap_scheduler": False, - "cuda_graph_config": None, - "cache_transceiver_config": { - "backend": "NIXL", - "max_tokens_in_buffer": 4096 - } - } - ctx_server_config["kv_cache_config"] = { - "max_attention_window": [512, 512, 512, 512, 512, 32768], - "enable_block_reuse": block_reuse, - "enable_partial_reuse": block_reuse, - } - gen_server_config["kv_cache_config"] = { - "max_attention_window": [512, 512, 512, 512, 512, 32768], - "enable_block_reuse": block_reuse, - "enable_partial_reuse": block_reuse, - } - disaggregated_server_config = { - "hostname": "localhost", - "backend": "pytorch", - "context_servers": { - "num_instances": 1 - }, - "generation_servers": { - "num_instances": 1 - } - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"]) - - @pytest.mark.skip_less_device(2) - @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], - ids=["cache_mgr_v1", "cache_mgr_v2"]) - @skip_pre_hopper - def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): - """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" - ctx_server_config = { - "disable_overlap_scheduler": True, - "cuda_graph_config": None, - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": use_kv_cache_manager_v2 - }, - "cache_transceiver_config": { - "backend": "NIXL", - "transceiver_runtime": "PYTHON" - } - } - gen_server_config = { - "disable_overlap_scheduler": True, - "cuda_graph_config": None, - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": use_kv_cache_manager_v2 - }, - "cache_transceiver_config": { - "backend": "NIXL", - "transceiver_runtime": "PYTHON" - } - } - disaggregated_server_config = { - "hostname": "localhost", - "port": 8000, - "backend": "pytorch", - "context_servers": { - "num_instances": 1, - "urls": ["localhost:8001"] - }, - "generation_servers": { - "num_instances": 1, - "urls": ["localhost:8002"] - } - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"]) - - @skip_pre_hopper - @pytest.mark.skip_less_device(2) - @parametrize_with_ids("enable_block_reuse", [True]) - @parametrize_with_ids("use_kv_cache_manager_v2", [True, False]) - def test_pipelined_kv_transfer_nixl_python_accuracy( - self, enable_block_reuse: bool, use_kv_cache_manager_v2: bool): - """Test pipelined Python transfer with Gemma 3 VSWA.""" - kv_cache_config = { - "use_kv_cache_manager_v2": use_kv_cache_manager_v2, - "enable_block_reuse": enable_block_reuse, - "enable_partial_reuse": enable_block_reuse, - "max_attention_window": [512, 512, 512, 512, 512, 32768], - } - cache_transceiver_config = { - "backend": "NIXL", - "transceiver_runtime": "PYTHON", - "max_tokens_in_buffer": 4096, - "enable_pipelined_transfer": True, - } - ctx_server_config = { - "max_num_tokens": 256, - "cuda_graph_config": None, - "kv_cache_config": dict(kv_cache_config), - "cache_transceiver_config": dict(cache_transceiver_config), - "enable_chunked_prefill": True, - } - gen_server_config = { - "cuda_graph_config": None, - "kv_cache_config": dict(kv_cache_config), - "cache_transceiver_config": dict(cache_transceiver_config), - "enable_chunked_prefill": True, - } - disaggregated_server_config = { - "hostname": "localhost", - "backend": "pytorch", - "schedule_style": "generation_first", - "context_servers": { - "num_instances": 1, - }, - "generation_servers": { - "num_instances": 1, - }, - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) - - @skip_pre_blackwell @pytest.mark.skip_less_device_memory(80000) class TestGPTOSS(LlmapiAccuracyTestHarness): diff --git a/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py b/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py deleted file mode 100644 index fd8c592b2308..000000000000 --- a/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -r"""Accuracy test for the KVCacheManagerV2 rebalance hook. - -Verifies that forcing the V2 auto-tuner to fire mid-generation does not -change greedy-decode outputs. Uses Gemma-3-1B with explicit VSWA so the -KV cache lands in >=2 pool groups and ``adjust()`` has real work to do -(a single pool group would make rebalance a no-op). - -Run as: - LLM_MODELS_ROOT=/path pytest \ - tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py -""" - -import pytest - -from tensorrt_llm import LLM -from tensorrt_llm.llmapi import KvCacheConfig, SamplingParams - -from ..conftest import llm_models_root, skip_pre_hopper - -# --------------------------------------------------------------------------- # -# Ratio injection -# --------------------------------------------------------------------------- # - - -def _inject_pool_ratio_mismatch(llm: LLM, *, skew: float = 2.0) -> None: - """Force the V2 auto-tuner to do real pool-resize work on the next rebalance call. - - Delegates to the backend-agnostic KVCacheManagerV2 introspection hook, which - bypasses the sample-count / cooldown gates and perturbs the target GPU ratio - past the auto-tuner's adjustment threshold. The hook requires a model with - >=2 pool groups (e.g. Gemma-3-1B with VSWA) and raises otherwise, so a future - model change can't silently turn this test into a no-op. - - Also drops the executor's rebalance-check throttle to every iteration, so - the test does not depend on how ``KV_POOL_REBALANCE_CHECK_INTERVAL`` compares - to the number of iterations this short prompt set happens to run. Raise that - interval above the iteration count and the hook would never fire, leaving the - token comparison below to pass vacuously; the ratio assertion in - ``_generate_tokens`` is the backstop that would catch it. - """ - from tensorrt_llm.runtime.kv_cache_manager_v2 import _introspection - - executor = llm._executor.engine - executor._rebalance_check_interval = 1 - kv_cache_manager = executor.kv_cache_manager - _introspection.force_rebalance_precondition(kv_cache_manager.impl, skew=skew) - - -# --------------------------------------------------------------------------- # -# Test -# --------------------------------------------------------------------------- # - -# A handful of prompts spanning short, medium, and long context lengths. -# The long prompt is intentionally repetitive so it occupies multiple KV -# blocks and creates enough pool pressure for rebalance to matter. -_PROMPTS = [ - "The capital of France is", - "Write one sentence about transformers.", - "List three prime numbers greater than 100:", - "The quick brown fox jumps over the lazy dog. " * 40, -] - -_SAMPLING = SamplingParams(max_tokens=64, temperature=0.0, top_k=1) - - -def _vswa_kv_cache_config(*, enable_rebalance: bool) -> KvCacheConfig: - """V2 manager + explicit VSWA pattern that yields multiple pool groups. - - Gemma-3-1B has 5 sliding-window layers : 1 full-attention layer. - """ - return KvCacheConfig( - use_kv_cache_manager_v2=True, - enable_kv_pool_rebalance=enable_rebalance, - max_attention_window=[512, 512, 512, 512, 512, 32768], - # Block reuse disabled per the standing Gemma3 WAR for non- - # inclusive sliding window kernel support. - enable_block_reuse=False, - enable_partial_reuse=False, - tokens_per_block=32, - free_gpu_memory_fraction=0.6, - ) - - -def _generate_tokens(*, model_path: str, disable_overlap: bool, enable_rebalance: bool): - """Run one LLM, return list[list[int]] of generated token ids. - - Note: the ratio-injection helper requires direct access to the - in-process PyExecutor, so the test runs in single-process worker - mode (``TLLM_WORKER_USE_SINGLE_PROCESS=1``). The caller is - responsible for setting that env var (via monkeypatch or otherwise) - before invoking this helper. - """ - from tensorrt_llm.runtime.kv_cache_manager_v2 import _introspection - - with LLM( - model_path, - disable_overlap_scheduler=disable_overlap, - kv_cache_config=_vswa_kv_cache_config(enable_rebalance=enable_rebalance), - ) as llm: - impl = llm._executor.engine.kv_cache_manager.impl - if enable_rebalance: - _inject_pool_ratio_mismatch(llm) - ratio_before = list(_introspection.current_gpu_ratio(impl)) - outputs = llm.generate(_PROMPTS, _SAMPLING) - ratio_after = list(_introspection.current_gpu_ratio(impl)) - - # Guard against a vacuous pass. Token equality between the rebalance - # and no-rebalance arms proves nothing if adjust() never ran, and - # nothing in the run logs at info level to tell us it did. The pool - # ratio moving is the observable signature that it happened. - if enable_rebalance: - assert ratio_after != ratio_before, ( - "rebalance never fired: GPU pool ratio unchanged at " - f"{ratio_before}. The token comparison would pass vacuously. " - "Check the executor's rebalance-check throttle and the V2 " - "auto-tuner's sample-count / cooldown gates." - ) - else: - assert ratio_after == ratio_before, ( - "pool ratio moved with enable_kv_pool_rebalance=False " - f"({ratio_before} -> {ratio_after}); the baseline arm is " - "supposed to hold pool ratios fixed." - ) - - return [list(o.outputs[0].token_ids) for o in outputs] - - -@skip_pre_hopper -class TestKvPoolRebalanceAccuracy: - """Token-exact greedy-decode equivalence under rebalance. - - Compares rebalance=off and rebalance=on with a forced mid-generation - adjust(). - """ - - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-1b-it/" - - @pytest.mark.parametrize("disable_overlap", [True, False], ids=["no_overlap", "overlap"]) - def test_rebalance_matches_baseline(self, disable_overlap, monkeypatch): - # Keep the PyExecutor in-process so the ratio-injection helper - # can reach .engine on the client side. - monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") - - baseline = _generate_tokens( - model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=False - ) - - treated = _generate_tokens( - model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=True - ) - - assert len(baseline) == len(treated) == len(_PROMPTS) - for i, (b, t) in enumerate(zip(baseline, treated)): - assert b == t, ( - f"prompt {i}: rebalance changed greedy-decode output\n" - f" baseline: {b[:16]}...\n" - f" treated: {t[:16]}..." - ) diff --git a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py index 1f2394634ea4..6d166da2c38e 100644 --- a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py +++ b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @@ -1438,8 +1438,6 @@ class TestModelRegistryAccuracy(LlmapiAccuracyTestHarness): pytest.param("nvidia/Llama-3.1-8B-Instruct-NVFP4", {}, [MMLU, GSM8K], marks=skip_pre_blackwell, id="nvidia_Llama-3.1-8B-Instruct-NVFP4"), - pytest.param("google/gemma-3-1b-it", {}, [MMLU, GSM8K], - id="google_gemma-3-1b-it"), pytest.param("mistralai/Ministral-8B-Instruct-2410", {}, [MMLU, GSM8K], id="mistralai_Ministral-8B-Instruct-2410"), pytest.param( diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index c8102d37d26c..a51f3bf9b59e 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1238,359 +1238,6 @@ def test_fp8(self): pytest.skip("FP8 pre-quantized Ministral-8B model not available") -@skip_post_blackwell -@skip_pre_hopper -class TestGemma3_27BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-27b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-27b-it/" - - def test_auto_dtype(self): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - free_gpu_memory_fraction=0.5, - ) - # We use FlashInfer as the attention backend for Gemma3 VLM to support custom mask for images. - # So, testing with it here. - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - attn_backend="FLASHINFER", - cuda_graph_config=None, - max_batch_size=128, - max_seq_len=4096) as llm: - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - - def test_fp8_prequantized(self): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig(enable_block_reuse=False, - enable_partial_reuse=False, - dtype="fp8") - # Note: This has only the LLM part quantized. Vision part is in bfloat16. - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-27b-it-fp8/" - with LLM(prequantized_model_path, - kv_cache_config=kv_cache_config, - attn_backend="FLASHINFER", - cuda_graph_config=None, - max_seq_len=4096) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - -@skip_pre_hopper -class TestGemma3_1BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-1b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-1b-it/" - - # NOTE: Disable block reuse for SWA window model. - kv_cache_config = KvCacheConfig(enable_block_reuse=True) - - def test_auto_dtype(self): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - ) - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - @parametrize_with_ids("torch_compile", [False, True]) - def test_fp8_prequantized(self, torch_compile): - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig(enable_block_reuse=False, - enable_partial_reuse=False, - dtype="fp8") - torch_compile_config = _get_default_torch_compile_config(torch_compile) - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-1b-it-fp8/" - with LLM(prequantized_model_path, - kv_cache_config=kv_cache_config, - torch_compile_config=torch_compile_config) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 - task = CnnDailymail(self.MODEL_NAME) - task.evaluate(llm) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_fp8_vswa_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-1b-it-fp8/" - with LLM(prequantized_model_path, - kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - @pytest.mark.parametrize("backend", ["xgrammar"]) - def test_fp8_guided_decoding_vswa_reuse(self, backend: str, mocker): - mocker.patch.dict(os.environ, {"TRTLLM_XGUIDANCE_LENIENT": "1"}) - prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-1b-it-fp8/" - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - cuda_graph_config = CudaGraphConfig(enable_padding=True) - llm = LLM(prequantized_model_path, - guided_decoding_backend=backend, - kv_cache_config=kv_cache_config, - cuda_graph_config=cuda_graph_config) - with llm: - task = JsonModeEval(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_without_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_without_reuse_low_memory_available(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - free_gpu_memory_fraction=0.1, - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - disable_overlap_scheduler=True) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_disable_overlap_scheduler(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - disable_overlap_scheduler=True) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_partial_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - enable_partial_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - free_gpu_memory_fraction=0.1, - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - enable_partial_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - free_gpu_memory_fraction=0.1, - ) - - with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_reuse_kv_cache_stats(self): - """Mirror of test_auto_dtype_vswa_reuse that collects per-iteration stats. - - Collects per-iteration KV cache statistics and writes them to a JSON - file for offline visualization with - ``scripts/visualize_kv_cache_stats.py``. - """ - import json - import time - from pathlib import Path - - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - iteration_stats_interval=1, - ) - - all_stats = [] - - def drain_stats(llm, phase_label): - """Drain the stats queue and tag each entry. - - Tags each entry with wall-clock time and a human-readable phase - label. - """ - stats = llm.get_stats(timeout=2) - ts = time.time() - for entry in stats: - entry["_collectedAt"] = ts - entry["_phase"] = phase_label - all_stats.extend(stats) - - with LLM( - self.MODEL_PATH, - kv_cache_config=kv_cache_config, - max_stats_len=-1, - enable_iter_perf_stats=True, - ) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - drain_stats(llm, "GSM8K") - - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - drain_stats(llm, "MMLU") - - # Write collected stats to JSON - out_dir = Path( - os.environ.get( - "KV_CACHE_STATS_OUTPUT_DIR", - "kv_cache_stats_output", - )) - out_dir.mkdir(parents=True, exist_ok=True) - timestamp = time.strftime("%Y%m%d_%H%M%S") - out_path = out_dir / f"kv_cache_stats_{timestamp}.json" - - payload = { - "model": self.MODEL_NAME, - "kv_cache_config": { - "enable_block_reuse": - kv_cache_config.enable_block_reuse, - "max_attention_window": - kv_cache_config.max_attention_window, - "iteration_stats_interval": - kv_cache_config.iteration_stats_interval, - }, - "num_entries": len(all_stats), - "stats": all_stats, - } - out_path.write_text(json.dumps(payload, indent=2)) - print(f"\n[kv_cache_stats] Wrote {len(all_stats)} entries to " - f"{out_path}") - - def test_auto_dtype_vswa_chunked_prefill_without_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - # chunked prefill case or more features - extra_llm_config = dict( - enable_chunked_prefill=True, - max_num_tokens=1024, - ) - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - **extra_llm_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - def test_auto_dtype_vswa_chunked_prefill_reuse(self): - # NOTE: Test with VSWA kv cache config. - kv_cache_config = KvCacheConfig( - enable_block_reuse=True, - max_attention_window=[512, 512, 512, 512, 512, 32768], - ) - - # chunked prefill case or more features - extra_llm_config = dict( - enable_chunked_prefill=True, - max_num_tokens=1024, - ) - with LLM(self.MODEL_PATH, - kv_cache_config=kv_cache_config, - **extra_llm_config) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) - - # This class has extensively parameterized test methods, which yield totally 200 test cases. # This is because this model requires high test coverage over the feature combinations. # Normally we should not parameterize test methods so extensively -- just test on the typical/important feature combinations. diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py index 07602993da95..c75cffdb842e 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_encode.py @@ -240,7 +240,6 @@ def test_qwen3_text_embedding_matches_huggingface(self, model_name, model_path): # # One representative per distinct TRT-LLM architecture class: # LlamaForCausalLM — TinyLlama (also covers Mistral, which aliases LlamaModel) -# Gemma3ForCausalLM — Gemma-3-1B (sliding window + global alternation) # Qwen3ForCausalLM — Qwen3-0.6B (QKNorm) DECODER_MODELS = [ # -- LlamaForCausalLM (covers Llama + Mistral family) -- @@ -249,12 +248,6 @@ def test_qwen3_text_embedding_matches_huggingface(self, model_name, model_path): f"{llm_models_root()}/llama-models-v2/TinyLlama-1.1B-Chat-v1.0", id="tinyllama-1.1b", ), - # -- Gemma3ForCausalLM -- - pytest.param( - "google/gemma-3-1b-it", - f"{llm_models_root()}/gemma/gemma-3-1b-it/", - id="gemma-3-1b", - ), # -- Qwen3ForCausalLM -- pytest.param( "Qwen/Qwen3-0.6B", @@ -274,8 +267,7 @@ class TestDecoderEncode(LlmapiAccuracyTestHarness): ] # Top-K size used for the argmax-in-top-K containment / overlap checks. - # Chosen to be robust to near-tie argmax flips under FP16/BF16 rounding - # on very large vocabularies (Gemma-3 has 262K tokens). + # This is robust to near-tie argmax flips under FP16/BF16 rounding. TOPK = 5 TOPK_MIN_OVERLAP = 3 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 1dd10b4c4ee8..e984117f758c 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -32,7 +32,6 @@ from tensorrt_llm.quantization import QuantAlgo from ..conftest import ( - get_sm_version, llm_models_root, skip_post_blackwell_ultra, skip_pre_blackwell, @@ -120,113 +119,6 @@ def test_auto_dtype(self, enable_chunked_prefill, max_num_tokens): ) -@skip_pre_hopper -class TestGemma3_27BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-27b-it" - # Note: This has only the LLM part quantized. Vision part is in bfloat16. - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-27b-it-fp8/" - MAX_NUM_TOKENS = 12800 - - sampling_params = SamplingParams( - max_tokens=MAX_NUM_TOKENS, truncate_prompt_tokens=MMMU.MAX_INPUT_LEN, stop="" - ) - - # Gemma3 VLM needs KV cache reuse disabled for custom mask support. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - free_gpu_memory_fraction=0.4, - dtype="fp8", - ) - - def _make_llm(self, model_path: str): - # Gemma3 VLM needs FlashInfer attention backend for custom mask support. - return LLM( - model_path, - max_batch_size=16, - max_num_tokens=self.MAX_NUM_TOKENS, - max_seq_len=8704, # 8192 + 512. - kv_cache_config=self.kv_cache_config, - attn_backend="FLASHINFER", - enable_chunked_prefill=False, - ) - - def test_fp8_prequantized(self): - # Blackwell FP8 numerics differ from Hopper at the cubin level - # (~5pt drop on MMMU). Route to a Blackwell-calibrated reference - # rather than relaxing the Hopper one. - extra_acc_spec = "sm100_fp8" if get_sm_version() >= 100 else None - with self._make_llm(self.MODEL_PATH) as llm: - task = MMMU(self.MODEL_NAME) - task.evaluate( - llm, - extra_acc_spec=extra_acc_spec, - sampling_params=self.sampling_params, - ) - - @skip_pre_blackwell - def test_nvfp4_prequantized(self): - model_path = f"{llm_models_root()}/gemma/gemma-3-27b-it-FP4" - with self._make_llm(model_path) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - -@skip_pre_hopper -class TestGemma3_12BInstruct(LlmapiAccuracyTestHarness): - MODEL_NAME = "google/gemma-3-12b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-12b-it" - MAX_NUM_TOKENS = 12800 - - sampling_params = SamplingParams( - max_tokens=MAX_NUM_TOKENS, truncate_prompt_tokens=MMMU.MAX_INPUT_LEN, stop="" - ) - - # Gemma3 VLM needs KV cache reuse disabled for custom mask support. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - free_gpu_memory_fraction=0.6, - ) - - kv_cache_config_fp8 = kv_cache_config.model_copy(update={"dtype": "fp8"}) - - def _make_llm(self, model_path: str, kv_cache_config: KvCacheConfig = None): - # Gemma3 VLM needs FlashInfer attention backend for custom mask support. - if kv_cache_config is None: - kv_cache_config = self.kv_cache_config - return LLM( - model_path, - max_batch_size=16, - max_num_tokens=self.MAX_NUM_TOKENS, - max_seq_len=8704, # 8192 + 512. - kv_cache_config=kv_cache_config, - attn_backend="FLASHINFER", - enable_chunked_prefill=False, - ) - - def test_auto_dtype(self): - with self._make_llm(self.MODEL_PATH) as llm: - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - def test_fp8_prequantized(self): - model_path = f"{llm_models_root()}/gemma/gemma-3-12b-it-fp8" - with self._make_llm(model_path, self.kv_cache_config_fp8) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8 - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - @skip_pre_blackwell - def test_nvfp4_prequantized(self): - model_path = f"{llm_models_root()}/gemma/gemma-3-12b-it-fp4" - with self._make_llm(model_path, self.kv_cache_config_fp8) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 - task = MMMU(self.MODEL_NAME) - task.evaluate(llm, sampling_params=self.sampling_params) - - @pytest.mark.skip_device_not_contain(["B200", "GB10"]) class TestGemma4_26B_A4B(LlmapiAccuracyTestHarness): MODEL_NAME = "google/gemma-4-26B-A4B-it" diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py deleted file mode 100644 index a853709a1479..000000000000 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py +++ /dev/null @@ -1,466 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Single-GPU integration and accuracy tests for LTX-2.""" - -import os -from dataclasses import dataclass - -import pytest -import torch -from defs import conftest -from defs.common import venv_check_call -from defs.examples.visual_gen.visual_gen_test_utils import ( - FeatureConfigState, - _assert_feature_quantization_installed, - _assert_lpips_below_threshold, - _assert_resolved_single_device_feature_config, - _assert_single_device_feature_executed, - _build_single_device_feature_args, - _cleanup_cuda, - _cleanup_single_device_feature_pipeline, - _disable_inductor_compile_worker_quiesce, - _fixed_nvfp4_quantization_backend, - _golden_media_path, - _lpips_deterministic_algorithms, - _lpips_model_path, - _preserve_lpips_candidate_on_failure, - _run_lpips_eval, - _run_reusable_video_lpips_eval, - _run_single_device_feature_generator, - _save_lpips_video_mp4, - _skip_if_missing, - _validate_single_feature_config, - _visual_gen_output_path, -) - -LTX2_LPIPS_NUM_FRAMES = 49 -LTX2_LPIPS_NUM_INFERENCE_STEPS = 8 -LTX2_LPIPS_THRESHOLD = 0.05 -LTX2_CUDA_GRAPH_LPIPS_THRESHOLD = 0.01 - -LTX2_FEATURE_LPIPS_THRESHOLD = 0.05 -LTX2_SUPPORTED_FEATURES = frozenset({"fp8-blockwise", "nvfp4", "cuda-graph"}) - - -@dataclass(frozen=True) -class LTX2AccuracyCase: - id: str - golden_file: str - features: FeatureConfigState - lpips_threshold: float - - -LTX2_FEATURE_PROFILES = ( - ("fp8-blockwise", FeatureConfigState(quantization="FP8_BLOCK_SCALES")), - ("nvfp4", FeatureConfigState(quantization="NVFP4")), - ("cuda-graph", FeatureConfigState(cuda_graph=True)), -) - - -def _build_ltx2_accuracy_cases(): - cases = [] - for profile_id, features in LTX2_FEATURE_PROFILES: - _validate_single_feature_config( - features, - LTX2_SUPPORTED_FEATURES, - "LTX-2", - ) - cases.append( - pytest.param( - LTX2AccuracyCase( - id=profile_id, - golden_file=(f"ltx2_{profile_id.replace('-', '_')}_lpips_golden_video.mp4"), - features=features, - lpips_threshold=LTX2_FEATURE_LPIPS_THRESHOLD, - ), - id=profile_id, - ) - ) - return cases - - -LTX2_ACCURACY_CASES = _build_ltx2_accuracy_cases() - - -# LTX-2 configuration -LTX2_MODEL_CHECKPOINT_PATH = "LTX-2/ltx-2-19b-dev.safetensors" -LTX2_TEXT_ENCODER_SUBPATH = "gemma-3-12b-it" -LTX2_T2V_PROMPT = ( - "A woman with long brown hair and light skin smiles at the camera while " - "standing in a sunlit park, her hair gently blowing in the breeze as she " - "tilts her head slightly to the side." -) -LTX2_T2V_HEIGHT = 512 -LTX2_T2V_WIDTH = 768 -LTX2_T2V_NUM_FRAMES = 121 -LTX2_T2V_STEPS = 40 -LTX2_T2V_GUIDANCE_SCALE = 4.0 -LTX2_T2V_MAX_SEQ_LEN = 1024 -LTX2_T2V_FRAME_RATE = 24.0 -LTX2_T2V_SEED = 42 -LTX2_T2V_NEGATIVE_PROMPT = "worst quality, inconsistent motion, blurry, jittery, distorted" - - -# LTX-2 Two-Stage configuration -LTX2_UPSAMPLER_SUBPATH = "LTX-2/ltx-2-spatial-upscaler-x2-1.0.safetensors" -LTX2_DISTILLED_LORA_SUBPATH = "LTX-2/ltx-2-19b-distilled-lora-384.safetensors" - - -def _ltx2_lpips_text_encoder_path(): - scratch_space = conftest.llm_models_root() - candidates = [ - os.path.join(scratch_space, LTX2_TEXT_ENCODER_SUBPATH), - os.path.join(scratch_space, "gemma", LTX2_TEXT_ENCODER_SUBPATH), - ] - for candidate in candidates: - if os.path.isdir(candidate): - return candidate - return candidates[0] - - -def _generate_ltx2_feature_video(case, output_path): - from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader - - checkpoint_path = _lpips_model_path("LTX-2", "ltx-2-19b-dev.safetensors") - text_encoder_path = _ltx2_lpips_text_encoder_path() - spatial_upsampler_path = _lpips_model_path("LTX-2", "ltx-2-spatial-upscaler-x2-1.0.safetensors") - distilled_lora_path = _lpips_model_path("LTX-2", "ltx-2-19b-distilled-lora-384.safetensors") - _skip_if_missing(checkpoint_path, "LTX-2 checkpoint") - _skip_if_missing(text_encoder_path, "LTX-2 text encoder", is_dir=True) - _skip_if_missing(spatial_upsampler_path, "LTX-2 spatial upsampler") - _skip_if_missing(distilled_lora_path, "LTX-2 distilled LoRA") - _disable_inductor_compile_worker_quiesce() - pipeline = None - with ( - _lpips_deterministic_algorithms(), - torch.compiler.set_stance("force_eager"), - _fixed_nvfp4_quantization_backend(case.features), - ): - args = _build_single_device_feature_args( - checkpoint_path, - case.features, - resolution=(LTX2_T2V_HEIGHT, LTX2_T2V_WIDTH), - num_frames=LTX2_LPIPS_NUM_FRAMES, - pipeline_config={ - "text_encoder_path": text_encoder_path, - "spatial_upsampler_path": spatial_upsampler_path, - "distilled_lora_path": distilled_lora_path, - }, - ) - try: - pipeline = PipelineLoader(args).load(skip_warmup=False) - _assert_resolved_single_device_feature_config( - pipeline, - case.features, - resolution=(LTX2_T2V_HEIGHT, LTX2_T2V_WIDTH), - num_frames=LTX2_LPIPS_NUM_FRAMES, - ) - _assert_feature_quantization_installed(pipeline, case.features) - result = pipeline.forward( - prompt=LTX2_T2V_PROMPT, - negative_prompt=LTX2_T2V_NEGATIVE_PROMPT, - height=LTX2_T2V_HEIGHT, - width=LTX2_T2V_WIDTH, - num_frames=LTX2_LPIPS_NUM_FRAMES, - num_inference_steps=LTX2_LPIPS_NUM_INFERENCE_STEPS, - guidance_scale=LTX2_T2V_GUIDANCE_SCALE, - seed=LTX2_T2V_SEED, - ) - assert result.video is not None, "LTX-2 feature run produced no video" - _assert_single_device_feature_executed(pipeline, case.features) - generated_video = result.video.detach().cpu() - finally: - try: - if pipeline is not None: - _cleanup_single_device_feature_pipeline(pipeline) - del pipeline - finally: - _cleanup_cuda() - - _save_lpips_video_mp4(generated_video, output_path, frame_rate=LTX2_T2V_FRAME_RATE) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -@pytest.mark.parametrize("case", LTX2_ACCURACY_CASES) -def test_ltx2_feature_accuracy_against_golden( - request, - tmp_path, - case, - _visual_gen_deps, - _visual_gen_lpips_scorer, -): - generated_path = tmp_path / f"ltx2_{case.id}_generated.mp4" - golden_path = _golden_media_path( - tmp_path, - case.golden_file, - f"LTX-2 {case.id} LPIPS golden video", - ) - _run_single_device_feature_generator( - case.features, _generate_ltx2_feature_video, case, generated_path - ) - score = _run_reusable_video_lpips_eval( - f"ltx2-{case.id}", - golden_path, - generated_path, - _visual_gen_lpips_scorer, - ) - _preserve_lpips_candidate_on_failure( - request, - score, - case.lpips_threshold, - generated_path, - f"ltx2_{case.id}_generated.mp4", - ) - _assert_lpips_below_threshold(score, case.lpips_threshold) - - -def _generate_ltx2_lpips_video(output_path, *, enable_cuda_graph=False): - from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader - from tensorrt_llm.visual_gen.args import CudaGraphConfig, TorchCompileConfig, VisualGenArgs - - checkpoint_path = _lpips_model_path("LTX-2", "ltx-2-19b-dev.safetensors") - text_encoder_path = _ltx2_lpips_text_encoder_path() - spatial_upsampler_path = _lpips_model_path("LTX-2", "ltx-2-spatial-upscaler-x2-1.0.safetensors") - distilled_lora_path = _lpips_model_path("LTX-2", "ltx-2-19b-distilled-lora-384.safetensors") - _skip_if_missing(checkpoint_path, "LTX-2 checkpoint") - _skip_if_missing(text_encoder_path, "LTX-2 text encoder", is_dir=True) - _skip_if_missing(spatial_upsampler_path, "LTX-2 spatial upsampler") - _skip_if_missing(distilled_lora_path, "LTX-2 distilled LoRA") - _disable_inductor_compile_worker_quiesce() - - # TorchCompileConfig(enable=False) does not suppress nested @torch.compile decorators. - # Wrapped here (not in the fixture) so the golden fixture and both sides of - # test_ltx2_cuda_graph_lpips_matches_eager run the same eager numerics. - with _lpips_deterministic_algorithms(), torch.compiler.set_stance("force_eager"): - args = VisualGenArgs( - model=checkpoint_path, - pipeline_config={ - "text_encoder_path": text_encoder_path, - "spatial_upsampler_path": spatial_upsampler_path, - "distilled_lora_path": distilled_lora_path, - }, - torch_compile_config=TorchCompileConfig(enable=False), - cuda_graph_config=CudaGraphConfig(enable=enable_cuda_graph), - ) - pipeline = PipelineLoader(args).load(skip_warmup=True) - try: - with torch.no_grad(): - result = pipeline.forward( - prompt=LTX2_T2V_PROMPT, - negative_prompt=LTX2_T2V_NEGATIVE_PROMPT, - height=LTX2_T2V_HEIGHT, - width=LTX2_T2V_WIDTH, - num_frames=LTX2_LPIPS_NUM_FRAMES, - num_inference_steps=LTX2_LPIPS_NUM_INFERENCE_STEPS, - guidance_scale=LTX2_T2V_GUIDANCE_SCALE, - seed=LTX2_T2V_SEED, - ) - generated_video = result.video.detach().cpu() - finally: - del pipeline - _cleanup_cuda() - - _save_lpips_video_mp4(generated_video, output_path, frame_rate=LTX2_T2V_FRAME_RATE) - - -def _generate_ltx2_cuda_graph_trtllm_backend_video(output_path): - from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams - from tensorrt_llm.visual_gen.args import ( - AttentionConfig, - CompilationConfig, - CudaGraphConfig, - ParallelConfig, - TorchCompileConfig, - ) - - scratch_space = conftest.llm_models_root() - checkpoint_path = os.path.join(scratch_space, LTX2_MODEL_CHECKPOINT_PATH) - text_encoder_path = _ltx2_lpips_text_encoder_path() - spatial_upsampler_path = os.path.join(scratch_space, LTX2_UPSAMPLER_SUBPATH) - distilled_lora_path = os.path.join(scratch_space, LTX2_DISTILLED_LORA_SUBPATH) - _skip_if_missing(checkpoint_path, "LTX-2 checkpoint") - _skip_if_missing(text_encoder_path, "LTX-2 text encoder", is_dir=True) - _skip_if_missing(spatial_upsampler_path, "LTX-2 spatial upsampler") - _skip_if_missing(distilled_lora_path, "LTX-2 distilled LoRA") - _disable_inductor_compile_worker_quiesce() - - visual_gen_args = VisualGenArgs( - model=checkpoint_path, - quant_config={"quant_algo": "NVFP4", "dynamic": True}, - attention_config=AttentionConfig(backend="TRTLLM"), - parallel_config=ParallelConfig( - cfg_size=1, - ulysses_size=1, - parallel_vae_size=1, - ), - compilation_config=CompilationConfig( - resolutions=[ - ( - LTX2_T2V_HEIGHT, - LTX2_T2V_WIDTH, - ) - ], - num_frames=[LTX2_LPIPS_NUM_FRAMES], - ), - cuda_graph_config=CudaGraphConfig(enable=True), - torch_compile_config=TorchCompileConfig( - enable=True, - enable_fullgraph=False, - enable_autotune=True, - ), - pipeline_config={ - "text_encoder_path": text_encoder_path, - "spatial_upsampler_path": spatial_upsampler_path, - "distilled_lora_path": distilled_lora_path, - }, - ) - - visual_gen = VisualGen(model=checkpoint_path, args=visual_gen_args) - try: - params = VisualGenParams( - height=LTX2_T2V_HEIGHT, - width=LTX2_T2V_WIDTH, - num_frames=LTX2_LPIPS_NUM_FRAMES, - num_inference_steps=LTX2_LPIPS_NUM_INFERENCE_STEPS, - guidance_scale=LTX2_T2V_GUIDANCE_SCALE, - max_sequence_length=LTX2_T2V_MAX_SEQ_LEN, - seed=LTX2_T2V_SEED, - frame_rate=LTX2_T2V_FRAME_RATE, - negative_prompt=LTX2_T2V_NEGATIVE_PROMPT, - ) - output = visual_gen.generate(inputs=LTX2_T2V_PROMPT, params=params) - _save_lpips_video_mp4(output.video, output_path, frame_rate=LTX2_T2V_FRAME_RATE) - finally: - visual_gen.shutdown() - del visual_gen - _cleanup_cuda() - - assert os.path.isfile(output_path), f"LTX-2 TRTLLM backend did not produce {output_path}" - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -def test_ltx2_lpips_against_golden(request, tmp_path, ltx2_two_stage_bf16_video_path): - golden_path = _golden_media_path( - tmp_path, "ltx2_lpips_golden_video.mp4", "LTX-2 LPIPS golden video" - ) - score = _run_lpips_eval( - tmp_path, - "ltx2", - "video", - LTX2_T2V_PROMPT, - golden_path, - ltx2_two_stage_bf16_video_path, - ) - _preserve_lpips_candidate_on_failure( - request, - score, - LTX2_LPIPS_THRESHOLD, - ltx2_two_stage_bf16_video_path, - "ltx2_lpips_golden_video.mp4", - ) - _assert_lpips_below_threshold(score, LTX2_LPIPS_THRESHOLD) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -def test_ltx2_cuda_graph_lpips_matches_eager(_visual_gen_deps, tmp_path): - eager_path = tmp_path / "ltx2_eager_generated.mp4" - cuda_graph_path = tmp_path / "ltx2_cuda_graph_generated.mp4" - - _generate_ltx2_lpips_video(eager_path, enable_cuda_graph=False) - _generate_ltx2_lpips_video(cuda_graph_path, enable_cuda_graph=True) - score = _run_lpips_eval( - tmp_path, - "ltx2_cuda_graph", - "video", - LTX2_T2V_PROMPT, - eager_path, - cuda_graph_path, - ) - _assert_lpips_below_threshold(score, LTX2_CUDA_GRAPH_LPIPS_THRESHOLD) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -def test_ltx2_cuda_graph_trtllm_backend(request, _visual_gen_deps, tmp_path): - generated_path = tmp_path / "ltx2_cuda_graph_trtllm_backend_generated.mp4" - golden_path = _golden_media_path( - tmp_path, "ltx2_lpips_golden_video.mp4", "LTX-2 LPIPS golden video" - ) - _generate_ltx2_cuda_graph_trtllm_backend_video(generated_path) - score = _run_lpips_eval( - tmp_path, - "ltx2_cuda_graph_trtllm_backend", - "video", - LTX2_T2V_PROMPT, - golden_path, - generated_path, - ) - _preserve_lpips_candidate_on_failure( - request, - score, - LTX2_LPIPS_THRESHOLD, - generated_path, - "ltx2_cuda_graph_trtllm_backend_generated.mp4", - ) - _assert_lpips_below_threshold(score, LTX2_LPIPS_THRESHOLD) - - -@pytest.fixture(scope="session") -def ltx2_two_stage_bf16_video_path(_visual_gen_deps, llm_venv): - """Generate LTX-2 two-stage BF16 video with the LPIPS config and return path.""" - output_path = _visual_gen_output_path(llm_venv, "ltx2_two_stage_bf16") - if os.path.isfile(output_path): - return output_path - _generate_ltx2_lpips_video(output_path) - return output_path - - -def test_ltx2_example(_visual_gen_deps, llm_root, llm_venv): - """Run examples/visual_gen/models/ltx2.py with NVFP4 config end-to-end. - - Validates that the LTX-2 example script and ``configs/ltx2-fp4-1gpu.yaml`` - work together as documented. The Gemma3 text encoder is passed separately via - ``--text_encoder_path`` because the shared YAML intentionally omits it to keep - the config model-path-agnostic. - """ - model_path = _lpips_model_path("LTX-2", "ltx-2-19b-dev.safetensors") - _skip_if_missing(model_path, "LTX-2 checkpoint") - text_encoder_path = _ltx2_lpips_text_encoder_path() - _skip_if_missing(text_encoder_path, "LTX-2 text encoder (gemma-3-12b-it)", is_dir=True) - - out_dir = os.path.join(llm_venv.get_working_directory(), "visual_gen_output", "ltx2_example") - os.makedirs(out_dir, exist_ok=True) - output_path = os.path.join(out_dir, "ltx2_output.mp4") - - script_path = os.path.join(llm_root, "examples", "visual_gen", "models", "ltx2.py") - config_path = os.path.join(llm_root, "examples", "visual_gen", "configs", "ltx2-fp4-1gpu.yaml") - assert os.path.isfile(script_path), f"Example script not found: {script_path}" - assert os.path.isfile(config_path), f"Config not found: {config_path}" - - venv_check_call( - llm_venv, - [ - script_path, - "--model", - model_path, - "--visual_gen_args", - config_path, - "--text_encoder_path", - text_encoder_path, - "--output_path", - output_path, - ], - ) - assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" diff --git a/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md b/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md index 916b25b2ff39..4599993cde20 100644 --- a/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md @@ -90,7 +90,6 @@ suffix, for example: - `wan21_t2v_14b_blackwell.yaml` - `wan22_i2v_a14b_blackwell.yaml` - `flux2_blackwell.yaml` -- `ltx2_blackwell.yaml` ### YAML Structure @@ -298,7 +297,7 @@ extraction in `build_visual_gen_db_entry` and treat it as a baseline reset. pytest perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] # Run all server recipes from one family YAML without uploading -pytest perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg-ltx2_blackwell] +pytest perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg-flux2_blackwell] ``` Set `LLM_MODELS_ROOT` to the shared model cache before running locally: diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index 4308d916b19c..206ec0d9f2a0 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -19,13 +19,6 @@ "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct", "llama_v3.1_8b_instruct_fp8": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8", "llama_v3.1_8b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4", - "gemma_3_27b_it": "gemma/gemma-3-27b-it", - "gemma_3_27b_it_fp8": "gemma/gemma-3-27b-it-fp8", - "gemma_3_27b_it_fp4": "gemma/gemma-3-27b-it-FP4", - "gemma_3_12b_it": "gemma/gemma-3-12b-it", - "gemma_3_12b_it_fp8": "gemma/gemma-3-12b-it-fp8", - "gemma_3_12b_it_fp4": "gemma/gemma-3-12b-it-fp4", - "gemma_3_1b_it": "gemma/gemma-3-1b-it", "gemma_4_26b_a4b_nvfp4": "gemma/nvidia-Gemma-4-26B-A4B-NVFP4", "gemma_4_31b_it_nvfp4": "gemma/nvidia-Gemma-4-31B-IT-NVFP4", "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", diff --git a/tests/integration/defs/perf/test_visual_gen_perf_sanity.py b/tests/integration/defs/perf/test_visual_gen_perf_sanity.py index c3e9e08f2d80..bddbd702caff 100644 --- a/tests/integration/defs/perf/test_visual_gen_perf_sanity.py +++ b/tests/integration/defs/perf/test_visual_gen_perf_sanity.py @@ -317,10 +317,6 @@ def _append_candidate(candidate: Path) -> None: if "/" in path_value: _append_candidate(models_root / path_value.split("/", 1)[1]) - if "gemma-3-12b-it" in path_value: - _append_candidate(models_root / "gemma" / "gemma-3-12b-it") - _append_candidate(models_root / "gemma-3-12b-it") - for candidate in candidates: if candidate.exists(): return str(candidate) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index b3d743f6766f..7d9d9177d534 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -19,10 +19,6 @@ accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[True] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] @@ -463,23 +459,6 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-aut accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-fp8] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto] accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_guided_decoding_vswa_reuse[xgrammar] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] -accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_vswa_reuse -accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[dep8] accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[ep8] accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] @@ -723,7 +702,6 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8_Flash_Next::test_fp8_1gpu_mtp3_trt accuracy/test_llm_api_pytorch.py::TestQwen3_8_Flash_Next::test_nvfp4_1gpu_mtp3_cutedsl_ple_offload accuracy/test_llm_api_pytorch.py::TestSeedOss_36B::test_auto_dtype accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b] -accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen3-0.6b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b] accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp] @@ -733,7 +711,6 @@ accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embe accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-8b] accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] -accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4 accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (120) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] diff --git a/tests/integration/test_lists/qa/llm_spark_func.yml b/tests/integration/test_lists/qa/llm_spark_func.yml index a59385112d68..0db4b20e4ece 100644 --- a/tests/integration/test_lists/qa/llm_spark_func.yml +++ b/tests/integration/test_lists/qa/llm_spark_func.yml @@ -31,11 +31,6 @@ llm_spark_func: - test_e2e.py::test_ptp_quickstart_advanced[Qwen3.6-35B-A3B-nvfp4-Qwen3.6-35B-A3B-NVFP4] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_nvfp4_prequantized - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_auto_dtype - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_fp8_prequantized - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_nvfp4_prequantized - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4_no_mtp - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4 - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] diff --git a/tests/integration/test_lists/qa/llm_spark_perf.yml b/tests/integration/test_lists/qa/llm_spark_perf.yml index 966db8a62ef9..7cbb2ef70e70 100644 --- a/tests/integration/test_lists/qa/llm_spark_perf.yml +++ b/tests/integration/test_lists/qa/llm_spark_perf.yml @@ -38,12 +38,6 @@ llm_spark_perf: - perf/test_perf.py::test_perf[qwen3_30b_a3b-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[qwen3_32b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[qwen3_32b-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_27b_it-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_27b_it_fp8-bench-pytorch-streaming-float8-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_27b_it_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_12b_it-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_12b_it_fp8-bench-pytorch-streaming-float8-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - - perf/test_perf.py::test_perf[gemma_3_12b_it_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] # Qwen3.6-35B-A3B NVFP4: baseline and the same checkpoint with MTP drafting - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,256-reqs:1-con:1] diff --git a/tests/integration/test_lists/test-db/l0_a100.yml b/tests/integration/test_lists/test-db/l0_a100.yml index 6a8a81066bd1..2b0c7fd7359c 100644 --- a/tests/integration/test_lists/test-db/l0_a100.yml +++ b/tests/integration/test_lists/test-db/l0_a100.yml @@ -33,7 +33,6 @@ l0_a100: - accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-cuda_graph] - accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp] - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b] - - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b] - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen3-0.6b] - accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b] # Moved from l0_a10.yml: Llama-7B + KV cache + spec resources OOM on A10's 22 GB. diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 407c04da0168..5120e440fadf 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -329,7 +329,6 @@ l0_b200: - examples/visual_gen/test_visual_gen_flux.py::test_flux1_example - examples/visual_gen/test_visual_gen_flux.py::test_flux2_example - examples/visual_gen/test_visual_gen_flux.py::test_flux2_reference_image_example - - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_example - examples/visual_gen/test_visual_gen_wan.py::test_wan_i2v_example - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_example - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_edge_policy_droid_example TIMEOUT (20) @@ -449,18 +448,12 @@ l0_b200: - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_against_golden[nvfp4] - examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[fp8-blockwise] - examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] - - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[fp8-blockwise] - - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[nvfp4] - - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[cuda-graph] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan21-fp8-blockwise] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan21-nvfp4] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan21-cuda-graph] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-fp8-blockwise] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-nvfp4] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-cuda-graph] - - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_lpips_against_golden - - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_cuda_graph_lpips_matches_eager - - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_cuda_graph_trtllm_backend - examples/visual_gen/test_visual_gen_wan.py::test_wan21_t2v_lpips_against_golden - examples/visual_gen/test_visual_gen_wan.py::test_wan22_t2v_lpips_against_golden # Measured on B200 with TRT-LLM commit 85665f5f from the staging main image: diff --git a/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml index 88ae6c735195..35596d119877 100644 --- a/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml @@ -20,6 +20,3 @@ l0_b200_visual_gen_perf_sanity: - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-flux2_blackwell-flux2_fp8_cfg1_ulysses4_teacache_on] TIMEOUT (120) - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] TIMEOUT (180) - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan22_i2v_a14b_blackwell-wan22_i2v_a14b_nvfp4_trtllm_cfg2_ulysses4] TIMEOUT (180) - - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_i2v_cfg2_ulysses4_compile_on] TIMEOUT (180) - - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_t2v_cfg2_ulysses4_compile_on] TIMEOUT (180) - - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_nvfp4_i2v_cfg2_ulysses4_compile_on] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 7c671ca75747..89a28d10e33e 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -38,10 +38,6 @@ l0_dgx_h100: - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[no_block_reuse] diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index 3e5214d689a0..2f9da878e252 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -127,19 +127,6 @@ l0_h100: - unittest/scaffolding - unittest/usage/test_transport.py - unittest/usage/test_e2e_capture.py - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler - - accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap] - - accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=TRTLLM] TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=TRTLLM] TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format @@ -166,7 +153,6 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format - - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized - accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8] - accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency] @@ -261,7 +247,6 @@ l0_h100: backend: pytorch orchestrator: ray tests: - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency] - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False] @@ -439,10 +424,6 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=vanilla-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=none-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False] - - accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] - - accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized - - accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 0aff51f06298..4e4803184eba 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -77,10 +77,6 @@ examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_ag examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux1-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux2-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6644450) -examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_cuda_graph_trtllm_backend SKIP (https://nvbugs/6668775) -examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[cuda-graph] SKIP (https://nvbugs/6655986) -examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6572800) -examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_lpips_against_golden SKIP (https://nvbugs/6655986) examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[attn2d_2x2] SKIP (https://nvbugs/6655990) examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2] SKIP (https://nvbugs/6535765) examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[ulysses4] SKIP (https://nvbugs/6535765) @@ -158,7 +154,6 @@ full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_ full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash SKIP (https://nvbugs/6316985) full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) -full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6479708) full:GB200/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:GB300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2] SKIP (https://nvbugs/6295740) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6432818) @@ -177,7 +172,6 @@ full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16 SKIP (htt full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_8_2_4T_A95B::test_fp8_tp16_mtp3_trtllm SKIP (https://nvbugs/6694922) full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) -full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6479708) full:GB300/disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6581064) full:GB300/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:GB300/test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] SKIP (https://nvbugs/6732110) @@ -343,7 +337,7 @@ unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfe unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) -unittest/llmapi/test_llm.py::test_generate_with_detokenization_stop_words_streaming[/scratch.trt_llm_data/llm-models/gemma/gemma-3-1b-it] SKIP (https://nvbugs/6566772) +unittest/llmapi/test_llm.py::test_chat_stream_post_processor_reuses_stream_metadata SKIP (https://nvbugs/6693989) unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp2[False-False-True] SKIP (https://nvbugs/6432826) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504) diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 1132748a33c0..000000000000 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -metadata: - model_name: gemma_3_1b_it - precision: bf16 - model_dir_name: gemma-3-1b-it - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '256' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - print_iter_log: true - max_batch_size: 256 - max_num_tokens: 512 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - kv_cache_config: - use_kv_cache_manager_v2: false - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: auto - max_attention_window: [512, 512, 512, 512, 512, 32768] - cache_transceiver_config: - max_tokens_in_buffer: 4096 - backend: NIXL - transceiver_runtime: PYTHON - disable_overlap_scheduler: false - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 32 - max_num_tokens: 8192 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: null - kv_cache_config: - use_kv_cache_manager_v2: false - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: auto - max_attention_window: [512, 512, 512, 512, 512, 32768] - cache_transceiver_config: - max_tokens_in_buffer: 4096 - backend: NIXL - transceiver_runtime: PYTHON - disable_overlap_scheduler: false diff --git a/tests/test_common/llm_data.py b/tests/test_common/llm_data.py index eb07cdad1d87..51f4555cb225 100644 --- a/tests/test_common/llm_data.py +++ b/tests/test_common/llm_data.py @@ -53,7 +53,6 @@ "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": "NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": "NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", # AutoDeploy accuracy tests - overlapping with model registry - "google/gemma-3-1b-it": "gemma/gemma-3-1b-it", "google/gemma-3n-E2B-it": "gemma/gemma-3n-E2B-it", "google/gemma-4-E2B-it": "gemma/gemma-4-E2B-it", "nvidia/Qwen3.5-397B-A17B-NVFP4": "Qwen3.5-397B-A17B-NVFP4", diff --git a/tests/unittest/_torch/visual_gen/test_cache_dit.py b/tests/unittest/_torch/visual_gen/test_cache_dit.py index e836cad6f995..35708e26e970 100644 --- a/tests/unittest/_torch/visual_gen/test_cache_dit.py +++ b/tests/unittest/_torch/visual_gen/test_cache_dit.py @@ -3,9 +3,9 @@ """Tests for Cache-DiT in visual generation. -Wan 2.2 step-split logic is covered with small CPU-side tests. Wan, FLUX, and LTX-2 -integration tests run on GPU only when cache_dit is installed, CUDA is available, and -checkpoints can be resolved (TRTLLM_CACHE_DIT_* env vars or the fallbacks inside each test). +Wan 2.2 step-split logic is covered with small CPU-side tests. Wan and FLUX +integration tests run on GPU only when cache_dit is installed, CUDA is available, +and checkpoints can be resolved. """ from __future__ import annotations @@ -41,20 +41,10 @@ _WAN_SUBPATH = "Wan2.1-T2V-1.3B-Diffusers" _FLUX_SUBPATH = "FLUX.1-dev" _QWEN_IMAGE_SUBPATH = "qwen-image" -_LTX2_DIR = "LTX-2" -_LTX2_WEIGHTS_FILE = "ltx-2-19b-dev.safetensors" -_LTX2_TEXT_ENCODER_SUBPATH = "gemma-3-12b-it" -# Same share as other Gemma checkpoints (e.g. google/gemma-3-1b-it -> gemma/gemma-3-1b-it). -_LTX2_TEXT_ENCODER_RELATIVE_PATHS = ( - _LTX2_TEXT_ENCODER_SUBPATH, - os.path.join("gemma", _LTX2_TEXT_ENCODER_SUBPATH), -) # Default NFS layout used on CI runners (override via TRTLLM_CACHE_DIT_*_CHECKPOINT). _CI_DEFAULT_LLM_MODELS = "/home/scratch.trt_llm_data_ci/llm-models" _DEFAULT_WAN_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _WAN_SUBPATH) _DEFAULT_FLUX_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _FLUX_SUBPATH) -_DEFAULT_LTX2_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _LTX2_DIR, _LTX2_WEIGHTS_FILE) -_DEFAULT_LTX2_TEXT_ENCODER = os.path.join(_CI_DEFAULT_LLM_MODELS, _LTX2_TEXT_ENCODER_SUBPATH) _DEFAULT_QWEN_IMAGE_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _QWEN_IMAGE_SUBPATH) @@ -106,47 +96,6 @@ def _resolve_flux_checkpoint() -> str | None: return None -def _resolve_ltx2_checkpoint() -> str | None: - """LTX-2 weights file: explicit env, LTX2_MODEL_PATH, CI default, then LLM_MODELS_ROOT (same tiers as Wan/Flux).""" - explicit = os.environ.get("TRTLLM_CACHE_DIT_LTX2_CHECKPOINT", "").strip() - if explicit: - if os.path.isfile(explicit): - return os.path.abspath(explicit) - if os.path.isdir(explicit): - cand = os.path.join(explicit, _LTX2_WEIGHTS_FILE) - if os.path.isfile(cand): - return os.path.abspath(cand) - ltx2_model = os.environ.get("LTX2_MODEL_PATH", "").strip() - if ltx2_model and os.path.isfile(ltx2_model): - return os.path.abspath(ltx2_model) - if os.path.isfile(_DEFAULT_LTX2_CHECKPOINT): - return os.path.abspath(_DEFAULT_LTX2_CHECKPOINT) - root = os.environ.get("LLM_MODELS_ROOT", "").strip() - if root: - cand = os.path.join(root, _LTX2_DIR, _LTX2_WEIGHTS_FILE) - if os.path.isfile(cand): - return os.path.abspath(cand) - return None - - -def _resolve_ltx2_text_encoder() -> str | None: - """Gemma text encoder directory: explicit env, CI default, then LLM_MODELS_ROOT.""" - explicit = os.environ.get("TRTLLM_CACHE_DIT_LTX2_TEXT_ENCODER", "").strip() - if explicit and os.path.isdir(explicit): - return os.path.abspath(explicit) - for rel in _LTX2_TEXT_ENCODER_RELATIVE_PATHS: - cand = os.path.join(_CI_DEFAULT_LLM_MODELS, rel) - if os.path.isdir(cand): - return os.path.abspath(cand) - root = os.environ.get("LLM_MODELS_ROOT", "").strip() - if root: - for rel in _LTX2_TEXT_ENCODER_RELATIVE_PATHS: - cand = os.path.join(root, rel) - if os.path.isdir(cand): - return os.path.abspath(cand) - return None - - @contextlib.contextmanager def _suppress_stdlib_logging_for_cache_dit(): """Silence cache_dit's stdlib logging under pytest capture (closed stdout/stderr). @@ -279,7 +228,7 @@ def test_boundary_one_all_timesteps_low(self): @requires_cache_dit @requires_cuda class TestCacheDiTRealPipelineForward: - """Wan, FLUX.1, and LTX-2 use the CI llm-models tree when checkpoints are present. + """Wan and FLUX.1 use the CI llm-models tree when checkpoints are present. Each test calls _teardown_cache_dit in finally; cache_dit treats caching as process-global, so a second loaded pipeline would otherwise skip setup. @@ -302,7 +251,6 @@ def _teardown_cache_dit(pipeline: object) -> None: def _load_visual_gen_pipeline( checkpoint_dir: str, *, - text_encoder_path: str = "", enable_torch_compile: bool = False, ): from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader @@ -313,18 +261,8 @@ def _load_visual_gen_pipeline( VisualGenArgs, ) - # text_encoder_path lives under pipeline_config and is only a - # legal key for the LTX2 family. Strict validation rejects it - # for Wan/Flux pipelines whose registry defaults are empty — - # so only emit the key when the caller supplied a non-empty - # value (LTX2 case). - pipeline_config: dict = {} - if text_encoder_path: - pipeline_config["text_encoder_path"] = text_encoder_path - args = VisualGenArgs( model=checkpoint_dir, - pipeline_config=pipeline_config, cache_config=CacheDiTConfig( max_warmup_steps=0, Fn_compute_blocks=1, @@ -480,62 +418,6 @@ def test_flux_cache_dit_skips_blocks_after_forward(self): if pipeline is not None: self._teardown_cache_dit(pipeline) - def test_ltx2_cache_dit_skips_blocks_after_forward(self): - ckpt = _resolve_ltx2_checkpoint() - text_enc = _resolve_ltx2_text_encoder() - if ckpt is None or text_enc is None: - missing = [] - if ckpt is None: - missing.append("LTX-2 checkpoint") - if text_enc is None: - missing.append("Gemma text encoder") - pytest.skip( - f"Missing {' and '.join(missing)}: set " - "TRTLLM_CACHE_DIT_LTX2_CHECKPOINT (file or directory with " - f"{_LTX2_WEIGHTS_FILE}) and TRTLLM_CACHE_DIT_LTX2_TEXT_ENCODER, " - "or LTX2_MODEL_PATH, or stage under CI tree " - f"{_DEFAULT_LTX2_CHECKPOINT} and one of " - f"{', '.join(os.path.join(_CI_DEFAULT_LLM_MODELS, r) for r in _LTX2_TEXT_ENCODER_RELATIVE_PATHS)} " - f"(same as Wan/Flux under {_CI_DEFAULT_LLM_MODELS}), " - f"or $LLM_MODELS_ROOT/{_LTX2_DIR}/{_LTX2_WEIGHTS_FILE} and " - f"$LLM_MODELS_ROOT/<{' or '.join(_LTX2_TEXT_ENCODER_RELATIVE_PATHS)}>" - ) - - pipeline = None - with _suppress_stdlib_logging_for_cache_dit(): - try: - pipeline = self._load_visual_gen_pipeline(ckpt, text_encoder_path=text_enc) - name = pipeline.__class__.__name__ - if name != "LTX2Pipeline": - pytest.skip(f"Checkpoint resolved to {name}, not LTX2Pipeline") - - assert pipeline.cache_accelerator is not None - assert pipeline.cache_accelerator.is_enabled() - - with torch.inference_mode(): - pipeline.forward( - prompt="cache dit validation", - negative_prompt="", - height=512, - width=768, - num_frames=33, - num_inference_steps=16, - guidance_scale=4.0, - seed=0, - max_sequence_length=256, - frame_rate=24.0, - ) - - stats = pipeline.cache_accelerator.get_stats() - cached = _total_accumulated_cached_steps(stats) - assert cached > 0, ( - "Expected Cache-DiT accumulated_cached_steps > 0 after forward; " - f"stats={stats!r}. Try more steps or a looser residual_diff_threshold." - ) - finally: - if pipeline is not None: - self._teardown_cache_dit(pipeline) - def test_qwen_image_cache_dit_skips_blocks_after_forward(self): ckpt = _resolve_qwen_image_checkpoint() if ckpt is None: diff --git a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py index f1064f780fa1..8c8bbe2dfbb1 100644 --- a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py @@ -51,7 +51,6 @@ _LTX2_BASE = os.path.join(str(llm_models_root(check=True)), "LTX-2") -_GEMMA3_DEFAULT = os.path.join(str(llm_models_root(check=True)), "gemma", "gemma-3-12b-it") CHECKPOINT_PATH_BF16 = os.environ.get( @@ -62,20 +61,6 @@ "LTX2_MODEL_PATH_FP8", os.path.join(_LTX2_BASE, "ltx-2-19b-dev-fp8.safetensors"), ) -GEMMA3_PATH = os.environ.get("LTX2_TEXT_ENCODER_PATH", _GEMMA3_DEFAULT) - - -def _ltx2_pipeline_config(**overrides): - """Build pipeline_config with the Gemma3 text_encoder_path LTX-2 needs. - - LTX-2's tokenizer + text encoder are loaded from a separate Gemma - directory (not the diffusion checkpoint), so every full-pipeline - load needs ``text_encoder_path`` set. Tests can pass extra keys via - ``overrides`` (e.g. ``spatial_upsampler_path`` for two-stage). - """ - cfg = {"text_encoder_path": GEMMA3_PATH} - cfg.update(overrides) - return cfg def _write_minimal_ltx2_native_checkpoint(tmp_path): @@ -206,7 +191,6 @@ def test_load_with_quantization(self, ltx2_bf16_checkpoint_exists, quant_algo: s args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, quant_config={"quant_algo": quant_algo, "dynamic": True}, - pipeline_config=_ltx2_pipeline_config(), ) pipeline = PipelineLoader(args).load(skip_warmup=True, skip_components=SKIP_COMPONENTS) @@ -261,7 +245,6 @@ def test_fp8_vs_bf16_single_layer(self, ltx2_bf16_checkpoint_exists, quant_algo: print(f"\n[Compare {quant_algo}] Loading BF16 pipeline...") args_bf16 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config=_ltx2_pipeline_config(), ) pipeline_bf16 = PipelineLoader(args_bf16).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -271,7 +254,6 @@ def test_fp8_vs_bf16_single_layer(self, ltx2_bf16_checkpoint_exists, quant_algo: args_fp8 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, quant_config={"quant_algo": quant_algo, "dynamic": True}, - pipeline_config=_ltx2_pipeline_config(), ) pipeline_fp8 = PipelineLoader(args_fp8).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -338,7 +320,6 @@ def get_module_memory_gb(module): args_bf16 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config=_ltx2_pipeline_config(), ) pipeline_bf16 = PipelineLoader(args_bf16).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -355,7 +336,6 @@ def get_module_memory_gb(module): args_fp8 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, quant_config={"quant_algo": "FP8", "dynamic": True}, - pipeline_config=_ltx2_pipeline_config(), ) pipeline_fp8 = PipelineLoader(args_fp8).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -392,7 +372,6 @@ def test_attention_backend_comparison(self, ltx2_bf16_checkpoint_exists): args_baseline = VisualGenArgs( model=CHECKPOINT_PATH_BF16, attention_config=AttentionConfig(backend="VANILLA"), - pipeline_config=_ltx2_pipeline_config(), ) pipeline_baseline = PipelineLoader(args_baseline).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -419,7 +398,6 @@ def test_attention_backend_comparison(self, ltx2_bf16_checkpoint_exists): args_trtllm = VisualGenArgs( model=CHECKPOINT_PATH_BF16, attention_config=AttentionConfig(backend="TRTLLM"), - pipeline_config=_ltx2_pipeline_config(), ) pipeline_trtllm = PipelineLoader(args_trtllm).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -1615,10 +1593,10 @@ def test_loads_as_two_stage_pipeline(self, ltx2_two_stage_assets_exist): args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config=_ltx2_pipeline_config( - spatial_upsampler_path=UPSAMPLER_PATH, - distilled_lora_path=LORA_PATH, - ), + pipeline_config={ + "spatial_upsampler_path": UPSAMPLER_PATH, + "distilled_lora_path": LORA_PATH, + }, ) pipeline = PipelineLoader(args).load(skip_warmup=True, skip_components=SKIP_COMPONENTS) @@ -1648,10 +1626,10 @@ def test_two_stage_lora_deltas_match_transformer(self, ltx2_two_stage_assets_exi args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config=_ltx2_pipeline_config( - spatial_upsampler_path=UPSAMPLER_PATH, - distilled_lora_path=LORA_PATH, - ), + pipeline_config={ + "spatial_upsampler_path": UPSAMPLER_PATH, + "distilled_lora_path": LORA_PATH, + }, ) pipeline = PipelineLoader(args).load(skip_warmup=True, skip_components=SKIP_COMPONENTS) @@ -1694,10 +1672,10 @@ def test_two_stage_with_quantization(self, ltx2_two_stage_assets_exist, quant_al args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config=_ltx2_pipeline_config( - spatial_upsampler_path=UPSAMPLER_PATH, - distilled_lora_path=LORA_PATH, - ), + pipeline_config={ + "spatial_upsampler_path": UPSAMPLER_PATH, + "distilled_lora_path": LORA_PATH, + }, quant_config={"quant_algo": quant_algo, "dynamic": True}, ) diff --git a/tests/unittest/llmapi/test_llm.py b/tests/unittest/llmapi/test_llm.py index fcea6626a701..823bd0c9b4ff 100644 --- a/tests/unittest/llmapi/test_llm.py +++ b/tests/unittest/llmapi/test_llm.py @@ -504,91 +504,6 @@ def test_generate_with_stop_words(): stop_reasons=["I J"]) -@force_ampere -@pytest.mark.part0 -@pytest.mark.parametrize("model_path", [ - get_model_path('gemma/gemma-3-1b-it'), -]) -def test_generate_with_detokenization_stop_words(model_path): - llm = LLM( - model=model_path, - kv_cache_config=global_kvcache_config, - ) - - # Format the prompt using chat template - messages = [{ - "role": "user", - "content": "Say exactly: Hello there! How can I help" - }] - - formatted_prompt = llm.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True) - - detokenization_prompts = [formatted_prompt] - - # Test case 1: Stop word "How" should be detected after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello there!"], - sampling_params=SamplingParams(stop="How", max_tokens=10), - finish_reasons=['stop'], - stop_reasons=["How"]) - - # Test case 2: Stop word "there" should be detected after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello"], - sampling_params=SamplingParams(stop="there", - max_tokens=10), - finish_reasons=['stop'], - stop_reasons=["there"]) - - # Test case 3: Stop word that should not be found after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello there! How can I help"], - sampling_params=SamplingParams(stop="XYZ", max_tokens=10), - finish_reasons=['length'], - stop_reasons=[None]) - - # Test case 4: Multiple stop words, one should be found after detokenization - llm_check_output(llm, - detokenization_prompts, ["Hello"], - sampling_params=SamplingParams(stop=["XYZ", "there"], - max_tokens=10), - finish_reasons=['stop'], - stop_reasons=["there"]) - - -@force_ampere -@pytest.mark.part0 -@pytest.mark.parametrize("model_path", [ - get_model_path('gemma/gemma-3-1b-it'), -]) -def test_generate_with_detokenization_stop_words_streaming(model_path): - llm = LLM( - model=model_path, - kv_cache_config=global_kvcache_config, - ) - - # Format the prompt using chat template - messages = [{ - "role": "user", - "content": "Say exactly: Hello there! How can I help" - }] - - formatted_prompt = llm.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True) - - sampling_params = SamplingParams(stop="How", max_tokens=10) - - for output in llm.generate_async(formatted_prompt, - sampling_params=sampling_params, - streaming=True): - if output.outputs[0].finish_reason == 'stop': - assert output.outputs[0].stop_reason == "How" - break - elif output.outputs[0].finish_reason == 'length': - assert False, f"Expected to find stop word 'How' but reached max_tokens. Generated: {output.outputs[0].text}" - - @force_ampere @pytest.mark.part0 def test_generate_with_bad_words(): diff --git a/tests/unittest/llmapi/test_llm_pytorch.py b/tests/unittest/llmapi/test_llm_pytorch.py index ef965bc5c04c..737141869e4e 100644 --- a/tests/unittest/llmapi/test_llm_pytorch.py +++ b/tests/unittest/llmapi/test_llm_pytorch.py @@ -36,12 +36,10 @@ run_llm_with_postprocess_parallel_and_result_handler, tinyllama_logits_processor_test_harness) from utils.util import (force_ampere, similar, skip_fp8_pre_ada, - skip_gpu_memory_less_than_40gb, skip_gpu_memory_less_than_80gb, skip_ray) from utils.llm_data import llm_models_root from tensorrt_llm._torch.peft.lora.config import LoraConfig from tensorrt_llm.executor.request import LoRARequest -import tempfile import torch from transformers.configuration_utils import PretrainedConfig @@ -58,9 +56,7 @@ register_checkpoint_weight_loader as _register_checkpoint_weight_loader, register_config_loader as _register_config_loader) -from peft import LoraConfig as PeftLoraConfig -from peft import get_peft_model -from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers import AutoTokenizer from dataclasses import replace # isort: on @@ -407,143 +403,6 @@ def test_llama_3_1_8b_fp8_with_bf16_lora(cuda_graph_config) -> None: assert similar(output.outputs[0].text, reference) -@pytest.mark.part2 -@test_lora_with_and_without_cuda_graph -def test_gemma3_1b_instruct_multi_lora(cuda_graph_config) -> None: - model_dir = f"{llm_models_root()}/gemma/gemma-3-1b-it" - - target_modules = ['attn_q', 'attn_k', 'attn_v'] - - # Set up temporary directory for LoRA adapters - with tempfile.TemporaryDirectory() as lora_dir: - print("Creating dummy LoRAs...") - - model = AutoModelForCausalLM.from_pretrained(model_dir, - dtype=torch.bfloat16, - device_map="auto") - hf_modules = ["q_proj", "k_proj", "v_proj"] - peft_lora_config = PeftLoraConfig(r=8, - target_modules=hf_modules, - bias="none", - task_type="CAUSAL_LM") - lora_paths = [] - for i in range(2): - lora_model = get_peft_model(model, peft_lora_config) - for param in lora_model.parameters(): - param.data.zero_() - lora_path = f"{lora_dir}/lora_{i}" - lora_model.save_pretrained(lora_path) - lora_paths.append(lora_path) - - trtllm_lora_config = LoraConfig(lora_dir=lora_paths, - lora_target_modules=target_modules, - max_lora_rank=8, - max_loras=2, - max_cpu_loras=2) - # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. - kv_cache_config = KvCacheConfig( - enable_block_reuse=False, - enable_partial_reuse=False, - ) - llm = LLM(model_dir, - lora_config=trtllm_lora_config, - kv_cache_config=kv_cache_config, - cuda_graph_config=cuda_graph_config) - - prompts = [ - "Is it ok to fill diesel in a petrol car?", - "What is the capital of France?", - ] - lora_req1 = LoRARequest("lora-1", 0, lora_paths[0]) - lora_req2 = LoRARequest("lora-2", 1, lora_paths[1]) - lora_requests = [lora_req1, lora_req2] - sampling_params = SamplingParams(max_tokens=200) - - outputs = llm.generate(prompts, - sampling_params, - lora_request=lora_requests) - - assert len(outputs) == 2 - - -@skip_gpu_memory_less_than_40gb -@pytest.mark.part3 -def test_lora_many_adapters_no_memory_leak() -> None: - """Verify GPU memory stays bounded when loading many unique LoRA adapters. - - Creates 20 dummy adapters but sets max_loras=2 and max_cpu_loras=4 to - force eviction. Without proper cleanup, _lora_weights can accumulate - GPU tensors for every loaded adapter, causing unbounded memory growth. - """ - model_dir = f"{llm_models_root()}/gemma/gemma-3-1b-it" - num_adapters = 20 - target_modules = ['attn_q', 'attn_k', 'attn_v'] - - with tempfile.TemporaryDirectory() as lora_dir: - model = AutoModelForCausalLM.from_pretrained(model_dir, - dtype=torch.bfloat16, - device_map="auto") - hf_modules = ["q_proj", "k_proj", "v_proj"] - peft_lora_config = PeftLoraConfig(r=8, - target_modules=hf_modules, - bias="none", - task_type="CAUSAL_LM") - lora_paths = [] - for i in range(num_adapters): - lora_model = get_peft_model(model, peft_lora_config) - for param in lora_model.parameters(): - param.data.zero_() - lora_path = f"{lora_dir}/lora_{i}" - lora_model.save_pretrained(lora_path) - lora_paths.append(lora_path) - - del model - torch.cuda.empty_cache() - - trtllm_lora_config = LoraConfig(lora_dir=lora_paths[:1], - lora_target_modules=target_modules, - max_lora_rank=8, - max_loras=2, - max_cpu_loras=4) - kv_cache_config = KvCacheConfig(enable_block_reuse=False, - enable_partial_reuse=False) - llm = LLM(model_dir, - lora_config=trtllm_lora_config, - kv_cache_config=kv_cache_config) - - sampling_params = SamplingParams(max_tokens=20) - warmup_count = 5 - - mem_samples = [] - for i in range(num_adapters): - lora_req = LoRARequest(f"lora-{i}", i, lora_paths[i]) - output = llm.generate("Hello, tell me a story.", - sampling_params, - lora_request=lora_req) - assert output.outputs[0].text != "" - - if i >= warmup_count: - mem_samples.append(torch.cuda.memory_allocated()) - - num_measured = len(mem_samples) - assert num_measured >= 2, "Not enough samples to measure growth" - - total_growth = mem_samples[-1] - mem_samples[0] - per_adapter_mb = (total_growth / (num_measured - 1)) / (1024 * 1024) - - # Each adapter is ~3 MB on GPU (r=8, 3 modules, 26 layers, bf16). - # The C++ PeftCacheManager handles eviction and _lora_weights - # stays empty, so per-adapter growth should be ~0. If GPU tensors - # leak, we would see ~3 MB/adapter of linear growth. Threshold - # of 1 MB/adapter catches leaks while tolerating noise from - # allocator fragmentation averaged over many samples. - max_per_adapter_mb = 1.0 - assert per_adapter_mb < max_per_adapter_mb, ( - f"GPU memory growing at {per_adapter_mb:.2f} MB/adapter over " - f"{num_measured} adapters (total {total_growth / (1024**2):.1f} MB). " - f"Possible _lora_weights leak.") - - @pytest.mark.parametrize( "lora_rank,max_lora_rank,description", [ From 64f6d67706d1e72236e8860b51d6c74b1bc40856 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:48:39 +0800 Subject: [PATCH 2/3] test: restore visual gen, KV rebalance, and Gemma-3 LoRA coverage Bring back LTX-2 visual gen tests, KV pool rebalance accuracy on gemma-3-1b-it, and Gemma-3 LoRA unittest coverage removed during the Gemma-3 checkpoint prune. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../test_kv_pool_rebalance_accuracy.py | 171 +++++++ .../visual_gen/test_visual_gen_ltx2.py | 466 ++++++++++++++++++ .../README_test_visual_gen_perf_sanity.md | 3 +- .../defs/perf/test_visual_gen_perf_sanity.py | 4 + .../test_lists/test-db/l0_b200.yml | 7 + .../l0_b200_visual_gen_perf_sanity.yml | 3 + .../test_lists/test-db/l0_dgx_b200.yml | 3 - .../test_lists/test-db/l0_h100.yml | 2 + tests/integration/test_lists/waives.txt | 4 + .../_torch/visual_gen/test_cache_dit.py | 126 ++++- .../_torch/visual_gen/test_ltx2_pipeline.py | 46 +- tests/unittest/llmapi/test_llm_pytorch.py | 143 +++++- 12 files changed, 957 insertions(+), 21 deletions(-) create mode 100644 tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py create mode 100644 tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py diff --git a/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py b/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py new file mode 100644 index 000000000000..fd8c592b2308 --- /dev/null +++ b/tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Accuracy test for the KVCacheManagerV2 rebalance hook. + +Verifies that forcing the V2 auto-tuner to fire mid-generation does not +change greedy-decode outputs. Uses Gemma-3-1B with explicit VSWA so the +KV cache lands in >=2 pool groups and ``adjust()`` has real work to do +(a single pool group would make rebalance a no-op). + +Run as: + LLM_MODELS_ROOT=/path pytest \ + tests/integration/defs/accuracy/test_kv_pool_rebalance_accuracy.py +""" + +import pytest + +from tensorrt_llm import LLM +from tensorrt_llm.llmapi import KvCacheConfig, SamplingParams + +from ..conftest import llm_models_root, skip_pre_hopper + +# --------------------------------------------------------------------------- # +# Ratio injection +# --------------------------------------------------------------------------- # + + +def _inject_pool_ratio_mismatch(llm: LLM, *, skew: float = 2.0) -> None: + """Force the V2 auto-tuner to do real pool-resize work on the next rebalance call. + + Delegates to the backend-agnostic KVCacheManagerV2 introspection hook, which + bypasses the sample-count / cooldown gates and perturbs the target GPU ratio + past the auto-tuner's adjustment threshold. The hook requires a model with + >=2 pool groups (e.g. Gemma-3-1B with VSWA) and raises otherwise, so a future + model change can't silently turn this test into a no-op. + + Also drops the executor's rebalance-check throttle to every iteration, so + the test does not depend on how ``KV_POOL_REBALANCE_CHECK_INTERVAL`` compares + to the number of iterations this short prompt set happens to run. Raise that + interval above the iteration count and the hook would never fire, leaving the + token comparison below to pass vacuously; the ratio assertion in + ``_generate_tokens`` is the backstop that would catch it. + """ + from tensorrt_llm.runtime.kv_cache_manager_v2 import _introspection + + executor = llm._executor.engine + executor._rebalance_check_interval = 1 + kv_cache_manager = executor.kv_cache_manager + _introspection.force_rebalance_precondition(kv_cache_manager.impl, skew=skew) + + +# --------------------------------------------------------------------------- # +# Test +# --------------------------------------------------------------------------- # + +# A handful of prompts spanning short, medium, and long context lengths. +# The long prompt is intentionally repetitive so it occupies multiple KV +# blocks and creates enough pool pressure for rebalance to matter. +_PROMPTS = [ + "The capital of France is", + "Write one sentence about transformers.", + "List three prime numbers greater than 100:", + "The quick brown fox jumps over the lazy dog. " * 40, +] + +_SAMPLING = SamplingParams(max_tokens=64, temperature=0.0, top_k=1) + + +def _vswa_kv_cache_config(*, enable_rebalance: bool) -> KvCacheConfig: + """V2 manager + explicit VSWA pattern that yields multiple pool groups. + + Gemma-3-1B has 5 sliding-window layers : 1 full-attention layer. + """ + return KvCacheConfig( + use_kv_cache_manager_v2=True, + enable_kv_pool_rebalance=enable_rebalance, + max_attention_window=[512, 512, 512, 512, 512, 32768], + # Block reuse disabled per the standing Gemma3 WAR for non- + # inclusive sliding window kernel support. + enable_block_reuse=False, + enable_partial_reuse=False, + tokens_per_block=32, + free_gpu_memory_fraction=0.6, + ) + + +def _generate_tokens(*, model_path: str, disable_overlap: bool, enable_rebalance: bool): + """Run one LLM, return list[list[int]] of generated token ids. + + Note: the ratio-injection helper requires direct access to the + in-process PyExecutor, so the test runs in single-process worker + mode (``TLLM_WORKER_USE_SINGLE_PROCESS=1``). The caller is + responsible for setting that env var (via monkeypatch or otherwise) + before invoking this helper. + """ + from tensorrt_llm.runtime.kv_cache_manager_v2 import _introspection + + with LLM( + model_path, + disable_overlap_scheduler=disable_overlap, + kv_cache_config=_vswa_kv_cache_config(enable_rebalance=enable_rebalance), + ) as llm: + impl = llm._executor.engine.kv_cache_manager.impl + if enable_rebalance: + _inject_pool_ratio_mismatch(llm) + ratio_before = list(_introspection.current_gpu_ratio(impl)) + outputs = llm.generate(_PROMPTS, _SAMPLING) + ratio_after = list(_introspection.current_gpu_ratio(impl)) + + # Guard against a vacuous pass. Token equality between the rebalance + # and no-rebalance arms proves nothing if adjust() never ran, and + # nothing in the run logs at info level to tell us it did. The pool + # ratio moving is the observable signature that it happened. + if enable_rebalance: + assert ratio_after != ratio_before, ( + "rebalance never fired: GPU pool ratio unchanged at " + f"{ratio_before}. The token comparison would pass vacuously. " + "Check the executor's rebalance-check throttle and the V2 " + "auto-tuner's sample-count / cooldown gates." + ) + else: + assert ratio_after == ratio_before, ( + "pool ratio moved with enable_kv_pool_rebalance=False " + f"({ratio_before} -> {ratio_after}); the baseline arm is " + "supposed to hold pool ratios fixed." + ) + + return [list(o.outputs[0].token_ids) for o in outputs] + + +@skip_pre_hopper +class TestKvPoolRebalanceAccuracy: + """Token-exact greedy-decode equivalence under rebalance. + + Compares rebalance=off and rebalance=on with a forced mid-generation + adjust(). + """ + + MODEL_PATH = f"{llm_models_root()}/gemma/gemma-3-1b-it/" + + @pytest.mark.parametrize("disable_overlap", [True, False], ids=["no_overlap", "overlap"]) + def test_rebalance_matches_baseline(self, disable_overlap, monkeypatch): + # Keep the PyExecutor in-process so the ratio-injection helper + # can reach .engine on the client side. + monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") + + baseline = _generate_tokens( + model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=False + ) + + treated = _generate_tokens( + model_path=self.MODEL_PATH, disable_overlap=disable_overlap, enable_rebalance=True + ) + + assert len(baseline) == len(treated) == len(_PROMPTS) + for i, (b, t) in enumerate(zip(baseline, treated)): + assert b == t, ( + f"prompt {i}: rebalance changed greedy-decode output\n" + f" baseline: {b[:16]}...\n" + f" treated: {t[:16]}..." + ) diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py new file mode 100644 index 000000000000..a853709a1479 --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_ltx2.py @@ -0,0 +1,466 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Single-GPU integration and accuracy tests for LTX-2.""" + +import os +from dataclasses import dataclass + +import pytest +import torch +from defs import conftest +from defs.common import venv_check_call +from defs.examples.visual_gen.visual_gen_test_utils import ( + FeatureConfigState, + _assert_feature_quantization_installed, + _assert_lpips_below_threshold, + _assert_resolved_single_device_feature_config, + _assert_single_device_feature_executed, + _build_single_device_feature_args, + _cleanup_cuda, + _cleanup_single_device_feature_pipeline, + _disable_inductor_compile_worker_quiesce, + _fixed_nvfp4_quantization_backend, + _golden_media_path, + _lpips_deterministic_algorithms, + _lpips_model_path, + _preserve_lpips_candidate_on_failure, + _run_lpips_eval, + _run_reusable_video_lpips_eval, + _run_single_device_feature_generator, + _save_lpips_video_mp4, + _skip_if_missing, + _validate_single_feature_config, + _visual_gen_output_path, +) + +LTX2_LPIPS_NUM_FRAMES = 49 +LTX2_LPIPS_NUM_INFERENCE_STEPS = 8 +LTX2_LPIPS_THRESHOLD = 0.05 +LTX2_CUDA_GRAPH_LPIPS_THRESHOLD = 0.01 + +LTX2_FEATURE_LPIPS_THRESHOLD = 0.05 +LTX2_SUPPORTED_FEATURES = frozenset({"fp8-blockwise", "nvfp4", "cuda-graph"}) + + +@dataclass(frozen=True) +class LTX2AccuracyCase: + id: str + golden_file: str + features: FeatureConfigState + lpips_threshold: float + + +LTX2_FEATURE_PROFILES = ( + ("fp8-blockwise", FeatureConfigState(quantization="FP8_BLOCK_SCALES")), + ("nvfp4", FeatureConfigState(quantization="NVFP4")), + ("cuda-graph", FeatureConfigState(cuda_graph=True)), +) + + +def _build_ltx2_accuracy_cases(): + cases = [] + for profile_id, features in LTX2_FEATURE_PROFILES: + _validate_single_feature_config( + features, + LTX2_SUPPORTED_FEATURES, + "LTX-2", + ) + cases.append( + pytest.param( + LTX2AccuracyCase( + id=profile_id, + golden_file=(f"ltx2_{profile_id.replace('-', '_')}_lpips_golden_video.mp4"), + features=features, + lpips_threshold=LTX2_FEATURE_LPIPS_THRESHOLD, + ), + id=profile_id, + ) + ) + return cases + + +LTX2_ACCURACY_CASES = _build_ltx2_accuracy_cases() + + +# LTX-2 configuration +LTX2_MODEL_CHECKPOINT_PATH = "LTX-2/ltx-2-19b-dev.safetensors" +LTX2_TEXT_ENCODER_SUBPATH = "gemma-3-12b-it" +LTX2_T2V_PROMPT = ( + "A woman with long brown hair and light skin smiles at the camera while " + "standing in a sunlit park, her hair gently blowing in the breeze as she " + "tilts her head slightly to the side." +) +LTX2_T2V_HEIGHT = 512 +LTX2_T2V_WIDTH = 768 +LTX2_T2V_NUM_FRAMES = 121 +LTX2_T2V_STEPS = 40 +LTX2_T2V_GUIDANCE_SCALE = 4.0 +LTX2_T2V_MAX_SEQ_LEN = 1024 +LTX2_T2V_FRAME_RATE = 24.0 +LTX2_T2V_SEED = 42 +LTX2_T2V_NEGATIVE_PROMPT = "worst quality, inconsistent motion, blurry, jittery, distorted" + + +# LTX-2 Two-Stage configuration +LTX2_UPSAMPLER_SUBPATH = "LTX-2/ltx-2-spatial-upscaler-x2-1.0.safetensors" +LTX2_DISTILLED_LORA_SUBPATH = "LTX-2/ltx-2-19b-distilled-lora-384.safetensors" + + +def _ltx2_lpips_text_encoder_path(): + scratch_space = conftest.llm_models_root() + candidates = [ + os.path.join(scratch_space, LTX2_TEXT_ENCODER_SUBPATH), + os.path.join(scratch_space, "gemma", LTX2_TEXT_ENCODER_SUBPATH), + ] + for candidate in candidates: + if os.path.isdir(candidate): + return candidate + return candidates[0] + + +def _generate_ltx2_feature_video(case, output_path): + from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader + + checkpoint_path = _lpips_model_path("LTX-2", "ltx-2-19b-dev.safetensors") + text_encoder_path = _ltx2_lpips_text_encoder_path() + spatial_upsampler_path = _lpips_model_path("LTX-2", "ltx-2-spatial-upscaler-x2-1.0.safetensors") + distilled_lora_path = _lpips_model_path("LTX-2", "ltx-2-19b-distilled-lora-384.safetensors") + _skip_if_missing(checkpoint_path, "LTX-2 checkpoint") + _skip_if_missing(text_encoder_path, "LTX-2 text encoder", is_dir=True) + _skip_if_missing(spatial_upsampler_path, "LTX-2 spatial upsampler") + _skip_if_missing(distilled_lora_path, "LTX-2 distilled LoRA") + _disable_inductor_compile_worker_quiesce() + pipeline = None + with ( + _lpips_deterministic_algorithms(), + torch.compiler.set_stance("force_eager"), + _fixed_nvfp4_quantization_backend(case.features), + ): + args = _build_single_device_feature_args( + checkpoint_path, + case.features, + resolution=(LTX2_T2V_HEIGHT, LTX2_T2V_WIDTH), + num_frames=LTX2_LPIPS_NUM_FRAMES, + pipeline_config={ + "text_encoder_path": text_encoder_path, + "spatial_upsampler_path": spatial_upsampler_path, + "distilled_lora_path": distilled_lora_path, + }, + ) + try: + pipeline = PipelineLoader(args).load(skip_warmup=False) + _assert_resolved_single_device_feature_config( + pipeline, + case.features, + resolution=(LTX2_T2V_HEIGHT, LTX2_T2V_WIDTH), + num_frames=LTX2_LPIPS_NUM_FRAMES, + ) + _assert_feature_quantization_installed(pipeline, case.features) + result = pipeline.forward( + prompt=LTX2_T2V_PROMPT, + negative_prompt=LTX2_T2V_NEGATIVE_PROMPT, + height=LTX2_T2V_HEIGHT, + width=LTX2_T2V_WIDTH, + num_frames=LTX2_LPIPS_NUM_FRAMES, + num_inference_steps=LTX2_LPIPS_NUM_INFERENCE_STEPS, + guidance_scale=LTX2_T2V_GUIDANCE_SCALE, + seed=LTX2_T2V_SEED, + ) + assert result.video is not None, "LTX-2 feature run produced no video" + _assert_single_device_feature_executed(pipeline, case.features) + generated_video = result.video.detach().cpu() + finally: + try: + if pipeline is not None: + _cleanup_single_device_feature_pipeline(pipeline) + del pipeline + finally: + _cleanup_cuda() + + _save_lpips_video_mp4(generated_video, output_path, frame_rate=LTX2_T2V_FRAME_RATE) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@pytest.mark.parametrize("case", LTX2_ACCURACY_CASES) +def test_ltx2_feature_accuracy_against_golden( + request, + tmp_path, + case, + _visual_gen_deps, + _visual_gen_lpips_scorer, +): + generated_path = tmp_path / f"ltx2_{case.id}_generated.mp4" + golden_path = _golden_media_path( + tmp_path, + case.golden_file, + f"LTX-2 {case.id} LPIPS golden video", + ) + _run_single_device_feature_generator( + case.features, _generate_ltx2_feature_video, case, generated_path + ) + score = _run_reusable_video_lpips_eval( + f"ltx2-{case.id}", + golden_path, + generated_path, + _visual_gen_lpips_scorer, + ) + _preserve_lpips_candidate_on_failure( + request, + score, + case.lpips_threshold, + generated_path, + f"ltx2_{case.id}_generated.mp4", + ) + _assert_lpips_below_threshold(score, case.lpips_threshold) + + +def _generate_ltx2_lpips_video(output_path, *, enable_cuda_graph=False): + from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader + from tensorrt_llm.visual_gen.args import CudaGraphConfig, TorchCompileConfig, VisualGenArgs + + checkpoint_path = _lpips_model_path("LTX-2", "ltx-2-19b-dev.safetensors") + text_encoder_path = _ltx2_lpips_text_encoder_path() + spatial_upsampler_path = _lpips_model_path("LTX-2", "ltx-2-spatial-upscaler-x2-1.0.safetensors") + distilled_lora_path = _lpips_model_path("LTX-2", "ltx-2-19b-distilled-lora-384.safetensors") + _skip_if_missing(checkpoint_path, "LTX-2 checkpoint") + _skip_if_missing(text_encoder_path, "LTX-2 text encoder", is_dir=True) + _skip_if_missing(spatial_upsampler_path, "LTX-2 spatial upsampler") + _skip_if_missing(distilled_lora_path, "LTX-2 distilled LoRA") + _disable_inductor_compile_worker_quiesce() + + # TorchCompileConfig(enable=False) does not suppress nested @torch.compile decorators. + # Wrapped here (not in the fixture) so the golden fixture and both sides of + # test_ltx2_cuda_graph_lpips_matches_eager run the same eager numerics. + with _lpips_deterministic_algorithms(), torch.compiler.set_stance("force_eager"): + args = VisualGenArgs( + model=checkpoint_path, + pipeline_config={ + "text_encoder_path": text_encoder_path, + "spatial_upsampler_path": spatial_upsampler_path, + "distilled_lora_path": distilled_lora_path, + }, + torch_compile_config=TorchCompileConfig(enable=False), + cuda_graph_config=CudaGraphConfig(enable=enable_cuda_graph), + ) + pipeline = PipelineLoader(args).load(skip_warmup=True) + try: + with torch.no_grad(): + result = pipeline.forward( + prompt=LTX2_T2V_PROMPT, + negative_prompt=LTX2_T2V_NEGATIVE_PROMPT, + height=LTX2_T2V_HEIGHT, + width=LTX2_T2V_WIDTH, + num_frames=LTX2_LPIPS_NUM_FRAMES, + num_inference_steps=LTX2_LPIPS_NUM_INFERENCE_STEPS, + guidance_scale=LTX2_T2V_GUIDANCE_SCALE, + seed=LTX2_T2V_SEED, + ) + generated_video = result.video.detach().cpu() + finally: + del pipeline + _cleanup_cuda() + + _save_lpips_video_mp4(generated_video, output_path, frame_rate=LTX2_T2V_FRAME_RATE) + + +def _generate_ltx2_cuda_graph_trtllm_backend_video(output_path): + from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams + from tensorrt_llm.visual_gen.args import ( + AttentionConfig, + CompilationConfig, + CudaGraphConfig, + ParallelConfig, + TorchCompileConfig, + ) + + scratch_space = conftest.llm_models_root() + checkpoint_path = os.path.join(scratch_space, LTX2_MODEL_CHECKPOINT_PATH) + text_encoder_path = _ltx2_lpips_text_encoder_path() + spatial_upsampler_path = os.path.join(scratch_space, LTX2_UPSAMPLER_SUBPATH) + distilled_lora_path = os.path.join(scratch_space, LTX2_DISTILLED_LORA_SUBPATH) + _skip_if_missing(checkpoint_path, "LTX-2 checkpoint") + _skip_if_missing(text_encoder_path, "LTX-2 text encoder", is_dir=True) + _skip_if_missing(spatial_upsampler_path, "LTX-2 spatial upsampler") + _skip_if_missing(distilled_lora_path, "LTX-2 distilled LoRA") + _disable_inductor_compile_worker_quiesce() + + visual_gen_args = VisualGenArgs( + model=checkpoint_path, + quant_config={"quant_algo": "NVFP4", "dynamic": True}, + attention_config=AttentionConfig(backend="TRTLLM"), + parallel_config=ParallelConfig( + cfg_size=1, + ulysses_size=1, + parallel_vae_size=1, + ), + compilation_config=CompilationConfig( + resolutions=[ + ( + LTX2_T2V_HEIGHT, + LTX2_T2V_WIDTH, + ) + ], + num_frames=[LTX2_LPIPS_NUM_FRAMES], + ), + cuda_graph_config=CudaGraphConfig(enable=True), + torch_compile_config=TorchCompileConfig( + enable=True, + enable_fullgraph=False, + enable_autotune=True, + ), + pipeline_config={ + "text_encoder_path": text_encoder_path, + "spatial_upsampler_path": spatial_upsampler_path, + "distilled_lora_path": distilled_lora_path, + }, + ) + + visual_gen = VisualGen(model=checkpoint_path, args=visual_gen_args) + try: + params = VisualGenParams( + height=LTX2_T2V_HEIGHT, + width=LTX2_T2V_WIDTH, + num_frames=LTX2_LPIPS_NUM_FRAMES, + num_inference_steps=LTX2_LPIPS_NUM_INFERENCE_STEPS, + guidance_scale=LTX2_T2V_GUIDANCE_SCALE, + max_sequence_length=LTX2_T2V_MAX_SEQ_LEN, + seed=LTX2_T2V_SEED, + frame_rate=LTX2_T2V_FRAME_RATE, + negative_prompt=LTX2_T2V_NEGATIVE_PROMPT, + ) + output = visual_gen.generate(inputs=LTX2_T2V_PROMPT, params=params) + _save_lpips_video_mp4(output.video, output_path, frame_rate=LTX2_T2V_FRAME_RATE) + finally: + visual_gen.shutdown() + del visual_gen + _cleanup_cuda() + + assert os.path.isfile(output_path), f"LTX-2 TRTLLM backend did not produce {output_path}" + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +def test_ltx2_lpips_against_golden(request, tmp_path, ltx2_two_stage_bf16_video_path): + golden_path = _golden_media_path( + tmp_path, "ltx2_lpips_golden_video.mp4", "LTX-2 LPIPS golden video" + ) + score = _run_lpips_eval( + tmp_path, + "ltx2", + "video", + LTX2_T2V_PROMPT, + golden_path, + ltx2_two_stage_bf16_video_path, + ) + _preserve_lpips_candidate_on_failure( + request, + score, + LTX2_LPIPS_THRESHOLD, + ltx2_two_stage_bf16_video_path, + "ltx2_lpips_golden_video.mp4", + ) + _assert_lpips_below_threshold(score, LTX2_LPIPS_THRESHOLD) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +def test_ltx2_cuda_graph_lpips_matches_eager(_visual_gen_deps, tmp_path): + eager_path = tmp_path / "ltx2_eager_generated.mp4" + cuda_graph_path = tmp_path / "ltx2_cuda_graph_generated.mp4" + + _generate_ltx2_lpips_video(eager_path, enable_cuda_graph=False) + _generate_ltx2_lpips_video(cuda_graph_path, enable_cuda_graph=True) + score = _run_lpips_eval( + tmp_path, + "ltx2_cuda_graph", + "video", + LTX2_T2V_PROMPT, + eager_path, + cuda_graph_path, + ) + _assert_lpips_below_threshold(score, LTX2_CUDA_GRAPH_LPIPS_THRESHOLD) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +def test_ltx2_cuda_graph_trtllm_backend(request, _visual_gen_deps, tmp_path): + generated_path = tmp_path / "ltx2_cuda_graph_trtllm_backend_generated.mp4" + golden_path = _golden_media_path( + tmp_path, "ltx2_lpips_golden_video.mp4", "LTX-2 LPIPS golden video" + ) + _generate_ltx2_cuda_graph_trtllm_backend_video(generated_path) + score = _run_lpips_eval( + tmp_path, + "ltx2_cuda_graph_trtllm_backend", + "video", + LTX2_T2V_PROMPT, + golden_path, + generated_path, + ) + _preserve_lpips_candidate_on_failure( + request, + score, + LTX2_LPIPS_THRESHOLD, + generated_path, + "ltx2_cuda_graph_trtllm_backend_generated.mp4", + ) + _assert_lpips_below_threshold(score, LTX2_LPIPS_THRESHOLD) + + +@pytest.fixture(scope="session") +def ltx2_two_stage_bf16_video_path(_visual_gen_deps, llm_venv): + """Generate LTX-2 two-stage BF16 video with the LPIPS config and return path.""" + output_path = _visual_gen_output_path(llm_venv, "ltx2_two_stage_bf16") + if os.path.isfile(output_path): + return output_path + _generate_ltx2_lpips_video(output_path) + return output_path + + +def test_ltx2_example(_visual_gen_deps, llm_root, llm_venv): + """Run examples/visual_gen/models/ltx2.py with NVFP4 config end-to-end. + + Validates that the LTX-2 example script and ``configs/ltx2-fp4-1gpu.yaml`` + work together as documented. The Gemma3 text encoder is passed separately via + ``--text_encoder_path`` because the shared YAML intentionally omits it to keep + the config model-path-agnostic. + """ + model_path = _lpips_model_path("LTX-2", "ltx-2-19b-dev.safetensors") + _skip_if_missing(model_path, "LTX-2 checkpoint") + text_encoder_path = _ltx2_lpips_text_encoder_path() + _skip_if_missing(text_encoder_path, "LTX-2 text encoder (gemma-3-12b-it)", is_dir=True) + + out_dir = os.path.join(llm_venv.get_working_directory(), "visual_gen_output", "ltx2_example") + os.makedirs(out_dir, exist_ok=True) + output_path = os.path.join(out_dir, "ltx2_output.mp4") + + script_path = os.path.join(llm_root, "examples", "visual_gen", "models", "ltx2.py") + config_path = os.path.join(llm_root, "examples", "visual_gen", "configs", "ltx2-fp4-1gpu.yaml") + assert os.path.isfile(script_path), f"Example script not found: {script_path}" + assert os.path.isfile(config_path), f"Config not found: {config_path}" + + venv_check_call( + llm_venv, + [ + script_path, + "--model", + model_path, + "--visual_gen_args", + config_path, + "--text_encoder_path", + text_encoder_path, + "--output_path", + output_path, + ], + ) + assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" diff --git a/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md b/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md index 4599993cde20..916b25b2ff39 100644 --- a/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_visual_gen_perf_sanity.md @@ -90,6 +90,7 @@ suffix, for example: - `wan21_t2v_14b_blackwell.yaml` - `wan22_i2v_a14b_blackwell.yaml` - `flux2_blackwell.yaml` +- `ltx2_blackwell.yaml` ### YAML Structure @@ -297,7 +298,7 @@ extraction in `build_visual_gen_db_entry` and treat it as a baseline reset. pytest perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] # Run all server recipes from one family YAML without uploading -pytest perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg-flux2_blackwell] +pytest perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg-ltx2_blackwell] ``` Set `LLM_MODELS_ROOT` to the shared model cache before running locally: diff --git a/tests/integration/defs/perf/test_visual_gen_perf_sanity.py b/tests/integration/defs/perf/test_visual_gen_perf_sanity.py index bddbd702caff..c3e9e08f2d80 100644 --- a/tests/integration/defs/perf/test_visual_gen_perf_sanity.py +++ b/tests/integration/defs/perf/test_visual_gen_perf_sanity.py @@ -317,6 +317,10 @@ def _append_candidate(candidate: Path) -> None: if "/" in path_value: _append_candidate(models_root / path_value.split("/", 1)[1]) + if "gemma-3-12b-it" in path_value: + _append_candidate(models_root / "gemma" / "gemma-3-12b-it") + _append_candidate(models_root / "gemma-3-12b-it") + for candidate in candidates: if candidate.exists(): return str(candidate) diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 5120e440fadf..407c04da0168 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -329,6 +329,7 @@ l0_b200: - examples/visual_gen/test_visual_gen_flux.py::test_flux1_example - examples/visual_gen/test_visual_gen_flux.py::test_flux2_example - examples/visual_gen/test_visual_gen_flux.py::test_flux2_reference_image_example + - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_example - examples/visual_gen/test_visual_gen_wan.py::test_wan_i2v_example - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_example - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_edge_policy_droid_example TIMEOUT (20) @@ -448,12 +449,18 @@ l0_b200: - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_against_golden[nvfp4] - examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[fp8-blockwise] - examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] + - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[fp8-blockwise] + - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[nvfp4] + - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[cuda-graph] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan21-fp8-blockwise] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan21-nvfp4] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan21-cuda-graph] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-fp8-blockwise] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-nvfp4] - examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-cuda-graph] + - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_lpips_against_golden + - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_cuda_graph_lpips_matches_eager + - examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_cuda_graph_trtllm_backend - examples/visual_gen/test_visual_gen_wan.py::test_wan21_t2v_lpips_against_golden - examples/visual_gen/test_visual_gen_wan.py::test_wan22_t2v_lpips_against_golden # Measured on B200 with TRT-LLM commit 85665f5f from the staging main image: diff --git a/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml index 35596d119877..88ae6c735195 100644 --- a/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml @@ -20,3 +20,6 @@ l0_b200_visual_gen_perf_sanity: - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-flux2_blackwell-flux2_fp8_cfg1_ulysses4_teacache_on] TIMEOUT (120) - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] TIMEOUT (180) - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan22_i2v_a14b_blackwell-wan22_i2v_a14b_nvfp4_trtllm_cfg2_ulysses4] TIMEOUT (180) + - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_i2v_cfg2_ulysses4_compile_on] TIMEOUT (180) + - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_t2v_cfg2_ulysses4_compile_on] TIMEOUT (180) + - perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_nvfp4_i2v_cfg2_ulysses4_compile_on] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 63755e48bd6e..1f8b32e2b389 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -17,9 +17,6 @@ l0_dgx_b200: tests: - unittest/_torch/misc/test_autotuner.py::test_autotuner_distributed_strategy - accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-TRTLLM] - # ------------- Disaggregated Serving: Pipelined KV Transfer (multi-GPU) --------------- - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_pipelined_kv_transfer_nixl_python_accuracy[use_kv_cache_manager_v2=False-enable_block_reuse=True] - - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_pipelined_kv_transfer_nixl_python_accuracy[use_kv_cache_manager_v2=True-enable_block_reuse=True] # ------------- KV Cache V2 Scheduler IT (multi-GPU) --------------- - kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_draft_tokens - kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_chunked_draft_tokens diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index 2f9da878e252..58e3d47a8962 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -127,6 +127,8 @@ l0_h100: - unittest/scaffolding - unittest/usage/test_transport.py - unittest/usage/test_e2e_capture.py + - accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap] + - accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=TRTLLM] TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=TRTLLM] TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 4e4803184eba..c1fa4c43f2e4 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -77,6 +77,10 @@ examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_ag examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux1-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux2-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6644450) +examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_cuda_graph_trtllm_backend SKIP (https://nvbugs/6668775) +examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[cuda-graph] SKIP (https://nvbugs/6655986) +examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6572800) +examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_lpips_against_golden SKIP (https://nvbugs/6655986) examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[attn2d_2x2] SKIP (https://nvbugs/6655990) examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2] SKIP (https://nvbugs/6535765) examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[ulysses4] SKIP (https://nvbugs/6535765) diff --git a/tests/unittest/_torch/visual_gen/test_cache_dit.py b/tests/unittest/_torch/visual_gen/test_cache_dit.py index 35708e26e970..e836cad6f995 100644 --- a/tests/unittest/_torch/visual_gen/test_cache_dit.py +++ b/tests/unittest/_torch/visual_gen/test_cache_dit.py @@ -3,9 +3,9 @@ """Tests for Cache-DiT in visual generation. -Wan 2.2 step-split logic is covered with small CPU-side tests. Wan and FLUX -integration tests run on GPU only when cache_dit is installed, CUDA is available, -and checkpoints can be resolved. +Wan 2.2 step-split logic is covered with small CPU-side tests. Wan, FLUX, and LTX-2 +integration tests run on GPU only when cache_dit is installed, CUDA is available, and +checkpoints can be resolved (TRTLLM_CACHE_DIT_* env vars or the fallbacks inside each test). """ from __future__ import annotations @@ -41,10 +41,20 @@ _WAN_SUBPATH = "Wan2.1-T2V-1.3B-Diffusers" _FLUX_SUBPATH = "FLUX.1-dev" _QWEN_IMAGE_SUBPATH = "qwen-image" +_LTX2_DIR = "LTX-2" +_LTX2_WEIGHTS_FILE = "ltx-2-19b-dev.safetensors" +_LTX2_TEXT_ENCODER_SUBPATH = "gemma-3-12b-it" +# Same share as other Gemma checkpoints (e.g. google/gemma-3-1b-it -> gemma/gemma-3-1b-it). +_LTX2_TEXT_ENCODER_RELATIVE_PATHS = ( + _LTX2_TEXT_ENCODER_SUBPATH, + os.path.join("gemma", _LTX2_TEXT_ENCODER_SUBPATH), +) # Default NFS layout used on CI runners (override via TRTLLM_CACHE_DIT_*_CHECKPOINT). _CI_DEFAULT_LLM_MODELS = "/home/scratch.trt_llm_data_ci/llm-models" _DEFAULT_WAN_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _WAN_SUBPATH) _DEFAULT_FLUX_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _FLUX_SUBPATH) +_DEFAULT_LTX2_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _LTX2_DIR, _LTX2_WEIGHTS_FILE) +_DEFAULT_LTX2_TEXT_ENCODER = os.path.join(_CI_DEFAULT_LLM_MODELS, _LTX2_TEXT_ENCODER_SUBPATH) _DEFAULT_QWEN_IMAGE_CHECKPOINT = os.path.join(_CI_DEFAULT_LLM_MODELS, _QWEN_IMAGE_SUBPATH) @@ -96,6 +106,47 @@ def _resolve_flux_checkpoint() -> str | None: return None +def _resolve_ltx2_checkpoint() -> str | None: + """LTX-2 weights file: explicit env, LTX2_MODEL_PATH, CI default, then LLM_MODELS_ROOT (same tiers as Wan/Flux).""" + explicit = os.environ.get("TRTLLM_CACHE_DIT_LTX2_CHECKPOINT", "").strip() + if explicit: + if os.path.isfile(explicit): + return os.path.abspath(explicit) + if os.path.isdir(explicit): + cand = os.path.join(explicit, _LTX2_WEIGHTS_FILE) + if os.path.isfile(cand): + return os.path.abspath(cand) + ltx2_model = os.environ.get("LTX2_MODEL_PATH", "").strip() + if ltx2_model and os.path.isfile(ltx2_model): + return os.path.abspath(ltx2_model) + if os.path.isfile(_DEFAULT_LTX2_CHECKPOINT): + return os.path.abspath(_DEFAULT_LTX2_CHECKPOINT) + root = os.environ.get("LLM_MODELS_ROOT", "").strip() + if root: + cand = os.path.join(root, _LTX2_DIR, _LTX2_WEIGHTS_FILE) + if os.path.isfile(cand): + return os.path.abspath(cand) + return None + + +def _resolve_ltx2_text_encoder() -> str | None: + """Gemma text encoder directory: explicit env, CI default, then LLM_MODELS_ROOT.""" + explicit = os.environ.get("TRTLLM_CACHE_DIT_LTX2_TEXT_ENCODER", "").strip() + if explicit and os.path.isdir(explicit): + return os.path.abspath(explicit) + for rel in _LTX2_TEXT_ENCODER_RELATIVE_PATHS: + cand = os.path.join(_CI_DEFAULT_LLM_MODELS, rel) + if os.path.isdir(cand): + return os.path.abspath(cand) + root = os.environ.get("LLM_MODELS_ROOT", "").strip() + if root: + for rel in _LTX2_TEXT_ENCODER_RELATIVE_PATHS: + cand = os.path.join(root, rel) + if os.path.isdir(cand): + return os.path.abspath(cand) + return None + + @contextlib.contextmanager def _suppress_stdlib_logging_for_cache_dit(): """Silence cache_dit's stdlib logging under pytest capture (closed stdout/stderr). @@ -228,7 +279,7 @@ def test_boundary_one_all_timesteps_low(self): @requires_cache_dit @requires_cuda class TestCacheDiTRealPipelineForward: - """Wan and FLUX.1 use the CI llm-models tree when checkpoints are present. + """Wan, FLUX.1, and LTX-2 use the CI llm-models tree when checkpoints are present. Each test calls _teardown_cache_dit in finally; cache_dit treats caching as process-global, so a second loaded pipeline would otherwise skip setup. @@ -251,6 +302,7 @@ def _teardown_cache_dit(pipeline: object) -> None: def _load_visual_gen_pipeline( checkpoint_dir: str, *, + text_encoder_path: str = "", enable_torch_compile: bool = False, ): from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader @@ -261,8 +313,18 @@ def _load_visual_gen_pipeline( VisualGenArgs, ) + # text_encoder_path lives under pipeline_config and is only a + # legal key for the LTX2 family. Strict validation rejects it + # for Wan/Flux pipelines whose registry defaults are empty — + # so only emit the key when the caller supplied a non-empty + # value (LTX2 case). + pipeline_config: dict = {} + if text_encoder_path: + pipeline_config["text_encoder_path"] = text_encoder_path + args = VisualGenArgs( model=checkpoint_dir, + pipeline_config=pipeline_config, cache_config=CacheDiTConfig( max_warmup_steps=0, Fn_compute_blocks=1, @@ -418,6 +480,62 @@ def test_flux_cache_dit_skips_blocks_after_forward(self): if pipeline is not None: self._teardown_cache_dit(pipeline) + def test_ltx2_cache_dit_skips_blocks_after_forward(self): + ckpt = _resolve_ltx2_checkpoint() + text_enc = _resolve_ltx2_text_encoder() + if ckpt is None or text_enc is None: + missing = [] + if ckpt is None: + missing.append("LTX-2 checkpoint") + if text_enc is None: + missing.append("Gemma text encoder") + pytest.skip( + f"Missing {' and '.join(missing)}: set " + "TRTLLM_CACHE_DIT_LTX2_CHECKPOINT (file or directory with " + f"{_LTX2_WEIGHTS_FILE}) and TRTLLM_CACHE_DIT_LTX2_TEXT_ENCODER, " + "or LTX2_MODEL_PATH, or stage under CI tree " + f"{_DEFAULT_LTX2_CHECKPOINT} and one of " + f"{', '.join(os.path.join(_CI_DEFAULT_LLM_MODELS, r) for r in _LTX2_TEXT_ENCODER_RELATIVE_PATHS)} " + f"(same as Wan/Flux under {_CI_DEFAULT_LLM_MODELS}), " + f"or $LLM_MODELS_ROOT/{_LTX2_DIR}/{_LTX2_WEIGHTS_FILE} and " + f"$LLM_MODELS_ROOT/<{' or '.join(_LTX2_TEXT_ENCODER_RELATIVE_PATHS)}>" + ) + + pipeline = None + with _suppress_stdlib_logging_for_cache_dit(): + try: + pipeline = self._load_visual_gen_pipeline(ckpt, text_encoder_path=text_enc) + name = pipeline.__class__.__name__ + if name != "LTX2Pipeline": + pytest.skip(f"Checkpoint resolved to {name}, not LTX2Pipeline") + + assert pipeline.cache_accelerator is not None + assert pipeline.cache_accelerator.is_enabled() + + with torch.inference_mode(): + pipeline.forward( + prompt="cache dit validation", + negative_prompt="", + height=512, + width=768, + num_frames=33, + num_inference_steps=16, + guidance_scale=4.0, + seed=0, + max_sequence_length=256, + frame_rate=24.0, + ) + + stats = pipeline.cache_accelerator.get_stats() + cached = _total_accumulated_cached_steps(stats) + assert cached > 0, ( + "Expected Cache-DiT accumulated_cached_steps > 0 after forward; " + f"stats={stats!r}. Try more steps or a looser residual_diff_threshold." + ) + finally: + if pipeline is not None: + self._teardown_cache_dit(pipeline) + def test_qwen_image_cache_dit_skips_blocks_after_forward(self): ckpt = _resolve_qwen_image_checkpoint() if ckpt is None: diff --git a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py index 8c8bbe2dfbb1..f1064f780fa1 100644 --- a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py @@ -51,6 +51,7 @@ _LTX2_BASE = os.path.join(str(llm_models_root(check=True)), "LTX-2") +_GEMMA3_DEFAULT = os.path.join(str(llm_models_root(check=True)), "gemma", "gemma-3-12b-it") CHECKPOINT_PATH_BF16 = os.environ.get( @@ -61,6 +62,20 @@ "LTX2_MODEL_PATH_FP8", os.path.join(_LTX2_BASE, "ltx-2-19b-dev-fp8.safetensors"), ) +GEMMA3_PATH = os.environ.get("LTX2_TEXT_ENCODER_PATH", _GEMMA3_DEFAULT) + + +def _ltx2_pipeline_config(**overrides): + """Build pipeline_config with the Gemma3 text_encoder_path LTX-2 needs. + + LTX-2's tokenizer + text encoder are loaded from a separate Gemma + directory (not the diffusion checkpoint), so every full-pipeline + load needs ``text_encoder_path`` set. Tests can pass extra keys via + ``overrides`` (e.g. ``spatial_upsampler_path`` for two-stage). + """ + cfg = {"text_encoder_path": GEMMA3_PATH} + cfg.update(overrides) + return cfg def _write_minimal_ltx2_native_checkpoint(tmp_path): @@ -191,6 +206,7 @@ def test_load_with_quantization(self, ltx2_bf16_checkpoint_exists, quant_algo: s args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, quant_config={"quant_algo": quant_algo, "dynamic": True}, + pipeline_config=_ltx2_pipeline_config(), ) pipeline = PipelineLoader(args).load(skip_warmup=True, skip_components=SKIP_COMPONENTS) @@ -245,6 +261,7 @@ def test_fp8_vs_bf16_single_layer(self, ltx2_bf16_checkpoint_exists, quant_algo: print(f"\n[Compare {quant_algo}] Loading BF16 pipeline...") args_bf16 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, + pipeline_config=_ltx2_pipeline_config(), ) pipeline_bf16 = PipelineLoader(args_bf16).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -254,6 +271,7 @@ def test_fp8_vs_bf16_single_layer(self, ltx2_bf16_checkpoint_exists, quant_algo: args_fp8 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, quant_config={"quant_algo": quant_algo, "dynamic": True}, + pipeline_config=_ltx2_pipeline_config(), ) pipeline_fp8 = PipelineLoader(args_fp8).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -320,6 +338,7 @@ def get_module_memory_gb(module): args_bf16 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, + pipeline_config=_ltx2_pipeline_config(), ) pipeline_bf16 = PipelineLoader(args_bf16).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -336,6 +355,7 @@ def get_module_memory_gb(module): args_fp8 = VisualGenArgs( model=CHECKPOINT_PATH_BF16, quant_config={"quant_algo": "FP8", "dynamic": True}, + pipeline_config=_ltx2_pipeline_config(), ) pipeline_fp8 = PipelineLoader(args_fp8).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -372,6 +392,7 @@ def test_attention_backend_comparison(self, ltx2_bf16_checkpoint_exists): args_baseline = VisualGenArgs( model=CHECKPOINT_PATH_BF16, attention_config=AttentionConfig(backend="VANILLA"), + pipeline_config=_ltx2_pipeline_config(), ) pipeline_baseline = PipelineLoader(args_baseline).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -398,6 +419,7 @@ def test_attention_backend_comparison(self, ltx2_bf16_checkpoint_exists): args_trtllm = VisualGenArgs( model=CHECKPOINT_PATH_BF16, attention_config=AttentionConfig(backend="TRTLLM"), + pipeline_config=_ltx2_pipeline_config(), ) pipeline_trtllm = PipelineLoader(args_trtllm).load( skip_warmup=True, skip_components=SKIP_COMPONENTS @@ -1593,10 +1615,10 @@ def test_loads_as_two_stage_pipeline(self, ltx2_two_stage_assets_exist): args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config={ - "spatial_upsampler_path": UPSAMPLER_PATH, - "distilled_lora_path": LORA_PATH, - }, + pipeline_config=_ltx2_pipeline_config( + spatial_upsampler_path=UPSAMPLER_PATH, + distilled_lora_path=LORA_PATH, + ), ) pipeline = PipelineLoader(args).load(skip_warmup=True, skip_components=SKIP_COMPONENTS) @@ -1626,10 +1648,10 @@ def test_two_stage_lora_deltas_match_transformer(self, ltx2_two_stage_assets_exi args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config={ - "spatial_upsampler_path": UPSAMPLER_PATH, - "distilled_lora_path": LORA_PATH, - }, + pipeline_config=_ltx2_pipeline_config( + spatial_upsampler_path=UPSAMPLER_PATH, + distilled_lora_path=LORA_PATH, + ), ) pipeline = PipelineLoader(args).load(skip_warmup=True, skip_components=SKIP_COMPONENTS) @@ -1672,10 +1694,10 @@ def test_two_stage_with_quantization(self, ltx2_two_stage_assets_exist, quant_al args = VisualGenArgs( model=CHECKPOINT_PATH_BF16, - pipeline_config={ - "spatial_upsampler_path": UPSAMPLER_PATH, - "distilled_lora_path": LORA_PATH, - }, + pipeline_config=_ltx2_pipeline_config( + spatial_upsampler_path=UPSAMPLER_PATH, + distilled_lora_path=LORA_PATH, + ), quant_config={"quant_algo": quant_algo, "dynamic": True}, ) diff --git a/tests/unittest/llmapi/test_llm_pytorch.py b/tests/unittest/llmapi/test_llm_pytorch.py index 737141869e4e..ef965bc5c04c 100644 --- a/tests/unittest/llmapi/test_llm_pytorch.py +++ b/tests/unittest/llmapi/test_llm_pytorch.py @@ -36,10 +36,12 @@ run_llm_with_postprocess_parallel_and_result_handler, tinyllama_logits_processor_test_harness) from utils.util import (force_ampere, similar, skip_fp8_pre_ada, + skip_gpu_memory_less_than_40gb, skip_gpu_memory_less_than_80gb, skip_ray) from utils.llm_data import llm_models_root from tensorrt_llm._torch.peft.lora.config import LoraConfig from tensorrt_llm.executor.request import LoRARequest +import tempfile import torch from transformers.configuration_utils import PretrainedConfig @@ -56,7 +58,9 @@ register_checkpoint_weight_loader as _register_checkpoint_weight_loader, register_config_loader as _register_config_loader) -from transformers import AutoTokenizer +from peft import LoraConfig as PeftLoraConfig +from peft import get_peft_model +from transformers import AutoModelForCausalLM, AutoTokenizer from dataclasses import replace # isort: on @@ -403,6 +407,143 @@ def test_llama_3_1_8b_fp8_with_bf16_lora(cuda_graph_config) -> None: assert similar(output.outputs[0].text, reference) +@pytest.mark.part2 +@test_lora_with_and_without_cuda_graph +def test_gemma3_1b_instruct_multi_lora(cuda_graph_config) -> None: + model_dir = f"{llm_models_root()}/gemma/gemma-3-1b-it" + + target_modules = ['attn_q', 'attn_k', 'attn_v'] + + # Set up temporary directory for LoRA adapters + with tempfile.TemporaryDirectory() as lora_dir: + print("Creating dummy LoRAs...") + + model = AutoModelForCausalLM.from_pretrained(model_dir, + dtype=torch.bfloat16, + device_map="auto") + hf_modules = ["q_proj", "k_proj", "v_proj"] + peft_lora_config = PeftLoraConfig(r=8, + target_modules=hf_modules, + bias="none", + task_type="CAUSAL_LM") + lora_paths = [] + for i in range(2): + lora_model = get_peft_model(model, peft_lora_config) + for param in lora_model.parameters(): + param.data.zero_() + lora_path = f"{lora_dir}/lora_{i}" + lora_model.save_pretrained(lora_path) + lora_paths.append(lora_path) + + trtllm_lora_config = LoraConfig(lora_dir=lora_paths, + lora_target_modules=target_modules, + max_lora_rank=8, + max_loras=2, + max_cpu_loras=2) + # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. + kv_cache_config = KvCacheConfig( + enable_block_reuse=False, + enable_partial_reuse=False, + ) + llm = LLM(model_dir, + lora_config=trtllm_lora_config, + kv_cache_config=kv_cache_config, + cuda_graph_config=cuda_graph_config) + + prompts = [ + "Is it ok to fill diesel in a petrol car?", + "What is the capital of France?", + ] + lora_req1 = LoRARequest("lora-1", 0, lora_paths[0]) + lora_req2 = LoRARequest("lora-2", 1, lora_paths[1]) + lora_requests = [lora_req1, lora_req2] + sampling_params = SamplingParams(max_tokens=200) + + outputs = llm.generate(prompts, + sampling_params, + lora_request=lora_requests) + + assert len(outputs) == 2 + + +@skip_gpu_memory_less_than_40gb +@pytest.mark.part3 +def test_lora_many_adapters_no_memory_leak() -> None: + """Verify GPU memory stays bounded when loading many unique LoRA adapters. + + Creates 20 dummy adapters but sets max_loras=2 and max_cpu_loras=4 to + force eviction. Without proper cleanup, _lora_weights can accumulate + GPU tensors for every loaded adapter, causing unbounded memory growth. + """ + model_dir = f"{llm_models_root()}/gemma/gemma-3-1b-it" + num_adapters = 20 + target_modules = ['attn_q', 'attn_k', 'attn_v'] + + with tempfile.TemporaryDirectory() as lora_dir: + model = AutoModelForCausalLM.from_pretrained(model_dir, + dtype=torch.bfloat16, + device_map="auto") + hf_modules = ["q_proj", "k_proj", "v_proj"] + peft_lora_config = PeftLoraConfig(r=8, + target_modules=hf_modules, + bias="none", + task_type="CAUSAL_LM") + lora_paths = [] + for i in range(num_adapters): + lora_model = get_peft_model(model, peft_lora_config) + for param in lora_model.parameters(): + param.data.zero_() + lora_path = f"{lora_dir}/lora_{i}" + lora_model.save_pretrained(lora_path) + lora_paths.append(lora_path) + + del model + torch.cuda.empty_cache() + + trtllm_lora_config = LoraConfig(lora_dir=lora_paths[:1], + lora_target_modules=target_modules, + max_lora_rank=8, + max_loras=2, + max_cpu_loras=4) + kv_cache_config = KvCacheConfig(enable_block_reuse=False, + enable_partial_reuse=False) + llm = LLM(model_dir, + lora_config=trtllm_lora_config, + kv_cache_config=kv_cache_config) + + sampling_params = SamplingParams(max_tokens=20) + warmup_count = 5 + + mem_samples = [] + for i in range(num_adapters): + lora_req = LoRARequest(f"lora-{i}", i, lora_paths[i]) + output = llm.generate("Hello, tell me a story.", + sampling_params, + lora_request=lora_req) + assert output.outputs[0].text != "" + + if i >= warmup_count: + mem_samples.append(torch.cuda.memory_allocated()) + + num_measured = len(mem_samples) + assert num_measured >= 2, "Not enough samples to measure growth" + + total_growth = mem_samples[-1] - mem_samples[0] + per_adapter_mb = (total_growth / (num_measured - 1)) / (1024 * 1024) + + # Each adapter is ~3 MB on GPU (r=8, 3 modules, 26 layers, bf16). + # The C++ PeftCacheManager handles eviction and _lora_weights + # stays empty, so per-adapter growth should be ~0. If GPU tensors + # leak, we would see ~3 MB/adapter of linear growth. Threshold + # of 1 MB/adapter catches leaks while tolerating noise from + # allocator fragmentation averaged over many samples. + max_per_adapter_mb = 1.0 + assert per_adapter_mb < max_per_adapter_mb, ( + f"GPU memory growing at {per_adapter_mb:.2f} MB/adapter over " + f"{num_measured} adapters (total {total_growth / (1024**2):.1f} MB). " + f"Possible _lora_weights leak.") + + @pytest.mark.parametrize( "lora_rank,max_lora_rank,description", [ From 1ac84d1b5b39b0bb2a79ebc1a4d338086071f308 Mon Sep 17 00:00:00 2001 From: "Xin He (SW-GPU)" <200704525+xinhe-nv@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:00:18 +0800 Subject: [PATCH 3/3] test: prune Gemma-2 and GPT-2 checkpoint tests Remove checkpoint-dependent CLI flow accuracy tests, reference thresholds, AutoDeploy registry entries, and orphaned gpt2 fixtures. Drop resolved waive for test_chat_stream_post_processor_reuses_stream_metadata. Signed-off-by: Xin He (SW-GPU) <200704525+xinhe-nv@users.noreply.github.com> --- .../auto_deploy/model_registry/models.yaml | 9 --- .../accuracy/references/cnn_dailymail.yaml | 33 ---------- .../defs/accuracy/references/mmlu.yaml | 2 - .../defs/accuracy/test_cli_flow.py | 60 ------------------- tests/integration/defs/conftest.py | 14 ----- .../defs/triton_server/conftest.py | 24 -------- tests/integration/test_lists/waives.txt | 1 - 7 files changed, 143 deletions(-) diff --git a/examples/auto_deploy/model_registry/models.yaml b/examples/auto_deploy/model_registry/models.yaml index 78a4ae2afa0b..f1e925a6d978 100644 --- a/examples/auto_deploy/model_registry/models.yaml +++ b/examples/auto_deploy/model_registry/models.yaml @@ -83,12 +83,6 @@ models: # - name: google/gemma-1.1-7b-it # config_id: default_ws_2 # yaml_extra: ['dashboard_default.yaml', 'world_size_2.yaml'] -- name: google/gemma-2-2b-it - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] -- name: google/gemma-2-9b-it - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # IndexError: list index out of range in AutoDeploy sharding path. See https://github.com/NVIDIA/TensorRT-LLM/issues/14681 # - name: google/codegemma-7b-it # config_id: default_ws_2 @@ -229,9 +223,6 @@ models: - name: Qwen/QwQ-32B config_id: default_ws_4 yaml_extra: ['dashboard_default.yaml', 'world_size_4.yaml'] -- name: google/gemma-2-27b-it - config_id: default_ws_1 - yaml_extra: ['dashboard_default.yaml', 'world_size_1.yaml'] # OOM during AutoDeploy run. # - name: perplexity-ai/r1-1776-distill-llama-70b # config_id: default_ws_4 diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index e576a74c4364..c15fab97de1e 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -1,27 +1,3 @@ -gpt2: - - accuracy: 18.408 - - quant_algo: W8A16 - accuracy: 18.408 - - quant_algo: W4A16 - accuracy: 15.251 - - kv_cache_quant_algo: INT8 - accuracy: 18.208 - - quant_algo: W8A8_SQ_PER_TENSOR_PLUGIN - accuracy: 18.377 - - quant_algo: W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN - accuracy: 18.408 - - extra_acc_spec: beam_width=4 - accuracy: 24.933 - - extra_acc_spec: beam_width=256 - num_samples: 32 - accuracy: 29.422 - - extra_acc_spec: beam_width=8;beam_width_array=[2,3,4,5] - num_samples: 32 - accuracy: 26.994 -gpt2-medium: - - accuracy: 22.730 - - quant_algo: FP8 - accuracy: 22.249 gpt-next: - accuracy: 25.516 state-spaces/mamba-130m-hf: @@ -126,15 +102,6 @@ google/gemma-7b: accuracy: 19.357 - quant_algo: W4A16_AWQ accuracy: 20.283 -google/gemma-2-9b-it: - - accuracy: 29.519 - - quant_algo: W8A16 - accuracy: 29.679 - - quant_algo: W4A16 - accuracy: 29.536 - - quant_algo: FP8 - kv_cache_quant_algo: FP8 - accuracy: 26.670 nvidia/Nemotron-Mini-4B-Instruct: - quant_algo: FP8 accuracy: 25.247 diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index d682a80068ab..251dc86e5e97 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -35,8 +35,6 @@ mistralai/Mistral-Small-3.1-24B-Instruct-2503: - quant_algo: FP8 kv_cache_quant_algo: FP8 accuracy: 81.1 -google/gemma-2-9b-it: - - accuracy: 73.05 google/gemma-4-26B-A4B-it: - accuracy: 71.296 google/gemma-3n-E2B-it: diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index cda65eec2cbb..39af5bf0aa96 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -30,40 +30,6 @@ allow_module_level=True) -class TestGpt2(CliFlowAccuracyTestHarness): - MODEL_NAME = "gpt2" - MODEL_PATH = f"{llm_models_root()}/gpt2" - EXAMPLE_FOLDER = "models/core/gpt" - - def test_auto_dtype(self): - # float16 - self.run(dtype='auto') - - @skip_post_blackwell - @pytest.mark.parametrize("precision", ["int8", "int4"]) - def test_weight_only(self, precision: str): - quant_algo = QuantAlgo.W8A16 if precision == "int8" else QuantAlgo.W4A16 - self.run(quant_algo=quant_algo) - - def test_beam_search(self): - self.run(extra_acc_spec="beam_width=4", - extra_build_args=["--max_beam_width=4"], - extra_summarize_args=["--num_beams=4", "--length_penalty=2.0"]) - - -class TestGpt2Medium(CliFlowAccuracyTestHarness): - MODEL_NAME = "gpt2-medium" - MODEL_PATH = f"{llm_models_root()}/gpt2-medium" - EXAMPLE_FOLDER = "models/core/gpt" - - def test_auto_dtype(self): - self.run(dtype='auto') - - @skip_pre_ada - def test_fp8(self): - self.run(quant_algo=QuantAlgo.FP8) - - class TestStarcoder2_15B(CliFlowAccuracyTestHarness): MODEL_NAME = "bigcode/starcoder2-15b" MODEL_PATH = f"{llm_models_root()}/starcoder2-model" @@ -315,29 +281,3 @@ def test_weight_only(self, precision: str): @skip_pre_ada def test_fp8(self): self.run(quant_algo=QuantAlgo.FP8, kv_cache_quant_algo=QuantAlgo.FP8) - - -@pytest.mark.skip_less_device_memory(40000) -class TestGemma2_9BIt(CliFlowAccuracyTestHarness): - MODEL_NAME = "google/gemma-2-9b-it" - MODEL_PATH = f"{llm_models_root()}/gemma/gemma-2-9b-it" - EXAMPLE_FOLDER = "models/core/gemma" - - @skip_post_blackwell - def test_auto_dtype(self): - self.run(tasks=[CnnDailymail(self.MODEL_NAME), - MMLU(self.MODEL_NAME)], - dtype='auto', - extra_convert_args=["--ckpt-type=hf"]) - - @skip_post_blackwell - @pytest.mark.parametrize("precision", ["int8", "int4"]) - def test_weight_only(self, precision: str): - quant_algo = QuantAlgo.W8A16 if precision == "int8" else QuantAlgo.W4A16 - self.run(quant_algo=quant_algo, extra_convert_args=["--ckpt-type=hf"]) - - @skip_pre_hopper - def test_fp8(self): - self.run(quant_algo=QuantAlgo.FP8, - kv_cache_quant_algo=QuantAlgo.FP8, - extra_convert_args=["--device_map=sequential"]) diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index d9f7fc882ff4..12650ea507f3 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -568,20 +568,6 @@ def llm_replit_code_v1_5_3b_model_root(): raise RuntimeError("replit-code-v1_5-3b must be cached") -@pytest.fixture(scope="module") -@cached_in_llm_models_root("gpt2", True) -def llm_gpt2_model_root(): - "Get gpt2 model root" - raise RuntimeError("gpt2 must be cached") - - -@pytest.fixture(scope="module") -@cached_in_llm_models_root("gpt2-medium", True) -def llm_gpt2_medium_model_root(): - "Get gpt2 medium model root" - raise RuntimeError("gpt2-medium must be cached") - - @pytest.fixture(scope="module") @cached_in_llm_models_root("GPT-2B-001_bf16_tp1.nemo", True) def llm_gpt2_next_model_root(): diff --git a/tests/integration/defs/triton_server/conftest.py b/tests/integration/defs/triton_server/conftest.py index 86fb46412bce..59188db72eec 100644 --- a/tests/integration/defs/triton_server/conftest.py +++ b/tests/integration/defs/triton_server/conftest.py @@ -295,18 +295,6 @@ def llama_v2_tokenizer_model_root(): return llama_v2_tokenizer_model_root -@pytest.fixture(scope="session") -def gpt_tokenizer_model_root(llm_backend_venv): - models_root = llm_models_root() - assert models_root, "Did you set LLM_MODELS_ROOT?" - gpt_tokenizer_model_root = os.path.join(models_root, "gpt2") - - assert os.path.exists( - gpt_tokenizer_model_root - ), f"{gpt_tokenizer_model_root} does not exist under NFS LLM_MODELS_ROOT dir" - return gpt_tokenizer_model_root - - @pytest.fixture(scope="session") def gptj_tokenizer_model_root(llm_backend_venv): models_root = llm_models_root() @@ -319,18 +307,6 @@ def gptj_tokenizer_model_root(llm_backend_venv): return gptj_tokenizer_model_root -@pytest.fixture(scope="session") -def gpt2_medium_tokenizer_model_root(): - models_root = llm_models_root() - assert models_root, "Did you set LLM_MODELS_ROOT?" - gpt_tokenizer_model_root = os.path.join(models_root, "gpt2-medium") - - assert os.path.exists( - gpt_tokenizer_model_root - ), f"{gpt_tokenizer_model_root} does not exist under NFS LLM_MODELS_ROOT dir" - return gpt_tokenizer_model_root - - @pytest.fixture(scope="session") def gpt_next_ptuning_model_root(): models_root = llm_models_root() diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index c1fa4c43f2e4..2cf9d8d8f611 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -341,7 +341,6 @@ unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfe unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) -unittest/llmapi/test_llm.py::test_chat_stream_post_processor_reuses_stream_metadata SKIP (https://nvbugs/6693989) unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp2[False-False-True] SKIP (https://nvbugs/6432826) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504)