diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index bcea8e086953..2f0e913a83e0 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -680,18 +680,6 @@ def llama_model_root(request): if request.param == "TinyLlama-1.1B-Chat-v1.0": llama_model_root = os.path.join(models_root, "llama-models-v2", "TinyLlama-1.1B-Chat-v1.0") - elif request.param == "llama-3.1-8b": - llama_model_root = os.path.join(models_root, "llama-3.1-model", - "Meta-Llama-3.1-8B") - elif request.param == "llama-3.1-8b-instruct-hf-fp8": - llama_model_root = os.path.join(models_root, "llama-3.1-model", - "Llama-3.1-8B-Instruct-FP8") - elif request.param == "llama-3.1-8b-instruct": - llama_model_root = os.path.join(models_root, "llama-3.1-model", - "Llama-3.1-8B-Instruct") - elif request.param == "llama-3.1-8b-hf-nvfp4": - llama_model_root = os.path.join(models_root, "nvfp4-quantized", - "Meta-Llama-3.1-8B") assert os.path.exists( llama_model_root ), f"{llama_model_root} does not exist under NFS LLM_MODELS_ROOT dir" diff --git a/tests/integration/defs/disaggregated/test_ad_disagg.py b/tests/integration/defs/disaggregated/test_ad_disagg.py index 24a3915475e7..e0c08b641b1d 100644 --- a/tests/integration/defs/disaggregated/test_ad_disagg.py +++ b/tests/integration/defs/disaggregated/test_ad_disagg.py @@ -67,32 +67,13 @@ def skip_b300(): AUTODEPLOY_DISAGG_SEED = 1234 REDUCED_TINYLLAMA_LAYERS = 2 REDUCED_DEEPSEEK_LAYERS = 2 -LLAMA_EAGLE3_EXPECTED_TEXT = " Berlin\nWhat is the capital of France? Paris\nWhat is the capital of" -LLAMA_EAGLE3_EXPECTED_TOKEN_IDS = [ - 20437, - 198, - 3923, - 374, - 279, - 6864, - 315, - 9822, - 30, - 12366, - 198, - 3923, - 374, - 279, - 6864, - 315, -] MODEL_PATHS = { - "EAGLE3-LLaMA3.1-Instruct-8B": "EAGLE3-LLaMA3.1-Instruct-8B", - "Llama-3.1-8B-Instruct": "llama-3.1-model/Llama-3.1-8B-Instruct/", "TinyLlama-1.1B-Chat-v1.0": "llama-models-v2/TinyLlama-1.1B-Chat-v1.0", "DeepSeek-V3-Lite": "DeepSeek-V3-Lite/bf16", + "Qwen3-8B-eagle3": "Qwen3/qwen3_8b_eagle3", + "Qwen3-8B": "Qwen3/Qwen3-8B", } @@ -657,14 +638,19 @@ def test_chunked_prefill_handoff(model): # --------------------------------------------------------------------------- -def llama_eagle3_config(): +def qwen3_eagle3_config(): return { "speculative_config": Eagle3DecodingConfig( max_draft_len=3, - speculative_model=model_path("EAGLE3-LLaMA3.1-Instruct-8B"), - eagle3_layers_to_capture={1, 15, 28}, + speculative_model=model_path("Qwen3-8B-eagle3"), + # TODO: these capture layers were carried over proportionally from + # the retired Llama-3.1-8B config (1/32, 15/32, 28/32 through the + # stack) and have NOT been validated against Qwen3-8B's actual + # layer count on GPU. Re-derive and confirm before relying on this + # test's output. + eagle3_layers_to_capture={1, 17, 31}, ), - # Force the Eagle3 draft to match the BF16 Llama 3.1 target. Shared KV + # Force the Eagle3 draft to match the BF16 Qwen3-8B target. Shared KV # cache management requires matching target and draft KV dtypes. "speculative_model_kwargs": {"torch_dtype": "bfloat16"}, } @@ -1039,18 +1025,39 @@ def test_async_sharded_generation_handoff(): @pytest.mark.skip_less_device(2) @pytest.mark.timeout(900) def test_async_eagle3_full_model_handoff(): + """Eagle3 one-model draft-token handoff, compared against an aggregate run. + + Unlike the retired Llama-3.1-8B version of this test, this compares + against a freshly-computed aggregate (non-disaggregated) generation using + the same speculative_config, instead of hardcoded golden text/token IDs. + That avoids needing pre-recorded goldens for the new model pairing, at the + cost of also exercising the aggregate Eagle3 one-model path as a + dependency. This still needs a real GPU run to confirm Qwen3-8B + + Qwen3/qwen3_8b_eagle3 actually produce matching, non-trivial draft-token + output under this config (see the eagle3_layers_to_capture TODO in + qwen3_eagle3_config). + """ + prompt = "What is the capital of Germany?" sampling_params_kwargs = { "max_tokens": 16, "ignore_eos": True, "top_k": 1, "seed": AUTODEPLOY_DISAGG_SEED, } - extra_config = llama_eagle3_config() + extra_config = qwen3_eagle3_config() + + aggregate_output = run_aggregate_generation( + "Qwen3-8B", + world_size=1, + prompt=prompt, + sampling_params_kwargs=sampling_params_kwargs, + extra_config=extra_config, + ) outputs = run_context_then_generation_handoff( - "Llama-3.1-8B-Instruct", + "Qwen3-8B", worker_world_sizes=(1, 1), generation_overlap=True, - prompt="What is the capital of Germany?", + prompt=prompt, sampling_params_kwargs=sampling_params_kwargs, extra_config=extra_config, ) @@ -1064,7 +1071,6 @@ def test_async_eagle3_full_model_handoff(): assert outputs["generation"].token_ids assert has_draft_tokens(outputs["context"]) assert has_draft_tokens(outputs["generation"]) - assert outputs["context"].text == " Berlin" - assert outputs["context"].token_ids == LLAMA_EAGLE3_EXPECTED_TOKEN_IDS[:1] - assert outputs["generation"].text == LLAMA_EAGLE3_EXPECTED_TEXT - assert outputs["generation"].token_ids == LLAMA_EAGLE3_EXPECTED_TOKEN_IDS + assert outputs["context"].token_ids == aggregate_output.token_ids[:1] + assert outputs["generation"].text == aggregate_output.text + assert outputs["generation"].token_ids == aggregate_output.token_ids diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_qwen3_8b.yaml similarity index 90% rename from tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b.yaml rename to tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_qwen3_8b.yaml index 487abd0d600e..78b3614a650e 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_qwen3_8b.yaml @@ -1,5 +1,5 @@ hostname: localhost -model: llama-3.1-model/Llama-3.1-8B-Instruct +model: Qwen3/Qwen3-8B free_gpu_memory_fraction: 0.25 backend: pytorch disable_overlap_scheduler: true diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 7f9497787554..6b9bb8469e1b 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -418,8 +418,8 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml", "cancel_stress_test": f"{test_configs_root}/disagg_config_cancel_stress_test.yaml", - "llama31_8b": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_llama31_8b.yaml", + "qwen3_8b": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_qwen3_8b.yaml", "mamba_conc_greater_than_mbs": f"{test_configs_root}/disagg_config_mamba_conc_greater_than_mbs.yaml", "mamba_bs1_concurrency2": @@ -2350,9 +2350,6 @@ def benchmark_model_root(request): model_path = os.path.join(models_root, "DeepSeek-V3-Lite", "fp8") elif (request.param == "DeepSeek-V3-Lite-bf16"): model_path = os.path.join(models_root, "DeepSeek-V3-Lite", "bf16") - elif request.param == "llama-3.1-8b-instruct-hf-fp8": - model_path = os.path.join(models_root, "llama-3.1-model", - "Llama-3.1-8B-Instruct-FP8") else: raise ValueError(f"Failed to find the model: {request.param}") return model_path @@ -4074,11 +4071,8 @@ def test_disaggregated_cancel_large_context_requests(disaggregated_test_root, @pytest.mark.skip_less_device(4) -@pytest.mark.parametrize("llama_model_root", ['llama-3.1-8b-instruct'], - indirect=True) def test_disaggregated_logprobs_serving(disaggregated_test_root, - disaggregated_example_root, llm_venv, - llama_model_root): + disaggregated_example_root, llm_venv): """Test logprobs via OpenAI API in disaggregated serving with multi-GPU TP. Covers the RCCA scenario (NVBug 5926823): disaggregated + streaming + logprobs, @@ -4140,10 +4134,11 @@ def extract_logprobs(result, api_type): logprobs = [item.get("logprob") for item in content] return tokens, logprobs - setup_model_symlink(llm_venv, llama_model_root, - "llama-3.1-model/Llama-3.1-8B-Instruct") + model_path = "Qwen3/Qwen3-8B" + model_dir = f"{llm_models_root()}/{model_path}" + setup_model_symlink(llm_venv, model_dir, model_path) - config_file = get_test_config("llama31_8b", disaggregated_example_root, + config_file = get_test_config("qwen3_8b", disaggregated_example_root, os.path.dirname(__file__)) env = llm_venv._new_env.copy() @@ -4151,13 +4146,13 @@ def extract_logprobs(result, api_type): ctx_workers, gen_workers, disagg_server, work_dir = [], [], None, None config, ctx_workers, gen_workers, disagg_server, server_port, work_dir = \ setup_disagg_cluster(config_file, env=env, - model_name=llama_model_root, + model_name=model_dir, cwd=llm_venv.get_working_directory(), server_start_timeout=600) server_host = config.get("hostname", "localhost") server_url = f"http://{server_host}:{server_port}" - model_name = "llama-3.1-model/Llama-3.1-8B-Instruct" + model_name = model_path max_tokens = 20 timeout = aiohttp.ClientTimeout(total=120) # Use emoji prompt to also stress-test multi-byte tokenizer handling diff --git a/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py b/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py index 6e8913ea67ab..41880fdb4410 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py +++ b/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py @@ -56,9 +56,9 @@ def get_ucx_tls(): MODEL_PATHS = { "DeepSeek-V3-Lite-fp8": "DeepSeek-V3-Lite/fp8", "TinyLlama-1.1B-Chat-v1.0": "llama-models-v2/TinyLlama-1.1B-Chat-v1.0", - "Llama-3.1-8B-Instruct": "llama-3.1-model/Llama-3.1-8B-Instruct/", - "EAGLE3-LLaMA3.1-Instruct-8B": "EAGLE3-LLaMA3.1-Instruct-8B", + "Qwen3-8B-eagle3": "Qwen3/qwen3_8b_eagle3", "Qwen3-8B-FP8": "Qwen3/Qwen3-8B-FP8", + "Qwen3-8B": "Qwen3/Qwen3-8B", } @@ -517,8 +517,9 @@ def test_disaggregated_llama_context_capacity(model, enable_cuda_graph, print("All workers terminated.") -@pytest.mark.parametrize("model", ["Llama-3.1-8B-Instruct"]) -@pytest.mark.parametrize("spec_dec_model_path", ["EAGLE3-LLaMA3.1-Instruct-8B"]) +@skip_pre_hopper +@pytest.mark.parametrize("model", ["Qwen3-8B"]) +@pytest.mark.parametrize("spec_dec_model_path", ["Qwen3-8B-eagle3"]) @pytest.mark.parametrize("generation_overlap", [False]) def test_disaggregated_spec_dec_batch_slot_limit(model, spec_dec_model_path, generation_overlap): diff --git a/tests/integration/defs/examples/test_ad_speculative_decoding.py b/tests/integration/defs/examples/test_ad_speculative_decoding.py index 40c535f4ebdd..37ef41ef9e32 100644 --- a/tests/integration/defs/examples/test_ad_speculative_decoding.py +++ b/tests/integration/defs/examples/test_ad_speculative_decoding.py @@ -14,1086 +14,14 @@ # limitations under the License. import json -import os import re -from dataclasses import dataclass from pathlib import Path -from typing import Optional, Set -import pytest import torch -import torch.nn as nn -from defs.conftest import llm_models_root from test_common.llm_data import hf_id_to_local_model_dir -from transformers import AutoModelForCausalLM, AutoTokenizer -from transformers.masking_utils import create_causal_mask -from transformers.modeling_outputs import BaseModelOutputWithPast -from transformers.models.llama.modeling_llama import LlamaModel -from transformers.utils.generic import ModelOutput -from tensorrt_llm import SamplingParams -from tensorrt_llm._torch.auto_deploy.llm import LLM -from tensorrt_llm._torch.auto_deploy.models.custom.modeling_eagle import ( - EagleDrafterForCausalLM, - EagleWrapper, - EagleWrapperConfig, -) +from tensorrt_llm._torch.auto_deploy.models.custom.modeling_eagle import EagleDrafterForCausalLM from tensorrt_llm._torch.auto_deploy.models.eagle import EagleDrafterFactory -from tensorrt_llm.llmapi import Eagle3DecodingConfig - -prompts = [ - "What is the capital of France?", - "Please explain the concept of gravity in simple words and a single sentence.", - "What are the main differences between Python and C++?", - "Summarize the plot of Romeo and Juliet in three sentences.", -] - -EAGLE_MODEL_SUBPATH = "EAGLE3-LLaMA3.1-Instruct-8B" -LLAMA_BASE_SUBPATH = "llama-3.1-model/Llama-3.1-8B-Instruct" -EAGLE_MAX_DRAFT_LEN = 3 - - -def get_model_paths(): - """Get model paths using llm_models_root().""" - models_root = llm_models_root() - base_model = os.path.join(models_root, LLAMA_BASE_SUBPATH) - eagle_model = os.path.join(models_root, EAGLE_MODEL_SUBPATH) - - print(f"Base model path: {base_model}") - print(f"EAGLE model path: {eagle_model}") - return base_model, eagle_model - - -@pytest.mark.parametrize( - ("attn_backend", "compile_backend"), - [ - ("trtllm", "torch-cudagraph"), - ("flashinfer", "torch-simple"), - ], -) -def test_autodeploy_eagle3_one_model_acceptance_rate(attn_backend: str, compile_backend: str): - """Test Eagle3 one-model acceptance rate with AutoDeploy engine. - - Runs Eagle3 one-model speculative decoding with streaming and verifies - that the acceptance rate is above a minimum threshold. - Parameterized over attention backend and compile backend. - """ - print("\n" + "=" * 80) - print( - f"Testing AutoDeploy Eagle3 One-Model Acceptance Rate " - f"(attn_backend={attn_backend}, compile_backend={compile_backend})" - ) - print("=" * 80) - - base_model, eagle_model = get_model_paths() - - print(f"\nBase Model: {base_model}") - print(f"Eagle3 Model: {eagle_model}") - - max_draft_len = EAGLE_MAX_DRAFT_LEN - - speculative_config = Eagle3DecodingConfig( - max_draft_len=max_draft_len, - speculative_model=eagle_model, - eagle3_layers_to_capture={1, 15, 28}, - ) - - with LLM( - model=base_model, - skip_loading_weights=False, - runtime="trtllm", - world_size=1, - speculative_config=speculative_config, - # Force the Eagle3 draft to match the target (Llama 3.1 8B is bfloat16). - # Shared KV cache requires matching dtypes between target and draft. - speculative_model_kwargs={"torch_dtype": "bfloat16"}, - compile_backend=compile_backend, - attn_backend=attn_backend, - transforms={"compile_model": {"piecewise_enabled": False}}, - max_num_tokens=512, - # max_batch_size must leave room for an extend-only sample batch during - # resize_kv_cache, i.e. max_num_tokens // max_batch_size >= 1 + max_draft_len. - # Otherwise the sample batch is classified as decode-only and the Eagle - # wrapper rejects it ("decode without drafting is not supported"). - # TODO: remove once resize_kv_cache is spec-aware. - # See: https://github.com/NVIDIA/TensorRT-LLM/issues/13348 - max_batch_size=128, - ) as llm: - _run_acceptance_rate_check(llm, max_draft_len) - - -def _run_acceptance_rate_check(llm, max_draft_len: int, min_acceptance_rate: float = 0.10): - """Common helper for acceptance rate tests. - - Submits all requests simultaneously so the executor processes them concurrently - (batch size > 1), then consumes streaming results to compute acceptance rates. - """ - batch_tok_ids = [llm.tokenizer.encode(p) for p in prompts] - sampling_params = SamplingParams(max_tokens=128, temperature=0, seed=42) - - print("\nRunning Eagle3 speculative decoding with streaming...") - print(f"Submitting all {len(batch_tok_ids)} requests simultaneously...") - - # Submit all requests before consuming any results so they are in-flight concurrently. - generators = [ - llm.generate_async(tok_ids, sampling_params, streaming=True) for tok_ids in batch_tok_ids - ] - - for i, gen in enumerate(generators): - num_tokens = 0 - num_drafted = 0 - num_accepted = 0 - - for output in gen: - new_tokens = output.outputs[0].token_ids - num_drafted += max_draft_len - num_accepted += len(new_tokens) - num_tokens - 1 - num_tokens = len(new_tokens) - - accept_rate = num_accepted / num_drafted - - generated_text = output.outputs[0].text - if not generated_text: - generated_text = llm.tokenizer.decode(output.outputs[0].token_ids) - print(f"\n[PROMPT {i}] {prompts[i]}") - print(f"[OUTPUT {i}] {generated_text}") - - print(f"\nRequest {i + 1} Acceptance Rate Statistics:") - print(f" Total tokens drafted: {num_drafted}") - print(f" Total tokens accepted: {num_accepted}") - print(f" Acceptance rate: {accept_rate:.2%}") - - assert accept_rate > min_acceptance_rate, ( - f"Request {i + 1}: Acceptance rate {accept_rate:.2%} is below minimum threshold " - f"{min_acceptance_rate:.0%}" - ) - - print("\n" + "=" * 80) - print("SUCCESS! All requests passed acceptance rate threshold") - print("=" * 80) - - -def load_weights(model_path: Path, model: torch.nn.Module): - """Load weights from checkpoint while applying the same _checkpoint_conversion_mapping that the factory uses. - - Returns: tuple of (loaded_keys, missing_keys, unexpected_keys) - """ - # 1. Load checkpoint keys - bin_path = model_path / "pytorch_model.bin" - safetensors_path = model_path / "model.safetensors" - - if safetensors_path.exists(): - from safetensors import safe_open - - with safe_open(safetensors_path, framework="pt") as f: - checkpoint_keys_original = list(f.keys()) - elif bin_path.exists(): - state_dict = torch.load(bin_path, map_location="cpu", weights_only=True) - checkpoint_keys_original = list(state_dict.keys()) - del state_dict - else: - raise FileNotFoundError(f"No checkpoint found at {model_path}") - - # 2. Apply _checkpoint_conversion_mapping (same logic as hf.py _remap_param_names_load_hook) - # This is the key part - the factory does this exact same thing in lines 496-512 of hf.py - conversion_mapping = getattr(model, "_checkpoint_conversion_mapping", None) - checkpoint_keys_remapped = [] - - for key in checkpoint_keys_original: - new_key = key - if conversion_mapping: - for pattern, replacement in conversion_mapping.items(): - new_key = re.sub(pattern, replacement, new_key) - checkpoint_keys_remapped.append(new_key) - - # 3. Get model's expected keys - model_keys = set(model.state_dict().keys()) - checkpoint_keys = set(checkpoint_keys_remapped) - - # 4. Calculate differences - loaded_keys = checkpoint_keys & model_keys - missing_in_checkpoint = model_keys - checkpoint_keys - unexpected_in_checkpoint = checkpoint_keys - model_keys - - return loaded_keys, missing_in_checkpoint, unexpected_in_checkpoint - - -def test_eagle_model_with_weights(): - """Test EagleModel forward pass with loaded weights using the EagleDrafterFactory. - - This test uses EagleDrafterFactory to initialize the model, which directly - builds the Eagle drafter model based on the checkpoint's model_type: - - 1. Factory creates config via AutoConfig.from_pretrained - 2. Factory selects EagleDrafterForCausalLM based on model_type="llama" - 3. Factory creates model via _from_config - 4. Factory loads weights via load_or_random_init -> _load_checkpoint - - This ensures the test validates the exact initialization path used in production. - """ - print("\n" + "=" * 80) - print("Test: EagleModel forward pass with loaded weights (via EagleDrafterFactory)") - print("=" * 80) - - _, eagle_model_path = get_model_paths() - eagle_path = Path(eagle_model_path) - - # 1. Setup Device - device = "cuda" if torch.cuda.is_available() else "cpu" - - # 2. Create factory - # EagleDrafterFactory directly builds the correct drafter model based on model_type - print("Creating EagleDrafterFactory...") - factory = EagleDrafterFactory( - model=eagle_model_path, - skip_loading_weights=False, # We want to test weight loading - ) - - # 3. Build model using factory - # Factory flow: - # build_model() -> prefetch_checkpoint() -> _build_model() - # _build_model() -> _get_model_config() (gets base LlamaConfig) - # _build_model() -> selects EagleDrafterForCausalLM for model_type="llama" - # _build_model() -> EagleDrafterForCausalLM._from_config(config) - print("Building model via factory.build_model('meta')...") - model = factory.build_model("meta") - print(f"Model type: {type(model).__name__}") - print(f"Model config type: {type(model.config).__name__}") - - # 4. Load weights from checkpoint and compare to model's expected keys - print("\n--- Weight Loading Analysis ---") - loaded_keys, missing_keys, unexpected_keys = load_weights(eagle_path, model) - - print(f"Total model parameters: {len(loaded_keys) + len(missing_keys)}") - print(f"Total checkpoint keys: {len(loaded_keys) + len(unexpected_keys)}") - print(f"✅ Weights to be loaded: {len(loaded_keys)}") - print(f"⚠️ Missing in checkpoint (will be random init): {len(missing_keys)}") - print(f"⚠️ Unexpected in checkpoint (will be ignored): {len(unexpected_keys)}") - - if unexpected_keys: - print("\nUnexpected keys (in checkpoint but model doesn't expect):") - for key in sorted(unexpected_keys): - if "t2d" in key: - print(f" - {key} (expected: not used in Eagle3 for Llama3.1-8B-Instruct)") - else: - print(f" - {key}") - - if loaded_keys: - print(f"\nLoaded keys ({len(loaded_keys)} total):") - for key in sorted(loaded_keys)[:20]: - print(f" - {key}") - if len(loaded_keys) > 20: - print(f" ... and {len(loaded_keys) - 20} more") - - print("--- End Weight Analysis ---\n") - - # Verify expected missing and unexpected keys - # These are the keys we expect based on Eagle3 architecture: - # - embed_tokens: shared from target model (not in Eagle checkpoint) - # - t2d: target-to-draft mapping, not used in Eagle3 (uses d2t instead) - expected_unexpected_keys = {"model.t2d"} - - assert len(missing_keys) == 0, ( - f"Expect all keys to be loaded.\nKeys that are missing: {missing_keys}\n" - ) - - assert unexpected_keys == expected_unexpected_keys, ( - f"Unexpected keys in checkpoint.\n" - f"Expected: {expected_unexpected_keys}\n" - f"Got: {unexpected_keys}\n" - f"Extra unexpected: {unexpected_keys - expected_unexpected_keys}\n" - f"Not unexpected (but expected): {expected_unexpected_keys - unexpected_keys}" - ) - - print("✅ Weight loading analysis matches expected missing/unexpected keys!") - - # 5. Load weights using factory (mimics actual pipeline) - # If tensor shapes do not match with how they are used in the forward() function, we will - # get an error. - print("Loading weights via factory.load_or_random_init()...") - factory.load_or_random_init(model, device) - print("Weights loaded successfully via factory interface!") - - model.eval() - - -############################################################################### -# Set up to test the prefill-only version of the EagleWrapper model in test_eagle_wrapper_forward(). -# This helps us guarantee that the EagleWrapper model, before it enters AutoDeploy, is working correctly, -# The test does not rely on any TRTLLM logic. -############################################################################### -class PrefillOnlyEagleResourceManager: - """Simple resource manager for Eagle speculative decoding (prefill-only variant). - - Stores hidden states for use by draft loop in EagleWrapper.forward(). - """ - - def __init__( - self, - hidden_size: int, - num_capture_layers: int, - max_batch_size: int, - max_seq_len: int, - max_draft_len: int, - target_dtype: torch.dtype, - ): - # Buffer for hidden states from target model: [max_tokens, hidden_size * num_capture_layers] - # Uses the same flattened 2D format as hidden_states_cache_* runtime buffers. - self.hidden_states = torch.empty( - max_batch_size * (max_seq_len + max_draft_len), - hidden_size * num_capture_layers, - device="cuda", - dtype=target_dtype, - ) - - -class LlamaModelWithCapture(LlamaModel): - """LlamaModel that captures un-normalized hidden states from specified layers. - - Overwrites the base model's forward method to capture hidden states from specified layers. - Base model's forward method is otherwise copied from LlamaModel in HuggingFace. - Takes PrefillOnlyEagleResourceManager as an argument to store captured hidden states. - """ - - def __init__( - self, - config, - layers_to_capture: Optional[Set[int]] = None, - resource_manager: Optional[PrefillOnlyEagleResourceManager] = None, - ): - super().__init__(config) - # layers_to_capture: set of layer indices (0-indexed) to capture - # If None, capture all layers - if layers_to_capture is None: - self.layers_to_capture = set(range(config.num_hidden_layers)) - else: - self.layers_to_capture = set(layers_to_capture) - - self.resource_manager = resource_manager - - # Validate layer indices - for idx in self.layers_to_capture: - if idx < 0 or idx >= config.num_hidden_layers: - raise ValueError( - f"Layer index {idx} out of range. " - f"Model has {config.num_hidden_layers} layers (0 to {config.num_hidden_layers - 1})" - ) - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs, - ) -> BaseModelOutputWithPast: - if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError("You must specify exactly one of input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - if cache_position is None: - # prefill only - no past key values. - cache_position = torch.arange(0, inputs_embeds.shape[1], device=inputs_embeds.device) - - if position_ids is None: - position_ids = cache_position.unsqueeze(0) - - causal_mask = create_causal_mask( - config=self.config, - input_embeds=inputs_embeds, - attention_mask=attention_mask, - cache_position=cache_position, - past_key_values=None, - position_ids=position_ids, - ) - - hidden_states = inputs_embeds - position_embeddings = self.rotary_emb(hidden_states, position_ids) - - # Buffer to collect captured hidden states - captured_hidden_states = [] - - for layer_idx, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]): - hidden_states = decoder_layer( - hidden_states, - attention_mask=causal_mask, - position_ids=position_ids, - cache_position=cache_position, - position_embeddings=position_embeddings, - **kwargs, - ) - - # Capture this layer's output if it's in our list - if layer_idx in self.layers_to_capture: - captured_hidden_states.append(hidden_states) - - # Apply final normalization for last_hidden_state - last_hidden_state = self.norm(hidden_states) - - # Store captured hidden states in resource manager if available - # Resource manager uses 2D flattened format: [max_tokens, hidden_size * num_capture_layers] - if self.resource_manager is not None and captured_hidden_states: - concatenated = torch.cat(captured_hidden_states, dim=-1) - batch_size, seq_len, total_hidden_size = concatenated.shape - assert self.resource_manager.hidden_states.shape[-1] == total_hidden_size, ( - f"Resource manager buffer last dim {self.resource_manager.hidden_states.shape[-1]} " - f"!= concatenated hidden states last dim {total_hidden_size}" - ) - # Flatten to [batch_size * seq_len, total_hidden_size] for 2D format - flattened = concatenated.view(batch_size * seq_len, total_hidden_size) - self.resource_manager.hidden_states[: (batch_size * seq_len), :].copy_(flattened) - - return BaseModelOutputWithPast( - last_hidden_state=last_hidden_state, - hidden_states=tuple(captured_hidden_states) if captured_hidden_states else None, - ) - - -@dataclass -class LlamaForCausalLMOutput(ModelOutput): - logits: Optional[torch.FloatTensor] = None - last_hidden_state: Optional[torch.FloatTensor] = None - hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None - - -class LlamaForCausalLMWithCapture(nn.Module): - """Wrapper combining LlamaModelWithCapture with lm_head for EagleWrapper testing. - - EagleWrapper.forward() expects target_model(input_ids, position_ids) to return logits. - This class wraps LlamaModelWithCapture (which captures hidden states to resource manager) - and adds the lm_head to produce logits. - """ - - def __init__(self, base_model, capture_model): - super().__init__() - self.model = capture_model # LlamaModelWithCapture with resource_manager - self.lm_head = base_model.lm_head - - def forward( - self, - input_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - **kwargs, - ): - outputs = self.model( - input_ids=input_ids, inputs_embeds=inputs_embeds, position_ids=position_ids, **kwargs - ) - logits = self.lm_head(outputs.last_hidden_state) - return LlamaForCausalLMOutput( - logits=logits, - last_hidden_state=outputs.last_hidden_state, - hidden_states=outputs.hidden_states, - ) - - def get_input_embeddings(self): - return self.model.embed_tokens - - def get_output_embeddings(self): - return self.model.lm_head - - @classmethod - def from_pretrained( - cls, - model_name: str, - resource_manager, - capture_layers, - dtype=torch.bfloat16, - ): - """Load a base model and create a LlamaForCausalLMWithCapture with shared weights.""" - print(f"Loading {model_name}...") - base_model = AutoModelForCausalLM.from_pretrained( - model_name, - torch_dtype=dtype, - device_map={"": 0}, - ) - base_model.eval() - - # Create LlamaModelWithCapture that shares weights with the base model - original_llama_model = base_model.model - - capture_model = LlamaModelWithCapture.__new__(LlamaModelWithCapture) - nn.Module.__init__(capture_model) - - capture_model.config = original_llama_model.config - capture_model.layers_to_capture = capture_layers - capture_model.resource_manager = resource_manager - - # Share all modules (no weight copying) - capture_model.embed_tokens = original_llama_model.embed_tokens - capture_model.layers = original_llama_model.layers - capture_model.norm = original_llama_model.norm - capture_model.rotary_emb = original_llama_model.rotary_emb - capture_model.gradient_checkpointing = original_llama_model.gradient_checkpointing - - return cls(base_model, capture_model) - - -def build_eagle_wrapper( - base_model_path: str, - eagle_model_path: str, - resource_manager: PrefillOnlyEagleResourceManager, - capture_layers: Set[int], - max_seq_len: int, - max_draft_len: int, - target_dtype: torch.dtype, - device: torch.device, -) -> tuple[EagleWrapper, nn.Module]: - """Build an EagleWrapper model for testing. - - This function encapsulates the model building logic using manual model building. - - Returns: - A tuple of (eagle_wrapper, target_model) where: - - eagle_wrapper: The EagleWrapper model ready for inference. - - target_model: The target model (for verification steps). - """ - # Build EagleWrapper manually. - print("\n" + "-" * 40) - print("Building EagleWrapper") - print("-" * 40) - - # Create target model with capture - target_model = LlamaForCausalLMWithCapture.from_pretrained( - base_model_path, resource_manager, capture_layers, target_dtype - ) - print("✓ Created target model with capture") - - # Create draft model using EagleDrafterFactory (mimics production pipeline) - # This ensures weights are loaded correctly via the same path as AutoDeploy - print("\nCreating draft model via EagleDrafterFactory...") - draft_factory = EagleDrafterFactory( - model=eagle_model_path, - skip_loading_weights=False, - ) - - # Build model on meta device first, then load weights - draft_model = draft_factory.build_model("meta") - print(f" Model type: {type(draft_model).__name__}") - - # Load weights via factory - print(" Loading weights via factory.load_or_random_init()...") - draft_factory.load_or_random_init(draft_model, device) - draft_model.eval() - - # Create EagleWrapper config - wrapper_config = EagleWrapperConfig( - max_draft_len=max_draft_len, - load_embedding_from_target=draft_model.load_embedding_from_target, - load_lm_head_from_target=draft_model.load_lm_head_from_target, - ) - - # Build EagleWrapper (this also loads weights from target into draft model where necessary) - eagle_wrapper = EagleWrapper( - config=wrapper_config, - target_model=target_model, - draft_model=draft_model, - resource_manager=resource_manager, - ) - eagle_wrapper.eval() - print("✓ Built EagleWrapper") - - return eagle_wrapper, target_model - - -def generate_target_outputs( - target_model: nn.Module, - input_ids: torch.Tensor, - num_iterations: int, -) -> torch.Tensor: - """Generate tokens from target model using greedy sampling. - - Runs target_model.forward() in a loop, taking the last logit from each output, - applying greedy sampling with torch.argmax, and appending to input_ids. - - Args: - target_model: Model that returns logits from forward(input_ids, position_ids). - input_ids: Initial input token ids of shape [batch_size, seq_len]. - num_iterations: Number of tokens to generate. - - Returns: - output_ids: Tensor of shape [batch_size, seq_len + num_iterations] containing - the original input_ids plus the generated tokens. - """ - device = input_ids.device - init_seq_len = input_ids.shape[1] - print(f"Initial sequence length: {init_seq_len}") - current_ids = input_ids.clone() - - with torch.no_grad(): - for _ in range(num_iterations): - # Generate position_ids from current sequence length - seq_len = current_ids.shape[1] - position_ids = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0) - position_ids = position_ids.expand(current_ids.shape[0], -1) - - # Forward pass - logits = target_model(current_ids, position_ids=position_ids).logits - - # Take the last logit and apply greedy sampling - last_logits = logits[:, -1, :] # [batch_size, vocab_size] - next_token = torch.argmax(last_logits, dim=-1, keepdim=True) # [batch_size, 1] - - # Append to current_ids - current_ids = torch.cat([current_ids, next_token], dim=1) - - return current_ids - - -def print_token_analysis( - input_ids: torch.Tensor, - num_previously_accepted: torch.Tensor, - target_output_ids: torch.Tensor, - tokenizer, -) -> None: - """Print debug analysis of accepted vs speculative tokens for each batch. - - Args: - input_ids: Current input token ids of shape [batch_size, seq_len]. - num_previously_accepted: Number of accepted tokens per batch [batch_size]. - target_output_ids: Reference output from target model [batch_size, total_seq_len]. - tokenizer: Tokenizer for decoding tokens to text. - """ - batch_size = input_ids.shape[0] - print("\n --- Token Analysis (per batch) ---") - - for i in range(batch_size): - prev_accepted_i = num_previously_accepted[i].item() - - # Accepted tokens (before speculation): input_ids[i, :num_previously_accepted[i]] - accepted_tokens = input_ids[i, :prev_accepted_i] - # Speculative tokens: input_ids[i, num_previously_accepted[i]:] - speculative_tokens = input_ids[i, prev_accepted_i:] - - # Target model's expected token at this position - if prev_accepted_i < target_output_ids.shape[1]: - target_token_at_pos = target_output_ids[i, prev_accepted_i] - else: - target_token_at_pos = None - - print(f"\n Batch {i}:") - print(f" num_previously_accepted: {prev_accepted_i}") - print( - f" Accepted tokens ({accepted_tokens.shape[0]} tokens): {accepted_tokens.tolist()}" - ) - accepted_text = tokenizer.decode(accepted_tokens, skip_special_tokens=True) - print(f' Accepted text: "{accepted_text}"') - print( - f" Speculative tokens ({speculative_tokens.shape[0]} tokens): {speculative_tokens.tolist()}" - ) - if speculative_tokens.shape[0] > 0: - spec_text = tokenizer.decode(speculative_tokens, skip_special_tokens=False) - print(f' Speculative text: "{spec_text}"') - if target_token_at_pos is not None: - target_tok_id = target_token_at_pos.item() - target_tok_str = tokenizer.decode([target_tok_id]) - print( - f' Target model\'s next token at pos {prev_accepted_i}: {target_tok_id} ("{target_tok_str}")' - ) - - -def manual_sample_and_verify( - next_target_inputs: list, - num_accepted_tokens: torch.Tensor, - target_model: nn.Module, - eagle_wrapper: nn.Module, - max_draft_len: int, - device: torch.device, -) -> list: - """Manually verify speculative tokens using sample_and_verify. - - This is used for batch_size > 1 where truncation prevents speculative tokens - from being fed back, so we verify them manually before truncation. - - Args: - next_target_inputs: List of tensors, one per batch element. - num_accepted_tokens: Number of tokens accepted so far per batch [batch_size]. - target_model: The target model for running forward pass. - eagle_wrapper: The EagleWrapper containing sample_and_verify. - max_draft_len: Maximum draft length (for capping counts). - device: Device to run on. - - Returns: - List of (num_accepted, num_speculative) tuples for each batch element. - """ - batch_size = len(next_target_inputs) - - # Due to our truncation trick, all sequences should have the same length - seq_lens = [seq.shape[0] for seq in next_target_inputs] - assert all(slen == seq_lens[0] for slen in seq_lens), ( - f"All sequences should have same length due to truncation, got {seq_lens}" - ) - verify_seq_len = seq_lens[0] - - # Stack into batched tensor - stacked_inputs = torch.stack(next_target_inputs, dim=0) # [batch_size, seq_len] - - # Run target model forward to get logits - verify_position_ids = ( - torch.arange(verify_seq_len, device=device, dtype=torch.long) - .unsqueeze(0) - .expand(batch_size, -1) - ) - with torch.no_grad(): - verify_target_logits = target_model(stacked_inputs, position_ids=verify_position_ids).logits - - # new_num_previously_accepted = num_accepted_tokens + 1 - # This represents the tokens accepted after target model's output from this iteration - new_num_previously_accepted = num_accepted_tokens + 1 - - # Call sample_and_verify to get acceptance counts - _, verify_newly_accepted, _, _ = eagle_wrapper.sample_and_verify( - stacked_inputs, verify_target_logits, new_num_previously_accepted - ) - - # Build results list - results = [] - for i in range(batch_size): - num_accepted_i = min(verify_newly_accepted[i].item(), max_draft_len) - num_speculative = next_target_inputs[i].shape[0] - new_num_previously_accepted[i].item() - results.append((num_accepted_i, num_speculative)) - - return results - - -def verify_eagle_wrapper_output(output, tokenizer, batch_size, num_previously_accepted): - """Verify the output structure and values from EagleWrapper forward pass. - - Args: - output: The output from EagleWrapper forward pass. - tokenizer: The tokenizer for decoding tokens. - batch_size: The batch size. - num_previously_accepted: Tensor of previously accepted token counts. - """ - # Verify output structure - print("\nOutput verification:") - assert output is not None, "Output should not be None" - assert hasattr(output, "new_tokens"), "Output should have new_tokens" - assert hasattr(output, "new_tokens_lens"), "Output should have new_tokens_lens" - - print(f" new_tokens: {type(output.new_tokens)} with {len(output.new_tokens)} items") - for i, tokens in enumerate(output.new_tokens): - new_tokens_text = tokenizer.decode(tokens, skip_special_tokens=True) - print(f" batch {i}: shape {tokens.shape}, tokens: {tokens.tolist()}") - print(f' batch {i}: decoded: "{new_tokens_text}"') - - # Compute num_accepted_tokens from new_tokens_lens + num_previously_accepted - num_accepted_tokens = num_previously_accepted + output.new_tokens_lens - - print(f" new_tokens_lens: {output.new_tokens_lens}") - print(f" num_accepted_tokens (computed): {num_accepted_tokens}") - - # Verify new_tokens_lens is within expected bounds - assert output.new_tokens_lens.shape == (batch_size,), ( - f"new_tokens_lens shape should be ({batch_size},), got {output.new_tokens_lens.shape}" - ) - - -@pytest.mark.skip( - reason="EagleWrapper interface was refactored (resource_manager removed from __init__, " - "sample_and_verify removed); test needs to be updated to match the new interface. " - "This test is valuable for validating Eagle3 correctness (acceptance ratio) directly " - "on the EagleWrapper model *before* the full export + transforms + KV-cache pipeline, " - "making it much easier to debug Eagle3 model issues in isolation. TODO: rewrite to " - "match the current EagleWrapper prefill-only and KV-cache forward interfaces." -) -@pytest.mark.parametrize("batch_size", [1, 2]) -def test_eagle_wrapper_forward(batch_size: int): - """Test EagleWrapper forward pass with target and draft models. - - This test validates the full speculative decoding loop: - 1. Target model processes input and captures hidden states - 2. Draft model generates speculative tokens - 3. EagleWrapper orchestrates verification and drafting - - For batch size 1, we call EagleWrapper forward in the expected way. Each iteration generates a "golden token" - (target output) and draft tokens. We input all of them to the wrapper model, - which verifies the draft tokens against the target output. It then outputs the accepted tokens - and newly generated draft tokens, along with numbers of accepted tokens, and the process repeats. - - For batch size > 1, we need to work around the fact that as we run the loop described above, the sequences lengths - in the batch will get out of sync. So instead, we do not provide validated draft tokens as input in each iteration - - we just input the first accepted token from the previous iteration - (which we know was generated by the target model), which keeps the batches in sync. - - To verify that the output draft tokens are reasonable, we run a manual target model verification step - after each iteration. We record how many of the output draft tokens were accepted. - - In the end, we test that the acceptance ratio of the draft tokens generated by the EagleWrapper is reasonable. - - Args: - batch_size: Number of prompts to process in parallel. - """ - print("\n" + "=" * 80) - print("Test: EagleWrapper forward pass") - print("=" * 80) - - # Set random seeds for reproducibility - torch.manual_seed(42) - - # Get model paths using integration test conventions - base_model_path, eagle_model_path = get_model_paths() - eagle_path = Path(eagle_model_path) - - # Configuration - capture_layers = {1, 15, 28} # Layers to capture for Eagle3 - num_capture_layers = len(capture_layers) - hidden_size = 4096 # Llama 3.1-8B hidden size - dtype = torch.bfloat16 - device = torch.device("cuda") - - # Test dimensions - max_batch_size = 4 - max_seq_len = 1024 - max_draft_len = 3 - - # Tokenize the test prompts - tokenizer = AutoTokenizer.from_pretrained(base_model_path) - # Llama uses left padding for batch inference - tokenizer.pad_token = tokenizer.eos_token - tokenizer.padding_side = "left" - - if batch_size == 1: - input_ids = tokenizer.encode(prompts[0], return_tensors="pt").to(device) - else: - tokenized = tokenizer( - prompts[:batch_size], - return_tensors="pt", - padding=True, - ) - input_ids = tokenized.input_ids.to(device) - - print(f"input_ids: {input_ids}") - seq_len = input_ids.shape[1] - init_seq_len = seq_len # Store initial sequence length for final comparison - - print("\nTest configuration:") - print(f" target_model: {base_model_path}") - print(f" eagle_model: {eagle_path}") - print(f" batch_size: {batch_size}, seq_len: {seq_len}") - print(f" max_draft_len: {max_draft_len}") - print(f" capture_layers: {capture_layers}") - print(f" prompts: {prompts[:batch_size]}") - print(f" input_ids: {input_ids}") - - # Create resource manager - resource_manager = PrefillOnlyEagleResourceManager( - hidden_size=hidden_size, - num_capture_layers=num_capture_layers, - max_batch_size=max_batch_size, - max_seq_len=max_seq_len, - max_draft_len=max_draft_len, - target_dtype=dtype, - ) - print("\n✓ Created resource manager") - print(f" target_hidden_states shape: {resource_manager.hidden_states.shape}") - - # Build eagle_wrapper and target_model using the refactored function - eagle_wrapper, target_model = build_eagle_wrapper( - base_model_path=base_model_path, - eagle_model_path=str(eagle_path), - resource_manager=resource_manager, - capture_layers=capture_layers, - max_seq_len=max_seq_len, - max_draft_len=max_draft_len, - target_dtype=dtype, - device=device, - ) - - # Create test inputs (input_ids already created from tokenizer above) - position_ids = ( - torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand(batch_size, -1) - ) - # Set previously_accepted_tokens to the length of input_ids (all context tokens are accepted) - # Shape should be [batch_size] - a 1D tensor with one value per batch - num_previously_accepted = torch.full((batch_size,), seq_len, device=device, dtype=torch.long) - - print("\nTest inputs:") - print(f" input_ids shape: {input_ids.shape}") - print(f" input_ids: {input_ids}") - print(f" position_ids shape: {position_ids.shape}") - print(f" num_previously_accepted: {num_previously_accepted}") - - # Generate target model outputs with greedy sampling - print("\nGenerating target model outputs with greedy sampling (for verification)...") - target_output_ids = generate_target_outputs(target_model, input_ids, num_iterations=100) - print(f" target_output_ids shape: {target_output_ids.shape}") - print(f" target_output_ids: {target_output_ids}") - - # Decode to text as sanity check - generated_text = tokenizer.decode(target_output_ids[0], skip_special_tokens=True) - print(f"\n Target model greedy generation decoded text:\n {generated_text}") - - print("\n✓ EagleWrapper forward pass completed successfully!") - print("✓ Output structure verified") - print("✓ new_tokens_lens within expected bounds") - print("✓ Target model greedy generation completed") - - print("\n================================================") - - num_iterations = 70 - - # Dictionary to track distribution of new_tokens_lens - # keys: 0 to max_draft_len - # newly_accepted_counts[i]: number of times the number of accepted draft tokens was i - newly_accepted_counts = {i: 0 for i in range(max_draft_len + 1)} - - for iteration in range(num_iterations): - print(f"\n{'=' * 40}") - print(f"EagleWrapper forward pass - Iteration {iteration + 1}/{num_iterations}") - print(f"{'=' * 40}") - - seq_len = input_ids.shape[1] - - # Debug: Print speculative tokens, accepted tokens, and target comparison - print_token_analysis(input_ids, num_previously_accepted, target_output_ids, tokenizer) - - kwargs = { - "num_previously_accepted": num_previously_accepted, - } - with torch.no_grad(): - output = eagle_wrapper( - input_ids=input_ids, - position_ids=position_ids, - **kwargs, - ) - - verify_eagle_wrapper_output(output, tokenizer, batch_size, num_previously_accepted) - - # Prepare next_target_inputs - # output.new_tokens[i] contains the full draft_input_ids tensor, but the valid prefix - # has length num_accepted_tokens[i] + max_draft_len. We slice to get only valid tokens. - # We then prepend the first token from the previous iteration's input_ids. - # This prepending is only needed for prefill-only mode, since in the cached case, the first token - # will always be in the KV cache. - # Compute num_accepted_tokens from num_previously_accepted + new_tokens_lens - num_accepted_tokens = num_previously_accepted + output.new_tokens_lens - valid_prefix_len = num_accepted_tokens + max_draft_len - next_target_inputs = [ - torch.cat( - (input_ids[i, 0].unsqueeze(0), output.new_tokens[i][: valid_prefix_len[i]]), - dim=0, - ) - for i in range(batch_size) - ] - - # Track distribution of newly accepted tokens by reading new_tokens_lens from the output. - # For batch size = 1, we are inputting draft tokens to the wrapper model, so new_tokens_lens - # gives the number of accepted tokens from drafts in the previous iteration. - if batch_size == 1: - for val in output.new_tokens_lens.tolist(): - newly_accepted_counts[val] += 1 - print(f" newly_accepted_counts so far: {newly_accepted_counts}") - - # For batch_size > 1, we use manual target model verification below instead to check which of the draft tokens - # generated in *this* iteration would be accepted by the target model. - else: - # For batch_size > 1, verify acceptance using sample_and_verify() - # before truncation (since truncation prevents speculative tokens from being fed back) - verify_results = manual_sample_and_verify( - next_target_inputs, - num_accepted_tokens, - target_model, - eagle_wrapper, - max_draft_len, - device, - ) - - # Update newly_accepted_counts map - for i, (num_accepted_i, num_speculative) in enumerate(verify_results): - newly_accepted_counts[num_accepted_i] += 1 - print( - f" [Batch {i}] sample_and_verify: {num_accepted_i}/{num_speculative} speculative accepted" - ) - - # Truncate to keep shapes consistent across batches in each iteration. - # We know that the first token that is generated in this iteration is accepted, so it is "safe". - # All speculative tokens are truncated regardless of whether they are accepted or not. - # This is a hack to prevent the sequence lengths from getting out of sync across batches in each iteration - # without needing to change the padding every iteration. - truncate_len = input_ids.shape[1] + 1 - next_target_inputs = [seq[:truncate_len] for seq in next_target_inputs] - - next_target_inputs = torch.stack(next_target_inputs, dim=0) - - print(f" next_target_inputs: {next_target_inputs}") - print(f" next_target_inputs.shape: {next_target_inputs.shape}") - - # Update for next iteration - input_ids = next_target_inputs - seq_len = input_ids.shape[1] - position_ids = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0) - position_ids = position_ids.expand(batch_size, -1) - - if batch_size > 1: - # For multi-batch: increment by 1 (we truncated, so just advance by one token) - num_previously_accepted = num_previously_accepted + 1 - else: - # For single batch: accept the tokens accepted in the previous iteration, plus one - # for the output token that was generated by the target. - num_previously_accepted = num_accepted_tokens + 1 - - print(f"\n{'=' * 40}") - print(f"Loop completed: {num_iterations} iterations") - print("Newly accepted tokens distribution:") - for k, v in newly_accepted_counts.items(): - print(f" {k}: {v}") - - # Calculate acceptance ratio - # For batch_size == 1: uses new_tokens_lens from eagle wrapper - # For batch_size > 1: uses manual verification against target model (since truncation - # prevents speculative tokens from being fed back) - total_accepted = sum(k * v for k, v in newly_accepted_counts.items()) - # First iteration has no tokens to newly accept, subsequent iterations have max_draft_len potential - - num_iterations_with_drafts = num_iterations - 1 if batch_size == 1 else num_iterations - total_potential = max_draft_len * (num_iterations_with_drafts) * batch_size - acceptance_ratio = total_accepted / total_potential if total_potential > 0 else 0.0 - print(f"\nAcceptance ratio: {total_accepted}/{total_potential} = {acceptance_ratio:.3f}") - if batch_size > 1: - print(" (batch_size > 1: measured via manual target model verification)") - assert acceptance_ratio > 0.1, ( - f"Acceptance ratio {acceptance_ratio:.3f} is too low (expected > 0.1)" - ) - - print("\n" + "=" * 80) - print("FINAL OUTPUT COMPARISON") - print("=" * 80) - for i in range(batch_size): - print(f"\n{'─' * 40}") - print(f"BATCH {i}") - print(f"{'─' * 40}") - print(f"\n[Target Model Output] ({target_output_ids[i].shape[0]} tokens):") - print(f" Tokens: {target_output_ids[i].tolist()}") - print(f' Text: "{tokenizer.decode(target_output_ids[i], skip_special_tokens=True)}"') - print(f"\n[Eagle Wrapper Output] ({input_ids[i].shape[0]} tokens):") - print(f" Tokens: {input_ids[i].tolist()}") - print(f' Text: "{tokenizer.decode(input_ids[i], skip_special_tokens=True)}"') - print("\n" + "=" * 80) - - # Verify that the first 10 generated tokens match between target model and eagle wrapper - # They seem to diverge after awhile but are semantically the same. - # Note that even running the target model in decode vs prefill mode, the outputs seem to diverge similarly, - # so this is not worrisome. This test provides a check that they are "similar enough" to each other. - num_tokens_to_check = 10 - print(f"\nVerifying first {num_tokens_to_check} generated tokens match...") - for i in range(batch_size): - target_generated = target_output_ids[i, init_seq_len : init_seq_len + num_tokens_to_check] - eagle_generated = input_ids[i, init_seq_len : init_seq_len + num_tokens_to_check] - - print(f" Batch {i}:") - print(f" Target: {target_generated.tolist()}") - print(f" Eagle: {eagle_generated.tolist()}") - - assert torch.equal(target_generated, eagle_generated), ( - f"Batch {i}: First {num_tokens_to_check} generated tokens do not match!\n" - f" Target: {target_generated.tolist()}\n" - f" Eagle: {eagle_generated.tolist()}" - ) - print(f"✓ First {num_tokens_to_check} generated tokens match for all batches!") def _load_valid_safetensors_index(index_path: Path): diff --git a/tests/integration/defs/llmapi/test_llm_examples.py b/tests/integration/defs/llmapi/test_llm_examples.py index 36f12ccd1905..b30d022a1893 100644 --- a/tests/integration/defs/llmapi/test_llm_examples.py +++ b/tests/integration/defs/llmapi/test_llm_examples.py @@ -61,9 +61,6 @@ def _run_llmapi_example(llm_root, engine_dir, llm_venv, script_name: str, # medusa-vicuna-7b-v1.3 f"{llm_models_root()}/medusa-vicuna-7b-v1.3": f"{llm_venv.get_working_directory()}/FasterDecoding/medusa-vicuna-7b-v1.3", - # llama3.1-medusa-8b-hf_v0.1 - f"{llm_models_root()}/llama3.1-medusa-8b-hf_v0.1": - f"{llm_venv.get_working_directory()}/nvidia/Llama-3.1-8B-Medusa-FP8", # Llama-3.1-8B-Instruct f"{llm_models_root()}/llama-3.1-model/Llama-3.1-8B-Instruct": f"{llm_venv.get_working_directory()}/meta-llama/Llama-3.1-8B-Instruct", diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index de9e62ad6e46..6af288ffcac4 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -23,10 +23,10 @@ import pytest import yaml -from defs.trt_test_alternative import (check_call, check_call_negative_test, - check_output, print_info, print_warning) +from defs.trt_test_alternative import (check_call, check_output, print_info, + print_warning) -from .common import get_mmlu_accuracy, venv_check_call +from .common import get_mmlu_accuracy from .conftest import (get_sm_version, llm_models_root, skip_post_blackwell, skip_pre_ada, skip_pre_blackwell, skip_pre_hopper, unittest_path) @@ -383,178 +383,6 @@ def temp_extra_llm_api_options_file(request): yield None -@pytest.mark.parametrize( - "model_name, llama_model_root, use_extra_config, pytorch_backend_config", - [('meta-llama/Llama-3.1-8B', 'llama-3.1-8b', False, False), - pytest.param('meta-llama/Llama-3.1-8B', - 'llama-3.1-8b-instruct-hf-fp8', - True, - False, - marks=skip_pre_hopper), - pytest.param('meta-llama/Llama-3.1-8B', - 'llama-3.1-8b-instruct-hf-fp8', - True, - True, - marks=skip_pre_hopper), - pytest.param('meta-llama/Llama-3.1-8B', - 'llama-3.1-8b-hf-nvfp4', - False, - False, - marks=skip_pre_blackwell)], - indirect=['llama_model_root']) -def test_trtllm_bench_pytorch_backend_sanity(llm_root, llm_venv, - llama_model_root, model_name, - use_extra_config, - pytorch_backend_config, - temp_extra_llm_api_options_file): - """Sanity check on latency benchmark for LLM API with PyTorch backend - """ - model_path, dataset_path = trtllm_bench_prolog(llm_root, llm_venv, - llama_model_root, model_name, - False, False) - - benchmark_cmd = \ - f"trtllm-bench --model {model_name} --model_path {model_path} " \ - f"throughput " \ - f"--dataset {dataset_path} --backend pytorch" - - mapping = { - "Meta-Llama-3.1-8B": 19.4, - "Llama-3.1-8B-Instruct-FP8": 12.0, - "Meta-Llama-3.1-8B-NVFP4": 10.2 - } - if use_extra_config: - benchmark_cmd += f" --config {temp_extra_llm_api_options_file}" - - model_id = llama_model_root.split(r"/")[-1] - if "nvfp4-quantized" in llama_model_root: - model_id += "-NVFP4" - - check_call(benchmark_cmd, shell=True) - - -def test_trtllm_bench_mgmn(llm_root, llm_venv): - model_name = "meta-llama/Llama-3.1-8B" - llama_model_dir = Path( - llm_models_root()) / "llama-3.1-model/Llama-3.1-8B-Instruct" - _, dataset_path = trtllm_bench_prolog(llm_root, - llm_venv, - model_subdir=llama_model_dir, - model_name=model_name, - quant=None, - streaming=False) - - benchmark_cmd = \ - f"mpirun --allow-run-as-root -n 2 trtllm-llmapi-launch trtllm-bench --model {model_name} " \ - f"--model_path {llama_model_dir} " \ - f"throughput " \ - f"--dataset {str(dataset_path)} --backend pytorch --tp 2" - - check_call(benchmark_cmd, shell=True, env=llm_venv._new_env) - - -@pytest.mark.parametrize( - "model_name", - [ - "meta-llama/Llama-3.1-8B", - ], -) -def test_trtllm_bench_help_sanity(model_name): - """Sanity check that the options are defined properly by printing out help - """ - check_call("trtllm-bench --help", shell=True) - check_call(f"trtllm-bench --model {model_name} throughput --help", - shell=True) - check_call(f"trtllm-bench --model {model_name} latency --help", shell=True) - - -@pytest.mark.parametrize("request_rate", [False, True], - ids=["", "enable_request_rate"]) -@pytest.mark.parametrize("concurrency", [False, True], - ids=["", "enable_concurrency"]) -def test_trtllm_bench_request_rate_and_concurrency(llm_root, llm_venv, - request_rate, concurrency): - """Sanity check on the trtllm-bench new request rate and concurrency API - """ - model_subdir = "llama-3.1-model/Meta-Llama-3.1-8B" - model_name = "meta-llama/Llama-3.1-8B" - - model_path, dataset_path = trtllm_bench_prolog(llm_root, - llm_venv, - model_subdir, - model_name, - quant=None, - streaming=False) - - benchmark_cmd = \ - f"trtllm-bench --model {model_name} --model_path {model_path} throughput " \ - f"--dataset {dataset_path} --backend pytorch" - - if request_rate: - benchmark_cmd += " --request_rate 100" - if concurrency: - benchmark_cmd += " --concurrency 100" - - print(f"cmd: {benchmark_cmd}") - - if request_rate and concurrency: - # negative test, request rate and concurrency should not be turned on at the same time - check_call_negative_test(benchmark_cmd, shell=True) - else: - check_call(benchmark_cmd, shell=True) - - -@pytest.mark.parametrize("model_subdir", [ - "llama-3.1-model/Meta-Llama-3.1-8B", -], - ids=lambda x: x.strip("-")) -@pytest.mark.parametrize( - "model_name", - [ - "meta-llama/Llama-3.1-8B", - ], -) -@pytest.mark.parametrize("streaming", [True, False], - ids=["non-streaming", "streaming"]) -@pytest.mark.parametrize("backend", ["pytorch"], ids=["PyTorch"]) -def test_trtllm_bench_iteration_log(llm_root, llm_venv, model_name, - model_subdir, streaming, backend): - """Test the iteration log functionality with necessary options - """ - iteration_log = None - - try: - iteration_log = tempfile.mkstemp(dir="/tmp", suffix=".txt")[1] - - model_path, dataset_path = trtllm_bench_prolog(llm_root, - llm_venv, - model_subdir, - model_name, - quant=None, - streaming=streaming) - - benchmark_cmd = \ - f"trtllm-bench --model {model_name} --model_path {model_path} " \ - f"throughput --dataset {dataset_path} --iteration_log {iteration_log}" - - if streaming: - benchmark_cmd += " --streaming" - - benchmark_cmd += f" --backend {backend}" - - check_call(benchmark_cmd, shell=True) - - assert os.path.exists( - iteration_log - ), f"Iteration log file {iteration_log} was not created." - if os.path.getsize(iteration_log) == 0: - raise AssertionError( - f"Iteration log file {iteration_log} is empty.") - finally: - if iteration_log: - shutil.rmtree(iteration_log, ignore_errors=True) - - def test_trtllm_serve_example(llm_root, llm_venv): example_root = Path(os.path.join(llm_root, "examples", "serve")) test_root = unittest_path() / "llmapi" / "apps" @@ -791,10 +619,8 @@ def test_openai_mmencoder_example(llm_root, llm_venv): str(test_root / "_test_openai_mmencoder.py")]) -@pytest.mark.parametrize("model_name", [ - "meta-llama/Llama-3.1-8B-Instruct", - pytest.param("openai/gpt-oss-120b", marks=skip_pre_hopper) -]) +@pytest.mark.parametrize( + "model_name", [pytest.param("openai/gpt-oss-120b", marks=skip_pre_hopper)]) def test_openai_chat_guided_decoding(llm_root, llm_venv, model_name: str): test_root = unittest_path() / "llmapi" / "apps" llm_venv.run_cmd([ @@ -805,10 +631,8 @@ def test_openai_chat_guided_decoding(llm_root, llm_venv, model_name: str): @pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize("model_name", [ - "llama-3.1-model/Meta-Llama-3.1-8B", - pytest.param("gpt_oss/gpt-oss-20b", marks=skip_pre_hopper) -]) +@pytest.mark.parametrize( + "model_name", [pytest.param("gpt_oss/gpt-oss-20b", marks=skip_pre_hopper)]) def test_trtllm_benchmark_serving(llm_venv, model_name): test_root = unittest_path() / "llmapi" / "apps" llm_venv.run_cmd([ @@ -828,8 +652,9 @@ def test_trtllm_multimodal_benchmark_serving(llm_root, llm_venv): ]) +@skip_pre_hopper @pytest.mark.skip_less_device(4) -@pytest.mark.skip_less_device_memory(40000) +@pytest.mark.skip_less_device_memory(80000) @pytest.mark.parametrize("service_discovery", ["etcd"]) def test_openai_disagg_multi_nodes_completion_service_discovery( llm_root, llm_venv, service_discovery): @@ -843,8 +668,9 @@ def test_openai_disagg_multi_nodes_completion_service_discovery( ]) +@skip_pre_hopper @pytest.mark.skip_less_device(4) -@pytest.mark.skip_less_device_memory(40000) +@pytest.mark.skip_less_device_memory(80000) @pytest.mark.parametrize("gen_config", ["gen_tp2pp1", "gen_tp1pp2", "gen_tp1pp1"]) @pytest.mark.parametrize("ctx_config", @@ -880,27 +706,9 @@ def parse_output(text): return results -def test_ptp_quickstart(llm_root, llm_venv): - example_root = Path(os.path.join(llm_root, "examples", "llm-api")) - - src = f"{llm_models_root()}/llama-3.1-model/Llama-3.1-8B-Instruct" - dst = f"{llm_venv.get_working_directory()}/meta-llama/Llama-3.1-8B-Instruct" - os.makedirs(os.path.dirname(dst), exist_ok=True) - os.symlink(src, dst, target_is_directory=True) - - venv_check_call(llm_venv, [str(example_root / "quickstart_example.py")]) - - @pytest.mark.parametrize("model_name,model_path", [ - ("Llama3.1-8B-BF16", "llama-3.1-model/Meta-Llama-3.1-8B"), ("Llama3.2-11B-BF16", "llama-3.2-models/Llama-3.2-11B-Vision"), ("Nemotron4_4B-BF16", "nemotron/Minitron-4B-Base"), - pytest.param('Llama3.1-8B-NVFP4', - 'nvfp4-quantized/Meta-Llama-3.1-8B', - marks=skip_pre_blackwell), - pytest.param('Llama3.1-8B-FP8', - 'llama-3.1-model/Llama-3.1-8B-Instruct-FP8', - marks=skip_pre_hopper), pytest.param('Qwen3-30B-A3B', 'Qwen3/Qwen3-30B-A3B', marks=pytest.mark.skip_less_device_memory(80000)), @@ -916,10 +724,6 @@ def test_ptp_quickstart(llm_root, llm_venv): marks=skip_pre_blackwell), pytest.param( 'GPT-OSS-120B', 'gpt_oss/gpt-oss-120b', marks=skip_pre_blackwell), - ("Llama3.1-8B-bf16-instruct", "llama-3.1-model/Llama-3.1-8B-Instruct"), - pytest.param('Llama3.1-8B-FP4', - 'modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4', - marks=skip_pre_blackwell), pytest.param( 'Qwen3-8b-fp8', 'Qwen3/nvidia-Qwen3-8B-FP8', marks=skip_pre_hopper), pytest.param('Qwen3-8b-nvfp4', @@ -958,11 +762,8 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): ]) else: mapping = { - "Llama3.1-8B-BF16": 18.60, "Llama3.2-11B-BF16": 18.88, "Nemotron4_4B-BF16": 12.50, - "Llama3.1-8B-FP8": 13.05, - "Llama3.1-8B-NVFP4": 10.2 } cmds = [ str(example_root / "quickstart_advanced.py"), @@ -1042,8 +843,6 @@ def test_ptp_quickstart_advanced_bs1(llm_root, llm_venv): @pytest.mark.parametrize("model_name,model_path,eagle_model_path", [ - ("Llama-3.1-8b-Instruct", "llama-3.1-model/Llama-3.1-8B-Instruct", - "EAGLE3-LLaMA3.1-Instruct-8B"), pytest.param('GPT-OSS-120B-Eagle3', 'gpt_oss/gpt-oss-120b', 'gpt_oss/gpt-oss-120b-Eagle3', @@ -1069,88 +868,6 @@ def test_ptp_quickstart_advanced_eagle3(llm_root, llm_venv, model_name, ]) -@pytest.mark.parametrize("model_name,model_path,eagle_model_path", [ - ("Llama-3.1-8b-Instruct", "llama-3.1-model/Llama-3.1-8B-Instruct", - "EAGLE3-LLaMA3.1-Instruct-8B"), -]) -def test_draft_token_tree_quickstart_advanced_eagle3(llm_root, llm_venv, - model_name, model_path, - eagle_model_path): - print(f"Testing {model_name}.") - example_root = Path(os.path.join(llm_root, "examples", "llm-api")) - llm_venv.run_cmd([ - str(example_root / "quickstart_advanced.py"), - "--prompt", - "You are a good assistant. Please tell me the capital of France is", - "--spec_decode_max_draft_len", - "3", - "--spec_decode_algo", - "eagle3", - "--model_dir", - f"{llm_models_root()}/{model_path}", - "--draft_model_dir", - f"{llm_models_root()}/{eagle_model_path}", - "--disable_kv_cache_reuse", - "--disable_overlap_scheduler", - "--eagle_choices", - "[[0], [1], [2], [0, 0], [0, 1], [0, 2], [1, 0], [1, 1], [2, 0], [0, 0, 0], [0, 1, 0], [1, 0, 0]]", - "--kv_cache_fraction", - "0.4", - ]) - - -@pytest.mark.parametrize("model_name,model_path,eagle_model_path", [ - ("Llama-3.1-8b-Instruct", "llama-3.1-model/Llama-3.1-8B-Instruct", - "EAGLE3-LLaMA3.1-Instruct-8B"), -]) -def test_draft_token_tree_quickstart_advanced_eagle3_depth_1_tree( - llm_root, llm_venv, model_name, model_path, eagle_model_path): - print(f"Testing {model_name}.") - example_root = Path(os.path.join(llm_root, "examples", "llm-api")) - llm_venv.run_cmd([ - str(example_root / "quickstart_advanced.py"), - "--prompt", - "You are a good assistant. Please tell me the capital of France is", - "--spec_decode_max_draft_len", - "3", - "--spec_decode_algo", - "eagle3", - "--model_dir", - f"{llm_models_root()}/{model_path}", - "--draft_model_dir", - f"{llm_models_root()}/{eagle_model_path}", - "--disable_kv_cache_reuse", - "--disable_overlap_scheduler", - "--eagle_choices", - "[[0], [1], [2]]", - "--kv_cache_fraction", - "0.4", - ]) - - -@pytest.mark.parametrize("model_name,model_path", [ - ("Llama-3.1-8B-Instruct", "llama-3.1-model/Llama-3.1-8B-Instruct"), -]) -def test_ptp_quickstart_advanced_ngram(llm_root, llm_venv, model_name, - model_path): - print(f"Testing {model_name}.") - example_root = Path(os.path.join(llm_root, "examples", "llm-api")) - llm_venv.run_cmd([ - str(example_root / "quickstart_advanced.py"), - "--model_dir", - f"{llm_models_root()}/{model_path}", - "--spec_decode_algo", - "NGRAM", - "--spec_decode_max_draft_len", - "4", - "--max_matching_ngram_size", - "2", - "--use_cuda_graph", - "--disable_kv_cache_reuse", - "--disable_overlap_scheduler", - ]) - - @skip_post_blackwell @pytest.mark.skip_less_device_memory(80000) @pytest.mark.skip_less_device(4) @@ -1390,23 +1107,6 @@ def test_ptp_quickstart_advanced_8gpus_chunked_prefill_sq_22k( llm_venv.run_cmd(cmd) -@skip_pre_blackwell -def test_ptp_quickstart_advanced_mixed_precision(llm_root, llm_venv): - example_root = Path(os.path.join(llm_root, "examples", "llm-api")) - model_path = "Llama-3_1-8B-Instruct_fp8_nvfp4_hf" - llm_venv.run_cmd([ - str(example_root / "quickstart_advanced.py"), - "--model_dir", - f"{llm_models_root()}/{model_path}", - ]) - - # NOTE: we deliberately do not check the LLM outputs with keyword matching ratios as in the - # other tests, as it can be brittle and cause flakiness in CI. - # This test now becomes a smoke / functional test. - # Proper accuracy tests should be added to - # `tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py`. - - @pytest.mark.parametrize("modality", ["image", "video"]) @pytest.mark.parametrize( "model_name,model_path,match_ratio", @@ -1839,58 +1539,3 @@ def test_get_ci_container_port(): assert container_port_start > 0 assert container_port_num > 0 assert container_port_start + container_port_num <= 60000 - - -@skip_pre_hopper -@pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize("model_name", ["meta/Meta-Llama-3.1-8B"], - ids=["llama3_1-8b"]) -@pytest.mark.parametrize("model_subdir", ["llama-3.1-model/Meta-Llama-3.1-8B"], - ids=["llama_v3_1"]) -def test_trtllm_bench_mig_launch(llm_root, llm_venv, model_name, model_subdir): - """Run benchmark in MIG mode, check if throughput increases with concurrency.""" - results = {} - concurrency_list = [1, 32, 64, 128] - - for concurrency in concurrency_list: - num_requests = concurrency * 10 - runner = BenchRunner(llm_root=llm_root, - llm_venv=llm_venv, - model_name=model_name, - model_subdir=model_subdir, - streaming=False, - use_mpirun=False, - tp_size=1, - concurrency=concurrency, - num_requests=num_requests) - - output = runner() - results[concurrency] = output - - print(f"\n=== Benchmark Results Comparison ===") - print(f"Model: {model_name}") - print( - f"{'Concurrency':<15} {'Throughput':<15} {'Latency':<15} {'Num Requests':<15}" - ) - print("-" * 60) - - for idx, val in enumerate(concurrency_list): - metrics = results.get(val) - if not isinstance(metrics, dict): - pytest.fail( - f"Unexpected benchmark result type for concurrency {val}: {type(metrics)}" - ) - try: - throughput = float(metrics.get('throughput', 0)) - latency = float(metrics.get('latency', 0)) - num_requests = int(metrics.get('num_requests', 0)) - except (ValueError, TypeError) as e: - pytest.fail( - f"Failed to parse benchmark results for concurrency {val}: {e}") - assert throughput > 0, f"Throughput is 0 for concurrency {val}" - assert latency > 0, f"Latency is 0 for concurrency {val}" - print(f"{val:<15} {throughput:<15} {latency:<15} {num_requests:<15}") - if idx > 0: - prev_throughput = float(results[concurrency_list[idx - 1]].get( - 'throughput', 0)) - assert throughput > prev_throughput * 1.3, f"Throughput is not increasing for concurrency {concurrency_list[idx]}" diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 24818d8a8ef4..2d81f1b5a78f 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -706,7 +706,7 @@ disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Ch disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] -disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] +disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving disaggregated/test_disaggregated.py::test_disaggregated_mamba_bs1_concurrency2 disaggregated/test_disaggregated.py::test_disaggregated_mamba_conc_greater_than_mbs[NVIDIA-Nemotron-3-Super-120B-A12B-FP8] disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] @@ -746,7 +746,7 @@ disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[ disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8] disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8] disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8] -disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct] +disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-Qwen3-8B-eagle3-Qwen3-8B] disaggregated/test_workers.py::test_workers_conditional_disaggregation[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] disaggregated/test_workers.py::test_workers_conversation_router[TinyLlama-1.1B-Chat-v1.0] @@ -766,7 +766,6 @@ test_e2e.py::test_eagle3_output_repetition_4gpus[Qwen3/saved_models_Qwen3-235B-A test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] test_e2e.py::test_openai_chat_harmony_perf_metrics test_e2e.py::test_openai_kv_cache_contamination -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B] test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B] test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1] test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] diff --git a/tests/integration/test_lists/qa/llm_spark_func.yml b/tests/integration/test_lists/qa/llm_spark_func.yml index 739455999a46..7f08ac243d3e 100644 --- a/tests/integration/test_lists/qa/llm_spark_func.yml +++ b/tests/integration/test_lists/qa/llm_spark_func.yml @@ -11,9 +11,6 @@ llm_spark_func: tests: - test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b] - test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-120B-gpt_oss/gpt-oss-120b] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-bf16-instruct-llama-3.1-model/Llama-3.1-8B-Instruct] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP4-modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4] - test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8b-fp8-Qwen3/nvidia-Qwen3-8B-FP8] - test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8b-nvfp4-Qwen3/nvidia-Qwen3-8B-NVFP4] - test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8B-bf16-Qwen3/Qwen3-8B] @@ -33,7 +30,6 @@ llm_spark_func: - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] - test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] - - test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct] - examples/serve/test_serve.py::test_nemotron3_super_120b_nvfp4 - examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_on] - examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_off] diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index 1af3adcf46f3..05a8c4836bb2 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -128,7 +128,6 @@ l0_a10: - test_e2e.py::test_openai_responses_entrypoint - test_e2e.py::test_openai_completions_example[pytorch] - test_e2e.py::test_openai_chat_example[pytorch] TIMEOUT (90) - - test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-] - test_e2e.py::test_trtllm_bench_invalid_token_pytorch[TinyLlama-1.1B-Chat-v1.0-TinyLlama-1.1B-Chat-v1.0] # visual_gen - unittest/_torch/visual_gen/test_profiler.py diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 1848511a29b4..0a96295bfaca 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -81,15 +81,9 @@ l0_b200: - accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] - accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp - disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0] # nvbugs 5300551 - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B] - test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] - test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16] - test_e2e.py::test_ptp_quickstart_advanced_mtp_eagle[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16] - - test_e2e.py::test_ptp_quickstart_advanced_mixed_precision - - test_e2e.py::test_ptp_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] - - test_e2e.py::test_ptp_quickstart_advanced_ngram[Llama-3.1-8B-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct] - - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False] - test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] # Covers tests/unittest/_torch/attention/. Two sub-trees moved in here from elsewhere # under tests/unittest/_torch/ and have no entry of their own on any list, so this entry diff --git a/tests/integration/test_lists/test-db/l0_dgx_h200.yml b/tests/integration/test_lists/test-db/l0_dgx_h200.yml index 90c8545a82e9..ddb75d9bcb87 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h200.yml @@ -106,7 +106,6 @@ l0_dgx_h200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] - - test_e2e.py::test_trtllm_bench_mgmn - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False] diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index d0ac04317b09..c1a9e541ccfb 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -163,8 +163,6 @@ l0_h100: - accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8 - accuracy/test_llm_api_pytorch.py::TestLagunaXS_2_1::test_fp8_dflash - accuracy/test_llm_api_pytorch.py::TestNemotron35Lightning::test_nvfp4_marlin_mtp3_chunked_prefill - - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False] - - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] @@ -179,7 +177,7 @@ l0_h100: - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8] - - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct] + - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-Qwen3-8B-eagle3-Qwen3-8B] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_cancel_gen_requests[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[False-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0] @@ -195,15 +193,10 @@ l0_h100: - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_transcribe_end_to_end - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[bf16-kv-v2-decoder-graphs-on-greedy] - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_beam_search[bf16-kv-v1-decoder-graphs-on-beam2] - - test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] - - test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] - - test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate] # negative test - - test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B] - test_e2e.py::test_openai_chat_harmony - test_e2e.py::test_openai_chat_harmony_perf_metrics - test_e2e.py::test_openai_responses - test_e2e.py::test_anthropic_messages - - test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct] - kv_cache/test_final_single_token_context_cuda_graph.py::test_final_token_reuse_cuda_graph[v1] - kv_cache/test_final_single_token_context_cuda_graph.py::test_final_token_reuse_cuda_graph[v2] - kv_cache/test_final_single_token_context_cuda_graph.py::test_changed_final_token_reuse_cuda_graph[v1] @@ -239,7 +232,6 @@ l0_h100: tests: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency] - - test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8] @@ -414,8 +406,6 @@ l0_h100: - accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill] - accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] - accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill] - - test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] - - test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3_depth_1_tree[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] # ---- moved to post-merge (MoE CI optimization) ---- - unittest/_torch/moe/test_moe_backend.py::test_moe_backend -k "CUTLASS" # ---- non-quantized (quant=None) moved to post-merge ---- diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml index 7e94f1147991..6cd4b222b5cb 100644 --- a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml +++ b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml @@ -29,9 +29,6 @@ l0_rtx_pro_6000: - unittest/_torch/thop/parallel/test_w4a8_linear.py - test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] - test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B] - - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] - test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B] # 3mins - test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_fp8_hf-Qwen3/saved_models_Qwen3-30B-A3B_fp8_hf] # 3mins - test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf] # 2mins diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 1d9e4076fa93..c970fa93d2a9 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -102,7 +102,6 @@ full:A10/unittest/scripts/test_perf_sanity_helpers.py::test_add_perf_metric_valu full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:A100/disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6581064) -full:A100/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:A100/disaggregated/test_disaggregated.py::test_disaggregated_python_transceiver_host_offload[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6758573) full:A100/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6758594) full:A100/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6758594) @@ -118,7 +117,6 @@ full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_a full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:B200/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6771023) full:B200/disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6435097) -full:B200/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:B200/disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6728119) full:B200/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] SKIP (https://nvbugs/6765807) full:B200/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_trtllm_stress] SKIP (https://nvbugs/6649384) @@ -135,7 +133,6 @@ full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_a full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6771023) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_gentp4[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6728119) -full:B300/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_mamba_conc_greater_than_mbs[NVIDIA-Nemotron-3-Super-120B-A12B-FP8] SKIP (https://nvbugs/6770978) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6728119) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_qwen3_32b_fp8[Qwen3/Qwen3-32B-FP8] SKIP (https://nvbugs/6770977) @@ -155,7 +152,6 @@ full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_b full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True] SKIP (https://nvbugs/5929339) full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) -full:GB200/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:GB300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2] SKIP (https://nvbugs/6295740) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6432818) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6661948) @@ -166,7 +162,6 @@ full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16 SKIP (htt full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:GB300/disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6581064) -full:GB300/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:GB300/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6758594) full:GB300/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6758594) full:GB300/unittest/_torch/modeling/test_modeling_gpt_oss.py::test_gpt_oss_trtllmgen[CUTLASS] SKIP (https://nvbugs/6633932) @@ -179,7 +174,6 @@ full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-C full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp SKIP (https://nvbugs/6768489) full:H100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:H100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_32b_fp8_stress] SKIP (https://nvbugs/6312828) full:H100/llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_logging SKIP (https://nvbugs/6727262) full:H100/llmapi/test_llm_api_qa.py::TestLlmDefaultBackend::test_llm_args_type_default SKIP (https://nvbugs/6727262) @@ -196,12 +190,10 @@ full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[t full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[bf16] SKIP (https://nvbugs/6618649) -full:H20/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6728119) full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6728119) full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6728119) full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6581064) -full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6728119) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True] SKIP (https://nvbugs/5929339) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False] SKIP (https://nvbugs/6616033) @@ -272,7 +264,6 @@ unittest/_torch/multi_gpu/test_linear.py::test_row_linear_norm_fusion[2-hidden:1 unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_update_weights_nemotron_h SKIP (https://nvbugs/6729495) unittest/_torch/speculative/hw_agnostic/test_dflash.py::test_dflash_qwen3_5_4b[False] SKIP (https://nvbugs/6535767) unittest/_torch/speculative/hw_agnostic/test_dflash.py::test_dflash_qwen3_5_4b[True] SKIP (https://nvbugs/6535767) -unittest/_torch/speculative/hw_agnostic/test_ngram.py::test_llama_ngram[True-True-TRTLLM] SKIP (https://nvbugs/6507102) unittest/_torch/speculative/test_eagle3.py SKIP (https://nvbugs/5461761) unittest/_torch/thop/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk2-cublaslt] SKIP (https://nvbugs/6581067) unittest/_torch/thop/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk2-cutlass] SKIP (https://nvbugs/6581067) diff --git a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py index 02b2b4421b1b..74b1119f787b 100644 --- a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py +++ b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py @@ -84,7 +84,7 @@ def env(): @pytest.fixture(scope="module") def model_name(): - return "llama-3.1-model/Llama-3.1-8B-Instruct" + return "Qwen3.5-4B" @pytest.fixture(scope="module", params=['pytorch'], ids=["pytorch"]) @@ -115,7 +115,7 @@ def worker(model_name: str, ctx_tp_pp_size: tuple, gen_tp_pp_size: tuple): "backend": "DEFAULT" }, "kv_cache_config": { - "free_gpu_memory_fraction": 0.5, + "free_gpu_memory_fraction": 0.8, "enable_block_reuse": False, }, "disable_overlap_scheduler": True, diff --git a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py index 780844f2a77d..2d78d05dff45 100644 --- a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py +++ b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py @@ -57,7 +57,7 @@ def env(): @pytest.fixture def model_name(): - return "llama-3.1-model/Llama-3.1-8B-Instruct" + return "Qwen3.5-4B" @pytest.fixture @@ -107,7 +107,7 @@ def worker(model_name: str, disagg_cluster_config: dict): "disagg_cluster": disagg_cluster_config, "cache_transceiver_config": {"backend": "DEFAULT"}, "kv_cache_config": { - "free_gpu_memory_fraction": 0.5, + "free_gpu_memory_fraction": 0.8, "enable_block_reuse": False, }, "disable_overlap_scheduler": True, diff --git a/tests/unittest/llmapi/apps/_test_openai_chat_guided_decoding.py b/tests/unittest/llmapi/apps/_test_openai_chat_guided_decoding.py index 22c04857bec0..ee8223f5a9a2 100644 --- a/tests/unittest/llmapi/apps/_test_openai_chat_guided_decoding.py +++ b/tests/unittest/llmapi/apps/_test_openai_chat_guided_decoding.py @@ -23,7 +23,6 @@ @pytest.fixture(scope="module", params=[ - "meta-llama/Llama-3.1-8B-Instruct", "openai/gpt-oss-120b", pytest.param("zai-org/GLM-5-FP8", marks=pytest.mark.skip_less_device(8)), @@ -58,9 +57,7 @@ def temp_extra_llm_api_options_file(model_name: str): @pytest.fixture(scope="module") def server(model_name: str, temp_extra_llm_api_options_file: str): - if model_name == "meta-llama/Llama-3.1-8B-Instruct": - model_path = get_model_path("llama-3.1-model/Llama-3.1-8B-Instruct") - elif model_name == "openai/gpt-oss-120b": + if model_name == "openai/gpt-oss-120b": model_path = get_model_path("gpt_oss/gpt-oss-120b") elif model_name == "zai-org/GLM-5-FP8": model_path = get_model_path("GLM-5-FP8") diff --git a/tests/unittest/llmapi/apps/_test_openai_multi_nodes.py b/tests/unittest/llmapi/apps/_test_openai_multi_nodes.py deleted file mode 100644 index 7413745e51a4..000000000000 --- a/tests/unittest/llmapi/apps/_test_openai_multi_nodes.py +++ /dev/null @@ -1,257 +0,0 @@ -import asyncio -import os -import re -import time - -import openai -import pytest -import torch -from utils.util import skip_num_gpus_less_than, skip_nvlink_inactive - -from ..test_llm import get_model_path, prompts -from .openai_server import RemoteOpenAIServer - -RANK = os.environ.get("SLURM_PROCID", 0) -MESSAGES = [{ - "role": "user", - "content": "Hello! How are you?" -}, { - "role": "assistant", - "content": "Hi! I am quite well, how can I help you today?" -}, { - "role": "user", - "content": "A song on old age?" -}] - - -@pytest.fixture(scope="module") -def model_name(): - return "llama-3.1-model/Llama-3.1-8B-Instruct" - - -@pytest.fixture(scope="module", params=['pytorch'], ids=["pytorch"]) -def backend(request): - return request.param - - -@pytest.fixture(scope="module", - params=[(16, 1), (8, 2)], - ids=lambda tp_pp_size: f'tp{tp_pp_size[0]}pp{tp_pp_size[1]}') -def tp_pp_size(request): - return request.param - - -@pytest.fixture(scope="module") -def server(model_name: str, backend: str, tp_pp_size: tuple): - os.environ["FORCE_DETERMINISTIC"] = "1" - model_path = get_model_path(model_name) - tp_size, pp_size = tp_pp_size - device_count = torch.cuda.device_count() - args = [ - "--tp_size", - f"{tp_size}", - "--pp_size", - f"{pp_size}", - "--gpus_per_node", - f"{device_count}", - "--kv_cache_free_gpu_memory_fraction", - "0.95", - "--backend", - backend, - ] - with RemoteOpenAIServer(model_path, args, llmapi_launch=True, - port=8001) as remote_server: - yield remote_server - - os.environ.pop("FORCE_DETERMINISTIC") - - -@pytest.fixture(scope="module") -def client(server: RemoteOpenAIServer): - return server.get_client() - - -@pytest.fixture(scope="module") -def async_client(server: RemoteOpenAIServer): - return server.get_async_client() - - -@skip_num_gpus_less_than(4) -def test_chat(client: openai.OpenAI, model_name: str): - if RANK == "0": - messages = [{ - "role": "system", - "content": "you are a helpful assistant" - }, { - "role": "user", - "content": "What is the result of 1+1? Answer in one word: " - }] - chat_completion = client.chat.completions.create( - model=model_name, - messages=messages, - max_tokens=1, - ) - assert chat_completion.id is not None - assert len(chat_completion.choices) == 1 - assert chat_completion.usage.completion_tokens == 1 - message = chat_completion.choices[0].message - - print(f"Output: {message.content}") - assert message.content == 'Two' - else: - time.sleep(30) - assert True - - -@skip_num_gpus_less_than(4) -def test_completion(client: openai.OpenAI, model_name: str): - if RANK == "0": - completion = client.completions.create( - model=model_name, - prompt=prompts, - max_tokens=5, - temperature=0.0, - ) - assert completion.choices[0].text == " D E F G H" - else: - time.sleep(30) - assert True - - -@skip_num_gpus_less_than(4) -@pytest.mark.asyncio(loop_scope="module") -async def test_chat_streaming(async_client: openai.AsyncOpenAI, - model_name: str): - if RANK == "0": - messages = [{ - "role": "system", - "content": "you are a helpful assistant" - }, { - "role": "user", - "content": "What is the result of 1+1? Answer in one word: " - }] - stream = await async_client.chat.completions.create( - model=model_name, - messages=messages, - max_tokens=1, - stream=True, - ) - async for chunk in stream: - delta = chunk.choices[0].delta - if delta.role: - assert delta.role == "assistant" - if delta.content: - assert delta.content == "Two" - else: - time.sleep(30) - assert True - - -@skip_num_gpus_less_than(4) -@pytest.mark.asyncio(loop_scope="module") -async def test_completion_streaming(async_client: openai.AsyncOpenAI, - model_name: str): - if RANK == "0": - completion = await async_client.completions.create( - model=model_name, - prompt=prompts, - max_tokens=5, - temperature=0.0, - stream=True, - ) - str_chunk = [] - async for chunk in completion: - str_chunk.append(chunk.choices[0].text) - assert "".join(str_chunk) == " D E F G H" - else: - time.sleep(30) - assert True - - -@skip_nvlink_inactive -@skip_num_gpus_less_than(4) -@pytest.mark.asyncio(loop_scope="module") -@pytest.mark.skip(reason="https://nvbugs/5112075") -async def test_multi_consistent_sync_chat(client: openai.OpenAI, - model_name: str): - """ - RCCA: https://nvbugs/4829393 - """ - if RANK == 0: - unique_content = set() - - async def send_request(messages=None): - try: - completion = client.chat.completions.create( - model=model_name, - messages=messages, - n=1, - max_tokens=1024, - temperature=0, - frequency_penalty=1.0, - stream=False, - stop=["hello"]) - unique_content.add(completion.choices[0].message.content) - except Exception as e: - print(f"Error: {e}") - - tasks = [] - for _ in range(50): - tasks.append(asyncio.create_task(send_request(MESSAGES))) - await asyncio.sleep(1) - - await asyncio.gather(*tasks) - - print(f"Number of unique responses: {len(unique_content)}") - assert len(unique_content) == 1, "Responses are not consistent" - content = list(unique_content)[0] - pattern = re.compile(r'[^a-zA-Z0-9\s\'\"]{5,}') - assert not bool(pattern.search(content)), content - else: - time.sleep(60) - assert True - - -@skip_nvlink_inactive -@skip_num_gpus_less_than(4) -@pytest.mark.asyncio(loop_scope="module") -@pytest.mark.skip(reason="https://nvbugs/5112075") -async def test_multi_consistent_async_chat(async_client: openai.AsyncOpenAI, - model_name: str): - """ - RCCA: https://nvbugs/4829393 - """ - - if RANK: - unique_content = set() - - async def send_request(messages=None): - try: - completion = await async_client.chat.completions.create( - model=model_name, - messages=messages, - n=1, - max_tokens=1024, - temperature=0, - frequency_penalty=1.0, - stream=False, - stop=["hello"]) - unique_content.add(completion.choices[0].message.content) - except Exception as e: - print(f"Error: {e}") - - tasks = [] - for _ in range(50): - tasks.append(asyncio.create_task(send_request(MESSAGES))) - await asyncio.sleep(1) - - await asyncio.gather(*tasks) - - print(f"Number of unique responses: {len(unique_content)}") - assert len(unique_content) == 1, "Responses are not consistent" - content = list(unique_content)[0] - pattern = re.compile(r'[^a-zA-Z0-9\s\'\"]{5,}') - assert not bool(pattern.search(content)), content - else: - time.sleep(60) - assert True diff --git a/tests/unittest/llmapi/apps/_test_trtllm_serve_benchmark.py b/tests/unittest/llmapi/apps/_test_trtllm_serve_benchmark.py index a9c1ecf2426a..7462711b0ef5 100644 --- a/tests/unittest/llmapi/apps/_test_trtllm_serve_benchmark.py +++ b/tests/unittest/llmapi/apps/_test_trtllm_serve_benchmark.py @@ -58,11 +58,9 @@ def dataset_path(dataset_name: str): @skip_gpu_memory_less_than_80gb -@pytest.mark.parametrize("model_name", [ - "llama-3.1-model/Meta-Llama-3.1-8B", - pytest.param("gpt_oss/gpt-oss-20b", marks=skip_pre_hopper) -], - indirect=True) +@pytest.mark.parametrize( + "model_name", [pytest.param("gpt_oss/gpt-oss-20b", marks=skip_pre_hopper)], + indirect=True) def test_trtllm_serve_benchmark(server: RemoteOpenAIServer, benchmark_root: str, model_path: str): model_name = model_path.split("/")[-1] diff --git a/tests/unittest/llmapi/test_llm.py b/tests/unittest/llmapi/test_llm.py index be45f0f08d5f..9ca16a859984 100644 --- a/tests/unittest/llmapi/test_llm.py +++ b/tests/unittest/llmapi/test_llm.py @@ -251,7 +251,6 @@ def test_llm_with_kv_cache_retention_config(): (get_model_path('codellama/CodeLlama-7b-Instruct-hf'), False, 0.95), (llama_model_path, False, 0.95), (get_model_path(qwen3_tokenizer_model_name), False, 0.95), - (get_model_path('llama-3.1-model/Meta-Llama-3.1-8B'), False, 0.95), (get_model_path('DeepSeek-R1/DeepSeek-R1'), False, 0.95) ]) @pytest.mark.part0 diff --git a/tests/unittest/llmapi/test_llm_pytorch.py b/tests/unittest/llmapi/test_llm_pytorch.py index 0457ac706ef5..5a5d80c99382 100644 --- a/tests/unittest/llmapi/test_llm_pytorch.py +++ b/tests/unittest/llmapi/test_llm_pytorch.py @@ -35,9 +35,7 @@ sampling_params_for_aborting_request, run_llm_with_postprocess_parallel_and_result_handler, tinyllama_logits_processor_test_harness) -from utils.util import (force_ampere, similar, skip_fp8_pre_ada, - skip_gpu_memory_less_than_40gb, - skip_gpu_memory_less_than_80gb, +from utils.util import (force_ampere, similar, skip_gpu_memory_less_than_40gb, skip_gpu_memory_less_than_138gb, skip_ray) from utils.llm_data import llm_models_root from tensorrt_llm._torch.peft.lora.config import LoraConfig @@ -420,35 +418,6 @@ def test_nemotron_nas_lora(cuda_graph_config) -> None: llm.shutdown() -@skip_gpu_memory_less_than_80gb -@pytest.mark.part0 -@test_lora_with_and_without_cuda_graph -def test_llama_3_1_8b_fp8_with_bf16_lora(cuda_graph_config) -> None: - skip_fp8_pre_ada(use_fp8=True) - model_dir = f"{llm_models_root()}/llama-3.1-model/Llama-3.1-8B-Instruct-FP8" - lora_dir = f"{llm_models_root()}/lora/llama-3-chinese-8b-instruct-v2-lora" - prompt = "美国的首都是哪里？" - reference = "华盛顿特区。华盛顿特区是美国的首都和一个行政区" - - lora_config = LoraConfig(lora_dir=[lora_dir], - max_lora_rank=64, - max_loras=2, - max_cpu_loras=2) - lora_req = LoRARequest("lora-chinese", 0, lora_dir) - - llm = LLM(model_dir, - lora_config=lora_config, - cuda_graph_config=cuda_graph_config) - - try: - output = llm.generate(prompt, - SamplingParams(max_tokens=20), - lora_request=[lora_req]) - finally: - llm.shutdown() - assert similar(output.outputs[0].text, reference) - - @pytest.mark.part2 @test_lora_with_and_without_cuda_graph def test_gemma3_1b_instruct_multi_lora(cuda_graph_config) -> None: diff --git a/tests/unittest/llmapi/test_session_reuse.py b/tests/unittest/llmapi/test_session_reuse.py index 88ec33dd7f77..b346d1c94041 100644 --- a/tests/unittest/llmapi/test_session_reuse.py +++ b/tests/unittest/llmapi/test_session_reuse.py @@ -497,16 +497,14 @@ def test_torch_compile_nodeids_are_private(): from test_common.session_reuse_hooks import _is_private_nodeid assert _is_private_nodeid( - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::" - "test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]" + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True]" ) assert _is_private_nodeid( "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::" "test_nvfp4_multi_gpus_piecewise_cuda_graph[baseline]" ) assert not _is_private_nodeid( - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::" - "test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]" + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False]" ) assert not _is_private_nodeid( "unittest/llmapi/test_llm_args.py::test_torch_compile_config_round_trip" diff --git a/tests/unittest/metrics/test_collector.py b/tests/unittest/metrics/test_collector.py index f3fe88d3ab27..8cb3addb9209 100644 --- a/tests/unittest/metrics/test_collector.py +++ b/tests/unittest/metrics/test_collector.py @@ -217,8 +217,8 @@ class TestConfigInfoMetrics: def test_model_config_info(self, collector): model_config = { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "served_model_name": "Llama-3.1-8B-Instruct", + "model": "Qwen/Qwen3-8B", + "served_model_name": "Qwen3-8B", "dtype": "float16", "quantization": "none", "max_model_len": "4096", diff --git a/tests/unittest/scripts/test_check_model_registry.py b/tests/unittest/scripts/test_check_model_registry.py index d6b9c743bd91..3659ba496402 100644 --- a/tests/unittest/scripts/test_check_model_registry.py +++ b/tests/unittest/scripts/test_check_model_registry.py @@ -38,9 +38,9 @@ def mod(): def test_validate_models_allows_same_name_with_different_config_id(mod): models = [ - {"name": "meta-llama/Llama-3.1-8B-Instruct", "yaml_extra": ["world_size_1.yaml"]}, + {"name": "Qwen/Qwen3-8B", "yaml_extra": ["world_size_1.yaml"]}, { - "name": "meta-llama/Llama-3.1-8B-Instruct", + "name": "Qwen/Qwen3-8B", "config_id": "fp8", "yaml_extra": ["world_size_1.yaml", "fp8.yaml"], },