From 8f45fd1affb756cd4430cc26211dd0c2621f44d2 Mon Sep 17 00:00:00 2001 From: Zheyu Fu Date: Mon, 14 Sep 2026 23:36:12 -0700 Subject: [PATCH 1/3] [None][test] Add InferenceX-style GSM8K accuracy eval mode Adds a `gsm8k_inferencex` lm-eval task and evaluator reproducing the InferenceX (formerly InferenceMAX) GSM8K protocol: chat template, 5-shot multiturn, a 12288-token generation budget, strict "#### N" extraction and lm-eval's default exemplar selection, so scores are comparable to inferencex.semianalysis.com/evaluation. Exposed as `trtllm-eval gsm8k_inferencex`. The accuracy harness gains a matching `GSM8KInferenceX` task and an optional explicit `threshold` on reference rows. Ported from the feat/m3_with_msa side branch (#16711). Signed-off-by: Zheyu Fu Co-Authored-By: Claude Fable 5.1 --- docs/source/commands/trtllm-eval.rst | 8 ++ tensorrt_llm/commands/eval.py | 5 +- tensorrt_llm/evaluate/__init__.py | 4 +- tensorrt_llm/evaluate/lm_eval.py | 134 +++++++++++++++++- .../gsm8k_inferencex/gsm8k_inferencex.yaml | 50 +++++++ tests/integration/defs/accuracy/README.md | 2 + .../defs/accuracy/accuracy_core.py | 39 +++-- 7 files changed, 226 insertions(+), 16 deletions(-) create mode 100644 tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencex/gsm8k_inferencex.yaml diff --git a/docs/source/commands/trtllm-eval.rst b/docs/source/commands/trtllm-eval.rst index d7511fbfb0a6..3dc62d862be8 100644 --- a/docs/source/commands/trtllm-eval.rst +++ b/docs/source/commands/trtllm-eval.rst @@ -34,6 +34,11 @@ The following tasks are currently supported: - accuracy - 4,096 - 256 + * - GSM8K (InferenceX protocol) + - QA; regex matching + - accuracy + - 4,096 + - 12,288 * - GPQA - QA; multiple choice - accuracy @@ -78,6 +83,9 @@ Here are some examples: # Evaluate Llama-3.1-8B-Instruct on GSM8K trtllm-eval --model meta-llama/Llama-3.1-8B-Instruct gsm8k + # Evaluate a model on GSM8K under the InferenceX protocol (12,288-token generation budget) + trtllm-eval --model --max_seq_len 16384 gsm8k_inferencex + # Evaluate Llama-3.3-70B-Instruct on GPQA Diamond trtllm-eval --model meta-llama/Llama-3.3-70B-Instruct gpqa_diamond diff --git a/tensorrt_llm/commands/eval.py b/tensorrt_llm/commands/eval.py index 5f33c00b9f39..4ed1efb6faea 100644 --- a/tensorrt_llm/commands/eval.py +++ b/tensorrt_llm/commands/eval.py @@ -22,8 +22,8 @@ from .. import LLM as PyTorchLLM from ..evaluate import (AALCR, AIME2025, AIME2026, GSM8K, HLE, MMLU, MMMU, ArenaHard, CnnDailymail, CoVoST2, GPQADiamond, - GPQAExtended, GPQAMain, GPQANemoSkills, IFBench, - ImageGenerationEval, JsonModeEval, LongBenchV1, + GPQAExtended, GPQAMain, GPQANemoSkills, GSM8KInferenceX, + IFBench, ImageGenerationEval, JsonModeEval, LongBenchV1, LongBenchV2, MMMUPro, SciCode) from ..llmapi import KvCacheConfig from ..llmapi.llm_args import TorchLlmArgs @@ -262,6 +262,7 @@ def main(ctx, model: str, tokenizer: Optional[str], main.add_command(CnnDailymail.command) main.add_command(MMLU.command) main.add_command(GSM8K.command) +main.add_command(GSM8KInferenceX.command) main.add_command(GPQADiamond.command) main.add_command(GPQAMain.command) main.add_command(GPQAExtended.command) diff --git a/tensorrt_llm/evaluate/__init__.py b/tensorrt_llm/evaluate/__init__.py index 7caf146632d8..39e02aeb3c1c 100755 --- a/tensorrt_llm/evaluate/__init__.py +++ b/tensorrt_llm/evaluate/__init__.py @@ -18,7 +18,8 @@ from .covost2 import CoVoST2 from .json_mode_eval import JsonModeEval from .lm_eval import (AIME2025, AIME2026, GSM8K, MMMU, GPQADiamond, - GPQAExtended, GPQAMain, LongBenchV1, MMMUPro) + GPQAExtended, GPQAMain, GSM8KInferenceX, LongBenchV1, + MMMUPro) from .longbench_v2 import LongBenchV2 from .mmlu import MMLU from .nemo_skills_eval import (AALCR, HLE, ArenaHard, GPQANemoSkills, IFBench, @@ -29,6 +30,7 @@ "CnnDailymail", "MMLU", "GSM8K", + "GSM8KInferenceX", "GPQADiamond", "GPQAMain", "GPQAExtended", diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index a533573ae90e..58a4b44b3bc2 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -855,7 +855,9 @@ def __init__(self, post_process_fn: Optional[Callable[[str], str]] = None, preserve_caller_max_tokens: bool = False, num_fewshot: Optional[int] = None, - stop_strings: Optional[List[str]] = None): + stop_strings: Optional[List[str]] = None, + shuffle_dataset: bool = True, + fewshot_random_seed: Optional[int] = None): try: import lm_eval except ImportError as e: @@ -889,6 +891,11 @@ def __init__(self, # larger than the lm-eval task's max_gen_toks. Used by thinking # models (e.g. Kimi K2.5) whose CoT output exceeds the task default. self.preserve_caller_max_tokens = preserve_caller_max_tokens + # shuffle_dataset=False and fewshot_random_seed reproduce the lm-eval + # CLI's exemplar selection (unshuffled data, few-shot sampler seed). + self.shuffle_dataset = shuffle_dataset + self.fewshot_random_seed = (random_seed if fewshot_random_seed is None + else fewshot_random_seed) task_manager = TaskManager( include_path=f"{os.path.dirname(__file__)}/lm_eval_tasks") @@ -909,7 +916,7 @@ def _adjust_config(task_dict, random_seed): } else: # NOTE: Few-shot random seed - task_obj.set_fewshot_seed(seed=random_seed) + task_obj.set_fewshot_seed(seed=self.fewshot_random_seed) # Caller override of the task yaml's shot count, the same # call lm-eval's own simple_evaluate makes. Without it a # 0-shot chat evaluation of a task whose yaml pins 5 shots @@ -925,9 +932,10 @@ def _adjust_config(task_dict, random_seed): adjusted_task_dict[task_name] = task_obj # NOTE: Shuffle dataset - data = adjusted_task_dict[task_name].dataset - for split in data.keys(): - data[split] = data[split].shuffle(random_seed) + if self.shuffle_dataset: + data = adjusted_task_dict[task_name].dataset + for split in data.keys(): + data[split] = data[split].shuffle(random_seed) return adjusted_task_dict @@ -1252,6 +1260,122 @@ def command(ctx, **kwargs) -> None: GSM8K.command_harness(ctx, **kwargs) +class GSM8KInferenceX(LmEvalEvaluator): + """GSM8K under the InferenceX (formerly InferenceMAX) protocol. + + Mirrors SemiAnalysisAI/InferenceX (infx/evals/gsm8k.yaml + run_lm_eval): + chat template, 5-shot multiturn, the documented 12288-token eval-only + generation budget, strict "#### N" extraction, and lm-eval's default + exemplar selection (unshuffled data, few-shot seed 1234), so scores are + comparable to inferencex.semianalysis.com/evaluation. Thinking mode + follows the chat template default unless chat_template_kwargs sets it. + The answer filter sees the raw generation, whereas a served InferenceX + run scores the reasoning-parsed content; this only matters if a model + writes "#### N" inside its thinking. + """ + + def __init__(self, **kwargs): + kwargs.setdefault("apply_chat_template", True) + kwargs.setdefault("fewshot_as_multiturn", True) + kwargs.setdefault("shuffle_dataset", False) + kwargs.setdefault("fewshot_random_seed", 1234) + super().__init__("gsm8k_inferencex", **kwargs) + + @click.command("gsm8k_inferencex") + @click.option("--dataset_path", + type=str, + default=None, + help="The path to GSM8K dataset. " + "If unspecified, the dataset is downloaded from HF hub.") + @click.option( + "--num_samples", + type=int, + default=None, + help="Number of samples to run the evaluation; None means full dataset." + ) + @click.option("--random_seed", + type=int, + default=0, + help="Random seed; data order is fixed under this protocol.") + @click.option("--apply_chat_template", + type=click.BOOL, + default=True, + help="Whether to apply chat template.") + @click.option( + "--chat_template_kwargs", + type=str, + default=None, + callback=lambda ctx, param, value: json.loads(value) if value else None, + help= + 'Chat template kwargs as JSON string, e.g., \'{"thinking_budget": 0}\'') + @click.option("--fewshot_as_multiturn", + type=click.BOOL, + default=True, + help="Apply fewshot as multiturn.") + @click.option("--num_fewshot", + type=int, + default=None, + help="Override the task yaml's shot count. Use 0 with " + "--apply_chat_template for a single-question chat " + "evaluation.") + @click.option("--system_prompt", + type=str, + default=None, + help="System prompt.") + @click.option("--max_input_length", + type=int, + default=4096, + help="Maximum prompt length.") + @click.option("--max_output_length", + type=int, + default=12288, + help="Maximum generation length.") + @click.option("--temperature", + type=float, + default=None, + help="Sampling temperature. Overrides task yaml gen_kwargs.") + @click.option( + "--top_p", + type=float, + default=None, + help="Nucleus sampling top_p. Overrides task yaml gen_kwargs.") + @click.option("--top_k", + type=int, + default=None, + help="Top-k sampling. Overrides task yaml gen_kwargs.") + @click.option("--sampling_seed", + type=int, + default=None, + help="Random seed for generation sampling.") + @click.option( + "--stop_strings", + type=str, + default=None, + callback=_parse_stop_strings, + help='Replace the task yaml\'s stop strings, as a JSON list, e.g. ' + '\'["", "<|im_end|>"]\'.') + @click.option("--log_samples", + is_flag=True, + default=False, + help="Log sample outputs for debugging.") + @click.option("--output_path", + type=str, + default=None, + help="Path to save evaluation results.") + @click.option("--output_dir", + type=str, + default=None, + help="Directory to save the task infos.") + @click.pass_context + @staticmethod + def command(ctx, **kwargs) -> None: + if kwargs.get("fewshot_as_multiturn", False): + assert kwargs.get( + "apply_chat_template", False + ), "apply_chat_template must be True when fewshot_as_multiturn is True" + GSM8KInferenceX.command_harness(ctx, **kwargs) + + class GPQADiamond(LmEvalEvaluator): def __init__(self, **kwargs): diff --git a/tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencex/gsm8k_inferencex.yaml b/tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencex/gsm8k_inferencex.yaml new file mode 100644 index 000000000000..f2f6fbc197e3 --- /dev/null +++ b/tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencex/gsm8k_inferencex.yaml @@ -0,0 +1,50 @@ +# SemiAnalysisAI/InferenceX infx/evals/gsm8k.yaml (upstream lm-eval gsm8k.yaml with a +# "#### [number]" answer-format instruction, lm-evaluation-harness#3411), renamed to +# avoid the registry clash and with max_gen_toks set to InferenceX's documented eval-only +# budget (16384 context minus 4096 prompt). +tag: + - math_word_problems +task: gsm8k_inferencex +dataset_path: openai/gsm8k +dataset_name: main +output_type: generate_until +training_split: train +fewshot_split: train +test_split: test +doc_to_text: "Question: {{question}}\nEnd your response with the answer on the last line, formatted as: #### [number]\nAnswer:" +doc_to_target: "{{answer}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + regexes_to_ignore: + - "," + - "\\$" + - "(?s).*#### " + - "\\.$" +generation_kwargs: + until: + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 + max_gen_toks: 12288 +repeats: 1 +num_fewshot: 5 +filter_list: + - name: "strict-match" + filter: + - function: "regex" + group_select: -1 + regex_pattern: "#### (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "regex" + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: "take_first" +metadata: + version: 3.0 diff --git a/tests/integration/defs/accuracy/README.md b/tests/integration/defs/accuracy/README.md index 33a3b9f06ba1..86f2acbcf43e 100644 --- a/tests/integration/defs/accuracy/README.md +++ b/tests/integration/defs/accuracy/README.md @@ -125,6 +125,8 @@ Model data type and quantization decide the precision in model computation, so a A direct implication is that multiple test cases with different features may share the same accuracy reference. This is by design. For example, we should expect a test case with tensor parallelism to have very similar accuracy to its single-GPU counterpart. +A reference row may set `threshold` to an explicit pass/fail floor that replaces the computed hypothesis-test threshold; see [references/gsm8k_inferencex.yaml](./references/gsm8k_inferencex.yaml). + #### Testing Logic As aforementioned, each test case evaluates the accuracy of a model with some specifications by running one or multiple tasks. diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index 6df6b549e316..2b6402bcb2ec 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -120,19 +120,21 @@ class HypothesisTestingParams: sigma: float = 50.0 higher_is_better: bool = True theta: float = field(init=False) - threshold: float = field(init=False) + # An explicit threshold replaces the one computed from the reference row. + threshold: Optional[float] = None def __post_init__(self) -> None: self.theta = compute_theta(self.num_samples, sigma=self.sigma, alpha=self.alpha, beta=self.beta) - self.threshold = compute_threshold( - self.num_samples, - self.ref_accuracy, - sigma=self.sigma, - alpha=self.alpha, - higher_is_better=self.higher_is_better) + if self.threshold is None: + self.threshold = compute_threshold( + self.num_samples, + self.ref_accuracy, + sigma=self.sigma, + alpha=self.alpha, + higher_is_better=self.higher_is_better) def report(self, accuracy: Optional[float] = None) -> str: metric_name = self.metric_name.upper() @@ -326,7 +328,8 @@ def get_hypothesis_testing_params(self, sigma=entry.get("sigma", self.SIGMA), num_samples=entry.get("num_samples", self.NUM_SAMPLES), higher_is_better=entry.get("higher_is_better", - self.HIGHER_IS_BETTER)) + self.HIGHER_IS_BETTER), + threshold=entry.get("threshold")) def evaluate(self, llm: Union[PyTorchLLM, AutoDeployLLM], @@ -550,6 +553,26 @@ class GSM8K(AccuracyTask): EVALUATE_KWARGS = dict(scores_filter=None) +class GSM8KInferenceX(AccuracyTask): + # InferenceX-protocol GSM8K, see tensorrt_llm.evaluate.GSM8KInferenceX. + DATASET = "gsm8k_inferencex" + DATASET_DIR = f"{llm_models_root()}/datasets/openai/gsm8k" + + ALPHA = 0.05 + BETA = 0.2 + SIGMA = 50 + NUM_SAMPLES = 1319 # Full sample + + MAX_INPUT_LEN = 4096 + MAX_OUTPUT_LEN = 12288 + + EVALUATOR_CLS = tensorrt_llm.evaluate.GSM8KInferenceX + EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, random_seed=0) + + # InferenceX reports the strict-match score. + EVALUATE_KWARGS = dict(scores_filter="exact_match,strict-match") + + class GPQADiamond(AccuracyTask): DATASET = "gpqa_diamond" DATASET_DIR = f"{llm_models_root()}/datasets/gpqa" From 21bbc5ccc58251f925176d5ab2b1d9c2db607d20 Mon Sep 17 00:00:00 2001 From: Zheyu Fu Date: Mon, 14 Sep 2026 23:36:12 -0700 Subject: [PATCH 2/3] [None][test] Run the MiniMax-M3 Eagle3 accuracy test under the InferenceX protocol `TestMiniMaxM3::test_nvfp4_eagle3` gains `eval_mode`: `default` keeps MMLU + completion-format GSM8K, `inferencex` runs InferenceX-protocol GSM8K with thinking enabled (as the InferenceX vLLM recipes serve M3), a 16k context and batch 64. The pre-merge and QA lists run `inferencex`; adds the matching reference row (published vLLM B200 FP4 Eagle3 thinking-enabled mean 97.0, explicit floor 94.0). Signed-off-by: Zheyu Fu Co-Authored-By: Claude Fable 5.1 --- .../accuracy/references/gsm8k_inferencex.yaml | 13 ++++ .../defs/accuracy/test_llm_api_pytorch.py | 59 +++++++++++++------ .../test_lists/qa/llm_function_core.txt | 2 +- .../test_lists/test-db/l0_dgx_b200.yml | 2 +- 4 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 tests/integration/defs/accuracy/references/gsm8k_inferencex.yaml diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencex.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencex.yaml new file mode 100644 index 000000000000..0890c3084a2b --- /dev/null +++ b/tests/integration/defs/accuracy/references/gsm8k_inferencex.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# GSM8K under the InferenceX protocol (GSM8KInferenceX in accuracy_core.py). +# accuracy: mean of the published InferenceX vLLM B200 FP4 rows run with thinking enabled +# (inferencex.semianalysis.com/api/v1/evaluations, 2026-08). threshold: explicit floor +# (InferenceX's own gate is 90.0). +nvidia/MiniMax-M3-NVFP4: + # MSA path, FP8 KV cache, one-model Eagle3, thinking_mode enabled (rows: 97.27, 96.74). + - quant_algo: MIXED_PRECISION + kv_cache_quant_algo: FP8 + spec_dec_algo: Eagle3 + accuracy: 97.0 + threshold: 94.0 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index d283757e9495..81f9cc3183a7 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -47,12 +47,15 @@ skip_no_mxfp4_swizzle, skip_no_sm120, skip_post_blackwell, skip_post_hopper, skip_pre_ada, skip_pre_blackwell, skip_pre_hopper, skip_ray, skip_x86) -from .accuracy_core import (GSM8K, MMLU, CnnDailymail, GPQADiamond, - JsonModeEval, LlmapiAccuracyTestHarness, - LongBenchV1, LongBenchV2, assert_acceptance_length, - assert_acceptance_length_for_llm, - assert_guided_decoding_regex, - compute_acceptance_length) + +# isort: off +from .accuracy_core import ( + GSM8K, MMLU, CnnDailymail, GPQADiamond, GSM8KInferenceX, JsonModeEval, + LlmapiAccuracyTestHarness, LongBenchV1, LongBenchV2, + assert_acceptance_length, assert_acceptance_length_for_llm, + assert_guided_decoding_regex, compute_acceptance_length) + +# isort: on # Keep helper definitions below imports so new imports do not need E402 @@ -7881,21 +7884,23 @@ def test_nvfp4(self, use_msa): @pytest.mark.skip_less_device(4) @pytest.mark.skip_less_device_memory(140000) + @parametrize_with_ids("eval_mode", ["default", "inferencex"]) @parametrize_with_ids("overlap_scheduler", [False, True]) @parametrize_with_ids("attention_dp", [False, True]) @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp, - overlap_scheduler): + overlap_scheduler, eval_mode): # One-model Eagle3 on the MSA backend with an FP8 KV cache and CUDA - # graphs; the GQA drafter shares the target KV cache. MMLU + GSM8K plus - # a chat-GSM8K acceptance probe, since accuracy alone does not notice a - # corrupted drafter KV. + # graphs; the GQA drafter shares the target KV cache. MMLU + GSM8K, or + # InferenceX GSM8K, plus a chat-GSM8K acceptance probe, since accuracy + # alone does not notice a corrupted drafter KV. from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.kernels.msa_utils import \ msa_package_available if not msa_package_available(): pytest.skip("MSA kernels (fmha_sm100) not available") model_name = "nvidia/MiniMax-M3-NVFP4" model_path = f"{llm_models_root()}/MiniMax-M3-NVFP4" + inferencex = eval_mode == "inferencex" max_draft_len = 3 spec_config = Eagle3DecodingConfig( max_draft_len=max_draft_len, @@ -7905,6 +7910,14 @@ def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp, kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.6, enable_block_reuse=False, dtype="fp8") + # InferenceX mode: 16k context for thinking output, batch 64 (the + # InferenceX default). Otherwise fmha_sm100 caps total_q x heads at + # 65536; with 4 verify tokens per row that is 512 (256 unsharded). + if inferencex: + max_seq_len, max_batch_size = 16384, 64 + else: + max_seq_len = 4096 + max_batch_size = 256 if attention_dp else 512 with LLM( model_path, tensor_parallel_size=tp_size, @@ -7913,14 +7926,12 @@ def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp, sparse_attention_config=MiniMaxM3SparseAttentionConfig( implementation="msa", indexer_kv_dtype="fp8"), moe_config=MoeConfig(backend="CUTLASS"), - max_seq_len=4096, - # fmha_sm100 caps total_q x heads at 65536; with 4 verify - # tokens per row that is 512 (256 with unsharded heads). - max_batch_size=256 if attention_dp else 512, + max_seq_len=max_seq_len, + max_batch_size=max_batch_size, speculative_config=spec_config, cuda_graph_config=CudaGraphConfig( enable_padding=True, - max_batch_size=64 if attention_dp else 128, + max_batch_size=64 if (inferencex or attention_dp) else 128, ), disable_overlap_scheduler=not overlap_scheduler, enable_attention_dp=attention_dp, @@ -7941,10 +7952,20 @@ def drain_spec_stats(llm): steps += sd.get("numRequestsWithDraftTokens", 0) return drafted, accepted, steps - task = MMLU(model_name) - task.evaluate(llm) - task = GSM8K(model_name) - task.evaluate(llm) + if inferencex: + # InferenceX serves M3 with thinking forced on. + task = GSM8KInferenceX(model_name) + task.evaluate(llm, + extra_evaluator_kwargs={ + "chat_template_kwargs": { + "thinking_mode": "enabled" + } + }) + else: + task = MMLU(model_name) + task.evaluate(llm) + task = GSM8K(model_name) + task.evaluate(llm) # Acceptance probe: 200 chat-format GSM8K questions, greedy, 512 tokens. questions = [ diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index f8055658d371..5311814fe057 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -489,7 +489,7 @@ accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] TIMEOUT (180) accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] TIMEOUT (60) -accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True] +accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True-eval_mode=inferencex] accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8 accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index eac8567fad07..cda9a03a675f 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -53,7 +53,7 @@ l0_dgx_b200: - disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp_tp4] - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] TIMEOUT (60) - - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True] + - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True-eval_mode=inferencex] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] TIMEOUT (60) - unittest/_torch/modeling/test_modeling_deepseekv4.py - accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_auto_dtype TIMEOUT (60) From a429afb0eb4142812623de0d3cf2dc2d1d7b924c Mon Sep 17 00:00:00 2001 From: Zheyu Fu Date: Tue, 15 Sep 2026 00:57:06 -0700 Subject: [PATCH 3/3] [None][test] Keep the InferenceX GSM8K exemplars in a single user turn InferenceX's run_lm_eval passes --apply_chat_template but not --fewshot_as_multiturn, so its 5-shot GSM8K prompt is one user turn that carries all exemplars. GSM8KInferenceX defaulted to multi-turn exemplars; switch the default (and the trtllm-eval option) to False so the mode mirrors the published protocol, and say so in the docstring. On 4x GB300 / 4x GB200 (TP4/EP4, MSA, FP8 KV, Eagle3, thinking enabled) the single-turn rendering scores 97.04 / 97.19 strict-match against the 97.0 reference; the multi-turn rendering scored 97.04 / 96.82 on the same day. Signed-off-by: Zheyu Fu --- tensorrt_llm/evaluate/lm_eval.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index 58a4b44b3bc2..bf1aa8f5a64b 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -1264,11 +1264,14 @@ class GSM8KInferenceX(LmEvalEvaluator): """GSM8K under the InferenceX (formerly InferenceMAX) protocol. Mirrors SemiAnalysisAI/InferenceX (infx/evals/gsm8k.yaml + run_lm_eval): - chat template, 5-shot multiturn, the documented 12288-token eval-only - generation budget, strict "#### N" extraction, and lm-eval's default - exemplar selection (unshuffled data, few-shot seed 1234), so scores are - comparable to inferencex.semianalysis.com/evaluation. Thinking mode - follows the chat template default unless chat_template_kwargs sets it. + chat template with the 5 exemplars in the single user turn (InferenceX + passes --apply_chat_template but not --fewshot_as_multiturn), the + documented 12288-token eval-only generation budget, strict "#### N" + extraction, and lm-eval's default exemplar selection (unshuffled data, + few-shot seed 1234), so scores are comparable to + inferencex.semianalysis.com/evaluation. Thinking mode follows the chat + template default unless chat_template_kwargs sets it; InferenceX serves + with thinking enabled where the framework offers a server-side default. The answer filter sees the raw generation, whereas a served InferenceX run scores the reasoning-parsed content; this only matters if a model writes "#### N" inside its thinking. @@ -1276,7 +1279,7 @@ class GSM8KInferenceX(LmEvalEvaluator): def __init__(self, **kwargs): kwargs.setdefault("apply_chat_template", True) - kwargs.setdefault("fewshot_as_multiturn", True) + kwargs.setdefault("fewshot_as_multiturn", False) kwargs.setdefault("shuffle_dataset", False) kwargs.setdefault("fewshot_random_seed", 1234) super().__init__("gsm8k_inferencex", **kwargs) @@ -1310,8 +1313,9 @@ def __init__(self, **kwargs): 'Chat template kwargs as JSON string, e.g., \'{"thinking_budget": 0}\'') @click.option("--fewshot_as_multiturn", type=click.BOOL, - default=True, - help="Apply fewshot as multiturn.") + default=False, + help="Apply fewshot as multiturn. InferenceX keeps the " + "exemplars in the single user turn.") @click.option("--num_fewshot", type=int, default=None,