[None][test] Add InferenceX-style GSM8K accuracy eval mode

docs/source/commands/trtllm-eval.rst

@@ -34,6 +34,11 @@ The following tasks are currently supported:
      - accuracy
      - 4,096
      - 256
+   * - GSM8K (InferenceX protocol)
+     - QA; regex matching
+     - accuracy
+     - 4,096
+     - 12,288
    * - GPQA
      - QA; multiple choice
      - accuracy
@@ -78,6 +83,9 @@ Here are some examples:
    # Evaluate Llama-3.1-8B-Instruct on GSM8K
    trtllm-eval --model meta-llama/Llama-3.1-8B-Instruct gsm8k
 
+   # Evaluate a model on GSM8K under the InferenceX protocol (12,288-token generation budget)
+   trtllm-eval --model <model> --max_seq_len 16384 gsm8k_inferencex
+
    # Evaluate Llama-3.3-70B-Instruct on GPQA Diamond
    trtllm-eval --model meta-llama/Llama-3.3-70B-Instruct gpqa_diamond
 

tensorrt_llm/commands/eval.py

@@ -22,8 +22,8 @@
 from .. import LLM as PyTorchLLM
 from ..evaluate import (AALCR, AIME2025, AIME2026, GSM8K, HLE, MMLU, MMMU,
                         ArenaHard, CnnDailymail, CoVoST2, GPQADiamond,
-                        GPQAExtended, GPQAMain, GPQANemoSkills, IFBench,
-                        ImageGenerationEval, JsonModeEval, LongBenchV1,
+                        GPQAExtended, GPQAMain, GPQANemoSkills, GSM8KInferenceX,
+                        IFBench, ImageGenerationEval, JsonModeEval, LongBenchV1,
                         LongBenchV2, MMMUPro, SciCode)
 from ..llmapi import KvCacheConfig
 from ..llmapi.llm_args import TorchLlmArgs
@@ -262,6 +262,7 @@ def main(ctx, model: str, tokenizer: Optional[str],
 main.add_command(CnnDailymail.command)
 main.add_command(MMLU.command)
 main.add_command(GSM8K.command)
+main.add_command(GSM8KInferenceX.command)
 main.add_command(GPQADiamond.command)
 main.add_command(GPQAMain.command)
 main.add_command(GPQAExtended.command)

tensorrt_llm/evaluate/__init__.py

@@ -18,7 +18,8 @@
 from .covost2 import CoVoST2
 from .json_mode_eval import JsonModeEval
 from .lm_eval import (AIME2025, AIME2026, GSM8K, MMMU, GPQADiamond,
-                      GPQAExtended, GPQAMain, LongBenchV1, MMMUPro)
+                      GPQAExtended, GPQAMain, GSM8KInferenceX, LongBenchV1,
+                      MMMUPro)
 from .longbench_v2 import LongBenchV2
 from .mmlu import MMLU
 from .nemo_skills_eval import (AALCR, HLE, ArenaHard, GPQANemoSkills, IFBench,
@@ -29,6 +30,7 @@
     "CnnDailymail",
     "MMLU",
     "GSM8K",
+    "GSM8KInferenceX",
     "GPQADiamond",
     "GPQAMain",
     "GPQAExtended",

tensorrt_llm/evaluate/lm_eval.py

@@ -855,7 +855,9 @@ def __init__(self,
                  post_process_fn: Optional[Callable[[str], str]] = None,
                  preserve_caller_max_tokens: bool = False,
                  num_fewshot: Optional[int] = None,
-                 stop_strings: Optional[List[str]] = None):
+                 stop_strings: Optional[List[str]] = None,
+                 shuffle_dataset: bool = True,
+                 fewshot_random_seed: Optional[int] = None):
         try:
             import lm_eval
         except ImportError as e:
@@ -889,6 +891,11 @@ def __init__(self,
         # larger than the lm-eval task's max_gen_toks. Used by thinking
         # models (e.g. Kimi K2.5) whose CoT output exceeds the task default.
         self.preserve_caller_max_tokens = preserve_caller_max_tokens
+        # shuffle_dataset=False and fewshot_random_seed reproduce the lm-eval
+        # CLI's exemplar selection (unshuffled data, few-shot sampler seed).
+        self.shuffle_dataset = shuffle_dataset
+        self.fewshot_random_seed = (random_seed if fewshot_random_seed is None
+                                    else fewshot_random_seed)
 
         task_manager = TaskManager(
             include_path=f"{os.path.dirname(__file__)}/lm_eval_tasks")
@@ -909,7 +916,7 @@ def _adjust_config(task_dict, random_seed):
                     }
                 else:
                     # NOTE: Few-shot random seed
-                    task_obj.set_fewshot_seed(seed=random_seed)
+                    task_obj.set_fewshot_seed(seed=self.fewshot_random_seed)
                     # Caller override of the task yaml's shot count, the same
                     # call lm-eval's own simple_evaluate makes. Without it a
                     # 0-shot chat evaluation of a task whose yaml pins 5 shots
@@ -925,9 +932,10 @@ def _adjust_config(task_dict, random_seed):
                     adjusted_task_dict[task_name] = task_obj
 
                     # NOTE: Shuffle dataset
-                    data = adjusted_task_dict[task_name].dataset
-                    for split in data.keys():
-                        data[split] = data[split].shuffle(random_seed)
+                    if self.shuffle_dataset:
+                        data = adjusted_task_dict[task_name].dataset
+                        for split in data.keys():
+                            data[split] = data[split].shuffle(random_seed)
 
             return adjusted_task_dict
 
@@ -1252,6 +1260,126 @@ def command(ctx, **kwargs) -> None:
         GSM8K.command_harness(ctx, **kwargs)
 
 
+class GSM8KInferenceX(LmEvalEvaluator):
+    """GSM8K under the InferenceX (formerly InferenceMAX) protocol.
+
+    Mirrors SemiAnalysisAI/InferenceX (infx/evals/gsm8k.yaml + run_lm_eval):
+    chat template with the 5 exemplars in the single user turn (InferenceX
+    passes --apply_chat_template but not --fewshot_as_multiturn), the
+    documented 12288-token eval-only generation budget, strict "#### N"
+    extraction, and lm-eval's default exemplar selection (unshuffled data,
+    few-shot seed 1234), so scores are comparable to
+    inferencex.semianalysis.com/evaluation. Thinking mode follows the chat
+    template default unless chat_template_kwargs sets it; InferenceX serves
+    with thinking enabled where the framework offers a server-side default.
+    The answer filter sees the raw generation, whereas a served InferenceX
+    run scores the reasoning-parsed content; this only matters if a model
+    writes "#### N" inside its thinking.
+    """
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault("apply_chat_template", True)
+        kwargs.setdefault("fewshot_as_multiturn", False)
+        kwargs.setdefault("shuffle_dataset", False)
+        kwargs.setdefault("fewshot_random_seed", 1234)
+        super().__init__("gsm8k_inferencex", **kwargs)
+
+    @click.command("gsm8k_inferencex")
+    @click.option("--dataset_path",
+                  type=str,
+                  default=None,
+                  help="The path to GSM8K dataset. "
+                  "If unspecified, the dataset is downloaded from HF hub.")
+    @click.option(
+        "--num_samples",
+        type=int,
+        default=None,
+        help="Number of samples to run the evaluation; None means full dataset."
+    )
+    @click.option("--random_seed",
+                  type=int,
+                  default=0,
+                  help="Random seed; data order is fixed under this protocol.")
+    @click.option("--apply_chat_template",
+                  type=click.BOOL,
+                  default=True,
+                  help="Whether to apply chat template.")
+    @click.option(
+        "--chat_template_kwargs",
+        type=str,
+        default=None,
+        callback=lambda ctx, param, value: json.loads(value) if value else None,
+        help=
+        'Chat template kwargs as JSON string, e.g., \'{"thinking_budget": 0}\'')
+    @click.option("--fewshot_as_multiturn",
+                  type=click.BOOL,
+                  default=False,
+                  help="Apply fewshot as multiturn. InferenceX keeps the "
+                  "exemplars in the single user turn.")
+    @click.option("--num_fewshot",
+                  type=int,
+                  default=None,
+                  help="Override the task yaml's shot count. Use 0 with "
+                  "--apply_chat_template for a single-question chat "
+                  "evaluation.")
+    @click.option("--system_prompt",
+                  type=str,
+                  default=None,
+                  help="System prompt.")
+    @click.option("--max_input_length",
+                  type=int,
+                  default=4096,
+                  help="Maximum prompt length.")
+    @click.option("--max_output_length",
+                  type=int,
+                  default=12288,
+                  help="Maximum generation length.")
+    @click.option("--temperature",
+                  type=float,
+                  default=None,
+                  help="Sampling temperature. Overrides task yaml gen_kwargs.")
+    @click.option(
+        "--top_p",
+        type=float,
+        default=None,
+        help="Nucleus sampling top_p. Overrides task yaml gen_kwargs.")
+    @click.option("--top_k",
+                  type=int,
+                  default=None,
+                  help="Top-k sampling. Overrides task yaml gen_kwargs.")
+    @click.option("--sampling_seed",
+                  type=int,
+                  default=None,
+                  help="Random seed for generation sampling.")
+    @click.option(
+        "--stop_strings",
+        type=str,
+        default=None,
+        callback=_parse_stop_strings,
+        help='Replace the task yaml\'s stop strings, as a JSON list, e.g. '
+        '\'["</s>", "<|im_end|>"]\'.')
+    @click.option("--log_samples",
+                  is_flag=True,
+                  default=False,
+                  help="Log sample outputs for debugging.")
+    @click.option("--output_path",
+                  type=str,
+                  default=None,
+                  help="Path to save evaluation results.")
+    @click.option("--output_dir",
+                  type=str,
+                  default=None,
+                  help="Directory to save the task infos.")
+    @click.pass_context
+    @staticmethod
+    def command(ctx, **kwargs) -> None:
+        if kwargs.get("fewshot_as_multiturn", False):
+            assert kwargs.get(
+                "apply_chat_template", False
+            ), "apply_chat_template must be True when fewshot_as_multiturn is True"
+        GSM8KInferenceX.command_harness(ctx, **kwargs)
+
+
 class GPQADiamond(LmEvalEvaluator):
 
     def __init__(self, **kwargs):

tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencex/gsm8k_inferencex.yaml

@@ -0,0 +1,50 @@
+# SemiAnalysisAI/InferenceX infx/evals/gsm8k.yaml (upstream lm-eval gsm8k.yaml with a
+# "#### [number]" answer-format instruction, lm-evaluation-harness#3411), renamed to
+# avoid the registry clash and with max_gen_toks set to InferenceX's documented eval-only
+# budget (16384 context minus 4096 prompt).
+tag:
+  - math_word_problems
+task: gsm8k_inferencex
+dataset_path: openai/gsm8k
+dataset_name: main
+output_type: generate_until
+training_split: train
+fewshot_split: train
+test_split: test
+doc_to_text: "Question: {{question}}\nEnd your response with the answer on the last line, formatted as: #### [number]\nAnswer:"
+doc_to_target: "{{answer}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: false
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+      - "(?s).*#### "
+      - "\\.$"
+generation_kwargs:
+  until:
+    - "</s>"
+    - "<|im_end|>"
+  do_sample: false
+  temperature: 0.0
+  max_gen_toks: 12288
+repeats: 1
+num_fewshot: 5
+filter_list:
+  - name: "strict-match"
+    filter:
+      - function: "regex"
+        group_select: -1
+        regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"
+  - name: "flexible-extract"
+    filter:
+      - function: "regex"
+        group_select: -1
+        regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+      - function: "take_first"
+metadata:
+  version: 3.0

tests/integration/defs/accuracy/README.md

@@ -125,6 +125,8 @@ Model data type and quantization decide the precision in model computation, so a
 
 A direct implication is that multiple test cases with different features may share the same accuracy reference. This is by design. For example, we should expect a test case with tensor parallelism to have very similar accuracy to its single-GPU counterpart.
 
+A reference row may set `threshold` to an explicit pass/fail floor that replaces the computed hypothesis-test threshold; see [references/gsm8k_inferencex.yaml](./references/gsm8k_inferencex.yaml).
+
 #### Testing Logic
 
 As aforementioned, each test case evaluates the accuracy of a model with some specifications by running one or multiple tasks.

tests/integration/defs/accuracy/accuracy_core.py

@@ -120,19 +120,21 @@ class HypothesisTestingParams:
     sigma: float = 50.0
     higher_is_better: bool = True
     theta: float = field(init=False)
-    threshold: float = field(init=False)
+    # An explicit threshold replaces the one computed from the reference row.
+    threshold: Optional[float] = None
 
     def __post_init__(self) -> None:
         self.theta = compute_theta(self.num_samples,
                                    sigma=self.sigma,
                                    alpha=self.alpha,
                                    beta=self.beta)
-        self.threshold = compute_threshold(
-            self.num_samples,
-            self.ref_accuracy,
-            sigma=self.sigma,
-            alpha=self.alpha,
-            higher_is_better=self.higher_is_better)
+        if self.threshold is None:
+            self.threshold = compute_threshold(
+                self.num_samples,
+                self.ref_accuracy,
+                sigma=self.sigma,
+                alpha=self.alpha,
+                higher_is_better=self.higher_is_better)
 
     def report(self, accuracy: Optional[float] = None) -> str:
         metric_name = self.metric_name.upper()
@@ -326,7 +328,8 @@ def get_hypothesis_testing_params(self,
             sigma=entry.get("sigma", self.SIGMA),
             num_samples=entry.get("num_samples", self.NUM_SAMPLES),
             higher_is_better=entry.get("higher_is_better",
-                                       self.HIGHER_IS_BETTER))
+                                       self.HIGHER_IS_BETTER),
+            threshold=entry.get("threshold"))
 
     def evaluate(self,
                  llm: Union[PyTorchLLM, AutoDeployLLM],
@@ -550,6 +553,26 @@ class GSM8K(AccuracyTask):
     EVALUATE_KWARGS = dict(scores_filter=None)
 
 
+class GSM8KInferenceX(AccuracyTask):
+    # InferenceX-protocol GSM8K, see tensorrt_llm.evaluate.GSM8KInferenceX.
+    DATASET = "gsm8k_inferencex"
+    DATASET_DIR = f"{llm_models_root()}/datasets/openai/gsm8k"
+
+    ALPHA = 0.05
+    BETA = 0.2
+    SIGMA = 50
+    NUM_SAMPLES = 1319  # Full sample
+
+    MAX_INPUT_LEN = 4096
+    MAX_OUTPUT_LEN = 12288
+
+    EVALUATOR_CLS = tensorrt_llm.evaluate.GSM8KInferenceX
+    EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, random_seed=0)
+
+    # InferenceX reports the strict-match score.
+    EVALUATE_KWARGS = dict(scores_filter="exact_match,strict-match")
+
+
 class GPQADiamond(AccuracyTask):
     DATASET = "gpqa_diamond"
     DATASET_DIR = f"{llm_models_root()}/datasets/gpqa"

tests/integration/defs/accuracy/references/gsm8k_inferencex.yaml

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# GSM8K under the InferenceX protocol (GSM8KInferenceX in accuracy_core.py).
+# accuracy: mean of the published InferenceX vLLM B200 FP4 rows run with thinking enabled
+# (inferencex.semianalysis.com/api/v1/evaluations, 2026-08). threshold: explicit floor
+# (InferenceX's own gate is 90.0).
+nvidia/MiniMax-M3-NVFP4:
+  # MSA path, FP8 KV cache, one-model Eagle3, thinking_mode enabled (rows: 97.27, 96.74).
+  - quant_algo: MIXED_PRECISION
+    kv_cache_quant_algo: FP8
+    spec_dec_algo: Eagle3
+    accuracy: 97.0
+    threshold: 94.0