add more scoring regex and create passing verifier config

examples/10_DeepSeekV4Pro_Example/working-config-aime.yaml

@@ -0,0 +1,51 @@
+name: deepseek-v4-pro-aime25-concurrency256-runrun_01
+version: '1.0'
+type: online
+timeout: 7200.0
+model_params:
+  name: /models/deepseek-ai/DeepSeek-V4-Pro
+  temperature: 1.0
+  max_new_tokens: 256000
+  top_k: -1
+  top_p: 1.0
+  streaming: 'on'
+  chat_template_kwargs:
+    thinking: true
+    reasoning_effort: max
+datasets:
+- name: perf-stub
+  type: performance
+  path: examples/10_DeepSeekV4Pro_Example/data/vllm_smoke.jsonl
+  samples: 1
+  parser:
+    prompt: text_input
+- name: aime25::deepseek_v4
+  type: accuracy
+  accuracy_config:
+    eval_method: pass_at_1
+    ground_truth: answer
+    extractor: boxed_math_extractor
+    num_repeats: 1
+settings:
+  runtime:
+    min_duration_ms: 100
+    max_duration_ms: 0
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    type: concurrency
+    target_qps: null
+    target_concurrency: 256
+  client:
+    num_workers: 32
+    worker_initialization_timeout: 600.0
+    warmup_connections: 0
+    min_required_connections: 0
+  warmup:
+    enabled: false
+endpoint_config:
+  endpoints:
+  - http://127.0.0.1:8000
+  api_key: null
+  api_type: openai
+report_dir: /lustre/fsw/coreai_mlperf_inference/zihaok/dsv4_investigation/AI_benchmark_run_guide/aime25/run_01/report

examples/10_DeepSeekV4Pro_Example/working-config-gpqa.yaml

@@ -0,0 +1,51 @@
+name: deepseek-v4-pro-gpqa-patch-concurrency256-256k-run_01
+version: '1.0'
+type: online
+timeout: 14400.0
+model_params:
+  name: /models/deepseek-ai/DeepSeek-V4-Pro
+  temperature: 1.0
+  max_new_tokens: 256000
+  top_k: -1
+  top_p: 1.0
+  streaming: 'on'
+  chat_template_kwargs:
+    thinking: true
+    reasoning_effort: max
+datasets:
+- name: perf-stub
+  type: performance
+  path: examples/10_DeepSeekV4Pro_Example/data/vllm_smoke.jsonl
+  samples: 1
+  parser:
+    prompt: text_input
+- name: gpqa::deepseek_v4
+  type: accuracy
+  accuracy_config:
+    eval_method: pass_at_1
+    extractor: abcd_extractor
+    ground_truth: ground_truth
+    num_repeats: 1
+settings:
+  runtime:
+    min_duration_ms: 100
+    max_duration_ms: 0
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    type: concurrency
+    target_qps: null
+    target_concurrency: 256
+  client:
+    num_workers: 32
+    worker_initialization_timeout: 1200.0
+    warmup_connections: 0
+    min_required_connections: 0
+  warmup:
+    enabled: false
+endpoint_config:
+  endpoints:
+  - http://127.0.0.1:8000
+  api_key: null
+  api_type: openai
+report_dir: /lustre/fsw/coreai_mlperf_inference/zihaok/dsv4_investigation/AI_benchmark_run_guide/gpqa_patch_256k/run_01/report

examples/10_DeepSeekV4Pro_Example/working-config-lcb.yaml

@@ -0,0 +1,50 @@
+name: deepseek-v4-pro-livecodebench-patch-concurrency256-256k-run_01
+version: '1.0'
+type: online
+timeout: 14400.0
+model_params:
+  name: /models/deepseek-ai/DeepSeek-V4-Pro
+  temperature: 1.0
+  max_new_tokens: 256000
+  top_k: -1
+  top_p: 1.0
+  streaming: 'on'
+  chat_template_kwargs:
+    thinking: true
+    reasoning_effort: max
+datasets:
+- name: perf-stub
+  type: performance
+  path: examples/10_DeepSeekV4Pro_Example/data/vllm_smoke.jsonl
+  samples: 1
+  parser:
+    prompt: text_input
+- name: livecodebench::deepseek_v4
+  type: accuracy
+  accuracy_config:
+    eval_method: code_bench_scorer
+    extractor: python_code_extractor
+    num_repeats: 1
+settings:
+  runtime:
+    min_duration_ms: 100
+    max_duration_ms: 0
+    scheduler_random_seed: 42
+    dataloader_random_seed: 42
+  load_pattern:
+    type: concurrency
+    target_qps: null
+    target_concurrency: 256
+  client:
+    num_workers: 32
+    worker_initialization_timeout: 1200.0
+    warmup_connections: 0
+    min_required_connections: 0
+  warmup:
+    enabled: false
+endpoint_config:
+  endpoints:
+  - http://127.0.0.1:8000
+  api_key: null
+  api_type: openai
+report_dir: /lustre/fsw/coreai_mlperf_inference/zihaok/dsv4_investigation/AI_benchmark_run_guide/livecodebench_patch_256k/run_01/report

src/inference_endpoint/commands/benchmark/execute.py

@@ -541,7 +541,15 @@ def _build_phases(
             )
         # Accuracy phases use bounded concurrency (not MAX_THROUGHPUT burst) so long
         # runs do not queue thousands of requests on the inference server at once.
-        acc_concurrency = max(1, ctx.config.settings.client.num_workers)
+        # If the top-level run is configured for fixed concurrency, keep that
+        # requested concurrency for accuracy instead of coupling it to the
+        # number of client worker processes.
+        lp = ctx.config.settings.load_pattern
+        if lp.type == LoadPatternType.CONCURRENCY and lp.target_concurrency is not None:
+            acc_concurrency = lp.target_concurrency
+        else:
+            acc_concurrency = ctx.config.settings.client.num_workers
+        acc_concurrency = max(1, acc_concurrency)
         acc_load_pattern: LoadPattern | None = LoadPattern(
             type=LoadPatternType.CONCURRENCY,
             target_concurrency=acc_concurrency,

src/inference_endpoint/evaluation/extractor.py

@@ -111,8 +111,70 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"):
         'choice4'
     """
 
+    CHOICE_MAP = {
+        "A": "choice1",
+        "B": "choice2",
+        "C": "choice3",
+        "D": "choice4",
+    }
+
+    # Each pattern below captures the answer letter as group 1.
+    FINAL_ANSWER_PATTERNS = [
+        # JSON-ish final responses.
+        # Examples: {"answer": "A"}, 'answer': 'C'
+        re.compile(r"""(?is)["']answer["']\s*:\s*["']?\s*([ABCD])\b"""),
+        # Explicit final-answer statements.
+        # Examples: "Final answer: B", "answer is (D)", "Answer = **C**"
+        re.compile(
+            r"""(?ix)
+            \b(?:final\s+answer|answer)\b
+            \s*(?:is|:|=)?\s*
+            (?:\\boxed\{\s*)?
+            (?:\*{1,2}|_{1,2})?
+            \(?\s*([ABCD])\b
+            """
+        ),
+        # Explicit option/choice statements near the end of the response.
+        # Examples: "option C", "Choice: (A)", "the correct choice is **B**"
+        re.compile(
+            r"""(?ix)
+            \b(?:option|choice)\b
+            \s*(?:is|:|=)?\s*
+            (?:\*{1,2}|_{1,2})?
+            \(?\s*([ABCD])\b
+            """
+        ),
+        # Boxed answers.
+        # Examples: "\\boxed{D}", "\\boxed{\\text{A}}", "\\boxed{\\textbf{C}}"
+        re.compile(
+            r"""(?is)\\boxed\{\s*(?:\\(?:text|textbf)\{\s*)?([ABCD])\b"""
+        ),
+        # A final standalone line such as "C", "**D**", or "(B)".
+        # Examples: final line "A", final line "**D**", final line "(B)"
+        re.compile(
+            r"""(?im)^\s*
+            (?:\*{1,2}|_{1,2})?
+            \(?\s*([ABCD])\s*\)?
+            (?:\*{1,2}|_{1,2})?
+            \s*[\.\)]?\s*$
+            """
+        ),
+        # A final answer line with the option text included, e.g. "(D) foo".
+        # Examples: "(D) all of the above", "**(B)** pressure increases"
+        re.compile(
+            r"""(?im)^\s*
+            (?:\*{1,2}|_{1,2})?
+            \(\s*([ABCD])\s*\)
+            (?:\*{1,2}|_{1,2})?
+            \s+\S
+            """
+        ),
+    ]
+
+    # Each fallback pattern below also captures the answer letter as group 1.
     PATTERNS = [
         # 0) "**Answer:** A" or "*Answers* – B", i.e. markdown-wrapped "Answer(s)" with an unwrapped letter.
+        # Examples: "**Answer:** A", "*Answers* - B", "__Answer__ C"
         re.compile(
             r"""(?ix)                   # case-insensitive, ignore-space
             (?:\*{1,2}|_{1,2})          # leading *…*  or _…_
@@ -125,6 +187,7 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"):
             re.X,
         ),
         # 0.1) Answer with optional markdown and colons
+        # Examples: "Answer: **D**", "**Answer:** C", "answer B"
         re.compile(
             r"""(?ix)           # ignore case, allow verbose mode
             ^\s*                      # optional leading whitespace
@@ -140,31 +203,41 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"):
             re.MULTILINE,
         ),
         # 1) Answer: (C)   or   Answers: (B)
+        # Examples: "Answer: (C)", "Answers - (B)"
         re.compile(r"(?ix)\bAnswer[s]?\b\s*[:\-–]?\s*\(\s*([ABCD])\s*\)"),
         # 2) Answer: C    or   Answers – D
+        # Examples: "Answer: C", "Answers - D"
         re.compile(r"(?ix)\bAnswer[s]?\b\s*[:\-–]?\s*([ABCD])\b"),
         # 3) Option B   or   Choice: C
+        # Examples: "Option B", "Choice: C"
         re.compile(r"(?ix)\b(?:Option|Choice)\b\s*[:\-–]?\s*([ABCD])\b"),
         # 7) LaTeX \boxed{...A...}, catches both \boxed{A} and
         #    \boxed{\text{A } 2.08\times10^{-6}\,\mathrm{m}} etc.
+        # Examples: "\\boxed{A}", "\\boxed{the answer is C}"
         re.compile(r"(?x)\\boxed\{[^}]*?([ABCD])[^}]*\}", re.MULTILINE),
         # 7.5) LaTeX \boxed{\textbf{...C...}}
+        # Examples: "\\boxed{\\textbf{C}}", "\\boxed{\\textbf{choice D}}"
         re.compile(
             r"(?x)\\boxed\{[^}]*?\\textbf\{[^}]*?([ABCD])[^}]*\}[^}]*\}", re.MULTILINE
         ),
         # 7.51) LaTeX \boxed{\text{...C...}}
+        # Examples: "\\boxed{\\text{B}}", "\\boxed{\\text{Answer: A}}"
         re.compile(
             r"(?x)\\boxed\{[^}]*?\\text\{[^}]*?([ABCD])[^}]*\}[^}]*\}", re.MULTILINE
         ),
         # 4) bare singletons:  (A)  [B]
+        # Examples: "(A)", "[B]"
         re.compile(r"(?x)(?<![A-Za-z0-9])[\(\[]\s*([ABCD])\s*[\)\]](?![A-Za-z0-9])"),
         # 5) Markdown-wrapped: *A*  **B**  _C_  __D__
+        # Examples: "*A*", "**B**", "_C_", "__D__"
         re.compile(
             r"(?x)(?<![A-Za-z0-9])(?:\*{1,2}|_{1,2})([ABCD])(?:\*{1,2}|_{1,2})(?![A-Za-z0-9])"
         ),
         # 6) LaTeX \textbf{...C...}
+        # Examples: "\\textbf{C}", "\\textbf{Answer D}"
         re.compile(r"(?x)\\textbf\{[^}]*?([ABCD])[^}]*\}"),
         # 8) markdown-wrapped answer plus ")" plus description, e.g. **D) …**
+        # Examples: "**D) all of the above**", "_B) pressure increases_"
         re.compile(r"""(?x)                        # ignore whitespace in pattern
             (?<![A-Za-z0-9])            # not preceded by word-char
             (?:\*{1,2}|_{1,2})          # opening ** or __ or * or _
@@ -174,6 +247,7 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"):
             (?![A-Za-z0-9])             # not followed by word-char
         """),
         # 9) final fallback: a line that's exactly "A", "B.", "C)", "**D**", etc.
+        # Examples: "A", "B.", "C)", "**D**", "A - because ..."
         re.compile(
             r"""(?x)^\s*
             (?:\*{1,2}|_{1,2})?     # optional markdown wrapper
@@ -188,6 +262,33 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"):
 
     @classmethod
     def extract(cls, text: str, default: str | None = None) -> str | None:
+        if not text or not isinstance(text, str):
+            return default if default is not None else ""
+
+        # Reasoning models often produce a long rationale followed by a concise
+        # final answer. Prefer explicit answer forms near the response tail before
+        # broad option-list fallbacks such as "(A)" or "(B)".
+        #
+        # Rightmost-match rule: scan only the last 6000 characters, collect every
+        # final-answer regex match as (start_offset, end_offset, letter), then
+        # select max(..., key=(start_offset, end_offset)). This guarantees that
+        # the latest regex match inside the scanned tail wins, e.g. an earlier
+        # "Choice: A" is overridden by a later "Final answer: D". This is still a
+        # regex-position guarantee, not a semantic guarantee: if the true answer
+        # is outside the tail, does not match these regexes, or is followed by a
+        # later false-positive answer-like string, the extractor can still choose
+        # the wrong letter.
+        tail = text.strip()[-6000:]
+        final_matches: list[tuple[int, int, str]] = []
+        for pat in cls.FINAL_ANSWER_PATTERNS:
+            for m in pat.finditer(tail):
+                letter = m.group(1).upper()
+                if letter in cls.CHOICE_MAP:
+                    final_matches.append((m.start(), m.end(), letter))
+        if final_matches:
+            _, _, letter = max(final_matches, key=lambda item: (item[0], item[1]))
+            return cls.CHOICE_MAP[letter]
+
         matches = []
         for prio, pat in enumerate(cls.PATTERNS):
             m = pat.search(text)
@@ -199,23 +300,16 @@ def extract(cls, text: str, default: str | None = None) -> str | None:
         # Sort by priority (lower is better) and then by match length (shorter is better)
         matches.sort(key=lambda triple: (triple[0], len(triple[1].group(0))))
 
-        choice_map = {
-            "A": "choice1",
-            "B": "choice2",
-            "C": "choice3",
-            "D": "choice4",
-        }
-
         # Return the best match
         for _, _, letter in matches:
-            return choice_map[letter]
+            return cls.CHOICE_MAP[letter]
 
         # Final fallback from OpenAI: take first character after stripping markdown
         # This is a last resort if no patterns matched
         stripped = text.removeprefix("**")
         if stripped and stripped[0].upper() in "ABCD":
             abcd_choice = stripped[0].upper()
-            return choice_map[abcd_choice]
+            return cls.CHOICE_MAP[abcd_choice]
 
         return default if default is not None else ""
 

src/inference_endpoint/evaluation/scoring.py

@@ -47,13 +47,24 @@
     _nltk = None
 
 from ..core.record import EventRecord, EventType, SampleEventType
+from ..core.types import TextModelOutput
 from ..dataset_manager.dataset import Dataset
 from ..dataset_manager.predefined.shopify_product_catalogue import ProductMetadata
 from .extractor import Extractor, PythonCodeExtractor
 
 logger = logging.getLogger(__name__)
 
 
+def _join_output_parts(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (tuple, list)):
+        return "".join(str(part) for part in value)
+    return str(value)
+
+
 class Scorer(ABC):
     """Scorers will read in a dataset and outputs from a log and compute an accuracy score.
     An optional extractor can be provided to post-process the output to extract values that
@@ -151,7 +162,10 @@ def get_outputs(self):
                     continue
                 record = decoder.decode(stripped)
                 if record.event_type == SampleEventType.COMPLETE:
-                    output_text = str(record.data) if record.data is not None else ""
+                    if isinstance(record.data, TextModelOutput):
+                        output_text = _join_output_parts(record.data.output)
+                    else:
+                        output_text = str(record.data) if record.data is not None else ""
                     outputs.append(
                         {"sample_uuid": record.sample_uuid, "output": output_text}
                     )