From 5a54dfc243caa2c18448c7908322522f470d332b Mon Sep 17 00:00:00 2001 From: Zihao K Date: Thu, 2 Jul 2026 23:19:02 -0700 Subject: [PATCH] add more scoring regex and create passing verifier config --- .../working-config-aime.yaml | 51 ++++++++ .../working-config-gpqa.yaml | 51 ++++++++ .../working-config-lcb.yaml | 50 ++++++++ .../commands/benchmark/execute.py | 10 +- .../evaluation/extractor.py | 112 ++++++++++++++++-- src/inference_endpoint/evaluation/scoring.py | 16 ++- 6 files changed, 279 insertions(+), 11 deletions(-) create mode 100644 examples/10_DeepSeekV4Pro_Example/working-config-aime.yaml create mode 100644 examples/10_DeepSeekV4Pro_Example/working-config-gpqa.yaml create mode 100644 examples/10_DeepSeekV4Pro_Example/working-config-lcb.yaml diff --git a/examples/10_DeepSeekV4Pro_Example/working-config-aime.yaml b/examples/10_DeepSeekV4Pro_Example/working-config-aime.yaml new file mode 100644 index 00000000..b84bf36f --- /dev/null +++ b/examples/10_DeepSeekV4Pro_Example/working-config-aime.yaml @@ -0,0 +1,51 @@ +name: deepseek-v4-pro-aime25-concurrency256-runrun_01 +version: '1.0' +type: online +timeout: 7200.0 +model_params: + name: /models/deepseek-ai/DeepSeek-V4-Pro + temperature: 1.0 + max_new_tokens: 256000 + top_k: -1 + top_p: 1.0 + streaming: 'on' + chat_template_kwargs: + thinking: true + reasoning_effort: max +datasets: +- name: perf-stub + type: performance + path: examples/10_DeepSeekV4Pro_Example/data/vllm_smoke.jsonl + samples: 1 + parser: + prompt: text_input +- name: aime25::deepseek_v4 + type: accuracy + accuracy_config: + eval_method: pass_at_1 + ground_truth: answer + extractor: boxed_math_extractor + num_repeats: 1 +settings: + runtime: + min_duration_ms: 100 + max_duration_ms: 0 + scheduler_random_seed: 42 + dataloader_random_seed: 42 + load_pattern: + type: concurrency + target_qps: null + target_concurrency: 256 + client: + num_workers: 32 + worker_initialization_timeout: 600.0 + warmup_connections: 0 + min_required_connections: 0 + warmup: + enabled: false +endpoint_config: + endpoints: + - http://127.0.0.1:8000 + api_key: null + api_type: openai +report_dir: /lustre/fsw/coreai_mlperf_inference/zihaok/dsv4_investigation/AI_benchmark_run_guide/aime25/run_01/report diff --git a/examples/10_DeepSeekV4Pro_Example/working-config-gpqa.yaml b/examples/10_DeepSeekV4Pro_Example/working-config-gpqa.yaml new file mode 100644 index 00000000..c58421e1 --- /dev/null +++ b/examples/10_DeepSeekV4Pro_Example/working-config-gpqa.yaml @@ -0,0 +1,51 @@ +name: deepseek-v4-pro-gpqa-patch-concurrency256-256k-run_01 +version: '1.0' +type: online +timeout: 14400.0 +model_params: + name: /models/deepseek-ai/DeepSeek-V4-Pro + temperature: 1.0 + max_new_tokens: 256000 + top_k: -1 + top_p: 1.0 + streaming: 'on' + chat_template_kwargs: + thinking: true + reasoning_effort: max +datasets: +- name: perf-stub + type: performance + path: examples/10_DeepSeekV4Pro_Example/data/vllm_smoke.jsonl + samples: 1 + parser: + prompt: text_input +- name: gpqa::deepseek_v4 + type: accuracy + accuracy_config: + eval_method: pass_at_1 + extractor: abcd_extractor + ground_truth: ground_truth + num_repeats: 1 +settings: + runtime: + min_duration_ms: 100 + max_duration_ms: 0 + scheduler_random_seed: 42 + dataloader_random_seed: 42 + load_pattern: + type: concurrency + target_qps: null + target_concurrency: 256 + client: + num_workers: 32 + worker_initialization_timeout: 1200.0 + warmup_connections: 0 + min_required_connections: 0 + warmup: + enabled: false +endpoint_config: + endpoints: + - http://127.0.0.1:8000 + api_key: null + api_type: openai +report_dir: /lustre/fsw/coreai_mlperf_inference/zihaok/dsv4_investigation/AI_benchmark_run_guide/gpqa_patch_256k/run_01/report diff --git a/examples/10_DeepSeekV4Pro_Example/working-config-lcb.yaml b/examples/10_DeepSeekV4Pro_Example/working-config-lcb.yaml new file mode 100644 index 00000000..6ff7ea0a --- /dev/null +++ b/examples/10_DeepSeekV4Pro_Example/working-config-lcb.yaml @@ -0,0 +1,50 @@ +name: deepseek-v4-pro-livecodebench-patch-concurrency256-256k-run_01 +version: '1.0' +type: online +timeout: 14400.0 +model_params: + name: /models/deepseek-ai/DeepSeek-V4-Pro + temperature: 1.0 + max_new_tokens: 256000 + top_k: -1 + top_p: 1.0 + streaming: 'on' + chat_template_kwargs: + thinking: true + reasoning_effort: max +datasets: +- name: perf-stub + type: performance + path: examples/10_DeepSeekV4Pro_Example/data/vllm_smoke.jsonl + samples: 1 + parser: + prompt: text_input +- name: livecodebench::deepseek_v4 + type: accuracy + accuracy_config: + eval_method: code_bench_scorer + extractor: python_code_extractor + num_repeats: 1 +settings: + runtime: + min_duration_ms: 100 + max_duration_ms: 0 + scheduler_random_seed: 42 + dataloader_random_seed: 42 + load_pattern: + type: concurrency + target_qps: null + target_concurrency: 256 + client: + num_workers: 32 + worker_initialization_timeout: 1200.0 + warmup_connections: 0 + min_required_connections: 0 + warmup: + enabled: false +endpoint_config: + endpoints: + - http://127.0.0.1:8000 + api_key: null + api_type: openai +report_dir: /lustre/fsw/coreai_mlperf_inference/zihaok/dsv4_investigation/AI_benchmark_run_guide/livecodebench_patch_256k/run_01/report diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index e193ee13..2d05f4f1 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -541,7 +541,15 @@ def _build_phases( ) # Accuracy phases use bounded concurrency (not MAX_THROUGHPUT burst) so long # runs do not queue thousands of requests on the inference server at once. - acc_concurrency = max(1, ctx.config.settings.client.num_workers) + # If the top-level run is configured for fixed concurrency, keep that + # requested concurrency for accuracy instead of coupling it to the + # number of client worker processes. + lp = ctx.config.settings.load_pattern + if lp.type == LoadPatternType.CONCURRENCY and lp.target_concurrency is not None: + acc_concurrency = lp.target_concurrency + else: + acc_concurrency = ctx.config.settings.client.num_workers + acc_concurrency = max(1, acc_concurrency) acc_load_pattern: LoadPattern | None = LoadPattern( type=LoadPatternType.CONCURRENCY, target_concurrency=acc_concurrency, diff --git a/src/inference_endpoint/evaluation/extractor.py b/src/inference_endpoint/evaluation/extractor.py index 99d07db4..7cae8566 100644 --- a/src/inference_endpoint/evaluation/extractor.py +++ b/src/inference_endpoint/evaluation/extractor.py @@ -111,8 +111,70 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"): 'choice4' """ + CHOICE_MAP = { + "A": "choice1", + "B": "choice2", + "C": "choice3", + "D": "choice4", + } + + # Each pattern below captures the answer letter as group 1. + FINAL_ANSWER_PATTERNS = [ + # JSON-ish final responses. + # Examples: {"answer": "A"}, 'answer': 'C' + re.compile(r"""(?is)["']answer["']\s*:\s*["']?\s*([ABCD])\b"""), + # Explicit final-answer statements. + # Examples: "Final answer: B", "answer is (D)", "Answer = **C**" + re.compile( + r"""(?ix) + \b(?:final\s+answer|answer)\b + \s*(?:is|:|=)?\s* + (?:\\boxed\{\s*)? + (?:\*{1,2}|_{1,2})? + \(?\s*([ABCD])\b + """ + ), + # Explicit option/choice statements near the end of the response. + # Examples: "option C", "Choice: (A)", "the correct choice is **B**" + re.compile( + r"""(?ix) + \b(?:option|choice)\b + \s*(?:is|:|=)?\s* + (?:\*{1,2}|_{1,2})? + \(?\s*([ABCD])\b + """ + ), + # Boxed answers. + # Examples: "\\boxed{D}", "\\boxed{\\text{A}}", "\\boxed{\\textbf{C}}" + re.compile( + r"""(?is)\\boxed\{\s*(?:\\(?:text|textbf)\{\s*)?([ABCD])\b""" + ), + # A final standalone line such as "C", "**D**", or "(B)". + # Examples: final line "A", final line "**D**", final line "(B)" + re.compile( + r"""(?im)^\s* + (?:\*{1,2}|_{1,2})? + \(?\s*([ABCD])\s*\)? + (?:\*{1,2}|_{1,2})? + \s*[\.\)]?\s*$ + """ + ), + # A final answer line with the option text included, e.g. "(D) foo". + # Examples: "(D) all of the above", "**(B)** pressure increases" + re.compile( + r"""(?im)^\s* + (?:\*{1,2}|_{1,2})? + \(\s*([ABCD])\s*\) + (?:\*{1,2}|_{1,2})? + \s+\S + """ + ), + ] + + # Each fallback pattern below also captures the answer letter as group 1. PATTERNS = [ # 0) "**Answer:** A" or "*Answers* – B", i.e. markdown-wrapped "Answer(s)" with an unwrapped letter. + # Examples: "**Answer:** A", "*Answers* - B", "__Answer__ C" re.compile( r"""(?ix) # case-insensitive, ignore-space (?:\*{1,2}|_{1,2}) # leading *…* or _…_ @@ -125,6 +187,7 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"): re.X, ), # 0.1) Answer with optional markdown and colons + # Examples: "Answer: **D**", "**Answer:** C", "answer B" re.compile( r"""(?ix) # ignore case, allow verbose mode ^\s* # optional leading whitespace @@ -140,31 +203,41 @@ class ABCDExtractor(Extractor, extractor_id="abcd_extractor"): re.MULTILINE, ), # 1) Answer: (C) or Answers: (B) + # Examples: "Answer: (C)", "Answers - (B)" re.compile(r"(?ix)\bAnswer[s]?\b\s*[:\-–]?\s*\(\s*([ABCD])\s*\)"), # 2) Answer: C or Answers – D + # Examples: "Answer: C", "Answers - D" re.compile(r"(?ix)\bAnswer[s]?\b\s*[:\-–]?\s*([ABCD])\b"), # 3) Option B or Choice: C + # Examples: "Option B", "Choice: C" re.compile(r"(?ix)\b(?:Option|Choice)\b\s*[:\-–]?\s*([ABCD])\b"), # 7) LaTeX \boxed{...A...}, catches both \boxed{A} and # \boxed{\text{A } 2.08\times10^{-6}\,\mathrm{m}} etc. + # Examples: "\\boxed{A}", "\\boxed{the answer is C}" re.compile(r"(?x)\\boxed\{[^}]*?([ABCD])[^}]*\}", re.MULTILINE), # 7.5) LaTeX \boxed{\textbf{...C...}} + # Examples: "\\boxed{\\textbf{C}}", "\\boxed{\\textbf{choice D}}" re.compile( r"(?x)\\boxed\{[^}]*?\\textbf\{[^}]*?([ABCD])[^}]*\}[^}]*\}", re.MULTILINE ), # 7.51) LaTeX \boxed{\text{...C...}} + # Examples: "\\boxed{\\text{B}}", "\\boxed{\\text{Answer: A}}" re.compile( r"(?x)\\boxed\{[^}]*?\\text\{[^}]*?([ABCD])[^}]*\}[^}]*\}", re.MULTILINE ), # 4) bare singletons: (A) [B] + # Examples: "(A)", "[B]" re.compile(r"(?x)(? str | None: + if not text or not isinstance(text, str): + return default if default is not None else "" + + # Reasoning models often produce a long rationale followed by a concise + # final answer. Prefer explicit answer forms near the response tail before + # broad option-list fallbacks such as "(A)" or "(B)". + # + # Rightmost-match rule: scan only the last 6000 characters, collect every + # final-answer regex match as (start_offset, end_offset, letter), then + # select max(..., key=(start_offset, end_offset)). This guarantees that + # the latest regex match inside the scanned tail wins, e.g. an earlier + # "Choice: A" is overridden by a later "Final answer: D". This is still a + # regex-position guarantee, not a semantic guarantee: if the true answer + # is outside the tail, does not match these regexes, or is followed by a + # later false-positive answer-like string, the extractor can still choose + # the wrong letter. + tail = text.strip()[-6000:] + final_matches: list[tuple[int, int, str]] = [] + for pat in cls.FINAL_ANSWER_PATTERNS: + for m in pat.finditer(tail): + letter = m.group(1).upper() + if letter in cls.CHOICE_MAP: + final_matches.append((m.start(), m.end(), letter)) + if final_matches: + _, _, letter = max(final_matches, key=lambda item: (item[0], item[1])) + return cls.CHOICE_MAP[letter] + matches = [] for prio, pat in enumerate(cls.PATTERNS): m = pat.search(text) @@ -199,23 +300,16 @@ def extract(cls, text: str, default: str | None = None) -> str | None: # Sort by priority (lower is better) and then by match length (shorter is better) matches.sort(key=lambda triple: (triple[0], len(triple[1].group(0)))) - choice_map = { - "A": "choice1", - "B": "choice2", - "C": "choice3", - "D": "choice4", - } - # Return the best match for _, _, letter in matches: - return choice_map[letter] + return cls.CHOICE_MAP[letter] # Final fallback from OpenAI: take first character after stripping markdown # This is a last resort if no patterns matched stripped = text.removeprefix("**") if stripped and stripped[0].upper() in "ABCD": abcd_choice = stripped[0].upper() - return choice_map[abcd_choice] + return cls.CHOICE_MAP[abcd_choice] return default if default is not None else "" diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index 84a6f655..32d6c774 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -47,6 +47,7 @@ _nltk = None from ..core.record import EventRecord, EventType, SampleEventType +from ..core.types import TextModelOutput from ..dataset_manager.dataset import Dataset from ..dataset_manager.predefined.shopify_product_catalogue import ProductMetadata from .extractor import Extractor, PythonCodeExtractor @@ -54,6 +55,16 @@ logger = logging.getLogger(__name__) +def _join_output_parts(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (tuple, list)): + return "".join(str(part) for part in value) + return str(value) + + class Scorer(ABC): """Scorers will read in a dataset and outputs from a log and compute an accuracy score. An optional extractor can be provided to post-process the output to extract values that @@ -151,7 +162,10 @@ def get_outputs(self): continue record = decoder.decode(stripped) if record.event_type == SampleEventType.COMPLETE: - output_text = str(record.data) if record.data is not None else "" + if isinstance(record.data, TextModelOutput): + output_text = _join_output_parts(record.data.output) + else: + output_text = str(record.data) if record.data is not None else "" outputs.append( {"sample_uuid": record.sample_uuid, "output": output_text} )