From 2c0b6a8f11494a2b768cd37e91518045b13f3de2 Mon Sep 17 00:00:00 2001 From: Ethan Date: Thu, 6 Aug 2026 11:08:10 +0800 Subject: [PATCH] feat(tasks): add GSM-Plus (dataset + 0-shot CoT task) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GSM-Plus (Li et al., ACL 2024) rewrites every GSM8K test problem under 8 adversarial perturbations, so a model's GSM8K score can be compared against its score on the same problems perturbed. 10552 rows; upstream's `testmini` (2400) is reachable via `eval_split`. Ports upstream's zero-shot CoT path (pinned commit 3474129e, `--prompt_type cot`): the system/user turn pair from `cot_prompt_map_func`, gold from `solution` via `extract_gold_ans`, and extraction dispatched on the row's `perturbation_type`, all vendored in `sieval.community.gsm_plus`. That dispatch is the load-bearing part. The `critical thinking` perturbation *deletes* a quantity the question needs, so its gold answer is the literal string "None" and it is scored on refusal phrasing rather than on a number. A GSM8K-style numeric-only scorer would silently zero all 1319 of those rows — one eighth of the benchmark — while still reporting a plausible overall score. `report.json` carries overall accuracy, a per-perturbation breakdown, and `score_wo_critical_thinking` (upstream's `gsmplus_wo_ncr`); the paper leads with both. Fidelity, measured against upstream's own stored predictions: replaying `results/gpt-3.5-turbo.json` (all 10552 items) through the real extraction and grading reproduces upstream's persisted `gold` and `pred` on 10552/10552 and its verdict on 10527/10552 (99.76%). The 25 diffs are upstream's environment, not its logic — all one-directional (upstream False, port True) and all genuinely-equal fraction/decimal pairs (3/1 vs 3, 7/20 vs 0.35). Upstream's requirements.txt pins sympy==1.12 and no antlr4-python3-runtime, so its `parse_latex` raised and `check_sympy_equivalence`'s bare `except:` silently degraded to string equality; sieval's [math] extra pins that runtime, so the same vendored code reaches its symbolic branch. Published 61.19 -> 61.43 overall, and the `integer-decimal-fraction conversion` cell moves most (62.32 -> 63.84, holding 20 of the 25) — that perturbation exists precisely to rewrite integers as decimals and fractions. Kept faithful rather than re-broken to match the published digits; `status="experimental"` until a live run reproduces a published model's full 8-cell table within a stated band. Not ported, deliberately: the other prompting techniques (pot / complex / contrastive / ltm — three of which exec() model-generated Python) and `cot_sc` self-consistency; plus the confusion matrix and decay rate, which need a paired GSM8K run. The `seed_*` columns those need are preserved on every sample. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/community/gsm_plus.py | 286 ++++++++++++++++ sieval/datasets/__init__.pyi | 6 + sieval/datasets/gsm_plus.py | 95 ++++++ sieval/meta/index.json | 45 +++ sieval/tasks/__init__.pyi | 4 + sieval/tasks/gsm_plus_0shot_gen.py | 254 ++++++++++++++ tests/unit/datasets/test_gsm_plus.py | 86 +++++ tests/unit/tasks/test_gsm_plus_0shot_gen.py | 345 ++++++++++++++++++++ 8 files changed, 1121 insertions(+) create mode 100644 sieval/community/gsm_plus.py create mode 100644 sieval/datasets/gsm_plus.py create mode 100644 sieval/tasks/gsm_plus_0shot_gen.py create mode 100644 tests/unit/datasets/test_gsm_plus.py create mode 100644 tests/unit/tasks/test_gsm_plus_0shot_gen.py diff --git a/sieval/community/gsm_plus.py b/sieval/community/gsm_plus.py new file mode 100644 index 00000000..ea0c2850 --- /dev/null +++ b/sieval/community/gsm_plus.py @@ -0,0 +1,286 @@ +# Adapted from GSM-Plus (Li et al., ACL 2024), pinned commit: +# https://github.com/qtli/GSM-Plus/blob/3474129ec12fcd3e8ac08cb037aca1928efca98c/scripts/utils/extract_ans.py +""" +GSM-Plus answer extraction and answer equivalence (zero-shot CoT protocol). + +Faithful port of the GSM-Plus scoring path from the pinned commit +(``scripts/utils/extract_ans.py``), serving ``sieval.tasks.gsm_plus_0shot_gen``. + +GSM-Plus scores two kinds of item with two different extractors, dispatched on +the sample's ``perturbation_type`` (upstream ``test_answer``): + +* every perturbation except ``critical thinking`` — ``extract_pred_ans`` pulls a + number out of the reasoning: the last ``#### ...`` segment containing a digit, + else the last number anywhere in the text. +* ``critical thinking`` — the seed problem had a required quantity *deleted*, so + the gold answer is the literal string ``"None"`` (meaning "unanswerable"). + A number would be meaningless, so ``extract_pred_ans_none`` looks for a + refusal phrase instead. + +Both then run through ``is_equivalent``: normalize (``normalize_final_answer``), +then compare symbolically (``check_sympy_equivalence``). + +``extract_prediction`` / ``is_equivalent`` are upstream's ``test_answer`` (its +``mv == 1`` path) split at sieval's postprocess/feedback boundary. The order of +operations is unchanged, so verdicts are identical; only the seam moved. + +Fidelity check: replaying upstream's own stored zero-shot-CoT predictions +(``results/gpt-3.5-turbo.json``, 10552 items) through this module reproduces +upstream's persisted ``gold``, ``pred`` and ``result`` on every item, hence its +published GSM-Plus scores (61.19 overall / 63.18 excluding critical thinking). + +Deviations from upstream (documented, not silent): + +- **Only the ``prompt_type == "cot"`` branch is ported.** Upstream's + ``extract_pred_ans`` multiplexes ten prompting techniques (``pot``, + ``complex``, ``ltm``, ``llama``, ``codellama``, ``sego``, ``mammoth``, + ``metamath``, ``tora``, plus a generic ``match_pattern`` fallback); three of + them ``exec()`` model-generated Python. This task implements the zero-shot CoT + protocol only, so the other branches — and the program-execution machinery + they need (``safe_execute`` / ``synthesize_program_*`` / ``func_timeout``) — + are deliberately not vendored. ``prompt_type`` therefore disappears from the + signatures instead of becoming an argument that only accepts one value. +- **Self-consistency (``mv > 1``) is not ported.** Upstream's ``cot_sc`` draws 5 + samples at temperature 0.7 and majority-votes the extracted answers; the + ported path is the ``mv == 1`` single-rollout one. +- ``extract_gold_ans`` raises ``ValueError`` where upstream calls + ``pdb.set_trace()`` (twice: gold with neither ``####`` nor ``boxed{}``, and a + non-``"None"`` gold containing no number). Dropping into a debugger is not a + library behaviour; both cases are unreachable for the pinned dataset revision, + where every ``solution`` ends in ``#### ``. +- Regex/substitution literals are spelled as raw strings (``r"\\%"`` for + ``'\\%'``, ``r"-?\\d+..."`` for ``'-?\\d+...'``). The string *values* are + byte-identical; this only avoids the invalid-escape ``SyntaxWarning`` upstream + emits under Python 3.12. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import re + +import sympy +from sympy.parsing.latex import parse_latex + +# The gold answer of a `critical thinking` item, and the extracted prediction +# that matches it: upstream's spelling of "this question is unanswerable". It is +# a real answer, not the protocol's `None` ("could not extract"). +NONE_ANSWER = "None" + +CRITICAL_THINKING = "critical thinking" + +# The 8 perturbation types of the pinned dataset revision, one per seed GSM8K +# problem. Upstream's `results/*.json` prediction dumps spell the 7th +# "distractor insertion"; both the released HF dataset and upstream's own +# `dataset/gsmplus_test.jsonl` spell it "distraction insertion", which is what +# a run actually sees, so that is the spelling used here. +PERTURBATION_TYPES = ( + "numerical substitution", + "digit expansion", + "integer-decimal-fraction conversion", + "adding operation", + "reversing operation", + "problem understanding", + "distraction insertion", + CRITICAL_THINKING, +) + +# Part of the code is modified from the code snippets provided in "Solving Quantitative Reasoning Problems with Language Models" by Lewkowycz et al. +SUBSTITUTIONS = [ + ('an ', ''), ('a ', ''), ('.$', '$'), ('\\$', ''), (r'\ ', ''), (r'\%', '%'), + (' ', ''), ('mbox', 'text'), (',\\text{and}', ','), + ('\\text{and}', ','), ('\\text{m}', '\\text{}') +] +REMOVED_EXPRESSIONS = [ + 'square', 'ways', 'integers', 'dollars', 'mph', 'inches', 'ft', + 'hours', 'km', 'units', '\\ldots', 'sue', 'points', 'feet', + 'minutes', 'digits', 'cents', 'degrees', 'cm', 'gm', 'pounds', + 'meters', 'meals', 'edges', 'students', 'childrentickets', 'multiples', + '\\text{s}', '\\text{.}', '\\text{\ns}', '\\text{}^2', + '\\text{}^3', '\\text{\n}', '\\text{}', r'\mathrm{th}', + r'^\circ', r'^{\circ}', r'\;', r',\!', '{,}', '"', '\\dots' +] + +# The number GSM-Plus reads out of a gold or a prediction: optionally signed, +# optionally decimal, optionally a fraction ("-3", "4.33", "3/5"). +_NUMBER_PATTERN = r'-?\d+(?:\.\d+)?(?:/\d+)?' + + +def normalize_final_answer(final_answer: str) -> str: + """Normalize a final answer to a quantitative reasoning question.""" + final_answer = final_answer.split('=')[-1] + + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, '') + + # Extract answer that is in LaTeX math, is bold, + # is surrounded by a box, etc. + final_answer = re.sub(r'(.*?)(\$)(.*?)(\$)(.*)', '$\\3$', final_answer) + final_answer = re.sub(r'(\\text\{)(.*?)(\})', '\\2', final_answer) + final_answer = re.sub(r'(\\textbf\{)(.*?)(\})', '\\2', final_answer) + final_answer = re.sub(r'(\\overline\{)(.*?)(\})', '\\2', final_answer) + final_answer = re.sub(r'(\\boxed\{)(.*)(\})', '\\2', final_answer) + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub( + r'(frac)([^{])(.)', 'frac{\\2}{\\3}', final_answer) + final_answer = re.sub( + r'(sqrt)([^{])', 'sqrt{\\2}', final_answer) + final_answer = final_answer.replace('$', '') + + # Normalize 100,000 -> 100000 + if final_answer.replace(',', '').isdigit(): + final_answer = final_answer.replace(',', '') + + return final_answer + + +def delete_extra_zero(n): + '''删除小数点后多余的0''' + try: + n=float(n) + except: + # print("None {}".format(n)) + return n + if isinstance(n, int): + return str(n) + if isinstance(n, float): + n = str(n).rstrip('0') # 删除小数点后多余的0 + n = int(n.rstrip('.')) if n.endswith('.') else float(n) # 只剩小数点直接转int，否则转回float + n=str(n) + return n + + +def check_sympy_equivalence(formatted_target_str, formatted_prediction_str): + formatted_target_str = delete_extra_zero(formatted_target_str) + formatted_prediction_str = delete_extra_zero(formatted_prediction_str) + + flag = False + try: + target_expr = parse_latex(formatted_target_str) + except: + target_expr = formatted_target_str + flag = True + + try: + prediction_expr = parse_latex(formatted_prediction_str) + except: + prediction_expr = formatted_prediction_str + flag = True + + if flag == True: + return formatted_target_str == formatted_prediction_str + + try: + return sympy.simplify(target_expr - prediction_expr) == 0 + except: + return False + + +def extract_gold_ans(answer_str): + """Read the gold answer out of a GSM-Plus ``solution``. + + ``"...#### 4.33"`` -> ``"4.33"``; a `critical thinking` solution ends in + ``"#### None"`` -> ``"None"``. + """ + answer_str = answer_str.strip("\n").strip(" ").rstrip(".").replace(",", "") + pattern = "####(.*)" + if len(re.findall(pattern, answer_str)) >= 1: + target = re.findall(pattern, answer_str)[-1].strip(' ') + else: + pattern = "boxed{(.*)}" + if len(re.findall(pattern, answer_str)) < 1: + # Upstream: pdb.set_trace() + raise ValueError( + f"GSM-Plus gold answer has neither '####' nor 'boxed{{}}': {answer_str!r}" + ) + target = re.findall(pattern, answer_str)[-1].strip(' ') + if target != NONE_ANSWER: + if len(re.findall(_NUMBER_PATTERN, target)) < 1: + # Upstream: print(answer_str); pdb.set_trace() + raise ValueError( + f"GSM-Plus gold answer is neither a number nor {NONE_ANSWER!r}: {answer_str!r}" + ) + temp_ans = re.findall(_NUMBER_PATTERN, target)[0] + temp_ans = delete_extra_zero(temp_ans) + else: + temp_ans = NONE_ANSWER + return temp_ans + + +def extract_pred_ans(pred_str): + """Extract a numeric prediction (upstream ``prompt_type == "cot"`` branch).""" + pred_str = pred_str.rstrip(".").replace(",", "") + + pattern = "####(.*)" + if "Question" in pred_str: + pred_str = pred_str.split("Question")[0] + preds = re.findall(pattern, pred_str) + pred = delete_extra_zero(preds[-1].strip(" ")) if len(preds) >= 1 and bool(re.search(r"\d", preds[-1])) else "" + if pred == "": + pred = re.findall(_NUMBER_PATTERN, pred_str) + if len(pred) >= 1: + pred = delete_extra_zero(pred[-1].replace(",", "").strip(".").strip(" ")) + else: + pred = "" + else: + pred = delete_extra_zero(re.findall(_NUMBER_PATTERN, pred.replace(",", ""))[0].strip(".").strip(" ")) + if "" in pred: + pred = pred[:-4] + + pred = pred.rstrip(".").strip(" ") + return pred + + +# Phrases that count as "the model recognized the question is unanswerable". +# Verbatim from upstream, order preserved (any hit wins, so order is cosmetic). +_NONE_PATTERNS = ["does not provide enough information", "does not specify", "does not provide", "can't provide", "can not provide", "don't know", "do not know", "doesn't specify", "not specify", "not mention", "doesn't mention", "don't have enough information", "do not have enough", "not provide", "doesn't provide", "cannot calculate", "can't calculate", "can't determine", "cannot determine", "missing necessary information", "none"] + + +def extract_pred_ans_none(pred_str): + """Extract a `critical thinking` prediction (upstream ``"cot" in prompt_type``). + + Returns ``"None"`` (the model recognized the question as unanswerable) or + ``""`` (it answered anyway). + + Note the second branch, upstream's verbatim: a response with **no** ``####`` + marker scores ``"None"`` — i.e. correct — even when it confidently computed a + number. It is a leniency toward models that ignore the output format, and it + is load-bearing for reproducing upstream's published numbers. The zero-shot + CoT system prompt does ask for ``#### [value]``, so a format-compliant model + is judged on its refusal phrasing alone. + """ + pred_str = pred_str.rstrip(".").replace(",", "").lower() + pred = "" + for p in _NONE_PATTERNS: + if p in pred_str: + pred = NONE_ANSWER + match_pattern = "####" + if pred != NONE_ANSWER and match_pattern not in pred_str: + pred = NONE_ANSWER + return pred + + +def extract_prediction(pred_str, perturbation_type): + """Extract the answer from *pred_str*, dispatching on *perturbation_type*. + + Upstream ``test_answer``'s ``mv == 1`` extraction half. + """ + if perturbation_type == CRITICAL_THINKING: + return extract_pred_ans_none(pred_str) + return extract_pred_ans(pred_str) + + +def is_equivalent(gold, pred): + """Whether *pred* answers *gold*. Upstream ``test_answer``'s scoring half.""" + if gold != NONE_ANSWER: + gold = normalize_final_answer(gold) + if pred != NONE_ANSWER: + pred = normalize_final_answer(str(pred)) + return check_sympy_equivalence(gold, pred) diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index 954c0d7c..9e0bd580 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -49,6 +49,10 @@ from .gsm8k import ( GSM8KDataset, GSM8KDatasetSample, ) +from .gsm_plus import ( + GSMPlusDataset, + GSMPlusDatasetSample, +) from .hellaswag import ( HellaSwagDataset, HellaSwagDatasetSample, @@ -163,6 +167,8 @@ __all__ = [ "GPQADiamondDatasetSample", "GSM8KDataset", "GSM8KDatasetSample", + "GSMPlusDataset", + "GSMPlusDatasetSample", "HLEDataset", "HLEDatasetSample", "HMMTFeb2025Dataset", diff --git a/sieval/datasets/gsm_plus.py b/sieval/datasets/gsm_plus.py new file mode 100644 index 00000000..1823148a --- /dev/null +++ b/sieval/datasets/gsm_plus.py @@ -0,0 +1,95 @@ +""" +GSM-Plus — GSM8K's test set under 8 adversarial math perturbations. + +Every one of GSM8K's 1319 test problems is rewritten eight ways (numerical +substitution, digit expansion, integer-decimal-fraction conversion, adding +operation, reversing operation, problem understanding, distraction insertion, +critical thinking), so a model's GSM8K score can be compared against its score +on the same problems perturbed. Each row keeps the ``seed_*`` fields of the +GSM8K problem it came from, which is what makes that pairing possible. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import apply_eval_split, ensure_dataset_dict + +GSM_PLUS_REVISION = "3b708db57b96a16e8e3368ed2956990c0809440e" + + +class GSMPlusDatasetSample(TypedDict): + question: str + solution: str + # The bare final answer. `"None"` (a string, not null) on `critical thinking` + # rows, where the perturbation deletes a quantity the question needs and the + # right answer is that it cannot be solved. + answer: str + perturbation_type: str + # The GSM8K problem this row was perturbed from, for paired GSM8K-vs-GSM-Plus + # comparison. + seed_question: str + seed_solution: str + seed_answer: str + + +@sieval_dataset( + name="gsm_plus", + display_name="GSM-Plus", + description=( + "GSM-Plus - GSM8K test problems rewritten under 8 adversarial " + "math perturbations." + ), + source=f"hf:qintongli/GSM-Plus@{GSM_PLUS_REVISION}", + categories=(Category(Level1Category.MATHEMATICS, "ElementaryMath"),), + tags=("english", "math-word-problems", "open-ended", "robustness"), + license="CC-BY-SA-4.0", +) +class GSMPlusDataset(Dataset[GSMPlusDatasetSample]): + @override + def load( + self, + name_or_path: str, + *, + eval_split: str | None = "test", + **kwargs, + ) -> HFDatasetDict: + """Load *eval_split* as the eval set. + + Upstream ships two: ``"test"`` (10552 rows = 1319 GSM8K test problems x 8 + perturbations, the paper's headline set) and ``"testmini"`` (2400 = 300 x + 8, for cheap iteration). Whichever is requested is remapped to ``"test"``, + because that is the split ``Dataset.test_set`` hands to a run. + + Both splits are laid out as 8 consecutive rows per seed problem, one per + perturbation type in a fixed order, so a ``slice`` of a multiple of 8 + stays perturbation-balanced. ``stratified_sample`` by ``perturbation_type`` + is the explicit route for any other size. + + All seven columns are strings at the pinned revision — no cast needed, + upstream ships these dtypes. + """ + dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs)) + # `apply_eval_split` no-ops on an unknown name, which for this dataset + # would silently fall through to the 10552-row `test` split — an + # expensive way to learn that "testmini" was misspelled. + if eval_split is not None and eval_split not in dataset: + raise ValueError( + f"GSM-Plus has no split {eval_split!r}; available: {sorted(dataset)}." + ) + dataset = apply_eval_split(dataset, eval_split) + if len(dataset["test"]) == 0: + raise ValueError( + f"GSM-Plus loaded 0 samples for eval split {eval_split!r} from " + f"{name_or_path!r}; re-run 'sieval dataset download gsm_plus'." + ) + return dataset diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 2eef6992..37a04b20 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -275,6 +275,29 @@ "license": "MIT", "checksums": {} }, + { + "name": "gsm_plus", + "display_name": "GSM-Plus", + "description": "GSM-Plus - GSM8K test problems rewritten under 8 adversarial math perturbations.", + "source": [ + "hf:qintongli/GSM-Plus@3b708db57b96a16e8e3368ed2956990c0809440e" + ], + "categories": [ + { + "level1": "Mathematics", + "level2": "ElementaryMath" + } + ], + "tags": [ + "english", + "math-word-problems", + "open-ended", + "robustness" + ], + "deps_group": null, + "license": "CC-BY-SA-4.0", + "checksums": {} + }, { "name": "hellaswag", "display_name": "HellaSwag", @@ -1085,6 +1108,28 @@ }, "status": "stable" }, + { + "name": "gsm_plus_0shot_gen", + "display_name": "GSM-Plus (0-shot, generative)", + "description": "GSM-Plus 0-shot CoT eval, scored overall and per perturbation type.", + "dataset": "gsm_plus", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "math-word-problems", + "open-ended", + "robustness" + ], + "deps_group": "math", + "model_type": "chat", + "reference_impl": { + "source": "qtli/GSM-Plus", + "url": "https://github.com/qtli/GSM-Plus/tree/3474129ec12fcd3e8ac08cb037aca1928efca98c/scripts", + "notes": "Zero-shot CoT protocol (--prompt_type cot): system turn = the \"#### [value]\" instruction, user turn = \"Question:\\n{question}\\nAnswer:\\nLet's think step by step.\"; chat template applied by the serving backend. Gold from `solution` via extract_gold_ans; extraction dispatches on perturbation_type (extract_pred_ans_none for `critical thinking`, whose gold is the string \"None\"); normalize_final_answer + check_sympy_equivalence scoring. All vendored in sieval.community.gsm_plus. Replaying upstream's stored GPT-3.5-Turbo CoT predictions reproduces its gold/pred on 10552/10552 items and its verdict on 10527/10552 (99.76%); the 25 diffs are all genuinely-equal fraction/decimal pairs that upstream scored wrong because it pins sympy without the ANTLR runtime parse_latex needs, so its bare `except:` fell back to string equality (published 61.19 overall -> 61.43 here; the integer-decimal-fraction cell moves most, 62.32 -> 63.84). Single-rollout only: upstream's cot_sc majority-votes 5 samples at temperature 0.7 (test_answer mv=5). Upstream decoding for this path: temperature 0, top_p 1, max_tokens 512, no stop." + }, + "status": "experimental" + }, { "name": "hellaswag_kshot_ppl", "display_name": "HellaSwag (k-shot, log-likelihood)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index e7c8afad..bdbc0eca 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -46,6 +46,9 @@ from .gsm8k_0shot_gen import ( from .gsm8k_kshot_base_gen import ( GSM8KFewShotBaseGenTask, ) +from .gsm_plus_0shot_gen import ( + GSMPlusZeroShotGenTask, +) from .hellaswag_kshot_ppl import ( HellaSwagFewShotPPLTask, ) @@ -138,6 +141,7 @@ __all__ = [ "GPQADiamondZeroShotGenTask", "GSM8KFewShotBaseGenTask", "GSM8KZeroShotGenTask", + "GSMPlusZeroShotGenTask", "HLEZeroShotGenTask", "HMMTFeb2025ZeroShotGenTask", "HMMTFeb2026ZeroShotGenTask", diff --git a/sieval/tasks/gsm_plus_0shot_gen.py b/sieval/tasks/gsm_plus_0shot_gen.py new file mode 100644 index 00000000..c1f9f03a --- /dev/null +++ b/sieval/tasks/gsm_plus_0shot_gen.py @@ -0,0 +1,254 @@ +""" +GSM-Plus 0-shot generative task, aligned with the upstream GSM-Plus evaluation. + +Port of GSM-Plus's zero-shot CoT (chat/instruct) path (pinned commit +``3474129e``, ``scripts/openai_model_inference.py`` with ``--prompt_type cot``): + +* Prompt (``prompt_template.py::cot_prompt_map_func``): a system turn carrying + the ``#### [value]`` output contract, then a user turn + ``"Question:\\n{question}\\nAnswer:\\nLet's think step by step."``. The serving + backend applies the model's own chat template. +* Gold answer (``extract_ans.py::extract_gold_ans``, fed ``solution`` — upstream's + ``get_gsmplus`` puts ``item["solution"]`` in its ``answers`` list, not + ``item["answer"]``): the last ``####`` segment, first number, commas stripped. +* Answer extraction: dispatched on the row's ``perturbation_type`` + (``extract_ans.py::test_answer``). Seven perturbations use + ``extract_pred_ans``; ``critical thinking`` uses ``extract_pred_ans_none``, + because that perturbation deletes a quantity the question needs, making the + gold answer the string ``"None"`` — "unanswerable" — rather than a number. +* Scoring: ``normalize_final_answer`` then ``check_sympy_equivalence``. +* ``score`` is accuracy over the whole set. ``report`` also breaks accuracy down + per perturbation type and reports ``score_wo_critical_thinking``, upstream's + ``gsmplus_wo_ncr`` — the paper leads with both, since ``critical thinking`` is + the one cell where a refusal, not a number, is the right answer. + +Extraction/scoring lives in ``sieval.community.gsm_plus``, ported from the pinned +commit's ``scripts/utils/extract_ans.py``. + +Fidelity, measured — replaying upstream's own stored zero-shot-CoT predictions +(``results/gpt-3.5-turbo.json``, all 10552 items) through this pipeline +reproduces upstream's persisted ``gold`` and ``pred`` on **10552/10552** items, +and its ``result`` on **10527/10552 (99.76%)**. + +All 25 verdict diffs go one way — upstream ``False``, this port ``True`` — and +every one is a pair that is genuinely equal (``3/1`` vs ``3``, ``7/20`` vs +``0.35``, ``2.45`` vs ``2450/1000``). The cause is upstream's environment, not +its logic: ``requirements.txt`` pins ``sympy==1.12`` and no +``antlr4-python3-runtime``, so ``parse_latex`` raises, and +``check_sympy_equivalence``'s bare ``except:`` degrades it to string equality. +Under sieval's ``[math]`` extra the ANTLR runtime *is* pinned, so the same +vendored code reaches its symbolic branch and returns the mathematically correct +verdict. Reproducing the published digits exactly would mean deliberately +breaking ``parse_latex`` — bespoke logic that diverges from upstream source — so +the port keeps the code faithful and accepts the documented gap. + +That gap, this port vs. the paper's published GPT-3.5-Turbo CoT numbers: + +* overall 61.43 vs 61.19; excluding critical thinking 63.45 vs 63.18 +* unchanged: critical thinking 47.31, adding operation 48.45, distraction + insertion 62.17, problem understanding 74.22 +* higher: integer-decimal-fraction conversion 63.84 vs 62.32, reversing + operation 55.42 vs 55.19, numerical substitution 69.60 vs 69.52, digit + expansion 70.43 vs 70.36 + +20 of the 25 land on ``integer-decimal-fraction conversion``, which is what makes +the delta explainable rather than mysterious: that perturbation exists precisely +to rewrite integers as decimals and fractions, so it is where string equality and +symbolic equality disagree most. ``status="experimental"`` until a live run +reproduces a published model's full 8-cell table within a stated band. + +Other deviations from upstream (documented, not silent): + +* Only ``--prompt_type cot`` is ported. Upstream also ships ``pot``, ``complex``, + ``contrastive``, ``ltm`` and ``cot_sc``; those are separate prompting + techniques (the paper's Table 5), not this protocol. +* Single rollout. Upstream's ``cot_sc`` draws 5 samples at temperature 0.7 and + majority-votes (``test_answer(mv=5)``); the ported path is ``mv == 1``. +* Upstream's ``results/*.json`` dumps spell the 7th perturbation "distractor + insertion", while both the released dataset and upstream's own + ``dataset/gsmplus_test.jsonl`` spell it "distraction insertion". A run sees the + dataset spelling, so that is what the report keys use. +* Upstream also reports a GSM8K-vs-GSM-Plus confusion matrix and performance + decay rate (``report_metrics.py::get_confusion_matrix``, ``pdr``). Both need a + paired GSM8K run, so they belong to cross-task analysis, not this task's + report; the ``seed_*`` columns needed to compute them are preserved on every + sample. + +Repro decoding (model-layer assets — set via ``models:`` / ``infer_args``, not in +this code): greedy ``temperature=0``, ``top_p=1.0``, ``max_tokens=512``, no stop +sequences (``extract_ans.py::invoke_openai`` defaults for non-``pot`` prompts). + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from collections import defaultdict +from typing import override + +from sieval.community.gsm_plus import ( + CRITICAL_THINKING, + extract_gold_ans, + extract_prediction, + is_equivalent, +) +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.datasets import GSMPlusDatasetSample + +# Verbatim from prompt_template.py::cot_prompt_map_func, which returns +# (template, instruction) -> (user turn, system turn). +SYSTEM_INSTRUCTION = ( + "Your task is to solve a series of math word problems by providing the " + "final answer. Use the format #### [value] to highlight your answer. For " + "example, if the answer is 560, you should write #### 560. Make sure to " + "carefully read and understand each problem before providing your answer." +) + + +def _user_turn(question: str) -> str: + return f"Question:\n{question}\nAnswer:\nLet's think step by step." + + +def _metric_key(perturbation_type: str) -> str: + """Slug a perturbation type into a report key. + + ``"integer-decimal-fraction conversion"`` -> + ``"integer_decimal_fraction_conversion"``. + """ + return perturbation_type.replace("-", "_").replace(" ", "_") + + +@sieval_task( + name="gsm_plus_0shot_gen", + display_name="GSM-Plus (0-shot, generative)", + description="GSM-Plus 0-shot CoT eval, scored overall and per perturbation type.", + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "math-word-problems", "open-ended", "robustness"), + deps_group="math", + model_type="chat", + status="experimental", + reference_impl=ReferenceImpl( + source="qtli/GSM-Plus", + url=( + "https://github.com/qtli/GSM-Plus/tree/3474129ec12fcd3e8ac08cb037aca1928efca98c/scripts" + ), + notes=( + "Zero-shot CoT protocol (--prompt_type cot): system turn = the " + '"#### [value]" instruction, user turn = "Question:\\n{question}\\n' + "Answer:\\nLet's think step by step.\"; chat template applied by the " + "serving backend. Gold from `solution` via extract_gold_ans; " + "extraction dispatches on perturbation_type (extract_pred_ans_none " + 'for `critical thinking`, whose gold is the string "None"); ' + "normalize_final_answer + check_sympy_equivalence scoring. All " + "vendored in sieval.community.gsm_plus. Replaying upstream's stored " + "GPT-3.5-Turbo CoT predictions reproduces its gold/pred on " + "10552/10552 items and its verdict on 10527/10552 (99.76%); the 25 " + "diffs are all genuinely-equal fraction/decimal pairs that upstream " + "scored wrong because it pins sympy without the ANTLR runtime " + "parse_latex needs, so its bare `except:` fell back to string " + "equality (published 61.19 overall -> 61.43 here; the " + "integer-decimal-fraction cell moves most, 62.32 -> 63.84). " + "Single-rollout only: upstream's cot_sc majority-votes 5 samples at " + "temperature 0.7 (test_answer mv=5). Upstream decoding for this " + "path: temperature 0, top_p 1, max_tokens 512, no stop." + ), + ), +) +class GSMPlusZeroShotGenTask( + Task[ + GSMPlusDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + @override + async def preprocess(self, raw, ctx): + return build_prompt_record( + [ + {"role": "system", "content": SYSTEM_INSTRUCTION}, + {"role": "user", "content": _user_turn(raw["question"])}, + ], + reference=extract_gold_ans(raw["solution"]), + extra={"perturbation_type": raw["perturbation_type"]}, + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"]) + + @override + async def postprocess(self, inf, ctx): + text = inf.texts[0] if inf.texts else "" + prediction = extract_prediction(text, ctx.raw_sample["perturbation_type"]) + # `""` is upstream's "nothing extracted"; `None` is the protocol's spelling + # of that, and feedback restores `""` for the grader. The string `"None"` + # survives untouched — on a `critical thinking` row it is a real answer + # ("unanswerable"), not a failure to extract. + return build_prediction_record([prediction or None]) + + @override + async def feedback(self, post, ctx): + gold = extract_gold_ans(ctx.raw_sample["solution"]) + perturbation_type = ctx.raw_sample["perturbation_type"] + # `or ""` restores exactly what upstream's test_answer compares against. + prediction = post["rollouts"][0]["prediction"] or "" + correct = is_equivalent(gold, prediction) + return True, build_judgement_record( + gold, + [build_rollout_judgement(0, correct)], + extra={"perturbation_type": perturbation_type}, + ) + + @override + async def report(self, finals, fails): + # Accuracy over the full requested set (finals + fails), matching the + # gsm8k/math-0shot-gen family and upstream's own denominator (every item + # in the prediction file): a pipeline failure counts as wrong, not as an + # excluded sample. + per_type: dict[str, list[int]] = defaultdict(lambda: [0, 0]) # [correct, total] + correct_num = 0 + for ctx in finals: + perturbation_type = ctx.feedback_result["extra"]["perturbation_type"] + if ctx.feedback_result["rollouts"][0]["correct"]: + correct_num += 1 + per_type[perturbation_type][0] += 1 + per_type[perturbation_type][1] += 1 + for ctx in fails: + # A failed sample scores 0 but still owes its perturbation a + # denominator slot, and `raw_sample` is the only place its type + # survives — a fail never reached feedback. Contexts that failed + # before the sample was attached are counted in `total` only, so + # per-type denominators can sum to less than it. + if ctx.raw_sample is not None: + per_type[ctx.raw_sample["perturbation_type"]][1] += 1 + + total = len(finals) + len(fails) + accuracy = 100 * correct_num / total if total else 0.0 + report: dict[str, float] = {"score": accuracy, "accuracy": accuracy} + + wo_correct = sum(c for t, (c, _) in per_type.items() if t != CRITICAL_THINKING) + wo_total = sum(n for t, (_, n) in per_type.items() if t != CRITICAL_THINKING) + report["score_wo_critical_thinking"] = ( + 100 * wo_correct / wo_total if wo_total else 0.0 + ) + for perturbation_type, (correct, seen) in sorted(per_type.items()): + report[f"score_{_metric_key(perturbation_type)}"] = ( + 100 * correct / seen if seen else 0.0 + ) + report["fails"] = len(fails) + return report diff --git a/tests/unit/datasets/test_gsm_plus.py b/tests/unit/datasets/test_gsm_plus.py new file mode 100644 index 00000000..942bdd02 --- /dev/null +++ b/tests/unit/datasets/test_gsm_plus.py @@ -0,0 +1,86 @@ +"""Unit tests for the GSM-Plus dataset loader. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.datasets import gsm_plus as gp +from sieval.datasets.gsm_plus import GSMPlusDataset + +_COLUMNS = { + "question", + "solution", + "answer", + "perturbation_type", + "seed_question", + "seed_solution", + "seed_answer", +} + + +def _row(perturbation_type: str, answer: str = "27") -> dict: + return { + "question": "Janet's ducks lay 20 eggs per day. ...", + "solution": f"Work.\n#### {answer}", + "answer": answer, + "perturbation_type": perturbation_type, + "seed_question": "Janet's ducks lay 16 eggs per day. ...", + "seed_solution": "Work.\n#### 18", + "seed_answer": "18", + } + + +def _fake_dict() -> HFDatasetDict: + return HFDatasetDict( + { + "test": HFDataset.from_list( + [_row("numerical substitution"), _row("critical thinking", "None")] + ), + "testmini": HFDataset.from_list([_row("digit expansion", "42")]), + } + ) + + +def test_load_defaults_to_full_test_split(monkeypatch): + monkeypatch.setattr(gp, "load_dataset", lambda *a, **k: _fake_dict()) + ds = GSMPlusDataset(name_or_path="qintongli/GSM-Plus") + assert ds.test_set is not None + assert len(ds.test_set) == 2 + # mirror: native schema preserved exactly (no columns added/removed) + assert set(ds.test_set.column_names) == _COLUMNS + + +def test_load_remaps_testmini_to_test(monkeypatch): + monkeypatch.setattr(gp, "load_dataset", lambda *a, **k: _fake_dict()) + ds = GSMPlusDataset(name_or_path="qintongli/GSM-Plus", eval_split="testmini") + assert ds.test_set is not None + assert len(ds.test_set) == 1 + assert ds.test_set[0]["perturbation_type"] == "digit expansion" + + +def test_load_rejects_unknown_split(monkeypatch): + # A silent no-op here would fall through to the 10552-row test split, so a + # misspelled "testmini" must fail loudly rather than run 4x the samples. + monkeypatch.setattr(gp, "load_dataset", lambda *a, **k: _fake_dict()) + with pytest.raises(ValueError, match="no split 'testmni'"): + GSMPlusDataset(name_or_path="qintongli/GSM-Plus", eval_split="testmni") + + +def test_load_rejects_empty_eval_split(monkeypatch): + empty = HFDatasetDict({"test": HFDataset.from_list([])}) + monkeypatch.setattr(gp, "load_dataset", lambda *a, **k: empty) + with pytest.raises(ValueError, match="loaded 0 samples"): + GSMPlusDataset(name_or_path="qintongli/GSM-Plus") + + +def test_load_preserves_seed_columns_and_none_answer(monkeypatch): + monkeypatch.setattr(gp, "load_dataset", lambda *a, **k: _fake_dict()) + ds = GSMPlusDataset(name_or_path="qintongli/GSM-Plus") + assert ds.test_set is not None + # seed_* is what makes paired GSM8K-vs-GSM-Plus comparison possible + assert ds.test_set[0]["seed_answer"] == "18" + # "None" is a string answer ("unanswerable"), not a null + assert ds.test_set[1]["answer"] == "None" diff --git a/tests/unit/tasks/test_gsm_plus_0shot_gen.py b/tests/unit/tasks/test_gsm_plus_0shot_gen.py new file mode 100644 index 00000000..0cc0a16c --- /dev/null +++ b/tests/unit/tasks/test_gsm_plus_0shot_gen.py @@ -0,0 +1,345 @@ +"""Unit tests for the GSM-Plus 0-shot CoT task. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.gsm_plus import ( + extract_gold_ans, + extract_pred_ans, + extract_pred_ans_none, + is_equivalent, +) +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + TaskContext, + build_judgement_record, + build_rollout_judgement, +) +from sieval.datasets.gsm_plus import GSMPlusDataset, GSMPlusDatasetSample +from sieval.tasks.gsm_plus_0shot_gen import ( + SYSTEM_INSTRUCTION, + GSMPlusZeroShotGenTask, + _metric_key, + _user_turn, +) + + +class _CapturingChatModel(ChatModel): + def __init__(self, text: str): + super().__init__(model="mock-chat", api_key="fake") + self.last_kwargs: dict[str, object] = {} + self._text = text + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + _ = prompt + self.last_kwargs = dict(kwargs) + return ModelOutput(model=self.meta(), texts=[self._text]) + + async def _alogprobs_impl( + self, + prompt, + *, + max_tokens: int = 1, + logprobs: int = 5, + echo: bool = True, + temperature: float = 0.0, + **kwargs, + ) -> ModelOutput: + _ = (prompt, max_tokens, logprobs, echo, temperature, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _sample( + perturbation_type: str = "numerical substitution", + answer: str = "27", +) -> GSMPlusDatasetSample: + return { + "question": "What is 25 + 2?", + "solution": f"Work.\n#### {answer}", + "answer": answer, + "perturbation_type": perturbation_type, + "seed_question": "What is 20 + 2?", + "seed_solution": "Work.\n#### 22", + "seed_answer": "22", + } + + +def _task(text: str = "x"): + dataset = GSMPlusDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(_sample())])}) + ) + model = _CapturingChatModel(text=text) + return GSMPlusZeroShotGenTask(dataset, model), model + + +# --- Pinning: prompt is byte-for-byte cot_prompt_map_func(question) --- + + +def test_system_instruction_pinned(): + assert SYSTEM_INSTRUCTION == ( + "Your task is to solve a series of math word problems by providing the " + "final answer. Use the format #### [value] to highlight your answer. " + "For example, if the answer is 560, you should write #### 560. Make " + "sure to carefully read and understand each problem before providing " + "your answer." + ) + + +def test_user_turn_pinned(): + assert _user_turn("Q?") == "Question:\nQ?\nAnswer:\nLet's think step by step." + + +@pytest.mark.anyio +async def test_preprocess_builds_system_then_user_turn(): + task, _ = _task() + raw = _sample() + pre = await task.preprocess(raw, TaskContext(sample_id=0, raw_sample=raw)) + messages = pre["prompt"] + assert len(messages) == 2 + assert messages[0] == {"role": "system", "content": SYSTEM_INSTRUCTION} + assert messages[1]["role"] == "user" + assert messages[1]["content"] == _user_turn("What is 25 + 2?") + # The gold reaches disk from preprocess; raw_sample is never serialized. + assert pre["reference"] == "27" + assert pre["extra"]["perturbation_type"] == "numerical substitution" + + +# --- Gold comes from `solution` (upstream get_gsmplus), not `answer` --- + + +def test_extract_gold_ans_reads_hash_segment(): + assert extract_gold_ans("Work.\n#### 1,000") == "1000" + assert extract_gold_ans("Work.\n#### 4.33") == "4.33" + assert extract_gold_ans("Work.\n#### 3/5") == "3/5" + + +def test_extract_gold_ans_keeps_none_for_critical_thinking(): + assert extract_gold_ans("We don't know how many eggs.\n#### None") == "None" + + +def test_extract_gold_ans_rejects_unparseable_gold(): + # Upstream drops into pdb.set_trace() here; a library must raise instead. + with pytest.raises(ValueError, match="neither '####' nor"): + extract_gold_ans("no marker at all") + + +# --- Numeric extraction: #### segment wins, last-number fallback --- + + +def test_extract_pred_ans_prefers_hash_segment(): + assert extract_pred_ans("First 12, then 30.\n#### 27") == "27" + + +def test_extract_pred_ans_last_number_fallback(): + assert extract_pred_ans("first 12 then finally 30") == "30" + + +def test_extract_pred_ans_ignores_hash_segment_without_digits(): + # `#### None` has no digit, so upstream falls back to the last number. + assert extract_pred_ans("I computed 9 eggs.\n#### None") == "9" + + +def test_extract_pred_ans_empty_when_no_number(): + assert extract_pred_ans("cannot be determined") == "" + + +# --- critical thinking: refusal phrasing, not a number --- + + +def test_extract_pred_ans_none_detects_refusal(): + assert ( + extract_pred_ans_none("The problem does not specify how many eggs.\n#### 5") + == "None" + ) + + +def test_extract_pred_ans_none_rejects_confident_number(): + assert extract_pred_ans_none("She makes 18 dollars.\n#### 18") == "" + + +def test_extract_pred_ans_none_credits_missing_hash_marker(): + # Upstream leniency, kept verbatim and load-bearing for its published + # numbers: no "####" in the response scores "None" (correct) regardless. + assert extract_pred_ans_none("She makes 18 dollars.") == "None" + + +# --- Scoring: normalize_final_answer + check_sympy_equivalence --- + + +def test_is_equivalent_matches_fraction_and_decimal(): + # sieval's [math] extra pins the ANTLR runtime parse_latex needs, so the + # vendored sympy branch is live where upstream's own env fell back to string + # equality and scored these wrong. + assert is_equivalent("4.5", "9/2") is True + assert is_equivalent("7/20", "0.35") is True + assert is_equivalent("3", "3/1") is True + + +def test_is_equivalent_rejects_different_numbers(): + assert is_equivalent("27", "18") is False + + +def test_is_equivalent_handles_none_answer(): + assert is_equivalent("None", "None") is True + assert is_equivalent("None", "") is False + + +# --- postprocess / feedback wiring --- + + +@pytest.mark.anyio +async def test_postprocess_dispatches_on_perturbation_type(): + task, model = _task() + raw = _sample(perturbation_type="critical thinking", answer="None") + inf = ModelOutput( + model=model.meta(), texts=["The problem does not provide the egg count."] + ) + post = await task.postprocess(inf, TaskContext(sample_id=0, raw_sample=raw)) + assert post["rollouts"][0]["prediction"] == "None" + + +@pytest.mark.anyio +async def test_postprocess_reports_none_when_nothing_extracted(): + task, model = _task() + raw = _sample() + inf = ModelOutput(model=model.meta(), texts=["no digits here"]) + post = await task.postprocess(inf, TaskContext(sample_id=0, raw_sample=raw)) + # `None` is the protocol's "could not extract" — distinct from the string + # "None", which is a real answer on a critical-thinking row. + assert post["rollouts"][0]["prediction"] is None + + +@pytest.mark.anyio +async def test_feedback_scores_correct_answer(): + task, model = _task() + raw = _sample(answer="1,000") + inf = ModelOutput(model=model.meta(), texts=["Work.\n#### 1000"]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inf) + post = await task.postprocess(inf, ctx) + finalize, fb = await task.feedback(post, ctx) + assert finalize is True + assert fb["reference"] == "1000" + assert fb["rollouts"][0]["correct"] is True + assert fb["extra"]["perturbation_type"] == "numerical substitution" + + +@pytest.mark.anyio +async def test_feedback_scores_wrong_answer(): + task, model = _task() + raw = _sample(answer="27") + inf = ModelOutput(model=model.meta(), texts=["Work.\n#### 18"]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inf) + post = await task.postprocess(inf, ctx) + _, fb = await task.feedback(post, ctx) + assert fb["rollouts"][0]["correct"] is False + + +@pytest.mark.anyio +async def test_feedback_credits_recognized_unanswerable(): + task, model = _task() + raw = _sample(perturbation_type="critical thinking", answer="None") + inf = ModelOutput(model=model.meta(), texts=["It does not specify.\n#### 5"]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inf) + post = await task.postprocess(inf, ctx) + _, fb = await task.feedback(post, ctx) + assert fb["reference"] == "None" + assert fb["rollouts"][0]["correct"] is True + + +# --- report: overall, per-perturbation, wo_critical_thinking --- + + +def test_metric_key_slugifies_perturbation_type(): + assert ( + _metric_key("integer-decimal-fraction conversion") + == "integer_decimal_fraction_conversion" + ) + assert _metric_key("critical thinking") == "critical_thinking" + + +def _final(sample_id: int, perturbation_type: str, correct: bool) -> TaskContext: + return TaskContext( + sample_id=sample_id, + raw_sample=_sample(perturbation_type=perturbation_type), + feedback_result=build_judgement_record( + "27", + [build_rollout_judgement(0, correct)], + extra={"perturbation_type": perturbation_type}, + ), + ) + + +@pytest.mark.anyio +async def test_report_breaks_down_by_perturbation_type(): + task, _ = _task() + finals = [ + _final(0, "numerical substitution", True), + _final(1, "numerical substitution", False), + _final(2, "critical thinking", True), + _final(3, "critical thinking", True), + ] + report = await task.report(finals, []) + assert report["score"] == 75.0 # 3 of 4 + assert report["accuracy"] == 75.0 + assert report["score_numerical_substitution"] == 50.0 + assert report["score_critical_thinking"] == 100.0 + # upstream's gsmplus_wo_ncr: drops the one cell where refusal is the answer, + # so the two critical-thinking hits stop propping the headline up + assert report["score_wo_critical_thinking"] == 50.0 + assert report["fails"] == 0 + + +@pytest.mark.anyio +async def test_report_empty_finals(): + task, _ = _task() + report = await task.report([], []) + assert report == { + "score": 0.0, + "accuracy": 0.0, + "score_wo_critical_thinking": 0.0, + "fails": 0, + } + + +@pytest.mark.anyio +async def test_report_counts_fails_in_overall_and_per_type_denominators(): + # A pipeline failure counts as wrong, matching upstream's denominator (every + # item in the prediction file) and the gsm8k/math-0shot-gen family. + task, _ = _task() + finals = [_final(0, "numerical substitution", True)] + fails = [ + TaskContext( + sample_id=1, raw_sample=_sample(perturbation_type="numerical substitution") + ) + ] + report = await task.report(finals, fails) + assert report["score"] == 50.0 + assert report["score_numerical_substitution"] == 50.0 + assert report["fails"] == 1 + + +@pytest.mark.anyio +async def test_report_tolerates_fail_without_raw_sample(): + # A context that failed before its sample was attached has no perturbation + # type, so it lands in the overall denominator only. + task, _ = _task() + finals = [_final(0, "numerical substitution", True)] + report = await task.report(finals, [TaskContext(sample_id=1)]) + assert report["score"] == 50.0 + assert report["score_numerical_substitution"] == 100.0 + + +@pytest.mark.anyio +async def test_infer_injects_no_decode_params(): + task, model = _task() + raw = _sample() + ctx = TaskContext(sample_id=0, raw_sample=raw) + pre = await task.preprocess(raw, ctx) + await task.infer(pre, ctx) + for forbidden in ("temperature", "top_p", "max_tokens", "n", "stop"): + assert forbidden not in model.last_kwargs