diff --git a/README.md b/README.md index 608f5ecc..0bd2b268 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ pdm install # or: pip install -e . Optional extras (per-benchmark dependencies): ```bash -pip install -e ".[math]" # AIME, GSM8K, HMMT, IMO-AnswerBench, MATH-500, TheoremQA (math-verify) +pip install -e ".[math]" # AIME, GSM1k, GSM8K, HMMT, IMO-AnswerBench, MATH-500, TheoremQA (math-verify) pip install -e ".[drop]" # DROP (numpy, scipy) pip install -e ".[ifbench]" # IFBench (emoji, nltk, setuptools, syllapy) pip install -e ".[ifeval]" # IFEval (absl, langdetect, nltk, immutabledict) diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index 9c6cc59b..d94ef7e0 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -45,6 +45,10 @@ from .gpqa_diamond import ( GPQADiamondDataset, GPQADiamondDatasetSample, ) +from .gsm1k import ( + GSM1KDataset, + GSM1KDatasetSample, +) from .gsm8k import ( GSM8KDataset, GSM8KDatasetSample, @@ -165,6 +169,8 @@ __all__ = [ "DROPDatasetSample", "GPQADiamondDataset", "GPQADiamondDatasetSample", + "GSM1KDataset", + "GSM1KDatasetSample", "GSM8KDataset", "GSM8KDatasetSample", "HLEDataset", diff --git a/sieval/datasets/gsm1k.py b/sieval/datasets/gsm1k.py new file mode 100644 index 00000000..05c39b4d --- /dev/null +++ b/sieval/datasets/gsm1k.py @@ -0,0 +1,83 @@ +""" +GSM1k loader — Scale AI's from-scratch re-do of GSM8K, built to detect overfitting. + +GSM1k is 1205 grade-school word problems written by human annotators with no LLM +or synthetic assistance, mirroring GSM8K's difficulty and answer-magnitude +distribution. The benchmark exists to be read as a *pair* with GSM8K, not on its +own: a model's GSM8K − GSM1k gap estimates how much of its GSM8K score is +memorization rather than reasoning. The paper measures drops of up to 8% and a +Spearman r² of 0.36 between a model's likelihood of generating a GSM8k example +and its gap. + +**Release history, because the pinned snapshot ships an empty dataset card while +the paper says the data is withheld.** The paper (Nov 2024) declined to publish +GSM1k "to prevent a similar problem of data contamination occurring in the +future" and precommitted to release on the earlier of two triggers — three +open-source models of different lineages reaching 95% accuracy, or June 2025 — +with its datasheet stating "The dataset (yet unreleased) will be released with +the MIT license." The pinned snapshot was uploaded to the ScaleAI org on +2025-03-31/04-01, i.e. that release. So `license="MIT"` here is the datasheet's +commitment for the *data*, not the eval repo's code license. + +**Provenance verified**, since an empty card is not evidence: every one of the 50 +questions in `gsm1k_public_50.csv` — the sample Scale published in its eval repo +while the full set was still withheld — appears verbatim in this snapshot's +`test` split with an identical answer (50/50 found, 0 answer mismatches). The row +count also matches the paper exactly: "GSM1k consists of 1205 problems". + +Schema, measured at the pinned revision: `question` and `answer` are both +strings and need no cast — no cast needed, upstream ships this dtype. All 1205 +answers are bare integers (1-6 characters, every one matching `-?[0-9]+`), with +no thousands separators, no `####` delimiter and **no worked solution** — the +released data carries final answers only. Two consequences for tasks: the gold +needs no `answer.split("####")` step (unlike `openai/gsm8k`), and GSM1k cannot +supply chain-of-thought few-shot exemplars of its own, which is why +`gsm1k_kshot_base_gen` borrows GSM8K's the way upstream's harness does. + +There is a `test` split and nothing else — no train set, so nothing to hold out. + +References: + +* Paper: +* Eval harness + public 50-example sample: + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +GSM1K_REVISION = "bc09569d09a614b9b530edc7f076fb214ac10493" + + +class GSM1KDatasetSample(TypedDict): + question: str + answer: str + + +@sieval_dataset( + name="gsm1k", + display_name="GSM1k", + description="Grade School Math 1k - 1205 human-written GSM8K mirror problems.", + source=f"hf:ScaleAI/gsm1k@{GSM1K_REVISION}", + categories=(Category(Level1Category.MATHEMATICS, "ElementaryMath"),), + tags=("english", "math-word-problems", "open-ended"), + license="MIT", +) +class GSM1KDataset(Dataset[GSM1KDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + # One unnamed config, unlike openai/gsm8k's "main" / "socratic", so there + # is no config argument to forward. + dataset = load_dataset(name_or_path, **kwargs) + return ensure_dataset_dict(dataset) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 5ffba622..200d1f87 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -253,6 +253,28 @@ "gpqa_diamond.csv": "sha256:41d1213cd7a4998605a26c2798500652572007161b3a92817ba46b35befcd305" } }, + { + "name": "gsm1k", + "display_name": "GSM1k", + "description": "Grade School Math 1k - 1205 human-written GSM8K mirror problems.", + "source": [ + "hf:ScaleAI/gsm1k@bc09569d09a614b9b530edc7f076fb214ac10493" + ], + "categories": [ + { + "level1": "Mathematics", + "level2": "ElementaryMath" + } + ], + "tags": [ + "english", + "math-word-problems", + "open-ended" + ], + "deps_group": null, + "license": "MIT", + "checksums": {} + }, { "name": "gsm8k", "display_name": "GSM8K", @@ -1064,6 +1086,49 @@ }, "status": "stable" }, + { + "name": "gsm1k_0shot_gen", + "display_name": "GSM1k (0-shot, generative)", + "description": "GSM1k 0-shot chat-model eval, prompt-paired with gsm8k_0shot_gen.", + "dataset": "gsm1k", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "math-word-problems", + "open-ended" + ], + "deps_group": "math", + "model_type": "chat", + "reference_impl": { + "source": "deepseek-ai/DeepSeek-Math", + "url": "https://github.com/deepseek-ai/DeepSeek-Math/tree/b8b0f8ce093d80bf8e9a641e44142f06d092c305/evaluation", + "notes": "Protocol borrowed from the sibling `gsm8k_0shot_gen`, so the two form a prompt-exact pair: user turn = question + \"Please reason step by step, and put your final answer within \\boxed{}.\", chat template applied by the serving backend; extract_answer(exhaust=False) (= extract_last_single_answer) and is_correct/math_equal (= eval_last_single_answer) are vendored byte-for-byte in sieval.community.deepseek_math. Gold is GSM1k's `answer` verbatim (already the bare final answer, so no '####' split). Scale AI published GSM1k at 5-shot raw completion only (see gsm1k_kshot_base_gen) — no published number matches this 0-shot chat protocol, so read the GSM8K - GSM1k diff, not the absolute score. Repeats: 1 rollout, greedy at temperature 0." + }, + "status": "experimental" + }, + { + "name": "gsm1k_kshot_base_gen", + "display_name": "GSM1k (few-shot, base generative)", + "description": "GSM1k few-shot eval on Scale AI's published lm-eval-harness protocol.", + "dataset": "gsm1k", + "eval_mode": "gen", + "n_shot": 5, + "tags": [ + "english", + "math-word-problems", + "open-ended", + "base-model" + ], + "deps_group": null, + "model_type": "gen", + "reference_impl": { + "source": "scaleapi/gsm1k_eval", + "url": "https://github.com/scaleapi/gsm1k_eval/blob/39294c6f31855aca8255b6174b22fc3a6311be0b/lm_eval/tasks/gsm1k/gsm1k_scale.yaml", + "notes": "Scale's own lm-evaluation-harness fork task `gsm1k`: prompt \"Question: {q}\\nAnswer:\" with 5 GSM8k-train exemplars, one `flexible-extract` filter (last numeric match) and `exact_match` with regexes_to_ignore [',', '$', '(?s).*#### ', '.$'] + ignore_case. Upstream resamples the 5 exemplars per question and raises max generation length from 256 to 1000 tokens; this task fixes one exemplar set (documented in the module docstring) and leaves max_tokens to the model layer, where 1000 matches upstream. Repeats: upstream runs `repeats: 1`, greedy at temperature 0 — match it with n=1 and temperature=0. GSM1k is a paired benchmark: read it as a diff against GSM8K on the same extraction rule, not as a standalone score." + }, + "status": "experimental" + }, { "name": "gsm8k_0shot_gen", "display_name": "GSM8K (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 42117391..5983755e 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -40,6 +40,12 @@ from .drop_kshot_gen import ( from .gpqa_diamond_0shot_gen import ( GPQADiamondZeroShotGenTask, ) +from .gsm1k_0shot_gen import ( + GSM1KZeroShotGenTask, +) +from .gsm1k_kshot_base_gen import ( + GSM1KFewShotBaseGenTask, +) from .gsm8k_0shot_gen import ( GSM8KZeroShotGenTask, ) @@ -151,6 +157,8 @@ __all__ = [ "CMMLUFewShotClpTask", "DROPFewShotGenTask", "GPQADiamondZeroShotGenTask", + "GSM1KFewShotBaseGenTask", + "GSM1KZeroShotGenTask", "GSM8KFewShotBaseGenTask", "GSM8KZeroShotGenTask", "HLEZeroShotGenTask", diff --git a/sieval/tasks/gsm1k_0shot_gen.py b/sieval/tasks/gsm1k_0shot_gen.py new file mode 100644 index 00000000..20c7807b --- /dev/null +++ b/sieval/tasks/gsm1k_0shot_gen.py @@ -0,0 +1,168 @@ +""" +GSM1k 0-shot generative task — the chat-side half of the GSM8K/GSM1k pair. + +Scale AI published GSM1k under one protocol only: 5-shot raw completion, ported +here as `gsm1k_kshot_base_gen`. That protocol needs a `gen` model, so on its own +it leaves GSM1k unrunnable for the chat endpoints this repo mostly verifies. This +task supplies the missing half by applying `gsm8k_0shot_gen`'s protocol — the +DeepSeek-Math zero-shot CoT path — to GSM1k, unchanged: + +* Prompt (DeepSeek's `run_subset_parallel.py::markup_question`, language="en", + task="cot"): the user turn is `{question}` followed by `"\\nPlease reason step + by step, and put your final answer within \\boxed{}."`, with the chat template + applied by the serving backend. +* Answer extraction: `extract_answer(reasoning, exhaust=False)` — DeepSeek's + `extract_last_single_answer`: last `\\boxed{...}` if present, else the text + after `"he answer is"`, else the last number, then `strip_string`. +* Scoring: `is_correct` — DeepSeek's `eval_last_single_answer` (numeric isclose + with %-variants, then a sympy symbolic fallback). `score` is this accuracy. + +Extraction and scoring live verbatim in `sieval.community.deepseek_math`, vendored +byte-faithfully from DeepSeek-Math at the pinned commit. Nothing about them is +GSM8K-specific: the gold is a bare integer either way. + +**This is a different measurement regime from upstream's, not a port of it — no +published GSM1k number corresponds to a 0-shot chat score.** What it buys is a +*prompt-exact pair*: run this task and `gsm8k_0shot_gen` against the same model +and the two differ only in which problem set the question came from — identical +prompt template, identical extractor, identical scorer, no few-shot exemplars to +vary. The GSM8K − GSM1k **diff** is the measurement, and it is the quantity GSM1k +exists to produce (the paper's Table 1 is a diff column first, an accuracy column +second). A single absolute number here aligns with nothing external. + +Deviation from the sibling it mirrors: the gold needs no `answer.split("####")` +step, because GSM1k's `answer` field already *is* the bare final answer — see +`sieval/datasets/gsm1k.py`. Both tasks divide `report()` by +`len(finals) + len(fails)`, so a pipeline failure counts as wrong on both sides +of the diff. + +`status="experimental"`: the extraction/scoring layer is the sibling's, verbatim +and already exercised, but this pairing has not yet been validated by a run, and +unlike the sibling it has no published column to be validated *against* — only +its own diff. + +Repro decoding (model-layer assets — set via `models:` / `infer_args`, not in +this code): greedy `temperature=0`, `top_p=1.0`, `max_tokens=1024`, stop = the +model's EOS only, matching `gsm8k_0shot_gen` so the pair stays comparable. + +References: + +* GSM1k paper: +* Protocol source: + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import override + +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.datasets import GSM1KDatasetSample + +# Verbatim from run_subset_parallel.py::markup_question (language="en", +# task="cot"): f"{content}\nPlease reason step by step, and put your final +# answer within " + "\\boxed{}." +COT_INSTRUCTION = ( + "\nPlease reason step by step, and put your final answer within \\boxed{}." +) + + +@sieval_task( + name="gsm1k_0shot_gen", + display_name="GSM1k (0-shot, generative)", + description="GSM1k 0-shot chat-model eval, prompt-paired with gsm8k_0shot_gen.", + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "math-word-problems", "open-ended"), + deps_group="math", + model_type="chat", + status="experimental", + reference_impl=ReferenceImpl( + source="deepseek-ai/DeepSeek-Math", + url=( + "https://github.com/deepseek-ai/DeepSeek-Math/tree/b8b0f8ce093d80bf8e9a641e44142f06d092c305/evaluation" + ), + notes=( + "Protocol borrowed from the sibling `gsm8k_0shot_gen`, so the two " + "form a prompt-exact pair: user turn = question + " + '"Please reason step by step, and put your final answer within ' + '\\boxed{}.", chat template applied by the serving backend; ' + "extract_answer(exhaust=False) (= extract_last_single_answer) and " + "is_correct/math_equal (= eval_last_single_answer) are vendored " + "byte-for-byte in sieval.community.deepseek_math. Gold is GSM1k's " + "`answer` verbatim (already the bare final answer, so no '####' " + "split). Scale AI published GSM1k at 5-shot raw completion only " + "(see gsm1k_kshot_base_gen) — no published number matches this " + "0-shot chat protocol, so read the GSM8K - GSM1k diff, not the " + "absolute score. Repeats: 1 rollout, greedy at temperature 0." + ), + ), +) +class GSM1KZeroShotGenTask( + Task[ + GSM1KDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + @override + async def preprocess(self, raw, ctx): + return build_prompt_record( + [ + {"role": "user", "content": raw["question"] + COT_INSTRUCTION}, + ], + reference=raw["answer"], + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"]) + + @override + async def postprocess(self, inf, ctx): + from sieval.community.deepseek_math import extract_answer + + text = inf.texts[0] if inf.texts else "" + # extract_answer returns "" when nothing was found; None is the protocol's + # spelling of that, and feedback restores "" for the grader. + return build_prediction_record([extract_answer(text, exhaust=False) or None]) + + @override + async def feedback(self, post, ctx): + from sieval.community.deepseek_math import is_correct + + gold = ctx.raw_sample["answer"] + # `or ""` gives the grader the same empty string a failed extraction + # produced upstream, rather than a None it has no branch for. + prediction = post["rollouts"][0].get("prediction") or "" + correct = is_correct({"prediction": prediction, "answer": gold}) + return True, build_judgement_record(gold, [build_rollout_judgement(0, correct)]) + + @override + async def report(self, finals, fails): + # Accuracy over the full requested set (finals + fails), matching + # `gsm8k_0shot_gen` so both sides of the paired diff count a pipeline + # failure as wrong rather than excluding it. + total = len(finals) + len(fails) + if total == 0: + return {"score": 0.0, "fails": len(fails), "accuracy": 0.0} + correct_num = sum( + 1 for ctx in finals if ctx.feedback_result["rollouts"][0]["correct"] + ) + accuracy = 100 * correct_num / total + return {"score": accuracy, "fails": len(fails), "accuracy": accuracy} diff --git a/sieval/tasks/gsm1k_kshot_base_gen.py b/sieval/tasks/gsm1k_kshot_base_gen.py new file mode 100644 index 00000000..e8d9e2d8 --- /dev/null +++ b/sieval/tasks/gsm1k_kshot_base_gen.py @@ -0,0 +1,354 @@ +""" +GSM1k few-shot generative task — the protocol Scale AI published GSM1k with. + +Port of Scale's `gsm1k` task from its lm-evaluation-harness fork (pinned commit +`39294c6f`, `lm_eval/tasks/gsm1k/gsm1k_scale.yaml`): + +* Prompt: `n_shot` exemplars of `"Question: {question}\\nAnswer: {answer}\\n\\n"` + followed by `"Question: {question}\\nAnswer:"` — upstream's `doc_to_text` with + its `doc_to_target` of `"{{answer}}"`, i.e. the exemplar answer verbatim, + calculator annotations (`<<9*7=63>>`) and `#### N` line included. +* Answer extraction: upstream declares exactly one filter, `flexible-extract` — + `regex` with `group_select: -1` over `(-?[$0-9.,]{2,})|(-?[0-9]+)`, then + `take_first`. That is lm-eval's `RegexFilter`: `findall`, take the **last** + match, first non-empty group, strip. It is also what the paper describes, + "extracts the last numeric answer in the response and compares this to the + correct answer". +* Scoring: `exact_match` with `ignore_case: true` and `regexes_to_ignore` + `[",", "\\$", "(?s).*#### ", "\\.$"]`, applied to prediction **and** gold in + that order, then lowercased. `_normalize_exact_match` reproduces the list + verbatim; the third entry cannot fire after extraction (both sides are bare + numbers by then) and is kept only so the normalizer reads as upstream's list + rather than a subset of it. +* Stop sequences: `until: ["Question:", "", "<|im_end|>"]`. + +Metric names say which extraction rule produced them, and there is deliberately +**no** bare `exact_match` key. GSM1k's whole use is a paired diff against GSM8K, +and `gsm8k_kshot_base_gen` spells its *strict* (`#### N`) metric `exact_match` +while upstream GSM1k's only metric is the flexible one — so a shared key would +let a reader diff two different extraction rules and see a gap that is pure +extraction. Both rules are therefore reported over the same response: +`flexible_exact_match` (upstream's filter, and the headline `score` / `correct`) +and `strict_exact_match` (the `#### N` rule, which on GSM1k also measures whether +the model followed the 5-shot format at all, since the gold carries no `####`). + +Deviations from Scale's harness (documented, not silent): + +* **Few-shot exemplars are fixed, not resampled per question.** Upstream draws + "five random examples from GSM8k to use as n-shot examples, which vary for each + new question"; `_GSM8K_FEWSHOT_EXAMPLES` is one fixed set of 5, so every sample + in a run shares one prompt prefix. This is the house pattern (`n_shot` + exemplars sampled once, as in `gsm8k_kshot_base_gen`), and for a paired + GSM8K/GSM1k read it is the stronger design: holding the prefix fixed keeps + exemplar variance out of the diff, which is what upstream's single shared + prompt is reaching for. It does move absolute scores relative to upstream's + published column. +* **Exemplar provenance.** The 5 pairs are rows of `openai/gsm8k` train (revision + `740312add88f781978c0658806c59bc2815b9866`, MIT) — GSM8k train is where + upstream's exemplars come from too, and GSM1k has no train split and ships no + worked solutions, so it cannot supply chain-of-thought exemplars of its own. + They were taken once as `shuffle(seed=1234)[:5]`, which is what + `gsm8k_kshot_base_gen` draws at `n_shot=5, fewshot_seed=1234` (verified + byte-identical with `datasets` 4.4.1): running that sibling at those settings + gives the same prompt prefix, so the pair can be run prompt-controlled. +* **`report()` divides by `len(finals) + len(fails)`** — a pipeline failure scores + wrong rather than being excluded, the convention across this repo's tasks. Note + `gsm8k_kshot_base_gen` divides by `len(finals)` instead, so a paired diff must + be read with both `fails` counts in view; the two rules agree when `fails == 0`. + +Comparison targets — the paper's Table 1, GSM8k → GSM1k (5-shot, temperature 0): +Meta-Llama-3-8B-Instruct 0.752 → 0.690 (diff 0.062), Meta-Llama-3-70B-Instruct +0.914 → 0.900 (0.014), Mistral-7B-Instruct-v0.2 0.428 → 0.419 (0.009), phi-2 +0.566 → 0.504 (0.063). Instruct checkpoints appear in a base-model task because +upstream applies **no** chat template: its `lm_eval` invocation passes no +`--apply_chat_template`, so every model was prompted with this raw 5-shot +completion. The **diff** is the measurement; treat a single absolute column as +alignment evidence only after checking `fails` and the exemplar deviation above. + +Repro decoding (model-layer assets — set via `models:` / `infer_args`, not here): +greedy `temperature=0`, and `max_tokens=1000` — upstream's one deliberate change +to lm-eval's defaults was raising the generation cap "from 256 to 1000" so chains +of thought are not truncated. This task forwards only the stop sequences, which +are coupled to the prompt format. + +`status="experimental"`: faithful to upstream's declared protocol by +construction, but the fixed-exemplar deviation changes every prompt and no run +has yet been checked against the published column. + +References: + +* Paper: +* Task config: + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import re +from typing import override + +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.datasets import GSM1KDatasetSample + +N_SHOT = 5 +STOP_SEQUENCES = ("Question:", "", "<|im_end|>") + +_STRICT_ANSWER_RE = re.compile(r"#### (\-?[0-9\.\,]+)") +_FLEXIBLE_ANSWER_RE = re.compile(r"(-?[$0-9.,]{2,})|(-?[0-9]+)") + +# `openai/gsm8k` train rows at revision 740312add88f781978c0658806c59bc2815b9866, +# taken as shuffle(seed=1234)[:5] — see the module docstring for why GSM1k borrows +# GSM8k's exemplars and why they are fixed. Verbatim, including the `<<...>>` +# calculator annotations and the `#### N` line, which is what upstream's +# `doc_to_target: "{{answer}}"` feeds the model. Upstream's en dash and right +# single quote are written as `\u2013` / `\u2019` so nobody can silently "fix" +# them to ASCII and break byte-fidelity. +_GSM8K_FEWSHOT_EXAMPLES: tuple[tuple[str, str], ...] = ( + ( + "Rodney has 35 dollars more than Ian. Ian has half as much money as Jessica " + "has. If Jessica has 100 dollars, how much more money does Jessica have than " + "Rodney?", + "Ian has 100/2 = <<100/2=50>>50 dollars.\n" + "Rodney has 50+35 = <<50+35=85>>85 dollars.\n" + "Jessica has 100-85 = <<100-85=15>>15 more dollars than Rodney.\n" + "#### 15", + ), + ( + "Lynne bought 7 books about cats and 2 books about the solar system. She also " + "bought 3 magazines. Each book cost 7$ and each magazine cost $4. How much did " + "Lynne spend in all?", + "Lynne bought a total of 7 + 2 = <<7+2=9>>9 books\n" + "The books cost Lynne 9 x 7 = $<<9*7=63>>63\n" + "For 3 magazines, Lynne spent 3 x 4 = $<<3*4=12>>12\n" + "In total, Lynne spent 63 + 12 = <<63+12=75>>75$\n" + "#### 75", + ), + ( + "Traci and Harris are baking cakes together. Traci has brought flour from her " + "own house and Harris has 400g of flour in his house. Each cake needs 100g of " + "flour and Traci and Harris have created 9 cakes each. How much flour, in " + "grams, did Traci bring from her own house?", + "To make the cakes, Traci and Harris used a total of 9 cakes * 100g of flour " + "per cake = <<9*100=900>>900g of flour.\n" + "Traci therefore brought 900g of needed flour \u2013 400g of flour from " + "Harris\u2019 flour = 500g of flour from her own house.\n" + "#### 500", + ), + ( + "Carly recently graduated and is looking for work in a field she studied for. " + "She sent 200 job applications to companies in her state, and twice that " + "number to companies in other states. Calculate the total number of job " + "applications she has sent so far.", + "If she sent 200 job applications to her state, she sent 200*2 = " + "<<200*2=400>>400 job applications to other states.\n" + "The total number of job applications she has sent is 400+200 = " + "<<400+200=600>>600\n" + "#### 600", + ), + ( + "Bert was able to sell 8 toy phones for $18 each, while Tory was able to sell " + "7 toy guns for $20 each. How much more did Bert earn than Tory?", + "Bert was able to earn 8 x $18 = $<<8*18=144>>144 for the toy phones.\n" + "While Tory was able to earn 7 x $20 = $<<7*20=140>>140 for the toy guns.\n" + "Therefore, Bert was able to earn $144 \u2013 $140 = $<<144-140=4>>4 more than " + "Tory.\n" + "#### 4", + ), +) + + +def _format_example(question: str, answer: str | None) -> str: + """Render upstream's `doc_to_text`, with the target appended for an exemplar.""" + prompt = f"Question: {question}\nAnswer:" + if answer is not None: + prompt += f" {answer}\n\n" + return prompt + + +def _normalize_exact_match(text: str) -> str: + # lm-eval `exact_match`: `regexes_to_ignore` in upstream's order, then + # `ignore_case`. The `#### ` entry is a no-op on an already-extracted number + # (both extractors return one) and is kept to mirror upstream's list. + text = re.sub(r",", "", text) + text = re.sub(r"\$", "", text) + text = re.sub(r"(?s).*#### ", "", text) + text = re.sub(r"\.$", "", text.strip()) + return text.strip().lower() + + +def _extract_strict_answer(text: str) -> str: + """The `#### N` rule. `""` when the response never emits that format.""" + match = _STRICT_ANSWER_RE.search(text) + return _normalize_exact_match(match.group(1)) if match else "" + + +def _extract_flexible_answer(text: str) -> str: + """Upstream's only filter: last regex match, first non-empty group, stripped.""" + matches = _FLEXIBLE_ANSWER_RE.findall(text) + if not matches: + return "" + last = next((part for part in matches[-1] if part), "") + return _normalize_exact_match(last) if last else "" + + +@sieval_task( + name="gsm1k_kshot_base_gen", + display_name="GSM1k (few-shot, base generative)", + description="GSM1k few-shot eval on Scale AI's published lm-eval-harness protocol.", + eval_mode=EvalMode.GEN, + n_shot=N_SHOT, + tags=("english", "math-word-problems", "open-ended", "base-model"), + model_type="gen", + status="experimental", + reference_impl=ReferenceImpl( + source="scaleapi/gsm1k_eval", + url=( + "https://github.com/scaleapi/gsm1k_eval/blob/39294c6f31855aca8255b6174b22fc3a6311be0b/lm_eval/tasks/gsm1k/gsm1k_scale.yaml" + ), + notes=( + "Scale's own lm-evaluation-harness fork task `gsm1k`: prompt " + '"Question: {q}\\nAnswer:" with 5 GSM8k-train exemplars, one ' + "`flexible-extract` filter (last numeric match) and `exact_match` " + "with regexes_to_ignore [',', '$', '(?s).*#### ', '.$'] + " + "ignore_case. Upstream resamples the 5 exemplars per question and " + "raises max generation length from 256 to 1000 tokens; this task " + "fixes one exemplar set (documented in the module docstring) and " + "leaves max_tokens to the model layer, where 1000 matches " + "upstream. Repeats: upstream runs `repeats: 1`, greedy at " + "temperature 0 — match it with n=1 and temperature=0. GSM1k is a " + "paired benchmark: read it as a diff against GSM8K on the same " + "extraction rule, not as a standalone score." + ), + ), +) +class GSM1KFewShotBaseGenTask( + Task[ + GSM1KDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + *, + n_shot: int = N_SHOT, + stop: tuple[str, ...] = STOP_SEQUENCES, + ): + if not 0 <= n_shot <= len(_GSM8K_FEWSHOT_EXAMPLES): + raise ValueError( + "n_shot must be between 0 and " + f"{len(_GSM8K_FEWSHOT_EXAMPLES)} (the number of vendored GSM8k " + f"exemplars), got {n_shot}" + ) + super().__init__(dataset=dataset, model=model, name=name) + self.n_shot = n_shot + self._stop = stop + + @override + async def preprocess(self, raw, ctx): + prefix = "".join( + _format_example(question, answer) + for question, answer in _GSM8K_FEWSHOT_EXAMPLES[: self.n_shot] + ) + return build_prompt_record( + prefix + _format_example(raw["question"], None), + reference=_normalize_exact_match(raw["answer"]), + ) + + @override + async def infer(self, pre, ctx): + # Keep `stop` out of the kwargs when unset so it can't clobber the + # model's configured stop via the `{**self._kwargs, **kwargs}` merge. + if self._stop: + return await self.model.agenerate(pre["prompt"], stop=list(self._stop)) + return await self.model.agenerate(pre["prompt"]) + + @override + async def postprocess(self, inf, ctx): + text = inf.texts[0] if inf.texts else "" + # The flexible rule is upstream's only filter, so it is the headline + # prediction; the strict `#### N` rule is a second extraction RULE over + # the same response, not a second rollout, so it rides in `extra`. `""` + # there means "this rule found nothing" (the sibling GSM8K task's + # convention) — a `None` would be dropped by serialization and read as + # "never measured" on resume. + return build_prediction_record( + [_extract_flexible_answer(text) or None], + extra={"strict_prediction": _extract_strict_answer(text)}, + ) + + @override + async def feedback(self, post, ctx): + gold = _normalize_exact_match(ctx.raw_sample["answer"]) + # Both extraction rules are recorded as co-equal metrics over one + # response; `correct` is DERIVED from the flexible one (upstream's single + # filter) so the headline and the metric cannot drift. + prediction = post["rollouts"][0].get("prediction") or "" + metrics: dict[str, bool | float] = { + "flexible_exact_match": prediction == gold, + "strict_exact_match": post["extra"]["strict_prediction"] == gold, + } + return True, build_judgement_record( + gold, + [ + build_rollout_judgement( + 0, bool(metrics["flexible_exact_match"]), metrics=metrics + ) + ], + metrics=metrics, + ) + + @override + async def report(self, finals, fails): + # Accuracy over the full requested set: a pipeline failure counts as + # wrong, not as an excluded sample. + total = len(finals) + len(fails) + if total == 0: + return { + "score": 0.0, + "fails": len(fails), + "flexible_exact_match": 0.0, + "strict_exact_match": 0.0, + } + flexible = ( + 100 + * sum( + 1 + for ctx in finals + if ctx.feedback_result["metrics"]["flexible_exact_match"] + ) + / total + ) + strict = ( + 100 + * sum( + 1 + for ctx in finals + if ctx.feedback_result["metrics"]["strict_exact_match"] + ) + / total + ) + return { + "score": flexible, + "fails": len(fails), + "flexible_exact_match": flexible, + "strict_exact_match": strict, + } diff --git a/tests/unit/tasks/test_gsm1k_0shot_gen.py b/tests/unit/tasks/test_gsm1k_0shot_gen.py new file mode 100644 index 00000000..ee617390 --- /dev/null +++ b/tests/unit/tasks/test_gsm1k_0shot_gen.py @@ -0,0 +1,125 @@ +"""Unit tests for the GSM1k 0-shot chat task (the GSM8K-paired half). + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import TaskContext +from sieval.datasets.gsm1k import GSM1KDataset, GSM1KDatasetSample +from sieval.tasks.gsm1k_0shot_gen import COT_INSTRUCTION, GSM1KZeroShotGenTask + + +class _CapturingChatModel(ChatModel): + def __init__(self, text: str): + super().__init__(model="mock-chat", api_key="fake") + self.last_kwargs: dict[str, object] = {} + self._text = text + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + _ = prompt + self.last_kwargs = dict(kwargs) + return ModelOutput(model=self.meta(), texts=[self._text]) + + async def _alogprobs_impl( + self, + prompt, + *, + max_tokens: int = 1, + logprobs: int = 5, + echo: bool = True, + temperature: float = 0.0, + **kwargs, + ) -> ModelOutput: + _ = (prompt, max_tokens, logprobs, echo, temperature, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _sample(answer: str = "42") -> GSM1KDatasetSample: + return {"question": "What is 40 + 2?", "answer": answer} + + +def _task(text: str) -> tuple[GSM1KZeroShotGenTask, _CapturingChatModel]: + dataset = GSM1KDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(_sample())])}) + ) + model = _CapturingChatModel(text=text) + return GSM1KZeroShotGenTask(dataset, model), model + + +@pytest.mark.anyio +async def test_prompt_is_the_deepseek_cot_user_turn(): + task, _ = _task("") + + pre = await task.preprocess(_sample(), TaskContext(sample_id=0)) + + assert pre["prompt"] == [ + {"role": "user", "content": "What is 40 + 2?" + COT_INSTRUCTION} + ] + # GSM1k's `answer` already IS the bare final answer — no `####` split, which + # is the one deviation from the gsm8k_0shot_gen sibling this pairs with. + assert pre["reference"] == "42" + + +@pytest.mark.anyio +async def test_boxed_answer_is_extracted_and_scored_correct(): + task, model = _task("Reasoning...\n\\boxed{42}") + raw = _sample() + inferred = ModelOutput(model=model.meta(), texts=["Reasoning...\n\\boxed{42}"]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inferred) + + post = await task.postprocess(inferred, ctx) + finalize, feedback = await task.feedback(post, ctx) + + assert finalize is True + assert post["rollouts"][0]["prediction"] == "42" + assert feedback["reference"] == "42" + assert feedback["rollouts"][0]["correct"] is True + + +@pytest.mark.anyio +async def test_unextractable_response_is_none_and_scores_wrong(): + task, model = _task("I decline to answer.") + raw = _sample() + inferred = ModelOutput(model=model.meta(), texts=["I decline to answer."]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inferred) + + post = await task.postprocess(inferred, ctx) + _, feedback = await task.feedback(post, ctx) + + assert post["rollouts"][0]["prediction"] is None + assert post["rollouts"][0]["extracted"] is False + assert feedback["rollouts"][0]["correct"] is False + + +@pytest.mark.anyio +async def test_report_counts_pipeline_failures_as_wrong(): + task, model = _task("\\boxed{42}") + raw = _sample() + inferred = ModelOutput(model=model.meta(), texts=["\\boxed{42}"]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inferred) + post = await task.postprocess(inferred, ctx) + _, feedback = await task.feedback(post, ctx) + + report = await task.report( + [TaskContext(sample_id=0, raw_sample=raw, feedback_result=feedback)], + [TaskContext(sample_id=1, raw_sample=raw)], + ) + + # Same denominator rule as gsm8k_0shot_gen, so both sides of the paired diff + # treat a pipeline failure identically. + assert report["fails"] == 1 + assert report["score"] == report["accuracy"] == 50.0 + + +@pytest.mark.anyio +async def test_report_on_empty_set_reports_zero(): + task, _ = _task("") + + report = await task.report([], []) + + assert report == {"score": 0.0, "fails": 0, "accuracy": 0.0} diff --git a/tests/unit/tasks/test_gsm1k_kshot_base_gen.py b/tests/unit/tasks/test_gsm1k_kshot_base_gen.py new file mode 100644 index 00000000..8e383110 --- /dev/null +++ b/tests/unit/tasks/test_gsm1k_kshot_base_gen.py @@ -0,0 +1,227 @@ +"""Unit tests for the GSM1k k-shot base generative task. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import ModelOutput +from sieval.core.models.gen_model import GenModel +from sieval.core.tasks import TaskContext +from sieval.datasets.gsm1k import GSM1KDataset, GSM1KDatasetSample +from sieval.tasks.gsm1k_kshot_base_gen import ( + _GSM8K_FEWSHOT_EXAMPLES, + N_SHOT, + STOP_SEQUENCES, + GSM1KFewShotBaseGenTask, + _extract_flexible_answer, + _extract_strict_answer, +) + + +class _CapturingGenModel(GenModel): + def __init__(self): + super().__init__(model="mock-gen", api_key="fake") + self.last_kwargs: dict[str, object] = {} + + async def _agenerate_impl(self, prompt: str, **kwargs) -> ModelOutput: + _ = prompt + self.last_kwargs = dict(kwargs) + return ModelOutput(model=self.meta(), texts=[" Work shown.\n#### 42"]) + + async def _alogprobs_impl( + self, + prompt: str, + *, + max_tokens: int = 1, + logprobs: int = 5, + echo: bool = True, + temperature: float = 0.0, + **kwargs, + ) -> ModelOutput: + _ = (prompt, max_tokens, logprobs, echo, temperature, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _sample(answer: str = "42") -> GSM1KDatasetSample: + return {"question": "What is 40 + 2?", "answer": answer} + + +def _task(n_shot: int = 0) -> tuple[GSM1KFewShotBaseGenTask, _CapturingGenModel]: + # GSM1k ships a `test` split only — a task needing a `train` split could not + # run on it at all, which is why the exemplars are vendored. + dataset = GSM1KDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(_sample())])}) + ) + model = _CapturingGenModel() + return GSM1KFewShotBaseGenTask(dataset, model, n_shot=n_shot), model + + +def test_vendored_exemplars_carry_gsm8k_cot_shape(): + # A GSM8k-train exemplar is what teaches the `#### N` final-answer format the + # strict rule reads; losing it would silently zero `strict_exact_match`. + assert len(_GSM8K_FEWSHOT_EXAMPLES) == N_SHOT == 5 + for question, answer in _GSM8K_FEWSHOT_EXAMPLES: + assert question.strip() == question + # A worked solution, then `#### N` on its own final line. + assert "\n#### " in answer + assert answer.split("\n")[-1].removeprefix("#### ").isdigit() + assert _extract_strict_answer(answer) == answer.rsplit("#### ", 1)[1] + + +def test_extractors_implement_upstreams_two_rules(): + # flexible-extract = last regex match, so a trailing restatement wins. + assert _extract_flexible_answer("First 7, then finally 42.") == "42" + # regexes_to_ignore: commas, dollar signs and a trailing period all dropped. + assert _extract_flexible_answer("It costs $1,234.") == "1234" + # strict-match needs the `#### ` delimiter and finds nothing without it. + assert _extract_strict_answer("Therefore the answer is 42.") == "" + assert _extract_strict_answer("Final.\n#### 1,234.") == "1234" + # Neither rule can extract from a response carrying no number. + assert _extract_flexible_answer("I cannot solve this.") == "" + assert _extract_strict_answer("I cannot solve this.") == "" + + +@pytest.mark.anyio +async def test_zero_shot_prompt_is_upstreams_doc_to_text(): + task, _ = _task(n_shot=0) + + pre = await task.preprocess(_sample(), TaskContext(sample_id=0)) + + assert pre["prompt"] == "Question: What is 40 + 2?\nAnswer:" + # GSM1k's gold is already the bare final answer: no `####` split, unlike + # openai/gsm8k. + assert pre["reference"] == "42" + + +@pytest.mark.anyio +async def test_fewshot_prompt_blocks_match_upstream_format(): + task, _ = _task(n_shot=2) + first_q, first_a = _GSM8K_FEWSHOT_EXAMPLES[0] + + pre = await task.preprocess(_sample(), TaskContext(sample_id=0)) + + assert pre["prompt"].startswith(f"Question: {first_q}\nAnswer: {first_a}\n\n") + assert pre["prompt"].endswith("Question: What is 40 + 2?\nAnswer:") + # n_shot exemplar blocks plus the unanswered question. + assert pre["prompt"].count("Question: ") == 3 + + +@pytest.mark.anyio +async def test_prompt_prefix_is_fixed_across_samples(): + # The documented deviation from upstream: exemplars do NOT vary per question. + task, _ = _task(n_shot=5) + + first = await task.preprocess(_sample(), TaskContext(sample_id=0)) + second = await task.preprocess( + {"question": "What is 1 + 1?", "answer": "2"}, TaskContext(sample_id=1) + ) + + tail = "Question: What is 40 + 2?\nAnswer:" + assert first["prompt"].removesuffix(tail) == second["prompt"].removesuffix( + "Question: What is 1 + 1?\nAnswer:" + ) + + +def test_n_shot_is_bounded_by_the_vendored_exemplar_count(): + dataset = GSM1KDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(_sample())])}) + ) + model = _CapturingGenModel() + + with pytest.raises(ValueError, match="n_shot must be between 0 and 5"): + GSM1KFewShotBaseGenTask(dataset, model, n_shot=6) + with pytest.raises(ValueError, match="n_shot must be between 0 and 5"): + GSM1KFewShotBaseGenTask(dataset, model, n_shot=-1) + + +@pytest.mark.anyio +async def test_infer_only_forwards_prompt_coupled_stop(): + task, model = _task() + + await task.infer( + {"prompt": "prompt"}, TaskContext(sample_id=0, raw_sample=_sample()) + ) + + assert model.last_kwargs == {"stop": list(STOP_SEQUENCES)} + + +@pytest.mark.anyio +async def test_headline_follows_the_flexible_rule_not_the_strict_one(): + task, model = _task() + raw = _sample() + inferred = ModelOutput( + model=model.meta(), + texts=["No delimiter here, but the final sentence says 42."], + ) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inferred) + + post = await task.postprocess(inferred, ctx) + finalize, feedback = await task.feedback(post, ctx) + report = await task.report( + [TaskContext(sample_id=0, raw_sample=raw, feedback_result=feedback)], [] + ) + + assert finalize is True + assert post["rollouts"][0]["prediction"] == "42" + assert post["extra"]["strict_prediction"] == "" + # Upstream's only filter is the flexible one, so it drives `correct`. + assert feedback["metrics"]["flexible_exact_match"] is True + assert feedback["metrics"]["strict_exact_match"] is False + assert feedback["rollouts"][0]["correct"] is True + assert report["score"] == report["flexible_exact_match"] == 100.0 + assert report["strict_exact_match"] == 0.0 + # No bare `exact_match` key: it means the STRICT rule in gsm8k_kshot_base_gen, + # so sharing the name would let a paired diff compare two different rules. + assert "exact_match" not in report + + +@pytest.mark.anyio +async def test_unextractable_response_is_none_and_scores_wrong(): + task, model = _task() + raw = _sample() + inferred = ModelOutput(model=model.meta(), texts=["I cannot solve this."]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inferred) + + post = await task.postprocess(inferred, ctx) + _, feedback = await task.feedback(post, ctx) + + assert post["rollouts"][0]["prediction"] is None + assert post["rollouts"][0]["extracted"] is False + assert feedback["metrics"]["flexible_exact_match"] is False + assert feedback["rollouts"][0]["correct"] is False + + +@pytest.mark.anyio +async def test_report_counts_pipeline_failures_as_wrong(): + task, model = _task() + raw = _sample() + inferred = ModelOutput(model=model.meta(), texts=["The answer is 42."]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inferred) + post = await task.postprocess(inferred, ctx) + _, feedback = await task.feedback(post, ctx) + + report = await task.report( + [TaskContext(sample_id=0, raw_sample=raw, feedback_result=feedback)], + [TaskContext(sample_id=1, raw_sample=raw)], + ) + + assert report["fails"] == 1 + # 1 correct out of (1 final + 1 fail), not out of 1 final. + assert report["score"] == 50.0 + + +@pytest.mark.anyio +async def test_report_on_empty_set_reports_zero_for_both_rules(): + task, _ = _task() + + report = await task.report([], []) + + assert report == { + "score": 0.0, + "fails": 0, + "flexible_exact_match": 0.0, + "strict_exact_match": 0.0, + }