feat(gsm1k): add GSM1k dataset + 5-shot base and 0-shot chat tasks

README.md

@@ -27,7 +27,7 @@ pdm install          # or: pip install -e .
 Optional extras (per-benchmark dependencies):
 
 ```bash
-pip install -e ".[math]"     # AIME, GSM8K, HMMT, IMO-AnswerBench, MATH-500, TheoremQA (math-verify)
+pip install -e ".[math]"     # AIME, GSM1k, GSM8K, HMMT, IMO-AnswerBench, MATH-500, TheoremQA (math-verify)
 pip install -e ".[drop]"     # DROP (numpy, scipy)
 pip install -e ".[ifbench]"  # IFBench (emoji, nltk, setuptools, syllapy)
 pip install -e ".[ifeval]"   # IFEval (absl, langdetect, nltk, immutabledict)

sieval/datasets/__init__.pyi

@@ -45,6 +45,10 @@ from .gpqa_diamond import (
     GPQADiamondDataset,
     GPQADiamondDatasetSample,
 )
+from .gsm1k import (
+    GSM1KDataset,
+    GSM1KDatasetSample,
+)
 from .gsm8k import (
     GSM8KDataset,
     GSM8KDatasetSample,
@@ -165,6 +169,8 @@ __all__ = [
     "DROPDatasetSample",
     "GPQADiamondDataset",
     "GPQADiamondDatasetSample",
+    "GSM1KDataset",
+    "GSM1KDatasetSample",
     "GSM8KDataset",
     "GSM8KDatasetSample",
     "HLEDataset",

sieval/datasets/gsm1k.py

@@ -0,0 +1,83 @@
+"""
+GSM1k loader — Scale AI's from-scratch re-do of GSM8K, built to detect overfitting.
+
+GSM1k is 1205 grade-school word problems written by human annotators with no LLM
+or synthetic assistance, mirroring GSM8K's difficulty and answer-magnitude
+distribution. The benchmark exists to be read as a *pair* with GSM8K, not on its
+own: a model's GSM8K − GSM1k gap estimates how much of its GSM8K score is
+memorization rather than reasoning. The paper measures drops of up to 8% and a
+Spearman r² of 0.36 between a model's likelihood of generating a GSM8k example
+and its gap.
+
+**Release history, because the pinned snapshot ships an empty dataset card while
+the paper says the data is withheld.** The paper (Nov 2024) declined to publish
+GSM1k "to prevent a similar problem of data contamination occurring in the
+future" and precommitted to release on the earlier of two triggers — three
+open-source models of different lineages reaching 95% accuracy, or June 2025 —
+with its datasheet stating "The dataset (yet unreleased) will be released with
+the MIT license." The pinned snapshot was uploaded to the ScaleAI org on
+2025-03-31/04-01, i.e. that release. So `license="MIT"` here is the datasheet's
+commitment for the *data*, not the eval repo's code license.
+
+**Provenance verified**, since an empty card is not evidence: every one of the 50
+questions in `gsm1k_public_50.csv` — the sample Scale published in its eval repo
+while the full set was still withheld — appears verbatim in this snapshot's
+`test` split with an identical answer (50/50 found, 0 answer mismatches). The row
+count also matches the paper exactly: "GSM1k consists of 1205 problems".
+
+Schema, measured at the pinned revision: `question` and `answer` are both
+strings and need no cast — no cast needed, upstream ships this dtype. All 1205
+answers are bare integers (1-6 characters, every one matching `-?[0-9]+`), with
+no thousands separators, no `####` delimiter and **no worked solution** — the
+released data carries final answers only. Two consequences for tasks: the gold
+needs no `answer.split("####")` step (unlike `openai/gsm8k`), and GSM1k cannot
+supply chain-of-thought few-shot exemplars of its own, which is why
+`gsm1k_kshot_base_gen` borrows GSM8K's the way upstream's harness does.
+
+There is a `test` split and nothing else — no train set, so nothing to hold out.
+
+References:
+
+* Paper: <https://arxiv.org/abs/2405.00332>
+* Eval harness + public 50-example sample: <https://github.com/scaleapi/gsm1k_eval>
+
+AI-Generated Code - Claude Opus 5 (1M context) (Anthropic)
+"""
+
+from typing import TypedDict, override
+
+from datasets import DatasetDict as HFDatasetDict
+from datasets import load_dataset
+
+from sieval.core.datasets import (
+    Category,
+    Dataset,
+    Level1Category,
+    sieval_dataset,
+)
+from sieval.core.utils.hf import ensure_dataset_dict
+
+GSM1K_REVISION = "bc09569d09a614b9b530edc7f076fb214ac10493"
+
+
+class GSM1KDatasetSample(TypedDict):
+    question: str
+    answer: str
+
+
+@sieval_dataset(
+    name="gsm1k",
+    display_name="GSM1k",
+    description="Grade School Math 1k - 1205 human-written GSM8K mirror problems.",
+    source=f"hf:ScaleAI/gsm1k@{GSM1K_REVISION}",
+    categories=(Category(Level1Category.MATHEMATICS, "ElementaryMath"),),
+    tags=("english", "math-word-problems", "open-ended"),
+    license="MIT",
+)
+class GSM1KDataset(Dataset[GSM1KDatasetSample]):
+    @override
+    def load(self, name_or_path: str, **kwargs) -> HFDatasetDict:
+        # One unnamed config, unlike openai/gsm8k's "main" / "socratic", so there
+        # is no config argument to forward.
+        dataset = load_dataset(name_or_path, **kwargs)
+        return ensure_dataset_dict(dataset)

sieval/meta/index.json

@@ -253,6 +253,28 @@
         "gpqa_diamond.csv": "sha256:41d1213cd7a4998605a26c2798500652572007161b3a92817ba46b35befcd305"
       }
     },
+    {
+      "name": "gsm1k",
+      "display_name": "GSM1k",
+      "description": "Grade School Math 1k - 1205 human-written GSM8K mirror problems.",
+      "source": [
+        "hf:ScaleAI/gsm1k@bc09569d09a614b9b530edc7f076fb214ac10493"
+      ],
+      "categories": [
+        {
+          "level1": "Mathematics",
+          "level2": "ElementaryMath"
+        }
+      ],
+      "tags": [
+        "english",
+        "math-word-problems",
+        "open-ended"
+      ],
+      "deps_group": null,
+      "license": "MIT",
+      "checksums": {}
+    },
     {
       "name": "gsm8k",
       "display_name": "GSM8K",
@@ -1064,6 +1086,49 @@
       },
       "status": "stable"
     },
+    {
+      "name": "gsm1k_0shot_gen",
+      "display_name": "GSM1k (0-shot, generative)",
+      "description": "GSM1k 0-shot chat-model eval, prompt-paired with gsm8k_0shot_gen.",
+      "dataset": "gsm1k",
+      "eval_mode": "gen",
+      "n_shot": 0,
+      "tags": [
+        "english",
+        "math-word-problems",
+        "open-ended"
+      ],
+      "deps_group": "math",
+      "model_type": "chat",
+      "reference_impl": {
+        "source": "deepseek-ai/DeepSeek-Math",
+        "url": "https://github.com/deepseek-ai/DeepSeek-Math/tree/b8b0f8ce093d80bf8e9a641e44142f06d092c305/evaluation",
+        "notes": "Protocol borrowed from the sibling `gsm8k_0shot_gen`, so the two form a prompt-exact pair: user turn = question + \"Please reason step by step, and put your final answer within \\boxed{}.\", chat template applied by the serving backend; extract_answer(exhaust=False) (= extract_last_single_answer) and is_correct/math_equal (= eval_last_single_answer) are vendored byte-for-byte in sieval.community.deepseek_math. Gold is GSM1k's `answer` verbatim (already the bare final answer, so no '####' split). Scale AI published GSM1k at 5-shot raw completion only (see gsm1k_kshot_base_gen) — no published number matches this 0-shot chat protocol, so read the GSM8K - GSM1k diff, not the absolute score. Repeats: 1 rollout, greedy at temperature 0."
+      },
+      "status": "experimental"
+    },
+    {
+      "name": "gsm1k_kshot_base_gen",
+      "display_name": "GSM1k (few-shot, base generative)",
+      "description": "GSM1k few-shot eval on Scale AI's published lm-eval-harness protocol.",
+      "dataset": "gsm1k",
+      "eval_mode": "gen",
+      "n_shot": 5,
+      "tags": [
+        "english",
+        "math-word-problems",
+        "open-ended",
+        "base-model"
+      ],
+      "deps_group": null,
+      "model_type": "gen",
+      "reference_impl": {
+        "source": "scaleapi/gsm1k_eval",
+        "url": "https://github.com/scaleapi/gsm1k_eval/blob/39294c6f31855aca8255b6174b22fc3a6311be0b/lm_eval/tasks/gsm1k/gsm1k_scale.yaml",
+        "notes": "Scale's own lm-evaluation-harness fork task `gsm1k`: prompt \"Question: {q}\\nAnswer:\" with 5 GSM8k-train exemplars, one `flexible-extract` filter (last numeric match) and `exact_match` with regexes_to_ignore [',', '$', '(?s).*#### ', '.$'] + ignore_case. Upstream resamples the 5 exemplars per question and raises max generation length from 256 to 1000 tokens; this task fixes one exemplar set (documented in the module docstring) and leaves max_tokens to the model layer, where 1000 matches upstream. Repeats: upstream runs `repeats: 1`, greedy at temperature 0 — match it with n=1 and temperature=0. GSM1k is a paired benchmark: read it as a diff against GSM8K on the same extraction rule, not as a standalone score."
+      },
+      "status": "experimental"
+    },
     {
       "name": "gsm8k_0shot_gen",
       "display_name": "GSM8K (0-shot, generative)",

sieval/tasks/__init__.pyi

@@ -40,6 +40,12 @@ from .drop_kshot_gen import (
 from .gpqa_diamond_0shot_gen import (
     GPQADiamondZeroShotGenTask,
 )
+from .gsm1k_0shot_gen import (
+    GSM1KZeroShotGenTask,
+)
+from .gsm1k_kshot_base_gen import (
+    GSM1KFewShotBaseGenTask,
+)
 from .gsm8k_0shot_gen import (
     GSM8KZeroShotGenTask,
 )
@@ -151,6 +157,8 @@ __all__ = [
     "CMMLUFewShotClpTask",
     "DROPFewShotGenTask",
     "GPQADiamondZeroShotGenTask",
+    "GSM1KFewShotBaseGenTask",
+    "GSM1KZeroShotGenTask",
     "GSM8KFewShotBaseGenTask",
     "GSM8KZeroShotGenTask",
     "HLEZeroShotGenTask",

sieval/tasks/gsm1k_0shot_gen.py

@@ -0,0 +1,168 @@
+"""
+GSM1k 0-shot generative task — the chat-side half of the GSM8K/GSM1k pair.
+
+Scale AI published GSM1k under one protocol only: 5-shot raw completion, ported
+here as `gsm1k_kshot_base_gen`. That protocol needs a `gen` model, so on its own
+it leaves GSM1k unrunnable for the chat endpoints this repo mostly verifies. This
+task supplies the missing half by applying `gsm8k_0shot_gen`'s protocol — the
+DeepSeek-Math zero-shot CoT path — to GSM1k, unchanged:
+
+* Prompt (DeepSeek's `run_subset_parallel.py::markup_question`, language="en",
+  task="cot"): the user turn is `{question}` followed by `"\\nPlease reason step
+  by step, and put your final answer within \\boxed{}."`, with the chat template
+  applied by the serving backend.
+* Answer extraction: `extract_answer(reasoning, exhaust=False)` — DeepSeek's
+  `extract_last_single_answer`: last `\\boxed{...}` if present, else the text
+  after `"he answer is"`, else the last number, then `strip_string`.
+* Scoring: `is_correct` — DeepSeek's `eval_last_single_answer` (numeric isclose
+  with %-variants, then a sympy symbolic fallback). `score` is this accuracy.
+
+Extraction and scoring live verbatim in `sieval.community.deepseek_math`, vendored
+byte-faithfully from DeepSeek-Math at the pinned commit. Nothing about them is
+GSM8K-specific: the gold is a bare integer either way.
+
+**This is a different measurement regime from upstream's, not a port of it — no
+published GSM1k number corresponds to a 0-shot chat score.** What it buys is a
+*prompt-exact pair*: run this task and `gsm8k_0shot_gen` against the same model
+and the two differ only in which problem set the question came from — identical
+prompt template, identical extractor, identical scorer, no few-shot exemplars to
+vary. The GSM8K − GSM1k **diff** is the measurement, and it is the quantity GSM1k
+exists to produce (the paper's Table 1 is a diff column first, an accuracy column
+second). A single absolute number here aligns with nothing external.
+
+Deviation from the sibling it mirrors: the gold needs no `answer.split("####")`
+step, because GSM1k's `answer` field already *is* the bare final answer — see
+`sieval/datasets/gsm1k.py`. Both tasks divide `report()` by
+`len(finals) + len(fails)`, so a pipeline failure counts as wrong on both sides
+of the diff.
+
+`status="experimental"`: the extraction/scoring layer is the sibling's, verbatim
+and already exercised, but this pairing has not yet been validated by a run, and
+unlike the sibling it has no published column to be validated *against* — only
+its own diff.
+
+Repro decoding (model-layer assets — set via `models:` / `infer_args`, not in
+this code): greedy `temperature=0`, `top_p=1.0`, `max_tokens=1024`, stop = the
+model's EOS only, matching `gsm8k_0shot_gen` so the pair stays comparable.
+
+References:
+
+* GSM1k paper: <https://arxiv.org/abs/2405.00332>
+* Protocol source: <https://github.com/deepseek-ai/DeepSeek-Math/tree/b8b0f8ce093d80bf8e9a641e44142f06d092c305/evaluation>
+
+AI-Generated Code - Claude Opus 5 (1M context) (Anthropic)
+"""
+
+from typing import override
+
+from sieval.core.models import ModelOutput
+from sieval.core.tasks import (
+    EvalMode,
+    JudgementRecord,
+    PredictionRecord,
+    PromptRecord,
+    ReferenceImpl,
+    Task,
+    build_judgement_record,
+    build_prediction_record,
+    build_prompt_record,
+    build_rollout_judgement,
+    sieval_task,
+)
+from sieval.datasets import GSM1KDatasetSample
+
+# Verbatim from run_subset_parallel.py::markup_question (language="en",
+# task="cot"): f"{content}\nPlease reason step by step, and put your final
+# answer within " + "\\boxed{}."
+COT_INSTRUCTION = (
+    "\nPlease reason step by step, and put your final answer within \\boxed{}."
+)
+
+
+@sieval_task(
+    name="gsm1k_0shot_gen",
+    display_name="GSM1k (0-shot, generative)",
+    description="GSM1k 0-shot chat-model eval, prompt-paired with gsm8k_0shot_gen.",
+    eval_mode=EvalMode.GEN,
+    n_shot=0,
+    tags=("english", "math-word-problems", "open-ended"),
+    deps_group="math",
+    model_type="chat",
+    status="experimental",
+    reference_impl=ReferenceImpl(
+        source="deepseek-ai/DeepSeek-Math",
+        url=(
+            "https://github.com/deepseek-ai/DeepSeek-Math/tree/b8b0f8ce093d80bf8e9a641e44142f06d092c305/evaluation"
+        ),
+        notes=(
+            "Protocol borrowed from the sibling `gsm8k_0shot_gen`, so the two "
+            "form a prompt-exact pair: user turn = question + "
+            '"Please reason step by step, and put your final answer within '
+            '\\boxed{}.", chat template applied by the serving backend; '
+            "extract_answer(exhaust=False) (= extract_last_single_answer) and "
+            "is_correct/math_equal (= eval_last_single_answer) are vendored "
+            "byte-for-byte in sieval.community.deepseek_math. Gold is GSM1k's "
+            "`answer` verbatim (already the bare final answer, so no '####' "
+            "split). Scale AI published GSM1k at 5-shot raw completion only "
+            "(see gsm1k_kshot_base_gen) — no published number matches this "
+            "0-shot chat protocol, so read the GSM8K - GSM1k diff, not the "
+            "absolute score. Repeats: 1 rollout, greedy at temperature 0."
+        ),
+    ),
+)
+class GSM1KZeroShotGenTask(
+    Task[
+        GSM1KDatasetSample,
+        PromptRecord,
+        ModelOutput,
+        PredictionRecord,
+        JudgementRecord,
+        dict[str, float],
+    ]
+):
+    @override
+    async def preprocess(self, raw, ctx):
+        return build_prompt_record(
+            [
+                {"role": "user", "content": raw["question"] + COT_INSTRUCTION},
+            ],
+            reference=raw["answer"],
+        )
+
+    @override
+    async def infer(self, pre, ctx):
+        return await self.model.agenerate(pre["prompt"])
+
+    @override
+    async def postprocess(self, inf, ctx):
+        from sieval.community.deepseek_math import extract_answer
+
+        text = inf.texts[0] if inf.texts else ""
+        # extract_answer returns "" when nothing was found; None is the protocol's
+        # spelling of that, and feedback restores "" for the grader.
+        return build_prediction_record([extract_answer(text, exhaust=False) or None])
+
+    @override
+    async def feedback(self, post, ctx):
+        from sieval.community.deepseek_math import is_correct
+
+        gold = ctx.raw_sample["answer"]
+        # `or ""` gives the grader the same empty string a failed extraction
+        # produced upstream, rather than a None it has no branch for.
+        prediction = post["rollouts"][0].get("prediction") or ""
+        correct = is_correct({"prediction": prediction, "answer": gold})
+        return True, build_judgement_record(gold, [build_rollout_judgement(0, correct)])
+
+    @override
+    async def report(self, finals, fails):
+        # Accuracy over the full requested set (finals + fails), matching
+        # `gsm8k_0shot_gen` so both sides of the paired diff count a pipeline
+        # failure as wrong rather than excluding it.
+        total = len(finals) + len(fails)
+        if total == 0:
+            return {"score": 0.0, "fails": len(fails), "accuracy": 0.0}
+        correct_num = sum(
+            1 for ctx in finals if ctx.feedback_result["rollouts"][0]["correct"]
+        )
+        accuracy = 100 * correct_num / total
+        return {"score": accuracy, "fails": len(fails), "accuracy": accuracy}