From e9aebe44b20f4637be96ca49a7771d2e996fb34a Mon Sep 17 00:00:00 2001 From: Ethan Date: Thu, 6 Aug 2026 11:07:22 +0800 Subject: [PATCH] feat(tasks): add AGIEval (21 subsets, 0-shot two-stage) with subset selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AGIEval v1.1 (Microsoft, arXiv:2304.06364) — 21 human-exam subsets, 19 MCQ + 2 math cloze, 7,272 problems. New `agieval` dataset + `agieval_0shot_gen` task, with upstream's prompt/parse/score layer vendored in `community/agieval`. Subset selection is the dataset's main knob, since nobody runs all 21 subsets by accident: args: {group: math} # 5 math subsets (1,943 problems) args: {group: en-mcq | zh-mcq | all} # upstream's leaderboard groups args: {subsets: [math, sat-math]} # exact names, any combination Selection always concatenates in canonical subset order, so the same choice spelled two ways yields the same sample ids. The task reproduces upstream's zero-shot protocol, which is TWO model calls: the model answers freely, then re-reads its own answer under an extraction cue and that short reply is what gets parsed (upstream's parser is "first A-F character", unusable on a chain of thought). Both ModelOutputs are returned from infer(), so both are profiled and stage 1's reasoning stays on disk. `extractor` pins a separate model for stage 2, as upstream did with gpt-35-turbo; unset, stage 2 runs on the model under test — the one protocol-level divergence, documented in reference_impl.notes. report() emits per-subset accuracy, `score` as the macro over the subsets that ran (upstream's "average for all datasets"), and `macro_en_mcq`/`macro_zh_mcq`/ `macro_math` only when a whole group ran — a partial macro is not the published number. status="experimental": faithful to upstream by construction but not yet validated against a run of our own. Targets are in the notes (GPT-4o zero-shot 62.3 all / 65.2 en / 63.3 zh). Verified: `sieval dataset download agieval` stages + checksums all 21 files; end-to-end run over real staged data produces both infer entries per sample, 2 profiled calls/sample, and the expected report keys; 43 new unit tests, full unit suite (2965) green; preflight all-pass. Co-Authored-By: Claude Opus 5 (1M context) --- examples/agieval-math.yaml | 66 +++++ sieval/community/agieval/__init__.py | 0 sieval/community/agieval/dataset_loader.py | 169 ++++++++++++ sieval/community/agieval/evaluation.py | 75 ++++++ sieval/community/agieval/math_equivalence.py | 176 +++++++++++++ sieval/community/agieval/post_process.py | 154 +++++++++++ sieval/datasets/__init__.pyi | 6 + sieval/datasets/agieval.py | 249 ++++++++++++++++++ sieval/meta/index.json | 87 +++++++ sieval/tasks/__init__.pyi | 4 + sieval/tasks/agieval_0shot_gen.py | 256 +++++++++++++++++++ tests/unit/community/test_agieval.py | 183 +++++++++++++ tests/unit/datasets/test_agieval.py | 159 ++++++++++++ tests/unit/tasks/test_agieval_0shot_gen.py | 241 +++++++++++++++++ 14 files changed, 1825 insertions(+) create mode 100644 examples/agieval-math.yaml create mode 100644 sieval/community/agieval/__init__.py create mode 100644 sieval/community/agieval/dataset_loader.py create mode 100644 sieval/community/agieval/evaluation.py create mode 100644 sieval/community/agieval/math_equivalence.py create mode 100644 sieval/community/agieval/post_process.py create mode 100644 sieval/datasets/agieval.py create mode 100644 sieval/tasks/agieval_0shot_gen.py create mode 100644 tests/unit/community/test_agieval.py create mode 100644 tests/unit/datasets/test_agieval.py create mode 100644 tests/unit/tasks/test_agieval_0shot_gen.py diff --git a/examples/agieval-math.yaml b/examples/agieval-math.yaml new file mode 100644 index 00000000..127eeacf --- /dev/null +++ b/examples/agieval-math.yaml @@ -0,0 +1,66 @@ +# ------------------------------------------------------------------------------ +# AGIEval — math subsets only (0-shot, two-stage) +# ------------------------------------------------------------------------------ +# AGIEval ships 21 subsets across two languages and two answer formats. This runs +# the 5 drawn from math exams — sat-math, aqua-rat, gaokao-mathqa, math, +# gaokao-mathcloze (1,943 problems) — selected with `group: math`. +# +# Other selections: +# args: {group: all} # all 21 (7,272 problems) +# args: {group: en-mcq} # AGIEval-en leaderboard group (8) +# args: {group: zh-mcq} # AGIEval-zh leaderboard group (11) +# args: {subsets: [math, sat-math]} # exact names, any combination +# +# Two-step flow: +# 1. sieval dataset download agieval +# 2. sieval eval agieval-math.yaml +# +# Each sample costs TWO model calls: the model answers, then re-reads its own +# answer under an extraction cue and that short reply is what gets scored. This +# is AGIEval's own zero-shot protocol, not an add-on — see the task's +# reference_impl.notes. +# +# Edit these fields to match your setup: +# models.local-model.infer.checkpoint — your model's on-disk path +# models.local-model.infer_meta.image — swap to your own image if you need +# custom deps, or drop the block to run +# without a container +# ------------------------------------------------------------------------------ + +result_dir: ./outputs/agieval-math + +models: + local-model: + args: + concurrency_limit: 32 + temperature: 0.0 + infer: + backend: sglang + checkpoint: /path/to/your/model # EDIT ME + infer_meta: + gpu: H100-80G + image: lmsysorg/sglang:latest + +datasets: + agieval_math: + class: AGIEvalDataset + path: "${SIEVAL_DATA_DIR}/agieval" # `sieval dataset download agieval` stages this + args: + group: math + +tasks: + agieval_0shot_gen: + class: AGIEvalZeroShotGenTask + dataset: agieval_math + model: local-model + # Answer extraction (stage 2) runs on the model under test by default. + # Upstream instead ran it on a fixed cheap model; pin one here to match, and + # note the score depends on which you choose: + # args: + # extractor: + # model: gpt-3.5-turbo + # api_base: https://api.openai.com/v1 + # temperature: 0.0 + infer_args: + # Cloze subsets need room to work; MCQ subsets stop long before this. + max_tokens: 4096 diff --git a/sieval/community/agieval/__init__.py b/sieval/community/agieval/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/community/agieval/dataset_loader.py b/sieval/community/agieval/dataset_loader.py new file mode 100644 index 00000000..8a2d3232 --- /dev/null +++ b/sieval/community/agieval/dataset_loader.py @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# adapted from https://github.com/ruixiangcui/AGIEval/blob/84ab72d94318290aad2e4ec820d535a95a1f7552/src/dataset_loader.py +"""AGIEval subset taxonomy + zero-shot prompt construction. + +Two things upstream keeps in ``src/dataset_loader.py`` and every other AGIEval +module keys off: + +* the **subset families** — four disjoint tuples (english/chinese × qa/cloze) + that decide prompt language and answer-parsing rules, plus the two scoring + overrides (``MULTI_CHOICE_SUBSETS``, ``MATH_OUTPUT_SUBSETS``); +* the **zero-shot prompt** (``convert_zero_shot``) and the **second-stage + answer-extraction prompt** (``generate_second_stage_input``). Upstream's + zero-shot protocol is two calls: the model answers freely, then a second call + re-reads its own answer under a "the answer is" cue so a short, parseable + letter/value can be extracted. Both stages are needed to reproduce AGIEval's + published zero-shot numbers. + +Kept out on purpose: the few-shot paths (``combine_prompt`` / ``concat_prompt`` +/ ``convert_few_shot``) and their ``tiktoken`` budget trimming — sieval ships +the zero-shot task only, and dead vendored code rots. + +Deltas from upstream, all deliberate: + +* ``convert_zero_shot`` / ``generate_second_stage_input`` are per-sample + functions here (upstream's operate on whole files) and take the subset name + first; the emitted strings are byte-identical. +* An unknown subset raises ``ValueError``. Upstream wraps the family dispatch in + ``try/except NameError`` and returns ``None`` for a name in no family, which + surfaces much later as a ``TypeError`` on the prompt. +* ``MATH_SUBSETS`` is a **sieval** grouping, not upstream's; see its comment. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from collections.abc import Mapping + +# Subset families, verbatim from upstream (order included: it is the order the +# families are declared in, and sieval reuses it as the canonical subset order). +# `gaokao-english` sits in the ENGLISH family because its *prompt* is English — +# the exam is Chinese, and upstream's own leaderboard averages count it as +# Chinese. Do not conflate the two groupings; see evaluation.py. +ENGLISH_QA_SUBSETS: tuple[str, ...] = ( + "lsat-ar", + "lsat-lr", + "lsat-rc", + "logiqa-en", + "sat-math", + "sat-en", + "aqua-rat", + "sat-en-without-passage", + "gaokao-english", +) +CHINESE_QA_SUBSETS: tuple[str, ...] = ( + "logiqa-zh", + "jec-qa-kd", + "jec-qa-ca", + "gaokao-chinese", + "gaokao-geography", + "gaokao-history", + "gaokao-biology", + "gaokao-chemistry", + "gaokao-physics", + "gaokao-mathqa", +) +ENGLISH_CLOZE_SUBSETS: tuple[str, ...] = ("math",) +CHINESE_CLOZE_SUBSETS: tuple[str, ...] = ("gaokao-mathcloze",) + +#: Answers are compared as *sets* of letters, not strings (upstream's +#: `multi_choice_datasets`). `gaokao-physics` stayed on this list after v1.1 +#: made its labels single-answer — kept, because set-vs-set and string-vs-string +#: agree on single letters. +MULTI_CHOICE_SUBSETS: tuple[str, ...] = ("jec-qa-kd", "jec-qa-ca", "gaokao-physics") + +#: Answers are compared by math equivalence, not string equality (upstream's +#: `math_output_datasets`). Same membership as the two cloze families. +MATH_OUTPUT_SUBSETS: tuple[str, ...] = ("gaokao-mathcloze", "math") + +#: All 21 data files under `data/v1_1`, in upstream's family-declaration order. +#: The AGIEval paper and README say "20 tasks" — they count `sat-en` and +#: `sat-en-without-passage` as one task in two prompt variants, while upstream's +#: own driver script evaluates and averages over all 21 files. +SUBSETS: tuple[str, ...] = ( + ENGLISH_QA_SUBSETS + CHINESE_QA_SUBSETS + ENGLISH_CLOZE_SUBSETS + CHINESE_CLOZE_SUBSETS +) + +#: sieval-defined grouping — upstream has no "math" group. The five subsets +#: drawn from mathematics exams: SAT math, AQuA-RAT algebraic word problems, +#: Gaokao math (MCQ + cloze), and MATH competition problems. Everything else in +#: AGIEval is language/logic/law/science. +MATH_SUBSETS: tuple[str, ...] = ( + "sat-math", + "aqua-rat", + "gaokao-mathqa", + "math", + "gaokao-mathcloze", +) + +_OPTION_LETTERS = "ABCDEFG" + +# Second-stage cue per family, verbatim from `generate_second_stage_input` +# (with_format_prompt=False, the setting upstream's run_prediction.py uses). +# The hardcoded "A through E" / "A到D" do not track the actual option count — +# upstream's text, kept as-is. +_SECOND_STAGE_CUES: tuple[tuple[tuple[str, ...], str], ...] = ( + (ENGLISH_QA_SUBSETS, "Therefore, among A through E, the answer is"), + (CHINESE_QA_SUBSETS, "因此，从A到D, 我们应选择"), + (ENGLISH_CLOZE_SUBSETS, "Therefore, the answer is"), + (CHINESE_CLOZE_SUBSETS, "因此，答案是"), +) + + +def zero_shot_prompt(subset: str, row: Mapping) -> str: + """Upstream ``convert_zero_shot(line, dataset_name)`` for one row. + + *row* needs ``passage`` / ``question`` / ``options`` (the AGIEval sample + fields); ``options`` may be empty for the cloze subsets, which do not use it. + """ + passage = row["passage"] if row["passage"] is not None else "" + question = row["question"] + options = row["options"] or [] + + if subset in ENGLISH_QA_SUBSETS: + count = len(options) + if count == 1: + count = 5 + return ( + passage + + "Q: " + + question + + " " + + "Answer Choices: " + + " ".join(options) + + "\n" + + "A: Among A through {}, the answer is".format(_OPTION_LETTERS[count - 1]) + ) + if subset in CHINESE_QA_SUBSETS: + count = len(options) + if count == 1: + count = 4 + return ( + passage + + "问题：" + + question + + " " + + "选项：" + + " ".join(options) + + "\n" + + "答案：从A到{}, 我们应选择".format(_OPTION_LETTERS[count - 1]) + ) + if subset in ENGLISH_CLOZE_SUBSETS: + return passage + "Q: " + question + "\nA: The answer is" + if subset in CHINESE_CLOZE_SUBSETS: + return passage + "问题：" + question + "\n答案：" + raise ValueError(f"Unknown AGIEval subset {subset!r}; expected one of {SUBSETS}") + + +def second_stage_prompt(subset: str, context: str, first_stage_output: str) -> str: + """Upstream ``generate_second_stage_input`` for one row. + + *context* is the first-stage prompt and *first_stage_output* the model's + reply to it; the cue that follows asks for the answer alone, which is what + :func:`sieval.community.agieval.post_process.post_process` parses. + """ + for subsets, cue in _SECOND_STAGE_CUES: + if subset in subsets: + return "{0}\n{1}\n{2}".format(context, first_stage_output, cue) + raise ValueError(f"Unknown AGIEval subset {subset!r}; expected one of {SUBSETS}") diff --git a/sieval/community/agieval/evaluation.py b/sieval/community/agieval/evaluation.py new file mode 100644 index 00000000..398e76ab --- /dev/null +++ b/sieval/community/agieval/evaluation.py @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# adapted from https://github.com/ruixiangcui/AGIEval/blob/84ab72d94318290aad2e4ec820d535a95a1f7552/src/evaluation.py +# leaderboard groupings from https://github.com/ruixiangcui/AGIEval/blob/84ab72d94318290aad2e4ec820d535a95a1f7552/post_process_and_evaluation.py +"""AGIEval per-sample verdict + the groupings its leaderboard averages over. + +Three comparison rules (upstream ``evaluate_single_sample``): set-of-letters for +the multi-answer MCQ subsets, math equivalence for the two cloze subsets, exact +string equality for everything else. + +The leaderboard groups here are **not** the prompt-language families in +:mod:`.dataset_loader`, and the difference is not cosmetic: ``gaokao-english`` is +prompted in English but averaged as Chinese (it is a Chinese Gaokao paper), and +``gaokao-mathqa`` is Chinese in both. Upstream's driver keeps two separate lists +for exactly this reason. + +Deltas from upstream: ``convert_to_set(None)`` returns an empty *set* (upstream +returns an empty *dict* — a typo, ``{}``); every comparison it feeds reaches the +same verdict either way. A ``None`` prediction (sieval's "could not extract") +needs no special case: it compares unequal under all three rules. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from .dataset_loader import MATH_OUTPUT_SUBSETS, MULTI_CHOICE_SUBSETS +from .math_equivalence import is_equiv + +#: The 8 English MCQ subsets upstream averages for its AGIEval-en leaderboard. +#: `gaokao-english` is excluded here and counted with the Chinese group below. +LEADERBOARD_EN_MCQ_SUBSETS: tuple[str, ...] = ( + "aqua-rat", + "logiqa-en", + "lsat-ar", + "lsat-lr", + "lsat-rc", + "sat-math", + "sat-en", + "sat-en-without-passage", +) + +#: The 11 Chinese MCQ subsets upstream averages for its AGIEval-zh leaderboard. +LEADERBOARD_ZH_MCQ_SUBSETS: tuple[str, ...] = ( + "logiqa-zh", + "jec-qa-kd", + "jec-qa-ca", + "gaokao-chinese", + "gaokao-english", + "gaokao-geography", + "gaokao-history", + "gaokao-biology", + "gaokao-chemistry", + "gaokao-physics", + "gaokao-mathqa", +) + + +def convert_to_set(item: str | list | None) -> set[str]: + if isinstance(item, list): + return set(item) + if isinstance(item, str): + return {item} + if item is None: + return set() + raise ValueError("Input can't parse:", item) + + +def evaluate_single_sample( + subset: str, prediction: str | list | None, label: str | None +) -> bool: + if subset in MULTI_CHOICE_SUBSETS: + return convert_to_set(prediction) == convert_to_set(label) + if subset in MATH_OUTPUT_SUBSETS: + return is_equiv(prediction, label) + return prediction == label diff --git a/sieval/community/agieval/math_equivalence.py b/sieval/community/agieval/math_equivalence.py new file mode 100644 index 00000000..b23e347c --- /dev/null +++ b/sieval/community/agieval/math_equivalence.py @@ -0,0 +1,176 @@ +# adapted from https://github.com/ruixiangcui/AGIEval/blob/84ab72d94318290aad2e4ec820d535a95a1f7552/src/math_equivalence.py +# which AGIEval vendors from https://github.com/hendrycks/math/blob/357963a7f5501a6c1708cf3f3fb0cdf525642761/modeling/math_equivalence.py +"""Hendrycks-MATH string equivalence, as AGIEval vendors it. + +Normalize both sides, then compare for equality — the grader AGIEval uses for its +two cloze subsets (``math``, ``gaokao-mathcloze``). + +Deliberately NOT reusing :mod:`sieval.community.math`, which holds a *trimmed* +variant of this same upstream file: sieval's ``strip_string`` drops the linebreak +strip, degree/dollar/percent removal, ``_remove_right_units``, the leading +``"k = "`` strip, space removal and the ``0.5 -> \\frac{1}{2}`` rewrite. Those +steps change verdicts, so AGIEval scores must come from AGIEval's own copy. + +Deltas from upstream: bare ``except:`` narrowed to ``except Exception``, and +``is_equiv`` no longer prints on the both-``None`` path (it still returns +``True``, unreachable here — a cloze sample always has a gold answer). + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + + +def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except Exception: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + +def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + a = int(a) + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except Exception: + return string + + +def _remove_right_units(string): + # "\\text{ " only ever occurs (at least in the val set) when describing units + if "\\text{ " in string: + splits = string.split("\\text{ ") + assert len(splits) == 2 + return splits[0] + else: + return string + + +def _fix_sqrt(string): + if "\\sqrt" not in string: + return string + splits = string.split("\\sqrt") + new_string = splits[0] + for split in splits[1:]: + if split[0] != "{": + a = split[0] + new_substr = "\\sqrt{" + a + "}" + split[1:] + else: + new_substr = "\\sqrt" + split + new_string += new_substr + return new_string + + +def _strip_string(string): + # linebreaks + string = string.replace("\n", "") + + # remove inverse spaces + string = string.replace("\\!", "") + + # replace \\ with \ + string = string.replace("\\\\", "\\") + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "") + string = string.replace("^\\circ", "") + + # remove dollar signs + string = string.replace("\\$", "") + + # remove units (on the right) + string = _remove_right_units(string) + + # remove percentage + string = string.replace("\\%", "") + # upstream writes this second line as `.replace("\%", "")`, which is the same + # two characters (Python keeps the backslash of an unknown escape) — i.e. a + # redundant repeat, not a bare-`%` strip. Kept, so a diff against upstream is + # line-for-line. + string = string.replace("\\%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split("=")) == 2: + if len(string.split("=")[0]) <= 2: + string = string.split("=")[1] + + # fix sqrt3 --> sqrt{3} + string = _fix_sqrt(string) + + # remove spaces + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # manually change 0.5 --> \frac{1}{2} + if string == "0.5": + string = "\\frac{1}{2}" + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + return string + + +def is_equiv(str1, str2, verbose=False): + if str1 is None and str2 is None: + return True + if str1 is None or str2 is None: + return False + + try: + ss1 = _strip_string(str1) + ss2 = _strip_string(str2) + if verbose: + print(ss1, ss2) + return ss1 == ss2 + except Exception: + return str1 == str2 diff --git a/sieval/community/agieval/post_process.py b/sieval/community/agieval/post_process.py new file mode 100644 index 00000000..67d4cbcc --- /dev/null +++ b/sieval/community/agieval/post_process.py @@ -0,0 +1,154 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# adapted from https://github.com/ruixiangcui/AGIEval/blob/84ab72d94318290aad2e4ec820d535a95a1f7552/src/post_process.py +"""AGIEval answer extraction from the second-stage model reply. + +Three parsers, picked by subset family (upstream ``post_process``): + +* MCQ, single answer -> ``find_first_capital_letter``: the first A-F character in + the reply. Only sane on the *second-stage* reply (a bare "A" / " D."), which is + exactly why upstream's zero-shot protocol has a second stage — run against a + free-form chain of thought it would happily return the "A" of an option label. +* MCQ, multi answer (``MULTI_CHOICE_SUBSETS``) -> ``parse_qa_multiple_answer``: + every A-F character, compared as a set. +* cloze (``MATH_OUTPUT_SUBSETS``) -> ``parse_math_answer``: last ``\\boxed{}``, + else last ``$...$``, else a trailing ``=``-expression or bare number. + +Deltas from upstream, all deliberate: + +* Zero-shot only: the ``setting_name`` parameter is gone, along with the + ``few-shot-CoT`` ``extract_last_line`` branches it gated and the few-shot + format-compliance helpers. ``remove_few_shot_prefix`` stays — upstream calls it + unconditionally inside ``parse_math_answer``, in every setting. +* A failed extraction returns ``None`` rather than ``""`` / ``[]``, which is + sieval's "could not extract" contract (``PredictionRecord``). Verdicts are + unchanged: neither ``""`` nor ``[]`` can equal a gold answer. +* Regexes are raw strings. The patterns are byte-identical — upstream's + ``"\\$(.*)\\$"`` and ``"(?:\\\\$)?\\d+..."`` are plain strings whose invalid + escapes Python leaves untouched. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import re + +from .dataset_loader import ( + CHINESE_CLOZE_SUBSETS, + CHINESE_QA_SUBSETS, + ENGLISH_CLOZE_SUBSETS, + ENGLISH_QA_SUBSETS, + MULTI_CHOICE_SUBSETS, + SUBSETS, +) + +_FEW_SHOT_PREFIXES = ("The answer is therefore", "答案是", "The answer is") +_CAPITAL_LETTERS = {"A", "B", "C", "D", "E", "F"} + + +def remove_few_shot_prefix(string: str) -> str: + for prefix in _FEW_SHOT_PREFIXES: + if string.startswith(prefix): + string = string[len(prefix) :].strip() + elif prefix in string: + index = string.rfind(prefix) + if index >= 0: + string = string[index + len(prefix) :].strip() + return string + + +def find_first_capital_letter(answer: str) -> str: + for c in answer: + if c in _CAPITAL_LETTERS: + return c + return "" + + +def parse_qa_multiple_answer(string: str) -> list[str]: + return re.findall(r"\(*([A-F])\)*", string) + + +def parse_math_answer(raw_string: str) -> str | None: + def remove_boxed(s): + left = "\\boxed{" + try: + assert s[: len(left)] == left + assert s[-1] == "}" + answer = s[len(left) : -1] + if "=" in answer: + answer = answer.split("=")[-1].lstrip(" ") + return answer + except Exception: + return None + + def last_boxed_only_string(string): + idx = string.rfind("\\boxed") + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx is None: + retval = None + else: + retval = string[idx : right_brace_idx + 1] + + return retval + + def get_answer_with_dollar_sign(s): + first_pattern = r"\$(.*)\$" + last_match = None + matches = re.findall(first_pattern, s) + if matches: + last_match = matches[-1] + if "=" in last_match: + last_match = last_match.split("=")[-1].lstrip(" ") + return last_match + + def get_answer_without_dollar_sign(s): + last_match = None + if "=" in s: + last_match = s.split("=")[-1].lstrip(" ").rstrip(".") + if "\\" in last_match: + last_match = last_match.split("\\")[0] + else: + pattern = r"(?:\$)?\d+(?:\.\d+)?(?![\w\d])" + matches = re.findall(pattern, s) + if matches: + last_match = matches[-1] + return last_match + + raw_string = remove_few_shot_prefix(raw_string) + if "\\boxed" in raw_string: + answer = remove_boxed(last_boxed_only_string(raw_string)) + else: + answer = get_answer_with_dollar_sign(raw_string) + if not answer: + answer = get_answer_without_dollar_sign(raw_string) + return answer + + +def post_process(subset: str, prediction: str) -> str | list | None: + """Extract the answer from a second-stage reply. ``None`` = nothing found.""" + if subset in ENGLISH_CLOZE_SUBSETS or subset in CHINESE_CLOZE_SUBSETS: + return parse_math_answer(prediction) or None + + if subset in MULTI_CHOICE_SUBSETS: + return parse_qa_multiple_answer(prediction) or None + + if subset in ENGLISH_QA_SUBSETS or subset in CHINESE_QA_SUBSETS: + return find_first_capital_letter(prediction) or None + + raise ValueError(f"Unknown AGIEval subset {subset!r}; expected one of {SUBSETS}") diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index 954c0d7c..6d66e478 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -5,6 +5,10 @@ from .aa_lcr import ( AALCRDataset, AALCRDatasetSample, ) +from .agieval import ( + AGIEvalDataset, + AGIEvalDatasetSample, +) from .aime_2024 import ( AIME2024Dataset, AIME2024DatasetSample, @@ -141,6 +145,8 @@ from .theoremqa import ( __all__ = [ "AALCRDataset", "AALCRDatasetSample", + "AGIEvalDataset", + "AGIEvalDatasetSample", "AIME2024Dataset", "AIME2024DatasetSample", "AIME2025Dataset", diff --git a/sieval/datasets/agieval.py b/sieval/datasets/agieval.py new file mode 100644 index 00000000..e50da8aa --- /dev/null +++ b/sieval/datasets/agieval.py @@ -0,0 +1,249 @@ +"""AGIEval dataset loader (v1.1) with per-subset selection. + +AGIEval is 21 files under ``data/v1_1`` of the official repo — 19 MCQ subsets and +2 cloze subsets (``math``, ``gaokao-mathcloze``) — drawn from human admission and +qualification exams (Gaokao, SAT, LSAT, LSAT-adjacent law exams, AQuA-RAT, MATH). +There is no combined config and no official HF mirror carrying the cloze subsets, +so each subset is fetched as its own commit-pinned, checksummed ``.jsonl``. + +Which subsets get loaded is the main knob, since the 21 span four languages × +formats and nobody runs all of them by accident: + +* ``subsets=["math", "sat-math"]`` — exact names (see :data:`SUBSETS`); +* ``group="math"`` — a named group (see :data:`SUBSET_GROUPS`); +* neither — all 21. + +Rows keep upstream's field names and nullability (``passage`` / ``question`` / +``options`` / ``label`` / ``answer`` / ``other``) and gain ``subset``, which is +absent from the raw rows but decides prompt, parsing and scoring downstream. Two +source-shape normalizations are unavoidable when concatenating the files, both +verdict-neutral and both guarded — see :meth:`AGIEvalDataset.load`. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import os +from typing import TypedDict, override + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict +from datasets import Features, List, Value, concatenate_datasets, load_dataset + +from sieval.community.agieval.dataset_loader import ( + MATH_SUBSETS, + SUBSETS, +) +from sieval.community.agieval.evaluation import ( + LEADERBOARD_EN_MCQ_SUBSETS, + LEADERBOARD_ZH_MCQ_SUBSETS, +) +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset + +# Pin the data to an immutable commit: `data/v1_1` is the current release (v1.0 +# is still in the repo under `data/v1`), and a bare branch URL would not survive +# the checksums below. +AGIEVAL_COMMIT = "84ab72d94318290aad2e4ec820d535a95a1f7552" +_DATA_BASE_URL = ( + f"https://raw.githubusercontent.com/ruixiangcui/AGIEval/{AGIEVAL_COMMIT}/data/v1_1" +) + +#: Named subset groups accepted by ``group=``. ``en-mcq`` / ``zh-mcq`` are +#: upstream's own leaderboard groupings (AGIEval-en / AGIEval-zh, MCQ only); +#: ``math`` is sieval's, and is the reason this loader exists in this shape. +SUBSET_GROUPS: dict[str, tuple[str, ...]] = { + "all": SUBSETS, + "math": MATH_SUBSETS, + "en-mcq": LEADERBOARD_EN_MCQ_SUBSETS, + "zh-mcq": LEADERBOARD_ZH_MCQ_SUBSETS, +} + +# One uniform schema for all 21 files. Needed because the per-file inferred +# schemas genuinely disagree: `options` / `label` / `answer` / `passage` are +# all-null in some subsets (inferred as `null`), `label` is a list in the two +# jec-qa files, `answer` is absent from the three sat files, and `other` is a +# struct whose members differ per subset. +_FEATURES = Features( + { + "subset": Value("string"), + "passage": Value("string"), + "question": Value("string"), + "options": List(Value("string")), + "label": Value("string"), + "answer": Value("string"), + # Upstream's provenance bag: `solution` (math / aqua-rat / sat-*), + # `source` (gaokao-*), `level` + `type` (math). Never read by AGIEval's + # own prompt/parse/score path; kept because the MATH solutions and + # difficulty levels have no other home. + "other": { + "solution": Value("string"), + "source": Value("string"), + "level": Value("string"), + "type": Value("string"), + }, + } +) + +_OTHER_KEYS = ("solution", "source", "level", "type") + + +class AGIEvalDatasetSample(TypedDict): + subset: str + passage: str | None + question: str + options: list[str] + label: str | None + answer: str | None + other: dict + + +@sieval_dataset( + name="agieval", + display_name="AGIEval", + description="AGIEval v1.1 — 21 human exam subsets (Gaokao, SAT, LSAT, MATH).", + source=tuple(f"url:{_DATA_BASE_URL}/{subset}.jsonl" for subset in SUBSETS), + checksums={ + "aqua-rat.jsonl": "sha256:cff42e946e6082dacb27285dae19cb4be98408ab760fe43c6c462e543be50572", # noqa: E501 + "gaokao-biology.jsonl": "sha256:789baadad69c998743302143a3e1c2022a2aca785bbe75644877a336242c7e69", # noqa: E501 + "gaokao-chemistry.jsonl": "sha256:4fb8a4b881f652a908545447a062c7ba458be2ba793a45fc11b892053406704e", # noqa: E501 + "gaokao-chinese.jsonl": "sha256:1ddcf8fa15e07a25589796dc1c72a341c2d874af8de41970262d66693f95285f", # noqa: E501 + "gaokao-english.jsonl": "sha256:2de1b1e5d9718d908ffb46665e949bff83bef956ad8b7de16ea412e058162b01", # noqa: E501 + "gaokao-geography.jsonl": "sha256:1170cb39171d6dfc35bd52a52505e5a120cf400abc697f282d0739db6181713b", # noqa: E501 + "gaokao-history.jsonl": "sha256:27350771d399814fc69dd6d7e6ce115b7913ae208e04e385a7e6fcdf51a6b8b7", # noqa: E501 + "gaokao-mathcloze.jsonl": "sha256:088675c147794970a3ed25c7147a3bbc59715d6813837d5f483f46dcb1b5008d", # noqa: E501 + "gaokao-mathqa.jsonl": "sha256:d246f12752d121289ef55cbf1bcf954243cefb65d110f71d09358e24065c808f", # noqa: E501 + "gaokao-physics.jsonl": "sha256:9f8f91b35b5cc2d3ba67b9c6f31bc72c51caaaa58a4a46375690d3b8b127a82b", # noqa: E501 + "jec-qa-ca.jsonl": "sha256:704efb9943cb827811d883163d893d483451ddaaf9f595f94866e51e70e20785", # noqa: E501 + "jec-qa-kd.jsonl": "sha256:fec1d0d85d480ee6c23371af59b2305732d8561296cd878b5367b65ebfc4a467", # noqa: E501 + "logiqa-en.jsonl": "sha256:63d0e8efaca1944e7eac5037903650f92ca9c036155a66b4e95bf2aa06da1702", # noqa: E501 + "logiqa-zh.jsonl": "sha256:0e5f6548932ce6cd388d8432d015db9baa468731c94409c98d20902832bb099c", # noqa: E501 + "lsat-ar.jsonl": "sha256:3b3e3fe09a07c695326adb82f38da0d67d5bbaadab41551f198c3102f1ea9dc8", # noqa: E501 + "lsat-lr.jsonl": "sha256:c6acb4d843db7b515da4d853e52bb8e6e5f776910f2e35a1805634db24bf94ea", # noqa: E501 + "lsat-rc.jsonl": "sha256:0eed491b3099d66b8110d4fabb98ce43285a6ac61ea5643285eec11b8abce202", # noqa: E501 + "math.jsonl": "sha256:43e783af2025318125a96a970a0df37941124a5c0dabea382a12ce1b04651a11", # noqa: E501 + "sat-en-without-passage.jsonl": "sha256:77eb57bb6f6f39466d5d169de5253e77755ef521ad6692f582307672affbf593", # noqa: E501 + "sat-en.jsonl": "sha256:33fe87b32ff16ae7ba52b27bb398190b082253b3617a3eb9737005262ba12dd4", # noqa: E501 + "sat-math.jsonl": "sha256:9cde4c0522b6196852a5562db6e72bf8817b4802822db0fc10013d1becdab3c3", # noqa: E501 + }, + categories=(Category(Level1Category.KNOWLEDGE, "Multi-domain"),), + tags=("english", "chinese", "multiple-choice", "open-ended"), + # The repo (code + data/v1_1) is MIT; per-subset source exams keep their own + # terms, which upstream's data/v1_1/LICENSE reproduces and defers to. + license="MIT", +) +class AGIEvalDataset(Dataset[AGIEvalDatasetSample]): + """AGIEval v1.1, one ``test`` split concatenated from the selected subsets.""" + + @override + def load( + self, + name_or_path: str, + subsets: list[str] | None = None, + group: str | None = None, + **kwargs, + ) -> HFDatasetDict: + """Load the selected subsets from ``/.jsonl``. + + Exactly one selection may be given: *subsets* (exact names) or *group* + (a :data:`SUBSET_GROUPS` key). Neither loads all 21. Selection order is + always :data:`SUBSETS` order, so the concatenation — and every sample id + derived from it — does not depend on how the argument was spelled. + + Two normalizations, both required to concatenate the files at all: + + * ``label`` is a 1-element **list** in ``jec-qa-kd`` / ``jec-qa-ca`` and + a string everywhere else; the list is unwrapped to its single element. + A longer list (AGIEval v1.0 had genuine multi-label rows) raises rather + than silently changing what gets compared. + * ``other.level`` is an ``int64`` in ``math.jsonl`` and absent elsewhere; + stringified so the struct has one dtype across subsets. No other column + is cast — upstream already ships them as strings. + """ + selected = self._select_subsets(subsets, group) + + parts: list[HFDataset] = [] + for subset in selected: + path = os.path.join(name_or_path, f"{subset}.jsonl") + if not os.path.isfile(path): + raise FileNotFoundError( + f"AGIEval subset file not found: {path}. Stage the data with " + "`sieval dataset download agieval`, and point the dataset's " + "`path` at the directory holding the .jsonl files." + ) + raw = ensure_dataset( + load_dataset("json", data_files=path, split="train", **kwargs) + ) + parts.append( + raw.map( + lambda row, s=subset: self._normalize(row, s), + features=_FEATURES, + remove_columns=raw.column_names, + ) + ) + + combined = concatenate_datasets(parts) + if len(combined) == 0: + raise ValueError( + f"AGIEval produced an empty test split for subsets={selected!r}; " + "check the staged .jsonl files." + ) + return HFDatasetDict({"test": combined}) + + @staticmethod + def _select_subsets(subsets: list[str] | None, group: str | None) -> list[str]: + if subsets is not None and group is not None: + raise ValueError( + "AGIEval: pass either `subsets` (exact names) or `group` " + f"(one of {sorted(SUBSET_GROUPS)}), not both." + ) + if group is not None: + if group not in SUBSET_GROUPS: + raise ValueError( + f"AGIEval: unknown group {group!r}; " + f"expected one of {sorted(SUBSET_GROUPS)}." + ) + chosen = set(SUBSET_GROUPS[group]) + elif subsets is not None: + unknown = [s for s in subsets if s not in SUBSETS] + if unknown: + raise ValueError( + f"AGIEval: unknown subset(s) {unknown!r}; " + f"expected names from {list(SUBSETS)}." + ) + if not subsets: + raise ValueError("AGIEval: `subsets` is empty; omit it to load all.") + chosen = set(subsets) + else: + chosen = set(SUBSETS) + return [subset for subset in SUBSETS if subset in chosen] + + @staticmethod + def _normalize(row: dict, subset: str) -> AGIEvalDatasetSample: + label = row.get("label") + if isinstance(label, list): + if len(label) != 1: + raise ValueError( + f"AGIEval subset {subset!r}: expected a single-answer label, " + f"got {label!r}. The pinned v1.1 data has none; a multi-label " + "row means the source changed and scoring must be revisited." + ) + label = label[0] + other = row.get("other") or {} + return { + "subset": subset, + "passage": row.get("passage"), + "question": row["question"], + # None for the cloze subsets, which have no options to show. + "options": row.get("options") or [], + "label": label, + "answer": row.get("answer"), + "other": { + key: None if other.get(key) is None else str(other[key]) + for key in _OTHER_KEYS + }, + } diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 2eef6992..7f8e3d46 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -24,6 +24,71 @@ "license": "apache-2.0", "checksums": {} }, + { + "name": "agieval", + "display_name": "AGIEval", + "description": "AGIEval v1.1 — 21 human exam subsets (Gaokao, SAT, LSAT, MATH).", + "source": [ + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/lsat-ar.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/lsat-lr.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/lsat-rc.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/logiqa-en.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/sat-math.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/sat-en.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/aqua-rat.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/sat-en-without-passage.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-english.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/logiqa-zh.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/jec-qa-kd.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/jec-qa-ca.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-chinese.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-geography.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-history.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-biology.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-chemistry.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-physics.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-mathqa.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/math.jsonl", + "url:https://raw.githubusercontent.com/ruixiangcui/AGIEval/84ab72d94318290aad2e4ec820d535a95a1f7552/data/v1_1/gaokao-mathcloze.jsonl" + ], + "categories": [ + { + "level1": "Knowledge", + "level2": "Multi-domain" + } + ], + "tags": [ + "english", + "chinese", + "multiple-choice", + "open-ended" + ], + "deps_group": null, + "license": "MIT", + "checksums": { + "aqua-rat.jsonl": "sha256:cff42e946e6082dacb27285dae19cb4be98408ab760fe43c6c462e543be50572", + "gaokao-biology.jsonl": "sha256:789baadad69c998743302143a3e1c2022a2aca785bbe75644877a336242c7e69", + "gaokao-chemistry.jsonl": "sha256:4fb8a4b881f652a908545447a062c7ba458be2ba793a45fc11b892053406704e", + "gaokao-chinese.jsonl": "sha256:1ddcf8fa15e07a25589796dc1c72a341c2d874af8de41970262d66693f95285f", + "gaokao-english.jsonl": "sha256:2de1b1e5d9718d908ffb46665e949bff83bef956ad8b7de16ea412e058162b01", + "gaokao-geography.jsonl": "sha256:1170cb39171d6dfc35bd52a52505e5a120cf400abc697f282d0739db6181713b", + "gaokao-history.jsonl": "sha256:27350771d399814fc69dd6d7e6ce115b7913ae208e04e385a7e6fcdf51a6b8b7", + "gaokao-mathcloze.jsonl": "sha256:088675c147794970a3ed25c7147a3bbc59715d6813837d5f483f46dcb1b5008d", + "gaokao-mathqa.jsonl": "sha256:d246f12752d121289ef55cbf1bcf954243cefb65d110f71d09358e24065c808f", + "gaokao-physics.jsonl": "sha256:9f8f91b35b5cc2d3ba67b9c6f31bc72c51caaaa58a4a46375690d3b8b127a82b", + "jec-qa-ca.jsonl": "sha256:704efb9943cb827811d883163d893d483451ddaaf9f595f94866e51e70e20785", + "jec-qa-kd.jsonl": "sha256:fec1d0d85d480ee6c23371af59b2305732d8561296cd878b5367b65ebfc4a467", + "logiqa-en.jsonl": "sha256:63d0e8efaca1944e7eac5037903650f92ca9c036155a66b4e95bf2aa06da1702", + "logiqa-zh.jsonl": "sha256:0e5f6548932ce6cd388d8432d015db9baa468731c94409c98d20902832bb099c", + "lsat-ar.jsonl": "sha256:3b3e3fe09a07c695326adb82f38da0d67d5bbaadab41551f198c3102f1ea9dc8", + "lsat-lr.jsonl": "sha256:c6acb4d843db7b515da4d853e52bb8e6e5f776910f2e35a1805634db24bf94ea", + "lsat-rc.jsonl": "sha256:0eed491b3099d66b8110d4fabb98ce43285a6ac61ea5643285eec11b8abce202", + "math.jsonl": "sha256:43e783af2025318125a96a970a0df37941124a5c0dabea382a12ce1b04651a11", + "sat-en-without-passage.jsonl": "sha256:77eb57bb6f6f39466d5d169de5253e77755ef521ad6692f582307672affbf593", + "sat-en.jsonl": "sha256:33fe87b32ff16ae7ba52b27bb398190b082253b3617a3eb9737005262ba12dd4", + "sat-math.jsonl": "sha256:9cde4c0522b6196852a5562db6e72bf8817b4802822db0fc10013d1becdab3c3" + } + }, { "name": "aime_2024", "display_name": "AIME 2024", @@ -790,6 +855,28 @@ }, "status": "experimental" }, + { + "name": "agieval_0shot_gen", + "display_name": "AGIEval (0-shot, generative)", + "description": "AGIEval v1.1 human exams — 21 subsets, two-stage 0-shot MCQ + cloze.", + "dataset": "agieval", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "chinese", + "multiple-choice", + "open-ended" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "AGIEval", + "url": "https://github.com/ruixiangcui/AGIEval/blob/84ab72d94318290aad2e4ec820d535a95a1f7552/src/dataset_loader.py", + "notes": "Port of AGIEval's own zero-shot pipeline (Microsoft, arXiv:2304.06364) on the pinned v1.1 data: prompts from src/dataset_loader.py, answer extraction from src/post_process.py, verdicts from src/evaluation.py, all vendored in community/agieval. TWO-STAGE: run_prediction.py's zero-shot path generates freely, then re-prompts with the reply plus generate_second_stage_input's cue (with_format_prompt=False) and parses THAT; post_process_and_evaluation.py scores the second-stage output, keeping the first only as an audit field. Both calls are returned from infer() so both land in profile.json. REPEATS: upstream samples each problem once (no n_repeats); its only retry re-queries replies that came back empty, which sieval covers with the model's max_retries. An empty first-stage reply still proceeds to stage 2 with an empty answer, as upstream's extract_answer does.\nDIVERGENCE (the one protocol-level choice): upstream runs stage 2 on gpt-35-turbo no matter which model produced stage 1; sieval defaults stage 2 to the model under test and takes the `extractor` task arg (model-config dict or Model) to pin a separate one. Matching upstream exactly means passing an extractor; leaving it unset measures the model's own answer-extraction, which for a weak model is not the same number. Pin whichever you choose — the score depends on it.\nSCORING: set-of-letters for jec-qa-kd / jec-qa-ca / gaokao-physics, math equivalence for math / gaokao-mathcloze, exact string compare otherwise. The math grader is AGIEval's own vendored hendrycks/math is_equiv, NOT sieval.community.math (a trimmed variant of the same file whose dropped normalizations change verdicts).\nUPSTREAM QUIRK, kept verbatim: 7 of 351 gaokao-mathqa rows carry multi-letter gold labels ('AD', 'ACD', 'A B D', ...) even in v1.1, but the subset is not on upstream's multi_choice list, so it is scored by exact single-letter compare and those 7 rows are unwinnable (~2% of that subset). Fixing it would silently diverge from every published AGIEval number.\nCOMPARISON TARGETS (upstream README leaderboard, v1.1 zero-shot, MCQ-only for the en/zh rows): GPT-4o 62.3 all / 65.2 en / 63.3 zh; GPT-3.5-Turbo 46.0 / 54.1 / 45.0. `score` is the macro over the subsets that ran (upstream's 'average for all datasets', denominator 21 when all are selected); macro_en_mcq / macro_zh_mcq mirror the two leaderboard groups — note gaokao-english is prompted in English but counted as Chinese there, matching upstream's driver. Asterisked leaderboard rows are v1.0 and not comparable to this v1.1 data. NOT YET VALIDATED against a run of our own, hence status=\"experimental\"." + }, + "status": "experimental" + }, { "name": "aime_2024_0shot_gen", "display_name": "AIME 2024 (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index e7c8afad..22126eb1 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -4,6 +4,9 @@ from .aa_lcr_0shot_gen import ( AALCRZeroShotGenTask, ) +from .agieval_0shot_gen import ( + AGIEvalZeroShotGenTask, +) from .aime_2024_0shot_gen import ( AIME2024ZeroShotGenTask, ) @@ -124,6 +127,7 @@ from .theoremqa_kshot_base_gen import ( __all__ = [ "AALCRZeroShotGenTask", + "AGIEvalZeroShotGenTask", "AIME2024ZeroShotGenTask", "AIME2025ZeroShotGenTask", "AIME2026ZeroShotGenTask", diff --git a/sieval/tasks/agieval_0shot_gen.py b/sieval/tasks/agieval_0shot_gen.py new file mode 100644 index 00000000..7bc4509f --- /dev/null +++ b/sieval/tasks/agieval_0shot_gen.py @@ -0,0 +1,256 @@ +"""AGIEval 0-shot generative task (chat models), all 21 subsets in one class. + +AGIEval's zero-shot protocol is **two model calls**, and reproducing its numbers +means running both: + +1. the model answers the exam question (upstream's ``convert_zero_shot`` prompt, + which ends in a "the answer is" cue); +2. that reply is fed back with a per-language extraction cue ("Therefore, among A + through E, the answer is"), and the short reply *that* produces is what gets + parsed. + +Stage 2 exists because stage 1's answer is prose: upstream's zero-shot parser is +"first A-F character in the reply", which on a chain of thought would return the +letter of the first option it happened to restate. Dropping stage 2 does not make +this a cheaper AGIEval — it makes it a different, worse one. + +Per-sample routing is by ``subset`` (four prompt families, three answer parsers, +three comparison rules); ``report()`` gives per-subset accuracy plus the macro +averages upstream's leaderboard publishes. Which subsets run is a *dataset* knob +(``subsets=`` / ``group="math"``), not a task knob — see +:mod:`sieval.datasets.agieval`. + +Both stages default to the model under test. Pass ``extractor`` (a model-config +dict or a Model) to run stage 2 on a fixed, cheap model instead, which is what +upstream did — see ``reference_impl.notes``. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from collections import defaultdict +from collections.abc import Mapping +from typing import cast, override + +from sieval.community.agieval.dataset_loader import ( + MATH_SUBSETS, + SUBSETS, + second_stage_prompt, + zero_shot_prompt, +) +from sieval.community.agieval.evaluation import ( + LEADERBOARD_EN_MCQ_SUBSETS, + LEADERBOARD_ZH_MCQ_SUBSETS, + evaluate_single_sample, +) +from sieval.community.agieval.post_process import post_process +from sieval.core.models import ChatModel, Model, ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.datasets import AGIEvalDatasetSample + +# Macro averages over named subset groups, reported as `macro_`. Distinct +# from the per-subset `score_` keys — and `macro_math` must stay distinct +# from `score_math`, the MATH subset's own accuracy. +_MACRO_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("en_mcq", LEADERBOARD_EN_MCQ_SUBSETS), + ("zh_mcq", LEADERBOARD_ZH_MCQ_SUBSETS), + ("math", MATH_SUBSETS), +) + + +@sieval_task( + name="agieval_0shot_gen", + display_name="AGIEval (0-shot, generative)", + description="AGIEval v1.1 human exams — 21 subsets, two-stage 0-shot MCQ + cloze.", + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "chinese", "multiple-choice", "open-ended"), + model_type="chat", + status="experimental", + reference_impl=ReferenceImpl( + source="AGIEval", + url="https://github.com/ruixiangcui/AGIEval/blob/84ab72d94318290aad2e4ec820d535a95a1f7552/src/dataset_loader.py", + notes=( + "Port of AGIEval's own zero-shot pipeline (Microsoft, arXiv:2304.06364) " + "on the pinned v1.1 data: prompts from src/dataset_loader.py, answer " + "extraction from src/post_process.py, verdicts from src/evaluation.py, " + "all vendored in community/agieval. TWO-STAGE: run_prediction.py's " + "zero-shot path generates freely, then re-prompts with the reply plus " + "generate_second_stage_input's cue (with_format_prompt=False) and " + "parses THAT; post_process_and_evaluation.py scores the second-stage " + "output, keeping the first only as an audit field. Both calls are " + "returned from infer() so both land in profile.json. REPEATS: upstream " + "samples each problem once (no n_repeats); its only retry re-queries " + "replies that came back empty, which sieval covers with the model's " + "max_retries. An empty first-stage reply still proceeds to stage 2 " + "with an empty answer, as upstream's extract_answer does.\n" + "DIVERGENCE (the one protocol-level choice): upstream runs stage 2 on " + "gpt-35-turbo no matter which model produced stage 1; sieval defaults " + "stage 2 to the model under test and takes the `extractor` task arg " + "(model-config dict or Model) to pin a separate one. Matching upstream " + "exactly means passing an extractor; leaving it unset measures the " + "model's own answer-extraction, which for a weak model is not the same " + "number. Pin whichever you choose — the score depends on it.\n" + "SCORING: set-of-letters for jec-qa-kd / jec-qa-ca / gaokao-physics, " + "math equivalence for math / gaokao-mathcloze, exact string compare " + "otherwise. The math grader is AGIEval's own vendored hendrycks/math " + "is_equiv, NOT sieval.community.math (a trimmed variant of the same " + "file whose dropped normalizations change verdicts).\n" + "UPSTREAM QUIRK, kept verbatim: 7 of 351 gaokao-mathqa rows carry " + "multi-letter gold labels ('AD', 'ACD', 'A B D', ...) even in v1.1, but " + "the subset is not on upstream's multi_choice list, so it is scored by " + "exact single-letter compare and those 7 rows are unwinnable (~2% of " + "that subset). Fixing it would silently diverge from every published " + "AGIEval number.\n" + "COMPARISON TARGETS (upstream README leaderboard, v1.1 zero-shot, " + "MCQ-only for the en/zh rows): GPT-4o 62.3 all / 65.2 en / 63.3 zh; " + "GPT-3.5-Turbo 46.0 / 54.1 / 45.0. `score` is the macro over the " + "subsets that ran (upstream's 'average for all datasets', denominator " + "21 when all are selected); macro_en_mcq / macro_zh_mcq mirror the two " + "leaderboard groups — note gaokao-english is prompted in English but " + "counted as Chinese there, matching upstream's driver. Asterisked " + "leaderboard rows are v1.0 and not comparable to this v1.1 data. " + "NOT YET VALIDATED against a run of our own, hence " + 'status="experimental".' + ), + ), +) +class AGIEvalZeroShotGenTask( + Task[ + AGIEvalDatasetSample, + PromptRecord, + list[ModelOutput], + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + """AGIEval zero-shot: answer, extract, score — routed per subset.""" + + def __init__( + self, + dataset, + model, + name: str | None = None, + extractor: Mapping | Model | None = None, + ): + super().__init__(dataset=dataset, model=model, name=name) + self._extractor = self._build_extractor(extractor, model) + + @staticmethod + def _build_extractor(extractor: Mapping | Model | None, model: Model) -> Model: + """Resolve the ``extractor`` task arg into the model for stage 2. + + ``None`` reuses the model under test (see the divergence note in + ``reference_impl.notes``); a Mapping is the YAML path, e.g. + ``{model: gpt-3.5-turbo, api_base: ..., temperature: 0}``. + """ + if extractor is None: + return model + if isinstance(extractor, Model): + return extractor + if isinstance(extractor, Mapping): + return ChatModel(**extractor) + raise ValueError( + "AGIEval `extractor` must be a model-config dict or a Model, got " + f"{type(extractor).__name__}. Omit it to run answer extraction on " + "the model under test." + ) + + @override + async def preprocess(self, raw, ctx): + return build_prompt_record( + [{"role": "user", "content": zero_shot_prompt(raw["subset"], raw)}], + # Upstream's load_dataset_as_result_schema: the label when set, the + # cloze answer otherwise. + reference=raw["label"] if raw["label"] else raw["answer"], + extra={"subset": raw["subset"]}, + ) + + @override + async def infer(self, pre, ctx): + """Answer, then re-read the answer under the extraction cue. + + Returns both calls: the runner profiles a ``list[ModelOutput]`` stage + value directly, so stage 2's spend needs no ``grader_output`` routing, + and stage 1's reply — the actual reasoning — stays on disk as evidence. + """ + subset = pre["extra"]["subset"] + first = await self.model.agenerate(pre["prompt"]) + # An empty reply is carried into stage 2 as "", matching upstream's + # extract_answer, rather than failing the sample. + answer = first.texts[0] if first.texts else "" + context = cast(list[dict[str, str]], pre["prompt"])[0]["content"] + second = await self._extractor.agenerate( + [ + { + "role": "user", + "content": second_stage_prompt(subset, context, answer), + } + ] + ) + return [first, second] + + @override + async def postprocess(self, inf, ctx): + extraction = inf[-1] + text = extraction.texts[0] if extraction.texts else "" + return build_prediction_record([post_process(ctx.raw_sample["subset"], text)]) + + @override + async def feedback(self, post, ctx): + raw = ctx.raw_sample + subset = raw["subset"] + reference = raw["label"] if raw["label"] else raw["answer"] + prediction = post["rollouts"][0]["prediction"] + correct = evaluate_single_sample(subset, prediction, reference) + return True, build_judgement_record( + reference, + [build_rollout_judgement(0, correct)], + extra={"subset": subset}, + ) + + @override + async def report(self, finals, fails): + """Per-subset accuracy, macro over subsets, and the leaderboard macros. + + ``score`` is the unweighted mean of the per-subset accuracies — upstream's + "average for all datasets" — over the subsets that ran, so a subset + selection reports the macro of *that* selection. A ``macro_`` key + appears only when **every** member of the group was evaluated: these keys + exist to be compared against published numbers, and a partial macro is + not that number. Infra failures are reported in ``fails``, not scored 0. + """ + by_subset: dict[str, list[bool]] = defaultdict(list) + for ctx in finals: + fb = ctx.feedback_result + by_subset[fb["extra"]["subset"]].append(fb["rollouts"][0]["correct"]) + + subset_acc = { + subset: 100 * sum(correct) / len(correct) + for subset, correct in by_subset.items() + } + overall = sum(subset_acc.values()) / len(subset_acc) if subset_acc else 0.0 + + metrics: dict[str, float] = {"score": overall, "fails": float(len(fails))} + # Canonical subset order, so two runs' reports line up key for key. + for subset in SUBSETS: + if subset in subset_acc: + metrics[f"score_{subset.replace('-', '_')}"] = subset_acc[subset] + for group_name, group in _MACRO_GROUPS: + if all(subset in subset_acc for subset in group): + metrics[f"macro_{group_name}"] = sum( + subset_acc[subset] for subset in group + ) / len(group) + return metrics diff --git a/tests/unit/community/test_agieval.py b/tests/unit/community/test_agieval.py new file mode 100644 index 00000000..ae091311 --- /dev/null +++ b/tests/unit/community/test_agieval.py @@ -0,0 +1,183 @@ +"""Unit tests for the vendored AGIEval prompt / parse / score layer. + +Pins the behaviours the port's fidelity claims rest on: the exact upstream prompt +strings, the family routing, and the three comparison rules (including the +gaokao-mathqa multi-letter-gold quirk that is kept on purpose). + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest + +from sieval.community.agieval.dataset_loader import ( + CHINESE_CLOZE_SUBSETS, + CHINESE_QA_SUBSETS, + ENGLISH_CLOZE_SUBSETS, + ENGLISH_QA_SUBSETS, + MATH_SUBSETS, + SUBSETS, + second_stage_prompt, + zero_shot_prompt, +) +from sieval.community.agieval.evaluation import ( + LEADERBOARD_EN_MCQ_SUBSETS, + LEADERBOARD_ZH_MCQ_SUBSETS, + evaluate_single_sample, +) +from sieval.community.agieval.math_equivalence import is_equiv +from sieval.community.agieval.post_process import post_process + + +def _row(**overrides) -> dict: + row = { + "passage": None, + "question": "Q?", + "options": ["(A)1", "(B)2", "(C)3", "(D)4"], + "label": "A", + "answer": None, + } + row.update(overrides) + return row + + +def test_subsets_cover_the_21_v1_1_files_without_overlap(): + families = ( + ENGLISH_QA_SUBSETS, + CHINESE_QA_SUBSETS, + ENGLISH_CLOZE_SUBSETS, + CHINESE_CLOZE_SUBSETS, + ) + assert len(SUBSETS) == 21 + assert len(set(SUBSETS)) == 21 + assert sum(len(family) for family in families) == 21 + + +def test_leaderboard_groups_are_not_the_prompt_families(): + # gaokao-english is prompted in English but averaged as Chinese upstream; + # a port that reused the prompt families for reporting would get both wrong. + assert "gaokao-english" in ENGLISH_QA_SUBSETS + assert "gaokao-english" in LEADERBOARD_ZH_MCQ_SUBSETS + assert "gaokao-english" not in LEADERBOARD_EN_MCQ_SUBSETS + assert len(LEADERBOARD_EN_MCQ_SUBSETS) == 8 + assert len(LEADERBOARD_ZH_MCQ_SUBSETS) == 11 + # The math group spans both languages and both formats. + assert set(MATH_SUBSETS) == { + "sat-math", + "aqua-rat", + "gaokao-mathqa", + "math", + "gaokao-mathcloze", + } + + +def test_zero_shot_prompt_english_qa_matches_upstream_string(): + prompt = zero_shot_prompt("sat-math", _row(passage="P. ")) + assert prompt == ( + "P. Q: Q? Answer Choices: (A)1 (B)2 (C)3 (D)4\n" + "A: Among A through D, the answer is" + ) + + +def test_zero_shot_prompt_letter_tracks_option_count(): + five = ["(A)1", "(B)2", "(C)3", "(D)4", "(E)5"] + assert zero_shot_prompt("aqua-rat", _row(options=five)).endswith( + "Among A through E, the answer is" + ) + + +def test_zero_shot_prompt_chinese_qa_matches_upstream_string(): + prompt = zero_shot_prompt("gaokao-mathqa", _row()) + assert prompt == "问题：Q? 选项：(A)1 (B)2 (C)3 (D)4\n答案：从A到D, 我们应选择" + + +def test_zero_shot_prompt_cloze_subsets_omit_options(): + assert zero_shot_prompt("math", _row(options=[], label=None, answer="7")) == ( + "Q: Q?\nA: The answer is" + ) + assert zero_shot_prompt( + "gaokao-mathcloze", _row(options=[], label=None, answer="7") + ) == ("问题：Q?\n答案：") + + +def test_zero_shot_prompt_rejects_unknown_subset(): + with pytest.raises(ValueError, match="Unknown AGIEval subset"): + zero_shot_prompt("mmlu", _row()) + + +def test_second_stage_prompt_appends_the_family_cue(): + assert second_stage_prompt("sat-math", "CTX", "reasoning") == ( + "CTX\nreasoning\nTherefore, among A through E, the answer is" + ) + assert second_stage_prompt("gaokao-mathqa", "CTX", "推理") == ( + "CTX\n推理\n因此，从A到D, 我们应选择" + ) + assert second_stage_prompt("math", "CTX", "r") == ( + "CTX\nr\nTherefore, the answer is" + ) + assert second_stage_prompt("gaokao-mathcloze", "CTX", "r") == "CTX\nr\n因此，答案是" + + +def test_post_process_single_answer_takes_first_capital_letter(): + assert post_process("logiqa-en", " C.") == "C" + # Upstream's parser is positional, not semantic: the first A-F character + # wins, even mid-word ("Among" -> A), which is why it only ever runs on the + # terse second-stage reply and not on a chain of thought. + assert post_process("logiqa-en", "Among the options, D is right") == "A" + assert post_process("logiqa-en", "no letters here") is None + + +def test_post_process_multi_answer_subsets_return_every_letter(): + assert post_process("jec-qa-kd", "(A) and (C)") == ["A", "C"] + assert post_process("gaokao-physics", " B") == ["B"] + assert post_process("jec-qa-ca", "no letters") is None + + +@pytest.mark.parametrize( + ("reply", "expected"), + [ + (" $\\boxed{\\frac{1}{2}}$", "\\frac{1}{2}"), + (" $x = 42$", "42"), + (" 42", "42"), + (" the value is 3.5", "3.5"), + ("The answer is therefore 8", "8"), + (" no answer at all", None), + ], +) +def test_post_process_cloze_extracts_the_math_answer(reply, expected): + assert post_process("math", reply) == expected + + +def test_post_process_rejects_unknown_subset(): + with pytest.raises(ValueError, match="Unknown AGIEval subset"): + post_process("mmlu", "A") + + +def test_evaluate_single_sample_compares_multi_answer_as_a_set(): + assert evaluate_single_sample("jec-qa-kd", ["A"], "A") is True + # Order-insensitive, and a superset is wrong rather than partially right. + assert evaluate_single_sample("jec-qa-ca", ["B", "A"], "A") is False + assert evaluate_single_sample("gaokao-physics", None, "A") is False + + +def test_evaluate_single_sample_uses_math_equivalence_for_cloze(): + assert evaluate_single_sample("math", "\\frac{1}{2}", "0.5") is True + assert evaluate_single_sample("gaokao-mathcloze", "2", "3") is False + assert evaluate_single_sample("math", None, "2") is False + + +def test_evaluate_single_sample_is_exact_elsewhere_including_the_quirk(): + assert evaluate_single_sample("sat-math", "D", "D") is True + # 7 gaokao-mathqa rows ship multi-letter gold while the subset is scored by + # exact compare, so no single-letter prediction can win them. Kept verbatim + # from upstream — see the task's reference_impl.notes. + assert evaluate_single_sample("gaokao-mathqa", "A", "AD") is False + + +def test_is_equiv_applies_the_normalizations_sieval_community_math_drops(): + # These four are exactly the steps sieval.community.math.strip_string omits, + # which is why AGIEval keeps its own copy. + assert is_equiv("2 cm", "2cm") is True # space removal + assert is_equiv("50\\%", "50") is True # percent removal + assert is_equiv("90^\\circ", "90") is True # degree removal + assert is_equiv("k = 5", "5") is True # leading "k = " strip + assert is_equiv(None, "5") is False diff --git a/tests/unit/datasets/test_agieval.py b/tests/unit/datasets/test_agieval.py new file mode 100644 index 00000000..95a2a465 --- /dev/null +++ b/tests/unit/datasets/test_agieval.py @@ -0,0 +1,159 @@ +"""Unit tests for the AGIEval dataset loader. + +Covers the two things this loader adds over "read some jsonl": subset selection +(the reason the loader exists in this shape) and the schema normalizations that +let 21 files with disagreeing per-file schemas concatenate at all. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +from pathlib import Path + +import pytest + +from sieval.datasets.agieval import SUBSET_GROUPS, AGIEvalDataset + +# Minimal rows in each source shape, keyed by subset. Field presence and dtypes +# mirror the pinned v1.1 files: sat-* have no `answer` key at all, cloze rows +# null out `options`/`label`, jec-qa wraps the label in a list, and only +# math.jsonl carries `level` (as an int). +_ROWS: dict[str, dict] = { + "sat-math": { + "passage": "", + "question": "sm?", + "options": ["(A)1", "(B)2", "(C)3", "(D)4"], + "label": "D", + "other": {"solution": "because"}, + }, + "aqua-rat": { + "passage": None, + "question": "ar?", + "options": ["(A)1", "(B)2", "(C)3", "(D)4", "(E)5"], + "label": "B", + "answer": None, + "other": {"solution": "because"}, + }, + "gaokao-mathqa": { + "passage": None, + "question": "gm?", + "options": ["(A)1", "(B)2", "(C)3", "(D)4"], + "label": "A", + "answer": None, + "other": {"source": "2021年浙江卷—数学"}, + }, + "math": { + "passage": None, + "question": "m?", + "options": None, + "label": None, + "answer": "(3,4]", + "other": {"solution": "s", "level": 5, "type": "Intermediate Algebra"}, + }, + "gaokao-mathcloze": { + "passage": None, + "question": "gc?", + "options": None, + "label": None, + "answer": "2", + "other": {"source": "2021年浙江卷—数学"}, + }, + "jec-qa-kd": { + "passage": None, + "question": "j?", + "options": ["(A)1", "(B)2", "(C)3", "(D)4"], + "label": ["B"], + "answer": None, + "other": None, + }, +} + + +def _stage(tmp_path: Path, *subsets: str, label_override=None) -> str: + """Write one .jsonl per subset, the way `sieval dataset download` stages them.""" + for subset in subsets: + row = dict(_ROWS[subset]) + if label_override is not None: + row["label"] = label_override + path = tmp_path / f"{subset}.jsonl" + path.write_text(json.dumps(row, ensure_ascii=False) + "\n", encoding="utf-8") + return str(tmp_path) + + +def test_group_math_loads_the_five_math_subsets(tmp_path): + path = _stage(tmp_path, *SUBSET_GROUPS["math"]) + test = AGIEvalDataset(path, group="math").dataset_dict["test"] + + assert len(test) == 5 + assert set(test["subset"]) == set(SUBSET_GROUPS["math"]) + + +def test_explicit_subsets_load_in_canonical_order(tmp_path): + path = _stage(tmp_path, "math", "sat-math") + # Argument order must not leak into row order, or the same selection spelled + # two ways would produce different sample ids. + forward = AGIEvalDataset(path, subsets=["sat-math", "math"]).dataset_dict["test"] + reverse = AGIEvalDataset(path, subsets=["math", "sat-math"]).dataset_dict["test"] + + assert forward["subset"] == reverse["subset"] == ["sat-math", "math"] + + +def test_row_keeps_upstream_fields_and_gains_subset(tmp_path): + path = _stage(tmp_path, "sat-math") + row = AGIEvalDataset(path, subsets=["sat-math"]).dataset_dict["test"][0] + + assert row["subset"] == "sat-math" + assert row["question"] == "sm?" + assert row["label"] == "D" + # `answer` is absent from the sat-* source rows entirely -> None, not "". + assert row["answer"] is None + assert row["other"]["solution"] == "because" + assert row["other"]["level"] is None + + +def test_cloze_rows_null_out_label_and_empty_options(tmp_path): + path = _stage(tmp_path, "math") + row = AGIEvalDataset(path, subsets=["math"]).dataset_dict["test"][0] + + assert row["label"] is None + assert row["answer"] == "(3,4]" + assert row["options"] == [] + # int64 upstream, stringified so the struct has one dtype across subsets. + assert row["other"]["level"] == "5" + + +def test_jec_qa_list_label_is_unwrapped(tmp_path): + path = _stage(tmp_path, "jec-qa-kd") + row = AGIEvalDataset(path, subsets=["jec-qa-kd"]).dataset_dict["test"][0] + + assert row["label"] == "B" + + +def test_multi_label_row_raises_instead_of_silently_rescoring(tmp_path): + # v1.0 had genuine multi-label jec-qa rows; if the pinned data ever grows one + # back, set-comparison semantics change and must be revisited deliberately. + path = _stage(tmp_path, "jec-qa-kd", label_override=["A", "B"]) + with pytest.raises(ValueError, match="single-answer label"): + AGIEvalDataset(path, subsets=["jec-qa-kd"]) + + +def test_subsets_and_group_together_is_rejected(tmp_path): + path = _stage(tmp_path, "math") + with pytest.raises(ValueError, match="not both"): + AGIEvalDataset(path, subsets=["math"], group="math") + + +def test_unknown_selection_is_rejected(tmp_path): + path = _stage(tmp_path, "math") + with pytest.raises(ValueError, match="unknown subset"): + AGIEvalDataset(path, subsets=["mmlu"]) + with pytest.raises(ValueError, match="unknown group"): + AGIEvalDataset(path, group="en") + with pytest.raises(ValueError, match="`subsets` is empty"): + AGIEvalDataset(path, subsets=[]) + + +def test_missing_subset_file_names_the_path_and_the_fix(tmp_path): + path = _stage(tmp_path, "math") + with pytest.raises(FileNotFoundError, match="sieval dataset download agieval"): + AGIEvalDataset(path, group="math") diff --git a/tests/unit/tasks/test_agieval_0shot_gen.py b/tests/unit/tasks/test_agieval_0shot_gen.py new file mode 100644 index 00000000..a57159ad --- /dev/null +++ b/tests/unit/tasks/test_agieval_0shot_gen.py @@ -0,0 +1,241 @@ +"""Unit tests for the AGIEval 0-shot generative task. + +The load-bearing behaviours here are the two-stage inference (what stage 2 is +prompted with, and that both calls survive into the stage value) and the report's +group macros. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import cast + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.community.agieval.dataset_loader import MATH_SUBSETS +from sieval.community.agieval.evaluation import LEADERBOARD_EN_MCQ_SUBSETS +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + TaskContext, + build_judgement_record, + build_prediction_record, + build_rollout_judgement, +) +from sieval.datasets.agieval import AGIEvalDataset, AGIEvalDatasetSample +from sieval.tasks.agieval_0shot_gen import AGIEvalZeroShotGenTask + + +class _ScriptedChatModel(ChatModel): + """Replies with ``reply`` and records every prompt it was given.""" + + def __init__(self, name: str = "mock-chat", reply: str = "The answer is D"): + super().__init__(model=name, api_key="fake") + self.reply = reply + self.prompts: list = [] + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + _ = kwargs + self.prompts.append(prompt) + return ModelOutput(model=self.meta(), texts=[self.reply]) + + async def _alogprobs_impl(self, prompt, **kwargs) -> ModelOutput: + _ = (prompt, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _sample(subset: str = "sat-math", **overrides) -> AGIEvalDatasetSample: + sample: dict = { + "subset": subset, + "passage": None, + "question": "Q?", + "options": ["(A)1", "(B)2", "(C)3", "(D)4"], + "label": "D", + "answer": None, + "other": {}, + } + sample.update(overrides) + return cast(AGIEvalDatasetSample, sample) + + +def _task(model=None, **kwargs) -> AGIEvalZeroShotGenTask: + dataset = AGIEvalDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(_sample())])}) + ) + return AGIEvalZeroShotGenTask(dataset, model or _ScriptedChatModel(), **kwargs) + + +@pytest.mark.anyio +async def test_preprocess_carries_subset_and_reference(): + task = _task() + pre = await task.preprocess(_sample(), TaskContext(sample_id=0)) + + assert pre["extra"]["subset"] == "sat-math" + assert pre["reference"] == "D" + assert pre["prompt"][0]["content"].startswith("Q: Q? Answer Choices:") + + +@pytest.mark.anyio +async def test_preprocess_uses_the_cloze_answer_as_reference(): + task = _task() + sample = _sample("math", options=[], label=None, answer="42") + pre = await task.preprocess(sample, TaskContext(sample_id=0)) + + assert pre["reference"] == "42" + + +@pytest.mark.anyio +async def test_infer_runs_two_stages_and_returns_both_outputs(): + model = _ScriptedChatModel(reply="I think it is D") + task = _task(model) + pre = await task.preprocess(_sample(), TaskContext(sample_id=0)) + + outputs = await task.infer(pre, TaskContext(sample_id=0)) + + # Both calls come back, so the runner profiles both and the first stage's + # reasoning stays on disk. + assert len(outputs) == 2 + assert len(model.prompts) == 2 + # Stage 2 re-sends the stage-1 prompt + reply + the family extraction cue. + stage2 = model.prompts[1][0]["content"] + assert stage2.startswith("Q: Q? Answer Choices:") + assert "I think it is D" in stage2 + assert stage2.endswith("Therefore, among A through E, the answer is") + + +@pytest.mark.anyio +async def test_infer_routes_stage_two_to_the_extractor_when_given(): + answerer = _ScriptedChatModel("answerer", reply="reasoning...") + extractor = _ScriptedChatModel("extractor", reply=" D") + task = _task(answerer, extractor=extractor) + pre = await task.preprocess(_sample(), TaskContext(sample_id=0)) + + outputs = await task.infer(pre, TaskContext(sample_id=0)) + + assert len(answerer.prompts) == 1 and len(extractor.prompts) == 1 + assert outputs[1].texts == [" D"] + + +@pytest.mark.anyio +async def test_infer_carries_an_empty_first_stage_reply_into_stage_two(): + class _Empty(_ScriptedChatModel): + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + _ = kwargs + self.prompts.append(prompt) + return ModelOutput(model=self.meta(), texts=[]) + + model = _Empty() + task = _task(model) + pre = await task.preprocess(_sample(), TaskContext(sample_id=0)) + + # Upstream feeds "" onward rather than dropping the sample. + outputs = await task.infer(pre, TaskContext(sample_id=0)) + assert len(outputs) == 2 + assert model.prompts[1][0]["content"].endswith( + "\n\nTherefore, among A through E, the answer is" + ) + + +def test_extractor_arg_rejects_a_non_model(): + with pytest.raises(ValueError, match="model-config dict or a Model"): + _task(extractor=42) + + +@pytest.mark.anyio +async def test_postprocess_parses_the_second_stage_reply(): + task = _task() + inf = [ + ModelOutput(model=None, texts=["long reasoning about A and B"]), # type: ignore[invalid-argument-type] + ModelOutput(model=None, texts=[" D."]), # type: ignore[invalid-argument-type] + ] + post = await task.postprocess(inf, TaskContext(sample_id=0, raw_sample=_sample())) + + # Parsed from stage 2 only — parsing stage 1 would have returned "A". + assert post["rollouts"][0]["prediction"] == "D" + assert post["rollouts"][0]["extracted"] is True + + +@pytest.mark.anyio +async def test_feedback_scores_per_subset_rule(): + task = _task() + ctx = TaskContext(sample_id=0, raw_sample=_sample("jec-qa-kd", label="B")) + finalize, fb = await task.feedback(build_prediction_record([["B"]]), ctx) + + assert finalize is True + assert fb["reference"] == "B" + assert fb["rollouts"][0]["correct"] is True + assert fb["extra"]["subset"] == "jec-qa-kd" + + +@pytest.mark.anyio +async def test_feedback_uses_math_equivalence_for_cloze(): + task = _task() + sample = _sample("math", options=[], label=None, answer="0.5") + ctx = TaskContext(sample_id=0, raw_sample=sample) + _, fb = await task.feedback(build_prediction_record(["\\frac{1}{2}"]), ctx) + + assert fb["rollouts"][0]["correct"] is True + + +def _final(subset: str, correct: bool) -> TaskContext: + return TaskContext( + sample_id=0, + raw_sample=_sample(subset), + feedback_result=build_judgement_record( + "D", [build_rollout_judgement(0, correct)], extra={"subset": subset} + ), + ) + + +@pytest.mark.anyio +async def test_report_macro_averages_over_subsets_not_samples(): + task = _task() + # sat-math: 1/2 correct, aqua-rat: 1/1 -> macro 75.0, micro would be 66.7. + finals = [ + _final("sat-math", True), + _final("sat-math", False), + _final("aqua-rat", True), + ] + metrics = await task.report(finals, []) + + assert metrics["score"] == pytest.approx(75.0) + assert metrics["score_sat_math"] == pytest.approx(50.0) + assert metrics["score_aqua_rat"] == pytest.approx(100.0) + assert metrics["fails"] == 0.0 + + +@pytest.mark.anyio +async def test_report_omits_group_macros_until_the_whole_group_ran(): + task = _task() + partial = [_final(subset, True) for subset in MATH_SUBSETS[:-1]] + assert "macro_math" not in await task.report(partial, []) + + full = [_final(subset, True) for subset in MATH_SUBSETS] + metrics = await task.report(full, []) + assert metrics["macro_math"] == pytest.approx(100.0) + # The MATH subset's own accuracy keeps its own key. + assert metrics["score_math"] == pytest.approx(100.0) + assert "macro_en_mcq" not in metrics + + +@pytest.mark.anyio +async def test_report_emits_the_leaderboard_macro_when_the_group_is_complete(): + task = _task() + finals = [_final(subset, True) for subset in LEADERBOARD_EN_MCQ_SUBSETS] + finals.append(_final("lsat-ar", False)) + + metrics = await task.report(finals, [1]) + + # lsat-ar is 1/2 -> the 8-subset macro is (7*100 + 50) / 8. + assert metrics["macro_en_mcq"] == pytest.approx(93.75) + assert metrics["fails"] == 1.0 + + +@pytest.mark.anyio +async def test_report_on_an_all_failed_run_is_zero_not_a_crash(): + task = _task() + metrics = await task.report([], [1, 2]) + + assert metrics["score"] == 0.0 + assert metrics["fails"] == 2.0