diff --git a/.claude/rules/tasks.md b/.claude/rules/tasks.md index 6b8eee52..0b7bf426 100644 --- a/.claude/rules/tasks.md +++ b/.claude/rules/tasks.md @@ -7,14 +7,31 @@ paths: ## Naming & Model Type -- File naming must follow `_shot_.py` pattern (authoritative table in `sieval/tasks/CLAUDE.md`): +- File naming must follow `_shot_[_].py` pattern (authoritative table in `sieval/tasks/CLAUDE.md`): - `_gen.py` → `model_type = "chat"` - `_base_gen.py` → `model_type = "gen"` (base model, uses GenModel) - `_ppl.py` → `model_type = "gen"` (perplexity, uses GenModel) - `_clp.py` → `model_type = "gen"` (conditional next-token log-prob, uses GenModel) -- Class naming: `Task` — words for shot count (`ZeroShot`, `FewShot`) +- Class naming: `[]Task` — words for shot count (`ZeroShot`, `FewShot`) - `ppl` vs `clp` distinction: see `sieval/tasks/CLAUDE.md`. +### Variants + +An optional trailing segment lets two readings of one benchmark coexist as +separate registered tasks (full rationale in `sieval/tasks/CLAUDE.md`). + +- The unqualified name means **what upstream measures, bugs included**. Never + repurpose it for a local change. +- `_fixed` requires a **defect** in upstream's data or grader, not a preference, + and owes both: every divergence in `reference_impl.notes`, and a **quantified** + score impact. +- A variant may not spell a mode — `..._clp_gen.py` is rejected. +- A different **measurement regime** is a mode, not a variant. +- A fix to **problem text or reference answers** is a `datasets/` concern: a + dataset variant applying a patch table over the same pinned revision, never a + forked copy. See `sieval/datasets/CLAUDE.md`. +- Do not coin a new variant name speculatively. + ## Checklist for New Benchmarks - Add benchmark-specific dependencies to `pyproject.toml` optional dependency groups (e.g., `[project.optional-dependencies.benchmark_name]`) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b328b06..389c7756 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,9 @@ See [tests/README.md](tests/README.md) for mock infrastructure and full command 1. Create dataset in `sieval/datasets/` — keep upstream field names, and cast a column's dtype only if the pinned revision requires it (each Task binds 1:1 to its own sample `TypedDict`, so uniformity with a sibling loader buys nothing) -2. Create task in `sieval/tasks/` — file naming: `_shot_.py` (see `sieval/tasks/CLAUDE.md`) +2. Create task in `sieval/tasks/` — file naming: `_shot_[_].py` + (see `sieval/tasks/CLAUDE.md`). The unqualified name tracks upstream, bugs included; a + local correction takes a `_fixed` variant and owes a quantified score impact 3. If the reference implementation repeats sampling (`n_repeats`, `--n 4`) and your task's default `n` differs, record that in `reference_impl.notes` with how to match it 4. Add unit tests under `tests/unit/datasets/` and `tests/unit/tasks/` mirroring the source layout diff --git a/pyproject.toml b/pyproject.toml index 681de131..dad2084f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -218,11 +218,23 @@ paths_to_mutate = ["sieval/core"] tests_dir = ["tests/unit"] exclude = ["sieval/core/**/__init__.py"] also_copy = [ + # `sieval/__init__.py` is load-bearing: without it `mutants/sieval` is a plain + # directory rather than a package, so every test in the copy resolves `sieval` + # through the editable install instead, and mutmut dies on an unrelated import + # before running a single mutant. + "sieval/__init__.py", + # `scripts/` is not a package, so `tests/unit/scripts/` puts it on `sys.path` + # by walking up from its own `__file__`. Inside the copy that resolves to + # `mutants/scripts`, so without this entry stats collection dies on + # `ModuleNotFoundError: No module named 'check_layer_imports'` — the same + # class of failure as the entry above, and just as invisible. + "scripts", "sieval/community", "sieval/datasets", "sieval/tasks", "sieval/infer", "sieval/cli", + "sieval/meta", "sieval/__main__.py", ] diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 117ebd5e..cd4f36a2 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -56,8 +56,15 @@ _MAX_DRIFT_DETAILS = 20 +# `_shot_[_].py`. The mode alternation is anchored right +# after the shot segment, so `model_type` stays readable off the name even when a +# variant follows. Multi-token modes lead the alternation, and a variant may not +# spell a mode (`..._clp_gen.py`) — that name has two readings, so it is rejected +# rather than resolved by regex precedence. +_TASK_MODE_ALTERNATION = "base_gen|llmjudge_gen|gen|ppl|clp" _TASK_FILE_PATTERN = re.compile( - r"^[a-z][a-z0-9_]*_(\d+|k)shot_(gen|base_gen|ppl|clp|llmjudge_gen)\.py$" + rf"^[a-z][a-z0-9_]*_(?:\d+|k)shot_(?:{_TASK_MODE_ALTERNATION})" + rf"(?:_(?!(?:{_TASK_MODE_ALTERNATION})\.py$)[a-z][a-z0-9_]*)?\.py$" ) _DATASET_SUFFIX_PATTERN = re.compile(r"(Dataset|DatasetSample|CSVSample)$") @@ -576,6 +583,7 @@ class PreflightRunner: "check_imports", "check_examples", "check_meta_index_sync", + "check_mutmut_config", "check_version", ] @@ -1914,6 +1922,75 @@ def _get_latest_git_tag(self) -> str | None: except (subprocess.CalledProcessError, FileNotFoundError): return None + def check_mutmut_config(self) -> list[CheckResult]: + """Verify ``[tool.mutmut]`` still produces an importable ``mutants/`` copy. + + mutmut runs the suite from a copy of the tree built out of + ``paths_to_mutate`` + ``also_copy``. Anything the suite imports that the + copy omits makes the run die during stats collection, before a single + mutant executes — and it surfaces as an unrelated broken test rather + than as a configuration error, which is why this went unnoticed twice. + + Two entries are load-bearing, for the same reason and with the same + symptom: + + * ``sieval/__init__.py`` — without it ``mutants/sieval`` is a plain + directory rather than a package, so every test in the copy resolves + ``sieval`` through the editable install instead. + * ``scripts`` — ``scripts/`` is not a package, so ``tests/unit/scripts/`` + puts it on ``sys.path`` by walking up from its own ``__file__``. In the + copy that resolves to ``mutants/scripts``. + + Cheap to assert, invisible otherwise: it made the mutation-score + requirement in ``sieval/core/CLAUDE.md`` unsatisfiable for as long as + either entry was missing. + """ + check = "check_mutmut_config" + pyproject = self.project_root / "pyproject.toml" + if not pyproject.exists(): + return [CheckResult("FAIL", check, "pyproject.toml not found")] + + import tomllib + + config = tomllib.loads(pyproject.read_text(encoding="utf-8")) + mutmut = config.get("tool", {}).get("mutmut") + if not mutmut: + return [CheckResult("PASS", check, "no [tool.mutmut] section to check")] + + copied = list(mutmut.get("also_copy", [])) + list( + mutmut.get("paths_to_mutate", []) + ) + # A parent entry ("sieval") carries the file; an exact entry is the + # normal case. Anything else means the path is not in the copy. + required = { + "sieval/__init__.py": "mutants/sieval is not an importable package", + "scripts": "tests/unit/scripts/ cannot import the module it tests", + } + missing = [ + f"{path!r} ({why})" + for path, why in required.items() + if not any( + path == entry or path.startswith(f"{entry}/") for entry in copied + ) + ] + if missing: + return [ + CheckResult( + "FAIL", + check, + "[tool.mutmut] omits a path the suite imports, so every " + "mutation run dies during stats collection", + [*missing, f"also_copy + paths_to_mutate = {copied}"], + ) + ] + return [ + CheckResult( + "PASS", + check, + "[tool.mutmut] copies every path the suite imports", + ) + ] + def check_version(self) -> list[CheckResult]: """Check CHANGELOG / git tag / Dockerfile version alignment.""" results: list[CheckResult] = [] diff --git a/sieval/community/ugmathbench.py b/sieval/community/ugmathbench.py new file mode 100644 index 00000000..104f64b9 --- /dev/null +++ b/sieval/community/ugmathbench.py @@ -0,0 +1,1133 @@ +# UGMathBench (YangLabHKUST/UGMathBench) is distributed under GPL-3.0, so none +# of its harness code is vendored here. What IS reproduced from the pinned +# commit is the benchmark's *protocol* — the query template and answer-type +# descriptions a model must be shown for the run to be UGMathBench at all: +# https://github.com/YangLabHKUST/UGMathBench/blob/df47bfa639bfb89bdb0220036a7b2f216e72b0b3/utils.py +""" +UGMathBench prompting, answer extraction, and answer-type-aware grading. + +UGMathBench problems carry a *sequence* of answers (a table to fill, several +sub-questions), each with its own answer type drawn from a fixed 10-type +vocabulary. Grading is therefore two-level: split the boxed response into as +many answers as the reference has, then compare each one under the rule its +type implies. A sample is correct only when every slot is. + +Three pieces: + +* :func:`build_prompt` — the upstream query template (single- vs multi-answer + wording, plus the per-slot type description), reproduced from the pinned + ``make_prompt``. This is what makes a run comparable to the paper. +* :func:`extract_predictions` — take the last ``\\boxed{...}``, normalize it, + split on commas that sit outside brackets. +* :func:`judge_answers` — per-slot dispatch over the 10 answer types. + +**The grader is an independent implementation, not a port.** Upstream's +``judge_rule.py`` is GPL-3.0 and cannot be carried into an Apache-2.0 +distribution, so the comparison rules here were written against the answer-type +semantics the prompt itself states, on top of ``math-verify`` (already a sieval +dependency). Known behavioural deltas versus the reference judge: + +* Symbolic equivalence starts from ``math-verify``'s parse/verify rather than + upstream's bespoke ``parse_latex`` + ``simplify`` chain, and is more permissive + on LaTeX shapes upstream's normalizer never learned. Numeric slots keep + upstream's *relative* tolerance as a second chance, since ``math-verify`` + compares floats at fixed decimal rounding. Where it used to be *stricter* than + upstream — no numeric sampling of free symbols — it no longer is: + :func:`_same_function` closes that gap with a fixed ladder of substitution + probes, arrived at independently rather than ported. See the parser note + below for why that pass is load-bearing here. +* **The dataset's gold is sympy source, not LaTeX, and only one parser reads it.** + ``math_verify.parse`` runs a LaTeX reader over both sides, and in LaTeX an + unescaped ``sin`` is the product s*i*n while ``pi`` is p*i — so the stored + ``7*sin(pi*x/5)+1`` becomes ``7*s*i*n*(i*p*x)/5 + 1`` and cannot match the + model's ``7\\sin(\\frac{\\pi}{5}x)+1`` by any route except exact string + equality. :func:`_parse_sympy_source` supplies the second reading. + The first live run measured what this had been costing: **716 of 15,183 + samples** were graded wrong purely for it (EAcc 34.46 -> 38.49, AAcc 40.87 -> + 45.59, CAcc 48.07 -> 53.53, with **zero** verdicts moving right-to-wrong). + Note the shape of the + bug — it was invisible to the reference-replay measurement below, because + replaying a gold as its own answer short-circuits on + ``_squash(pred) == _squash(gold)`` and never reaches the symbolic path at + all. **A self-replay canary exercises the fast path and is silent about the + comparison logic it appears to certify.** +* No answer-type *inference*: upstream's ``is_equal`` retries every judgement + method until one accepts, which lets a slot be graded by a rule its declared + type did not ask for. Here the declared type decides, except inside OL/UOL + elements (whose own types the dataset does not record). +* Extraction is strict, as ``eval_rule.py`` grades with + ``Judger(strict_extract=True)``: no "guess the last LaTeX formula / the last + number in the response" fallback, and the ``answer is`` / ``answer:`` hand-off + is kept because upstream keeps it too (its no-box branch runs the same + ``elif`` chain). Two smaller differences remain inside that branch and are + *not* repairs — just shapes this reads differently: + + - upstream splits on ``herefore`` and keeps only the tail **before** looking + for a box, so a response whose last box sits before its last "Therefore" + loses the box entirely there, where :func:`_last_boxed_content` always + takes the last box in the whole response; + - upstream returns the **entire response** when the box is empty + (``if not content: return text``), where :func:`extract_answer` treats an + empty box as "no box" and falls through to the marker search. + + Both sit inside the 95.51% live agreement measured against upstream's judge, + so neither is worth a divergence of its own — they are listed because a + divergence list that quietly rounds off is not one. +* **Commas are split at a different bracket depth**, in three ways. All three + are deliberate, and all three run in the same direction: a row upstream + miscounts — and therefore grades wrong in every slot, whatever the model + answered — this one counts correctly. + + - Upstream's ``split_by_comma`` counts ``<`` and ``>`` as brackets. + :data:`_OPENERS` does not. In this dataset they are overwhelmingly the + *relational operators*, a slot whose entire answer is ``<``, so an opening + ``<`` swallows the comma after it; the grouping form ``\\langle`` / + ``\\rangle`` is folded to parentheses before the scan and keeps working. + Worth **6 rows** across ``Financial_mathematics_0300`` and + ``Calculus_-_single_variable_0824`` (all three versions each). + - Upstream lets its bracket depth go *negative*, so an unmatched closer parks + everything after it below zero, where no comma splits again. This clamps at + zero, which is worth **3 more rows** — ``Arithmetic_0071``, whose second + slot is ``>``: upstream splits the first comma, drops to -1 on the ``>``, + and never splits again. + - This also tracks ``{}``, which keeps a comma inside a LaTeX group + (``\\frac{a,b}{c}``) from splitting a slot in two. Worth **0 rows** on the + references — they are plain sympy source — so it earns its place only on + the prediction side, where the model writes LaTeX. + + Nine rows total, all of them ours-accepts / upstream-rejects, and they are + counted inside the 552 below. The only ``<...>`` *pairs* in the 42,064 gold + slots are ``
`` markup, so the grouping reading buys back nothing. +* Both sides are normalized *by the same pass*, references included. Upstream + normalizes its references too — ``judge()`` runs ``norm_ans_str`` over the + gold — but its extraction-time pass, ``Judger.normalize_answer``, runs on the + prediction alone and rewrites shapes the reference never sees, notably + ``sqrt(x)`` into ``sqrt{(}x)``. A plain-sympy reference such as + ``sqrt(1.83985)`` therefore cannot match itself upstream. This asymmetry, not + any single answer type, is the largest divergence in this module; it is + measured below. +* A ``TF`` slot whose reference is not a boolean at all (9 of 1665, e.g. ``-22``, + ``not real``) is compared as a value rather than graded wrong outright, which + is what upstream's assert-then-swallow does. Those slots are otherwise + unwinnable regardless of what the model answers. +* A numeric slot whose reference is exactly zero is compared absolutely + (``|pred| <= tolerance``). Upstream divides by the reference, so a zero + reference raises into a bare ``except`` and the slot falls through to the + symbolic path instead. +* Per-slot verdicts are returned rather than only the sample-level ``all()``, + so a wrong answer can be located without re-running the grader. +* **Some answers are refused rather than graded**, because the text being parsed + is model output and ``parse_expr`` evaluates what it parses. Three shapes: + + - A boxed ``__import__('os').system(...)`` would otherwise run. + :func:`_sympy_globals` removes the builtins from the parse namespace. + - A boxed ``eval("...")`` — or ``sympify``/``S``/``N``, or any name at all, + since ``auto_symbol`` makes every unknown one callable — hands a *string* + back to sympy, which re-sympifies it with its own default namespace and so + gets the builtins back. :func:`_quotes_free` refuses the quote instead of + the callee, which is the only end of it that can be enumerated. + - A boxed ``9^9^9^9`` asks for a 370-million-digit integer that never + returns. :func:`_evaluable` screens it out with an unevaluated pre-parse. + Grading runs in a worker process, so this occupies one worker rather than + the shared event loop, but a worker held forever is still a worker lost. + + A refused answer grades wrong. Upstream has none of these guards; they are + not a divergence in what the benchmark *measures*, and no reachable + comparison changes — the largest exponent in the pinned references is three + digits, and not one of the 42,064 gold slots contains a quote. + +23 of the 15,183 pinned rows (0.15%) cannot be graded correctly here even when +the reference is replayed verbatim as the answer. 19 are unwinnable upstream +too: the "comma-separated answers in one box" protocol cannot express them — a +comma inside an open-ended phrase, or an unbalanced bracket left by upstream's +own answer splitting. The other 4 are a shape this module loses and upstream +wins — a ``UOL`` reference whose top-level commas sit outside any bracket, which +:func:`split_answers` reads as separate slots +(``Calculus_-_single_variable_0384`` v1-v3, ``Complex_analysis_0035`` v3). + +The prompt builder, by contrast, IS exact: it reproduces upstream's ``raw`` +template byte-for-byte on all 15,183 rows. + +Scores are nonetheless *not* the paper's, which is why the only task built on +this module is the ``_fixed`` variant (``ugmathbench_0shot_gen_fixed``) and the +unqualified name is left vacant. + +How far from upstream, measured rather than argued. Upstream's judge was run as +a local *instrument* — GPL-3.0 restricts distribution, not use, and no upstream +code is vendored or redistributed here — over all 15,183 pinned rows, with each +row's own reference replayed back as a boxed answer: + +* upstream accepts its own reference on 14,616 rows (96.27%); +* this module accepts 15,160 (99.85%); +* the 552 rows that disagree (3.64%) span 192 of 5,061 problems, an EAcc + **ceiling** difference of **3.79 pp** — roughly five times the 0.70 pp + binomial standard error at this sample size, so the divergence is material + rather than noise; +* per answer slot, running upstream's own dispatch loop without its + short-circuit: 788 of 42,064 slots disagree (1.87%), concentrated in ``EX`` + (350) and ``NV`` (322), with ``TF`` contributing exactly the 9 non-boolean + references described above. + +Direction matters more than magnitude here: **548 of those 552 are rows where +upstream rejects its own reference**, i.e. slots no model could win there. That +is repair, not drift, and the prediction-only normalization above is its +dominant cause; the non-boolean ``TF`` references contribute 9 and the +comma-splitting rules below 9. Only 4 rows go the other way (the ``UOL`` note +above). A *ceiling*, not an expectation: the +difference is realized only on a problem a model would otherwise answer +correctly in all three versions. + +The measurement is a replay of stored references, so it bounds the grader's +divergence, not a model's score. What it cannot show is how the grader behaves +on real model prose — see the promotion criteria on the task class. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import math +import re + +#: The 16 subject configs of the HF dataset, in the order the benchmark lists +#: them. Doubles as the default load order, so a sliced run is reproducible. +SUBJECTS: tuple[str, ...] = ( + "Abstract_algebra", + "Algebra", + "Arithmetic", + "Calculus_-_multivariable", + "Calculus_-_single_variable", + "Combinatorics", + "Complex_analysis", + "Differential_equations", + "Financial_mathematics", + "Geometry", + "Linear_algebra", + "Number_theory", + "Probability", + "Set_theory_and_logic", + "Statistics", + "Trigonometry", +) + +#: Randomized versions per problem. EAcc is defined over all of them, so this +#: is a property of the benchmark, not a knob. +VERSIONS: int = 3 + +#: Answer-type code -> the description the prompt shows the model. Reproduced +#: from upstream ``make_prompt``'s ``type2descriptions``; ``{options}`` is +#: filled with the slot's option list for the two multiple-choice types. +TYPE_DESCRIPTIONS: dict[str, str] = { + "UOL": ( + "an unordered list of answers surrounded by parentheses with any answer " + 'types, for example, (1, x^2, True), where "unordered list" means ' + "changing the order of elements results in the same answer" + ), + "OL": ( + "an ordered list of answers surrounded by parentheses with any answer " + 'types, for example, (1, x^2, True), where "ordered list" means changing ' + "the order of elements results in different answers" + ), + "INT": "a range inteval", + "TF": "either True or False", + "EX": "an expression", + "EQ": "an equation", + "MCS": "one option of a multiple choice question with options {options}", + "MCM": ( + "more than one option concatenated without space or commas of a multiple " + "choice question with options {options}, for example: BD" + ), + "NV": "a numerical value without units", + "OE": ( + "a word, phrase, term or string that satisfies the requirements of the problem" + ), +} + +_PROMPT_HEAD = ( + "The following is an undergraduate-level mathematical problem in {subject}. " + "You need to solve the problem by completing all placeholders [ANS].\n\n" +) +_PROMPT_SINGLE_TYPES = ( + "This problem involves only one placeholders [ANS] to be completed. " + "The answer type is {descriptions}.\n\n" +) +_PROMPT_MULTI_TYPES = ( + "This problem involves {count} placeholders [ANS] to be completed. " + "Their answer types are, in order, {descriptions}.\n\n" +) +_PROMPT_TAIL_SINGLE = ( + "Problem:\n{problem}\n\n" + "All mathematical formulas and symbols you output should be represented with " + 'LaTeX. Please end your response with: "The final answer is \\boxed{ANSWER}", ' + "where ANSWER should be your final answer." +) +_PROMPT_TAIL_MULTI = ( + "Problem:\n{problem}\n\n" + "All mathematical formulas and symbols you output should be represented with " + 'LaTeX. Please end your response with: "The final answers are \\boxed{ANSWER}"' + ", where ANSWER should be the sequence of your final answers, separated by " + "commas." +) + + +def describe_answer_type(answer_type: str, options: list[str] | None = None) -> str: + """Render one slot's type description, filling MC options where the type asks. + + Upstream interpolates the option list with Python's ``list`` repr, so the + model sees ``['A', 'B', 'C', 'D', 'E']``; kept as-is for prompt fidelity. + """ + description = TYPE_DESCRIPTIONS.get(answer_type) + if description is None: + raise KeyError( + f"unknown UGMathBench answer type {answer_type!r}; " + f"expected one of {sorted(TYPE_DESCRIPTIONS)}" + ) + if "{options}" in description: + return description.format(options=list(options or [])) + return description + + +def _pad_options( + answer_types: list[str], options: list[list[str]] | None +) -> list[list[str]]: + """One option list per declared type, padded when the dataset is short. + + A handful of pinned rows declare fewer ``options`` entries than + ``answer_type`` entries. Options only carry meaning for the two + multiple-choice types, so a missing entry is a data gap, not a different + question — pad rather than refuse to build the prompt. + """ + padded = [list(entry) for entry in (options or [])][: len(answer_types)] + padded.extend([] for _ in range(len(answer_types) - len(padded))) + return padded + + +def build_prompt( + subject: str, + problem: str, + n_answers: int, + answer_types: list[str], + options: list[list[str]] | None = None, +) -> str: + """Build the UGMathBench query for one problem version. + + The single- and multi-answer wordings differ upstream (down to "The final + answer is" vs "The final answers are"), and the count drives which one is + used, so both are reproduced rather than unified. + + *n_answers* is the length of the reference answer sequence and is passed + separately from *answer_types* on purpose: a few pinned rows declare fewer + types than answers, and upstream's template takes the count from the answers + while describing only the declared types. Reproducing that keeps the prompt + byte-identical to the one the paper's numbers came from. + + The two branches read the type list differently, and that is upstream's + shape rather than a simplification: its single-answer branch describes + ``answer_type[0]`` alone, while its multi-answer branch joins every declared + type. The two agree on every pinned row, since a row with one answer + declares one type — but building the joined string once and using it in both + branches would make this port's fidelity depend on that coincidence holding + after a re-cut of the data, so each branch derives its own. + """ + per_slot_options = _pad_options(answer_types, options) + + head = _PROMPT_HEAD.format(subject=subject) + if n_answers == 1: + first_type = answer_types[:1] + descriptions = ", ".join( + describe_answer_type(answer_type, slot_options) + for answer_type, slot_options in zip( + first_type, per_slot_options[:1], strict=True + ) + ) + types = _PROMPT_SINGLE_TYPES.format(descriptions=descriptions) + tail = _PROMPT_TAIL_SINGLE + else: + descriptions = ", ".join( + describe_answer_type(answer_type, slot_options) + for answer_type, slot_options in zip( + answer_types, per_slot_options, strict=True + ) + ) + types = _PROMPT_MULTI_TYPES.format(count=n_answers, descriptions=descriptions) + tail = _PROMPT_TAIL_MULTI + # `ANSWER` is a literal in the template, not a field to fill. + return head + types + tail.replace("{problem}", problem) + + +# --- answer extraction ----------------------------------------------------- + +_BOXED_MARKERS = ("\\boxed", "\\fbox") +# Ordered by specificity: a response saying "the final answer is X" should not +# be cut at the bare "answer is". +_ANSWER_MARKERS = ("final answers are", "final answer is", "answer is", "answer:") +_LATEX_WRAPPERS = re.compile(r"\\(?:text|mathrm|mathbf|textbf|mbox)\s*\{([^{}]*)\}") +_SIMPLE_REMOVALS = ("\\left", "\\right", "\\!", "\\,", "\\;", "\\quad", "\\qquad") + + +def _last_boxed_content(text: str) -> str | None: + """Return the content of the last ``\\boxed{...}`` / ``\\fbox{...}``. + + Brace-balanced rather than regex-greedy, so a nested ``\\frac{a}{b}`` inside + the box survives intact. + """ + start = max(text.rfind(marker) for marker in _BOXED_MARKERS) + if start < 0: + return None + open_brace = text.find("{", start) + if open_brace < 0: + return None + depth = 0 + for index in range(open_brace, len(text)): + char = text[index] + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return text[open_brace + 1 : index] + return None + + +def normalize_answer(text: str) -> str: + """Strip LaTeX decoration that never carries meaning for a comparison.""" + normalized = text.replace("∶", ":").replace("，", ",") + normalized = normalized.replace("\\approx", "=").replace("\\simeq", "=") + normalized = _LATEX_WRAPPERS.sub(r"\1", normalized) + for token in _SIMPLE_REMOVALS: + normalized = normalized.replace(token, "") + # Percent signs and degree marks: the dataset stores bare values ("a + # numerical value without units"), so a trailing unit is decoration too. + normalized = normalized.replace("^{\\circ}", "").replace("^\\circ", "") + normalized = normalized.replace("\\%", "").replace("%", "") + return normalized.replace("$", "").strip() + + +def extract_answer(response: str) -> str | None: + """Pull the answer segment out of a full model response. + + Strict, as upstream's ``eval_rule.py`` grades: the last box, else an + explicit "the answer is" hand-off, else ``None``. There is deliberately no + "take the last number in the response" fallback — the prompt mandates a box, + and guessing inflates scores for models that ignored the format. + """ + boxed = _last_boxed_content(response) + if boxed is not None: + normalized = normalize_answer(boxed) + return normalized or None + + haystack = response.lower() + for marker in _ANSWER_MARKERS: + position = haystack.rfind(marker) + if position >= 0: + tail = response[position + len(marker) :].strip() + # One line only: whatever follows a blank line is new prose. + tail = tail.split("\n\n")[0].strip().rstrip(".") + normalized = normalize_answer(tail) + if normalized: + return normalized + return None + + +#: ``<`` and ``>`` are deliberately absent, unlike upstream's ``split_by_comma`` +#: which counts them as brackets. In this dataset they are overwhelmingly the +#: *relational operators* — a slot whose whole answer is ``<`` — so treating an +#: opening ``<`` as a bracket swallows the comma after it and the slot count +#: comes out short, which grades every slot in the row wrong however good the +#: answer. Angle brackets as grouping earn nothing back: of the 42,064 gold +#: slots on the pinned revision the only ``<...>`` pairs are ``
`` markup, +#: and ``\langle`` / ``\rangle`` are folded to parentheses below. +_OPENERS = {"(": ")", "[": "]", "{": "}"} +_CLOSERS = set(_OPENERS.values()) +_SET_DELIMITERS = { + "\\{": "(", + "\\}": ")", + "\\langle": "(", + "\\rangle": ")", + "\\lbrace": "(", + "\\rbrace": ")", +} + + +def split_answers(text: str) -> list[str]: + """Split a boxed answer into one string per ``[ANS]`` slot. + + Splits on commas at bracket depth zero, so a slot that is itself a list + (``(1, 2, 3)``) or an interval (``(-\\infty, 5)``) stays whole. LaTeX set + delimiters are folded to plain parentheses first so they nest like brackets. + + Three deliberate differences from upstream's ``split_by_comma``, all + enumerated with their measured cost in the module docstring: ``{}`` counts + as a bracket here and does not upstream; ``<`` and ``>`` do not count here + and do upstream; and the depth clamps at zero instead of going negative. + """ + folded = text + for latex_delimiter, plain in _SET_DELIMITERS.items(): + folded = folded.replace(latex_delimiter, plain) + + parts: list[str] = [] + depth = 0 + start = 0 + for index, char in enumerate(folded): + if char in _OPENERS: + depth += 1 + elif char in _CLOSERS: + depth = max(0, depth - 1) + elif char == "," and depth == 0: + parts.append(folded[start:index]) + start = index + 1 + parts.append(folded[start:]) + return [part.strip().strip("$").strip() for part in parts] + + +# --- answer comparison ----------------------------------------------------- + +#: Upstream's ``norm_str2bool`` tests the single letters *before* lowercasing +#: (``if s in ['T', 'Y']``), so only the capitals are booleans; the word forms +#: are matched after a ``.lower()`` and so are case-insensitive. The asymmetry +#: looks like an oversight but it is load-bearing: ``t`` and ``y`` are ordinary +#: parameter names, and reading them as booleans marks a wrong answer right. +_TRUE_LETTERS = {"T", "Y"} +_FALSE_LETTERS = {"F", "N"} +_TRUE_WORDS = {"true", "yes"} +_FALSE_WORDS = {"false", "no"} +_WHITESPACE = re.compile(r"\s+") + + +def _squash(text: str) -> str: + """Whitespace- and decoration-insensitive form, for the cheap equality path.""" + return _WHITESPACE.sub("", text.replace("$", "").replace("\\", "")).lower() + + +def _to_bool(text: str) -> bool | None: + """Read a ``TF`` answer as a boolean, or ``None`` if it is not one. + + Only reachable from the ``TF`` branch of :func:`judge_answer`. Upstream + gates the same conversion on the declared answer type — ``norm_ans_str`` + calls ``norm_str2bool`` only ``if ans_type == "TF"`` — and leaves the + elements of a list alone, under a standing ``TODO: deal with OL with + boolean``. Applying it to list elements instead reads the parameter names + the dataset actually uses as truth values. + """ + stripped = text.strip().strip(".") + if stripped in _TRUE_LETTERS: + return True + if stripped in _FALSE_LETTERS: + return False + word = stripped.lower() + if word in _TRUE_WORDS: + return True + if word in _FALSE_WORDS: + return False + return None + + +def _to_float(text: str) -> float | None: + try: + return float(text.replace(",", "").replace(" ", "")) + except ValueError: + return None + + +def _parse_math(text: str) -> list: + from math_verify import parse + + # `$`-wrapping steers the LaTeX extractor at a bare answer string. It does + # NOT reliably read the plain-sympy shapes the dataset stores: `parse` runs + # a LaTeX reader, where an unescaped `sin` is the product s*i*n and `pi` is + # p*i, so the stored gold `7*sin(pi*x/5)+1` comes back as + # `7*s*i*n*(i*p*x)/5 + 1`. Bare algebra ("x^3+2*x^2+6") survives; anything + # naming a function does not. `_parse_sympy_source` is the other half. + return parse(f"${text}$") + + +#: Deterministic probe points for :func:`_same_function`. A *fixed* ladder, not +#: a seeded RNG: a grader has to return the same verdict for the same pair on +#: every run and in every process, and a module-level RNG would make the answer +#: depend on how many comparisons preceded it. Off the integers and away from +#: 0 and 1, so the usual poles and fixed points are avoided, yet deliberately +#: kept small: an answer like ``e^{\cosh(4x)}`` overflows to infinity above +#: x ~ 2, and every overflowed probe is discarded, so a ladder reaching into +#: the twenties would leave too few usable points and mark a correct +#: exponential answer wrong. +_PROBES: tuple[float, ...] = (0.41, 0.73, 1.19, 1.57, 2.11) +_MIN_CLEAN_PROBES = 3 +_MAX_FREE_SYMBOLS = 4 + +#: Largest integer exponent :func:`_parse_sympy_source` will evaluate. Sympy +#: computes ``a**b`` eagerly, so a boxed ``9^9^9^9`` asks for a 370-million-digit +#: integer and never returns. Nothing this benchmark asks for comes close — the +#: largest exponent in the pinned references is three digits — so the cap costs +#: no reachable comparison. See :func:`_evaluable`. +_MAX_EXPONENT = 10_000 + + +def _sympy_globals() -> dict: + """Namespace for :func:`parse_expr`, with the builtins removed. + + ``parse_expr`` evaluates its input, and its *default* global namespace is + built by ``exec("from sympy import *", ...)`` — which also injects + ``__builtins__``. Since the string being parsed is model output, that hands + a model ``__import__`` and ``open``: a boxed + ``__import__('os').system(...)`` runs, and the grader still reports the slot + wrong, so nothing in the run looks unusual. + + Clearing ``__builtins__`` closes that without narrowing the dialect. The + sympy names have to stay: ``auto_symbol`` rewrites an unknown callable into + ``Function('sin')``, so a namespace holding *only* the answer aliases fails + every legitimate ``sin(pi*x/5)`` with ``NameError: name 'Function' is not + defined``. + + This is a namespace restriction, not a sandbox — attribute access on sympy + objects still resolves, and it only covers the namespace *this* parse runs + in. It does not survive a nested parse, which is why :func:`_quotes_free` + exists. + """ + namespace: dict = {} + exec("from sympy import *", namespace) # noqa: S102 - fixed literal, not input + namespace["__builtins__"] = {} + return namespace + + +def _quotes_free(text: str) -> bool: + """Is *text* free of the string literals that reopen the interpreter? + + :func:`_sympy_globals` sanitizes the namespace the top-level parse runs in, + and that is not enough on its own. Sympy re-sympifies a *string* argument + with its own default namespace, which has the builtins back, so a call + carrying a string literal escapes the restriction and runs:: + + eval("__import__('os').system(...)") + + ``sympify``, ``S`` and ``N`` do the same thing, and ``auto_symbol`` turns + any unrecognized name into a ``Function``, so the callee cannot be + allowlisted — every function call is a potential carrier. What can be + refused is the payload: without a quote there is no string literal for the + nested parse to read, and the argument comes back as a sympy object + (``eval(chr(112))`` evaluates ``chr`` symbolically and does nothing). + + Nothing legitimate is lost. Not one of the 42,064 gold slots on the pinned + revision contains a quote — the dialect is sympy source, where quotes have + no meaning — and a refused prediction only loses this one reading, with the + LaTeX and literal-equality paths still offered to the comparison. + """ + return "'" not in text and '"' not in text + + +def _evaluable(cleaned: str, local: dict, transformations) -> bool: + """Is *cleaned* free of the two exponent shapes that never finish? + + Deliberately narrower than "would this terminate". ``parse_expr`` evaluates + as it parses, so the check cannot run afterwards — by then the process is + already computing. Parsing with ``evaluate=False`` first builds the tree + without doing the arithmetic (microseconds even for the pathological cases), + which is cheap enough to screen on. + + Rejected: a power whose exponent is itself a power (``9**9**9``, the tower + shape), and an integer exponent above :data:`_MAX_EXPONENT`. A left-nested + ``(x**2)**3`` is fine and stays — only the right-nested tower explodes. + + What it does **not** screen: an eagerly-evaluating sympy callable that needs + no exponent to be expensive. ``from sympy import *`` puts ``factorial``, + ``prime`` and ``primepi`` in the parse namespace, and the early return below + lets anything without ``**`` straight through — measured, ``primepi(10**12)`` + takes 48 s and ``factorial(1000000)`` 3.8 s. Enumerating those callees is + the same losing game as allowlisting them in :func:`_quotes_free`, so the + bound that actually holds is the caller's: grading runs in a worker process + under :data:`~sieval.core.utils.offload.GRADE_TIMEOUT`, which scores the slot + wrong and moves on. This screen only buys back the two shapes common enough + to be worth not spending a worker on. + + A rejected answer grades wrong rather than hanging the run. That is the + correct trade for a grader: this pass only ever *upgrades* a verdict + (:func:`math_equal` reaches it after every other strategy said "not equal"), + so refusing to run it can lose a point but can never invent one. + """ + import sympy + from sympy.parsing.sympy_parser import parse_expr + + if "**" not in cleaned: + return True + try: + tree = parse_expr( + cleaned, + local_dict=local, + global_dict=_sympy_globals(), + transformations=transformations, + evaluate=False, + ) + except Exception: + # Unparseable under this reading; the real parse will fail the same way + # and is harmless. Screening is not the place to decide that. + return True + for node in sympy.preorder_traversal(tree): + if not isinstance(node, sympy.Pow): + continue + exponent = node.exp + if exponent.has(sympy.Pow): + return False + if exponent.is_Integer and abs(int(exponent)) > _MAX_EXPONENT: + return False + return True + + +def _parse_sympy_source(text: str) -> list: + """Parse UGMathBench's plain-sympy answer syntax as sympy source. + + The dataset stores gold answers as sympy-ish source rather than LaTeX + (``x^3+2*x^2+6``, ``pi/6*(4^3-2^3)``, ``9*[sin(x)]^8*cos(x)``), while a + model answers in LaTeX. Reading the gold with the LaTeX parser mangles it + (see :func:`_parse_math`), so it is read here with sympy's own parser too + and both readings are offered to the comparison. + + Square brackets are grouping in this dialect, ``^`` is exponentiation, and + ``e`` / ``pi`` / ``ln`` / ``infinity`` are the constants and functions they + look like — none of which sympy assumes by default. + + The text reaching this function is model output, and ``parse_expr`` + evaluates what it parses, so all three halves of that are guarded: + :func:`_sympy_globals` takes the interpreter out of the parse namespace, + :func:`_quotes_free` keeps a nested parse from handing it back, and + :func:`_evaluable` refuses arithmetic that would not finish. + """ + import sympy + from sympy.parsing.sympy_parser import parse_expr + + cleaned = ( + text.replace("[", "(") + .replace("]", ")") + .replace("{", "(") + .replace("}", ")") + .replace("^", "**") + .replace("infinity", "oo") + .replace("$", "") + .strip() + ) + if not cleaned or not _quotes_free(cleaned): + return [] + local = { + "e": sympy.E, + "E": sympy.E, + "pi": sympy.pi, + "ln": sympy.log, + "log": sympy.log, + "oo": sympy.oo, + "I": sympy.I, + # The dataset spells inverse trig `arcsin`; sympy calls it `asin`. + # Without the alias `arcsin(3/10)` parses as a *symbol* times a number, + # which silently changes the free-symbol set and loses the comparison. + "arcsin": sympy.asin, + "arccos": sympy.acos, + "arctan": sympy.atan, + "arcsec": sympy.asec, + "arccsc": sympy.acsc, + "arccot": sympy.acot, + "arcsinh": sympy.asinh, + "arccosh": sympy.acosh, + "arctanh": sympy.atanh, + } + out: list = [] + globals_ = _sympy_globals() + for transformations in _source_transformations(): + if not _evaluable(cleaned, local, transformations): + continue + try: + out.append( + parse_expr( + cleaned, + local_dict=local, + global_dict=globals_, + transformations=transformations, + ) + ) + except Exception: + # Not sympy source under this reading (LaTeX, prose, an unbalanced + # bracket) — the other readings are expected to handle those. + continue + return out + + +def _source_transformations(): + """Strict sympy source first, then the same with implicit multiplication. + + Both readings are kept rather than just the permissive one. Implicit + multiplication is what a *prediction* sometimes needs (``5 - 5c`` is + ``5 - 5*c``, and neither the LaTeX parser nor strict sympy reads it that + way), but it also happily reinterprets a single symbol as a product, so it + is offered as an extra candidate and never as a replacement. + """ + from sympy.parsing.sympy_parser import ( + implicit_multiplication_application, + standard_transformations, + ) + + return ( + standard_transformations, + standard_transformations + (implicit_multiplication_application,), + ) + + +def _sympy_candidates(text: str) -> list: + """Every plausible sympy reading of *text*, from both parsers. + + Both are tried on both sides on purpose: the gold is usually sympy source + and the prediction usually LaTeX, but neither is guaranteed, and committing + a parser to a side would manufacture disagreements of its own. + """ + import sympy + + out: list = [] + try: + candidates = _parse_math(text) + _parse_sympy_source(text) + except Exception: + return out + for item in candidates: + if isinstance(item, sympy.Basic) and not any(item == seen for seen in out): + out.append(item) + return out + + +def _same_function(pred_expr, gold_expr, precision: float) -> bool: + """Do two expressions denote the same function, by numeric substitution? + + Feeds identical values to identically *named* free symbols and compares the + results. This is what catches equivalence that survives no string + normalization — ``3\\cos(2\\sqrt{35}t)`` against ``3*cos(sqrt(980/7)*t)``, + or ``x^0*e^(-8*x)`` against ``e^{-8x}``. + + Substitution is keyed on symbol *name*, never on the symbol object: the two + parsers build ``Symbol('x')`` with different assumptions, so the objects + compare unequal and a set union would yield two distinct ``x`` that then + receive two different values — making every equivalent pair look unequal. + + Conservative by construction, because the only judgement this can change is + wrong-to-right: + + * the two sides must involve the *same* set of symbol names, so an answer + in ``x`` never matches one in ``t``; + * every probe that evaluates cleanly must agree, and at least + ``_MIN_CLEAN_PROBES`` of them must evaluate cleanly, so a pair that only + survives at a single lucky point is rejected; + * non-finite and complex results are discarded rather than compared. + """ + import sympy + + pred_names = {symbol.name for symbol in pred_expr.free_symbols} + gold_names = {symbol.name for symbol in gold_expr.free_symbols} + if pred_names != gold_names or len(gold_names) > _MAX_FREE_SYMBOLS: + return False + + if not gold_names: + try: + pred_value, gold_value = ( + complex(pred_expr.evalf()), + complex(gold_expr.evalf()), + ) + except Exception: + return False + if abs(pred_value.imag) > 1e-9 or abs(gold_value.imag) > 1e-9: + return False + return _close(pred_value.real, gold_value.real, precision) + + ordered = sorted(gold_names) + clean = 0 + for probe in _PROBES: + values = { + name: sympy.Float(probe + 0.17 * index) + for index, name in enumerate(ordered) + } + try: + pred_value = complex( + pred_expr.subs( + {s: values[s.name] for s in pred_expr.free_symbols} + ).evalf() + ) + gold_value = complex( + gold_expr.subs( + {s: values[s.name] for s in gold_expr.free_symbols} + ).evalf() + ) + except Exception: + continue + parts = (pred_value.real, pred_value.imag, gold_value.real, gold_value.imag) + # Reject only NaN and infinity. A magnitude ceiling would be wrong here: + # the comparison below is *relative*, and an exponential answer is + # legitimately enormous at these probes (e^cosh(4x) is ~1e40 at the + # first one), so capping magnitude discards every probe and silently + # marks a correct answer wrong. + if any(not math.isfinite(part) for part in parts): + continue # undefined at this probe — it says nothing either way + clean += 1 + if abs(pred_value.imag) > 1e-9 or abs(gold_value.imag) > 1e-9: + if abs(pred_value - gold_value) > abs(gold_value) * precision * 1.01: + return False + continue + if not _close(pred_value.real, gold_value.real, precision): + return False + return clean >= _MIN_CLEAN_PROBES + + +def _equivalent_by_substitution(pred: str, gold: str, precision: float) -> bool: + """Last-chance equivalence check, over every parse of both sides.""" + try: + pred_exprs = _sympy_candidates(pred) + gold_exprs = _sympy_candidates(gold) + for pred_expr in pred_exprs: + for gold_expr in gold_exprs: + if _same_function(pred_expr, gold_expr, precision): + return True + except Exception: + # Same contract as the rest of the module: an ungradeable answer is a + # wrong answer, not a crashed run. + return False + return False + + +def _numeric_value(parsed: list) -> float | None: + """Best-effort real value of a parsed expression, for relative tolerance.""" + for item in parsed: + is_number = getattr(item, "is_number", False) + if not is_number: + continue + try: + value = complex(item.evalf()) + except (TypeError, ValueError, AttributeError): + continue + if abs(value.imag) < 1e-12: + return value.real + return None + + +def math_equal(pred: str, gold: str, precision: float = 1e-3) -> bool: + """Compare two mathematical answers. + + Four chances, cheapest first: squashed string equality, plain-float + comparison at *precision* (relative, as upstream), ``math-verify`` symbolic + equivalence with a relative-tolerance retry on the parsed numeric values + (``math-verify`` compares floats at fixed decimal rounding, which rejects + pairs upstream's relative tolerance accepts), and finally equivalence by + numeric substitution. + + The substitution pass exists because the first three all compare a LaTeX + prediction against a gold that ``math-verify`` has read as LaTeX too — and + the dataset does not store LaTeX. A gold naming any function comes back + mangled (``sin`` as s*i*n, ``pi`` as p*i), so ``7\\sin(\\frac{\\pi}{5}x)+1`` + could not match the stored ``7*sin(pi*x/5)+1`` by any route but exact string + equality. The first live run of this task measured the cost: of 1,634 wrong + slots where extraction and reference agreed on slot count, 570 (34.9%) were + this function's error rather than the model's, all of them in the free-form + types and none in the structured ones. + + Because the new pass runs only after the others have said "not equal", the + only verdict it can change is wrong-to-right; it cannot break a comparison + that already succeeded. + """ + if _squash(pred) == _squash(gold): + return True + + pred_float, gold_float = _to_float(pred), _to_float(gold) + if pred_float is not None and gold_float is not None: + return _close(pred_float, gold_float, precision) + + try: + from math_verify import verify + + parsed_gold, parsed_pred = _parse_math(gold), _parse_math(pred) + if parsed_gold and parsed_pred and verify(parsed_gold, parsed_pred): + return True + gold_value, pred_value = ( + _numeric_value(parsed_gold), + _numeric_value(parsed_pred), + ) + # Accept on success, but do NOT reject on failure: both values come from + # the LaTeX reading, which is the one known to mangle this dataset's + # gold. Returning its verdict here made the substitution pass below + # unreachable for every pair the mangling happens to turn into a + # *number* — `2**100` reads as `2`, so a correct `2^100` was compared + # against 2 and graded wrong without the pass ever running. + if ( + gold_value is not None + and pred_value is not None + and _close(pred_value, gold_value, precision) + ): + return True + except Exception: + # math-verify raises on pathological input (unbalanced LaTeX, runaway + # simplify). Fall through rather than return: a crash in one comparison + # strategy is not evidence about the answer, and the substitution pass + # below carries its own exception contract. + pass + return _equivalent_by_substitution(pred, gold, precision) + + +def _close(pred: float, gold: float, precision: float) -> bool: + tolerance = precision * 1.01 + if gold == 0.0: + return abs(pred) <= tolerance + return abs((pred - gold) / gold) <= tolerance + + +def _option_letters(text: str, options: list[str]) -> list[str]: + allowed = {option.lower() for option in options} or { + chr(code) for code in range(ord("a"), ord("z") + 1) + } + return [char for char in text.lower() if char in allowed] + + +def _judge_multiple_choice_single(pred: str, gold: str, options: list[str]) -> bool: + # Options are usually bare letters, but some problems label choices with + # expressions ("Q(X)"), so compare the whole answer before touching brackets. + if _squash(pred) == _squash(gold): + return True + target = gold.strip().lower() + candidate = pred.strip().strip("[]().").strip() + if candidate.lower() == target: + return True + # "D: 1/2" / "D. 1/2" / "D) 1/2" — the letter is the answer, the rest is echo. + match = re.match(r"^([A-Za-z])\s*[:.)]", candidate) + if match: + return match.group(1).lower() == target + letters = _option_letters(candidate, options) + return len(letters) == 1 and letters[0] == target + + +def _judge_multiple_choice_multiple(pred: str, gold: str, options: list[str]) -> bool: + # Same first move as the single-choice rule: options are usually bare + # letters, but a problem may label its choices with words, and + # `_option_letters` only ever matches single characters. Without this the + # slot would be unwinnable whenever the options are not letters. + if _squash(pred) == _squash(gold): + return True + gold_letters = sorted(_option_letters(gold, options)) + pred_letters = sorted(_option_letters(pred, options)) + return bool(gold_letters) and gold_letters == pred_letters + + +def _list_elements(text: str) -> list[str]: + stripped = text.strip() + while len(stripped) >= 2 and stripped[0] in "([<" and stripped[-1] in ")]>": + stripped = stripped[1:-1].strip() + return [element for element in split_answers(stripped) if element] + + +def _element_equal(pred: str, gold: str, precision: float) -> bool: + """Compare one OL/UOL element, whose own answer type the dataset omits. + + Deliberately *not* boolean-aware. Upstream converts booleans only for a slot + the dataset typed ``TF`` and never for the elements inside a list, and the + elements here are overwhelmingly parameter names: reading ``t``, ``y``, + ``f`` and ``n`` as truth values makes ``(x, t)`` match a gold ``(x, y)``. + Thirteen OL/UOL references on the pinned data carry a bare ``t`` or ``y``. + """ + return math_equal(pred, gold, precision) + + +def _judge_ordered_list(pred: str, gold: str, precision: float) -> bool: + pred_items, gold_items = _list_elements(pred), _list_elements(gold) + if len(pred_items) != len(gold_items): + return False + return all( + _element_equal(p, g, precision) + for p, g in zip(pred_items, gold_items, strict=True) + ) + + +def _judge_unordered_list(pred: str, gold: str, precision: float) -> bool: + pred_items, gold_items = _list_elements(pred), _list_elements(gold) + if len(pred_items) != len(gold_items): + return False + remaining = list(pred_items) + for gold_item in gold_items: + for index, pred_item in enumerate(remaining): + if _element_equal(pred_item, gold_item, precision): + remaining.pop(index) + break + else: + return False + return True + + +def judge_answer( + pred: str, + gold: str, + answer_type: str, + options: list[str] | None = None, + precision: float = 1e-3, +) -> bool: + """Grade one ``[ANS]`` slot under the rule its declared *answer_type* implies. + + Both sides are normalized first. The reference needs it as much as the + prediction does: some stored answers carry the very decoration the answer + type says they should not (a percent sign on a "numerical value without + units"), and comparing a normalized prediction against a raw reference marks + a right answer wrong. + """ + pred, gold = normalize_answer(pred), normalize_answer(gold) + slot_options = list(options or []) + match answer_type: + case "TF": + gold_bool = _to_bool(gold) + if gold_bool is None: + # A handful of slots are typed TF but store a number or a word. + # Upstream's TF judge asserts the reference is True/False and + # grades the slot wrong when it is not, which makes it + # unwinnable; compare the stored value instead. + return math_equal(pred, gold, precision) + return _to_bool(pred) is gold_bool + case "MCS": + return _judge_multiple_choice_single(pred, gold, slot_options) + case "MCM": + return _judge_multiple_choice_multiple(pred, gold, slot_options) + case "OE": + # A word or phrase: compare as text, never as mathematics. + return _squash(pred) == _squash(gold) + case "OL": + return _judge_ordered_list(pred, gold, precision) + case "UOL": + return _judge_unordered_list(pred, gold, precision) + case _: + # NV / EX / EQ / INT — all mathematical values. + return math_equal(pred, gold, precision) + + +def extract_predictions(response: str) -> list[str] | None: + """Extract one predicted answer per ``[ANS]`` slot from a full response. + + ``None`` when no answer could be recovered at all — the caller records that + as "not extracted" rather than as an empty answer. + """ + extracted = extract_answer(response) + if extracted is None: + return None + return split_answers(extracted) + + +def judge_answers( + predictions: list[str] | None, + golds: list[str], + answer_types: list[str], + options: list[list[str]] | None = None, + precision: float = 1e-3, +) -> list[bool]: + """Grade extracted answers against a problem's reference sequence. + + Returns one verdict per reference answer. A missing extraction, or a slot + count that disagrees with the reference, grades every slot wrong: upstream + rejects a miscounted answer outright rather than aligning a prefix. + + Every reference answer is graded. Where a row declares fewer types than + answers, the undeclared slots fall through to the mathematical rule; that + diverges from upstream, which silently truncates both sides to the declared + types and so never looks at the trailing answers at all. + """ + if predictions is None or len(predictions) != len(golds): + return [False] * len(golds) + + effective_types = list(answer_types[: len(golds)]) + effective_types.extend("NV" for _ in range(len(golds) - len(effective_types))) + per_slot_options = _pad_options(effective_types, options) + + return [ + judge_answer(pred, gold, answer_type, slot_options, precision) + for pred, gold, answer_type, slot_options in zip( + predictions, golds, effective_types, per_slot_options, strict=True + ) + ] diff --git a/sieval/core/CLAUDE.md b/sieval/core/CLAUDE.md index 77c45633..c7264bd1 100644 --- a/sieval/core/CLAUDE.md +++ b/sieval/core/CLAUDE.md @@ -23,6 +23,14 @@ Hierarchical: global (MultiTaskRunner) → task (TaskRunner) → stage → model ## Test Requirements -* Coverage ≥ 95%: `python -m pytest tests/unit/ tests/integration/ --cov -v` -* Mutation score ≥ 70% for modified modules: `mutmut run --paths-to-mutate=sieval/core/.py` +* **Coverage ≥ 95%** — gated in CI (`fail_under = 95` over `sieval/core`). Locally `pytest --cov` + dies on a pyarrow double-registration in some environments; it reproduces on untouched modules, + so use `python -m coverage run --source=sieval -m pytest ` + `coverage report -m`. +* **Mutation score ≥ 70%** for modified modules — **currently unobtainable, and not in CI.** + `mutmut run` (never `python -m mutmut`, which double-executes its `__main__` and dies on the + first mutant) fails during stats collection: a test that spawns a fresh interpreter re-imports + mutmut's injected trampoline and fails on it. Scope by mutant name + (`mutmut run "sieval.core.utils.offload.*"`), never by narrowing `paths_to_mutate`. The copy + paths are asserted by `check_preflight.py --check check_mutmut_config` — necessary, not + sufficient. **Do not quote a mutation score until this is fixed.** * Disk persistence tests: use fresh `TaskLoader` from disk, not `runner._contexts` diff --git a/sieval/core/utils/offload.py b/sieval/core/utils/offload.py new file mode 100644 index 00000000..14e8443e --- /dev/null +++ b/sieval/core/utils/offload.py @@ -0,0 +1,224 @@ +"""Run CPU-bound stage work off the event loop, in a worker process. + +Every runner in a session shares one event loop — :meth:`MultiTaskRunner.arun` +starts each :class:`TaskRunner` with ``tg.start_soon`` inside a single +``anyio.run`` — so a stage that computes synchronously stalls *every* other +task. Measured, a co-running benchmark dropped to 0.4% of its solo throughput. + +``anyio.to_thread.run_sync`` is the house pattern for that, called directly at +the site (``core/tasks/loader.py``, ``infer/deployer.py``, +``cli/leaderboard/session.py``, scicode's target reads). Reach for a *process* +only when one of these holds: + +1. **A thread changes the answer.** ``math-verify`` bounds ``parse``/``verify`` + with ``signal.SIGALRM``, which only arms on the main thread; off it the call + raises, the callers' broad ``except`` swallows it, and verdicts flip + (``\\frac{1}{2}`` against ``0.5`` goes True -> False). Disabling its timeout + makes it thread-safe but hands the caller a bound it cannot enforce. +2. **The work has no bound of its own.** A thread cannot be cancelled, so an + input that never finishes holds its anyio token for the rest of the session + until enough accumulate to wedge every other offload — surfacing as a session + that stops progressing, never as a wrong answer in testing. Only a process + can be given up on, which is what :data:`GRADE_TIMEOUT` does. The two + DeepSeek-Math graders are here on this criterion alone: thread-safe, but + reached with ``math_equal(..., timeout=False)``, so nothing else bounds them. + +Not ``anyio.to_process.run_sync``, the obvious way to avoid hand-rolling a pool: +its worker runs ``del sys.modules["__main__"]`` before re-importing the parent's +main module, and ``dill`` (pulled in by HuggingFace ``datasets``) does +``import __main__`` at import time, so a bare ``import sieval`` fails every +worker's init. ``spawn`` *replaces* ``sys.modules["__main__"]``, never deletes it. + +Degrades rather than fails: with no pool, work runs inline — slow but correct. +``SIEVAL_OFFLOAD_WORKERS=0`` forces that path. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import atexit +import os +import threading +from collections.abc import Callable +from concurrent.futures import BrokenExecutor, ProcessPoolExecutor +from functools import partial + +import anyio +import anyio.to_thread +from loguru import logger + +#: Worker count. Grading is CPU-bound, so this tracks cores rather than the +#: sample concurrency — queueing beyond the cores available buys nothing. +#: ``SIEVAL_OFFLOAD_WORKERS=0`` disables offloading entirely, which is the +#: escape hatch for an environment where spawning is not allowed. +_ENV_WORKERS = "SIEVAL_OFFLOAD_WORKERS" + +#: Default ceiling for grading one rollout. Generous against the tens of +#: milliseconds a symbolic comparison normally costs and the 5 s ``math-verify`` +#: allows itself per parse/verify, so reaching it means an input that got past +#: the caller's own guards — worth surfacing rather than a silent slow sample. +GRADE_TIMEOUT = 30.0 + +#: Extra admissions beyond the worker count (see :data:`_limiter`). Enough that +#: a worker never idles waiting for the next caller to be let in, small enough +#: that the queue a caller can sit behind stays a fixed multiple of the pool. +_QUEUE_SLACK = 2 + +_pool: ProcessPoolExecutor | None = None +_pool_failed = False +_lock = threading.Lock() + +#: Admission control, sized to the pool rather than to the sample concurrency — +#: the stage limiters upstream bound *samples in flight*, a different quantity. +#: +#: 1. ``timeout`` measures the grade, not the backlog. ``future.result(timeout)`` +#: starts counting when the caller begins waiting, so it waits out the queue +#: ahead of it too. Capping the waiters caps that queue at ``_QUEUE_SLACK``, +#: keeping the worst case a small multiple of one grade instead of a function +#: of how many samples the session happens to run. Unbounded, an ordinary 1 s +#: grade "times out" purely from queueing. +#: 2. It keeps grading off anyio's shared thread tokens: passing ``limiter=`` to +#: ``run_sync`` substitutes this one for the default 40, which the loader, the +#: deployer and scicode's reads are also drawing on. +_limiter: anyio.CapacityLimiter | None = None + + +def _worker_count() -> int: + configured = os.environ.get(_ENV_WORKERS) + if configured is not None: + try: + return max(0, int(configured)) + except ValueError: + logger.warning( + "{} is not an integer ({!r}); using the default worker count.", + _ENV_WORKERS, + configured, + ) + return max(1, min(8, (os.cpu_count() or 2) - 1)) + + +def _get_pool() -> ProcessPoolExecutor | None: + """The shared pool, created on first use. ``None`` means "run inline". + + Call from the event loop: the admission limiter is an anyio primitive and + needs a running async context to be constructed. + """ + global _pool, _pool_failed, _limiter + if _pool is not None or _pool_failed: + return _pool + with _lock: + if _pool is not None or _pool_failed: + return _pool + workers = _worker_count() + if workers == 0: + _pool_failed = True + return None + try: + import multiprocessing + + # `spawn`, not `fork`: the parent is an async process with live + # worker threads, and forking one of those risks inheriting a held + # lock and deadlocking the child. + _pool = ProcessPoolExecutor( + max_workers=workers, mp_context=multiprocessing.get_context("spawn") + ) + _limiter = anyio.CapacityLimiter(workers + _QUEUE_SLACK) + except Exception as exc: + # Both objects or neither. A pool that outlived a failed limiter + # would still be handed out below (the guard above returns `_pool` + # whenever it is set), and would then run against anyio's shared + # 40-token default — silently undoing the admission control that + # makes `timeout` mean "one grade" rather than "grade plus queue". + if _pool is not None: + _pool.shutdown(wait=False, cancel_futures=True) + _pool = None + _limiter = None + _pool_failed = True + logger.warning( + "Could not start the offload pool ({}); CPU-bound stage work " + "will run on the event loop, which slows every task sharing it.", + exc, + ) + return _pool + + +def shutdown() -> None: + """Tear the pool down. Registered with :mod:`atexit`; safe to call twice.""" + global _pool, _limiter + with _lock: + pool, _pool = _pool, None + _limiter = None + if pool is not None: + pool.shutdown(wait=False, cancel_futures=True) + + +# Registered once, at import. Registering alongside each pool would add another +# handler every time a pool is rebuilt after `shutdown()`. +atexit.register(shutdown) + + +async def run_cpu_bound[T]( + func: Callable[..., T], *args, timeout: float | None = None +) -> T: + """Run *func(\\*args)* in a worker process, leaving the event loop free. + + *func* must be picklable by name (module-level, not a lambda or a closure), + as must its arguments and return value — the grading entry points take + strings and return bools, which is the shape this is for. + + Raises :exc:`TimeoutError` when *timeout* elapses. The worker is left to + finish on its own: a pool cannot interrupt a running call, and tearing the + pool down would punish every other in-flight sample for one bad input. + *timeout* bounds the call, not the backlog in front of it — see + :data:`_limiter`. + + Falls back to running inline when no pool is available: degraded (slow) + rather than broken, and unbounded, since on the event loop there is nothing + left to interrupt it with. + """ + pool = _get_pool() + if pool is None: + return func(*args) + try: + future = pool.submit(func, *args) + except Exception as exc: + # Pool died (a worker segfaulted), is shutting down, or could not start + # a worker at all — ENOMEM and a blocked `clone` surface here as + # OSError and PermissionError, not as BrokenExecutor. Every failure to + # submit means the same thing: do not retry into it, take the slow path + # for the rest of the run. + _mark_unusable(exc) + return func(*args) + try: + # `_limiter` gates how many callers may be *waiting*, which is what puts + # a ceiling on `timeout`. Submissions and token handoffs are both FIFO, + # so a caller holding a token is within `_QUEUE_SLACK` of the front of + # the pool queue however long the backlog behind it grows. + return await anyio.to_thread.run_sync( + partial(future.result, timeout), limiter=_limiter + ) + except TimeoutError: + future.cancel() + raise + except BrokenExecutor as exc: + _mark_unusable(exc) + return func(*args) + + +def _mark_unusable(exc: Exception) -> None: + global _pool, _pool_failed + with _lock: + # Drop the handle as well as setting the flag. `_get_pool` returns + # `_pool` whenever it is set, so the flag alone only stops the pool + # being *rebuilt* — it never stops the dead one being handed out, and + # every later sample would pay another failed `submit` before falling + # back. Not shut down here: callers already awaiting a future from it + # still need it alive, and its workers are gone in the case that + # brought us here anyway. + _pool = None + if not _pool_failed: + _pool_failed = True + logger.warning( + "Offload pool became unusable ({}); CPU-bound stage work falls " + "back to the event loop for the rest of this run.", + exc, + ) diff --git a/sieval/datasets/CLAUDE.md b/sieval/datasets/CLAUDE.md index 02ae8c97..a5cc8335 100644 --- a/sieval/datasets/CLAUDE.md +++ b/sieval/datasets/CLAUDE.md @@ -14,6 +14,19 @@ - `hf:` sources are revision-pinned; `url:` sources carry per-file `checksums` (sha256). Regenerate the meta index (`scripts/sync_meta_index.py`) after editing either. +## Corrected variants + +A dataset ships upstream's rows as they are. Repairing a genuinely broken row is +a `datasets/` concern rather than a task one: a separate registered +`_fixed` dataset over the **same pinned revision**, applying a patch table +rather than forking the data — a patch table shrinks to empty when upstream +fixes the row, which is the only exit condition a local fix can have. The +unqualified name keeps tracking upstream, as it does for tasks +(`sieval/tasks/CLAUDE.md`). + +No such dataset exists yet. The first one settles the details — do not design +the patch-table format in advance. + ## Subpackages A multi-module benchmark gets a subdirectory; `datasets/__init__.py` lazy-loads it. diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index e5cd3cbe..0e98a428 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -161,6 +161,10 @@ from .theoremqa import ( TheoremQADataset, TheoremQADatasetSample, ) +from .ugmathbench import ( + UGMathBenchDataset, + UGMathBenchDatasetSample, +) __all__ = [ "AALCRDataset", @@ -243,4 +247,6 @@ __all__ = [ "TEvalBeforeCallingDatasetSample", "TheoremQADataset", "TheoremQADatasetSample", + "UGMathBenchDataset", + "UGMathBenchDatasetSample", ] diff --git a/sieval/datasets/ugmathbench.py b/sieval/datasets/ugmathbench.py new file mode 100644 index 00000000..604ff8e4 --- /dev/null +++ b/sieval/datasets/ugmathbench.py @@ -0,0 +1,119 @@ +""" +UGMathBench dataset loader — 5,061 undergraduate problems x 3 randomized versions. + +The HF repo ships one config per subject, and each row packs all three +randomized versions of a problem side by side (``problem_v1`` / ``answer_v1`` +/ ... / ``options_v3``). This loader concatenates the 16 subjects and unpacks +each row into three samples carrying a ``version`` field, the same shape the +upstream generation script materializes before inference. + +Ordering is **problem-major**: a problem's three versions are adjacent, so +``slice(n)`` keeps whole problems and the effective-accuracy metric stays +defined on a truncated run. (Upstream emits version-major, which only matters +for file layout — the score is order-independent.) + +Rows are mirrored as-is, including two upstream-corrupt versions whose problem +text is an error message and whose answer sequence is empty +(``Financial_mathematics_0132`` v2, ``Linear_algebra_0306`` v3). Dropping them +would quietly inflate effective accuracy for those two problems, which can +never satisfy "correct in all three versions" upstream either. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.community.ugmathbench import SUBJECTS, VERSIONS +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset, ensure_dataset_dict + +UGMATHBENCH_REVISION = "8ab16f0c131a9b3b52195e64175cb5a8f3881bbf" + + +class UGMathBenchDatasetSample(TypedDict): + id: str + subject: str + topic: str + subtopic: str + level: str + keywords: list[str] + version: int + problem: str + answer: list[str] + answer_type: list[str] + options: list[list[str]] + + +@sieval_dataset( + name="ugmathbench", + display_name="UGMathBench", + description="Undergraduate math, 16 subjects, 3 randomized versions per problem.", + source=f"hf:UGMathBench/ugmathbench@{UGMATHBENCH_REVISION}", + categories=(Category(Level1Category.MATHEMATICS, "AdvancedMath"),), + tags=("english", "open-ended"), + license="GPL-3.0", +) +class UGMathBenchDataset(Dataset[UGMathBenchDatasetSample]): + SUBJECTS = SUBJECTS + + @override + def load( + self, + name_or_path: str, + subjects: list[str] | None = None, + **kwargs, + ) -> HFDatasetDict: + # `None` means "unspecified, load everything"; `[]` is a caller asking + # for nothing, which silently loading all 16 subjects would misread. + selected = self.SUBJECTS if subjects is None else tuple(subjects) + if not selected: + raise ValueError( + "UGMathBench `subjects` is empty; omit it to load all " + f"{len(self.SUBJECTS)} subjects." + ) + unknown = [subject for subject in selected if subject not in self.SUBJECTS] + if unknown: + raise ValueError( + f"Unknown UGMathBench subject(s) {unknown}; " + f"expected a subset of {list(self.SUBJECTS)}." + ) + + rows: list[UGMathBenchDatasetSample] = [] + for subject in selected: + split = ensure_dataset( + load_dataset(name_or_path, subject, split="test", **kwargs) + ) + for row in split: + rows.extend(_unpack_versions(row)) + return ensure_dataset_dict( + HFDatasetDict({"test": HFDataset.from_list([dict(row) for row in rows])}) + ) + + +def _unpack_versions(row: dict) -> list[UGMathBenchDatasetSample]: + """Split one packed row into one sample per randomized version.""" + return [ + { + "id": row["id"], + "subject": row["subject"], + "topic": row["topic"], + "subtopic": row["subtopic"], + "level": row["level"], + "keywords": list(row["keywords"]), + "version": version, + "problem": row[f"problem_v{version}"], + "answer": list(row[f"answer_v{version}"]), + "answer_type": list(row[f"answer_type_v{version}"]), + "options": [list(entry) for entry in row[f"options_v{version}"]], + } + for version in range(1, VERSIONS + 1) + ] diff --git a/sieval/meta/index.json b/sieval/meta/index.json index d77ab81d..d9fd998d 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -892,6 +892,27 @@ "deps_group": null, "license": "MIT", "checksums": {} + }, + { + "name": "ugmathbench", + "display_name": "UGMathBench", + "description": "Undergraduate math, 16 subjects, 3 randomized versions per problem.", + "source": [ + "hf:UGMathBench/ugmathbench@8ab16f0c131a9b3b52195e64175cb5a8f3881bbf" + ], + "categories": [ + { + "level1": "Mathematics", + "level2": "AdvancedMath" + } + ], + "tags": [ + "english", + "open-ended" + ], + "deps_group": null, + "license": "GPL-3.0", + "checksums": {} } ], "tasks": [ @@ -1934,6 +1955,26 @@ "notes": "Prompt follows official short-form examples by default; n_shot can select any prefix of the built-in examples. answer_clean and numeric matching mirror official utils.py/number_utils.py." }, "status": "stable" + }, + { + "name": "ugmathbench_0shot_gen_fixed", + "display_name": "UGMathBench (0-shot, generative, corrected)", + "description": "Undergraduate math, 3 randomized versions per problem; EAcc + gap.", + "dataset": "ugmathbench", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "open-ended" + ], + "deps_group": "math", + "model_type": "chat", + "reference_impl": { + "source": "UGMathBench", + "url": "https://github.com/YangLabHKUST/UGMathBench/blob/df47bfa639bfb89bdb0220036a7b2f216e72b0b3/eval_rule.py", + "notes": "Prompt and metric definitions mirror upstream's `raw` template and `eval_file` (aacc / eacc / cacc / Delta / RE). The grader does NOT, and cannot: upstream's `judge_rule.py` is GPL-3.0 and would not ship in an Apache-2.0 distribution, so answer comparison is an independent math-verify-based implementation of the same 10 answer types. The `_fixed` variant therefore does not reproduce the paper's numbers, and the unqualified name stays vacant rather than reserved. MEASURED DIVERGENCE: upstream's judge was run as a local instrument (GPL-3.0 restricts distribution, not use; nothing is vendored) over all 15,183 pinned rows, each row's own reference replayed back as a boxed answer. Upstream accepts its own reference on 14,616 rows (96.27%), this task on 15,160 (99.85%); the 552 rows that disagree span 192 of 5,061 problems — an EAcc ceiling difference of 3.79 pp, about 5x the 0.70 pp binomial standard error. 548 of the 552 are rows upstream rejects its OWN reference on, so the gap is repair, not drift; only 4 go the other way (a UOL reference whose top-level commas sit outside any bracket). Dominant cause: upstream normalizes the prediction and the reference by different passes — its extraction-time normalize_answer rewrites sqrt(x) into sqrt{(}x) on the prediction only, which a plain-sympy reference never survives. Contributing: a TF slot whose reference is not a boolean (9 of 1665; upstream asserts it is and swallows the AssertionError into a False, so no answer wins), a zero-valued numeric reference, the declared answer type deciding the rule where upstream's `is_equal` retries every method until one accepts, and three deliberate differences in where commas are split (9 rows): `<` and `>` are the relational operators here, not brackets, so counting them as brackets — which upstream does — swallows the comma after them and the row comes out a slot short (6 rows); the bracket depth clamps at zero instead of going negative, where upstream loses every remaining slot after one unmatched closer (3 rows); and `{}` counts as a bracket, which costs 0 rows on the references but keeps a comma inside a LaTeX group in the model's answer from splitting a slot in two. All enumerated in sieval/community/ugmathbench.py. The figure bounds the GRADER, not a model's score: it is a replay of stored references, so it says nothing about extraction on real model prose. LIVE HEAD-TO-HEAD (the stronger measurement, and the one that promoted this task): over a full 15,183-version run, upstream's judge re-graded the same stored responses and agreed on 95.51% of samples, disagreeing 591 to 91 in this task's favour; upstream scores EAcc 35.55 where this task scores 38.49, and the residual misses are 0.60% of samples (LaTeX interval notation, absolute-value bars, a `y = ` prefix on an EX answer). Note that the replay figure could NOT see the largest defect the live run found — replaying a gold as its own answer short-circuits on string equality and never reaches the symbolic path. GUARDS: since the parsed text is model output, three shapes are refused rather than evaluated — the parse namespace has its builtins removed; an answer containing a quote is refused outright, because a quoted string handed to any callable (eval, sympify, S, N, or any name at all, since auto_symbol makes unknown names callable) is re-sympified with sympy's own default namespace and gets the builtins back; and an answer requiring unbounded arithmetic (a power tower) is screened out by an unevaluated pre-parse. All three grade the answer wrong, and none is reachable by any pinned reference — the largest exponent is three digits and not one of the 42,064 gold slots contains a quote. SAMPLING: upstream generates greedily, one sample per version (temperature 0, max_tokens 2048), and this task issues one rollout per version to match; set temperature via the model config. Upstream also offers a model-as-judge variant (eval_marj.py) which it now recommends over the rule-based path — not implemented here. METRICS: `relative_delta` is upstream's RE scaled by 100 — upstream prints the bare ratio (0.1667) where this reports 16.67; do not compare the two directly." + }, + "status": "stable" } ] } diff --git a/sieval/tasks/CLAUDE.md b/sieval/tasks/CLAUDE.md index 3bcb3dc2..d0e7d924 100644 --- a/sieval/tasks/CLAUDE.md +++ b/sieval/tasks/CLAUDE.md @@ -2,16 +2,43 @@ ## Naming Conventions -File: `_shot_.py` — suffix determines `model_type`: +File: `_shot_[_].py` — the mode determines `model_type`: -| Suffix | `model_type` | +| Mode | `model_type` | | --- | --- | | `_gen.py` | `"chat"` | | `_base_gen.py` | `"gen"` | | `_ppl.py` | `"gen"` | | `_clp.py` | `"gen"` | -Class: `Task` — words for shot count (`ZeroShot`, `FewShot`). +Class: `[]Task` — words for shot count +(`ZeroShot`, `FewShot`). + +### Variants + +An optional trailing segment lets two readings of one benchmark coexist as +separate registered tasks. The name is the registry key *and* the run-directory +name, so it is the only place the distinction can live. + +| Variant | Means | +| --- | --- | +| *(none)* | Tracks upstream — its protocol, its grader, its defects | +| `_fixed` | Ours, diverging to repair a defect in upstream's grader or data | + +- **The unqualified name always tracks upstream, bugs included**, and is never + repurposed by a local change. It stays free even if nothing will occupy it — + `ugmathbench`'s faithful grader cannot ship at all (upstream is GPL-3.0). +- **`_fixed` is licensed by a defect, not a preference**, and owes two things: + every divergence enumerated in `reference_impl.notes`, and its score impact + **quantified**. An unmeasured fork is not a fix. +- The mode is read positionally, so a variant may not spell one: + `foo_0shot_clp_gen.py` has two readings and is rejected. +- The table is the current vocabulary, not the limit — a new variant earns a row + when a second real case arrives. Do not coin one speculatively. + +Not variants: a different **measurement regime** (that is a mode — +`arc_challenge_kshot_clp` vs `_ppl`), and a fix to **problem text or reference +answers** (a `datasets/` concern — see `sieval/datasets/CLAUDE.md`). ### Constructor knobs: `n_shot` vs `k` diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 2136c931..3a7f027f 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -151,6 +151,9 @@ from .t_eval_before_calling_0shot_gen import ( from .theoremqa_kshot_base_gen import ( TheoremQAKShotBaseGenTask, ) +from .ugmathbench_0shot_gen_fixed import ( + UGMathBenchZeroShotGenFixedTask, +) __all__ = [ "AALCRZeroShotGenTask", @@ -203,4 +206,5 @@ __all__ = [ "SimpleQAVerifiedZeroShotGenTask", "TEvalBeforeCallingZeroShotGenTask", "TheoremQAKShotBaseGenTask", + "UGMathBenchZeroShotGenFixedTask", ] diff --git a/sieval/tasks/_math_verify.py b/sieval/tasks/_math_verify.py new file mode 100644 index 00000000..9de4a6c5 --- /dev/null +++ b/sieval/tasks/_math_verify.py @@ -0,0 +1,24 @@ +"""The math-verify comparison shared by the math-competition tasks. + +Extracted rather than copied twelve times because the twelve call sites were +already byte-identical and share a contract that has to change together — gold +first, both sides ``$``-wrapped. The cost of that is worth stating plainly: +**editing this function rotates the verdicts of all twelve benchmarks at once** +(AIME x3, HMMT x3, Apex x2, BRUMO, CMIMC, SMT, MATH-500), where the duplication +it replaced let them drift apart deliberately. A change here needs the same +before/after count a scorer change in any one of them would need. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + + +def verify_answer(gold: str, pred: str) -> bool: + """Compare one answer pair. Module-level so a worker process can pickle it. + + Must run in a process: math-verify's ``signal.SIGALRM`` bound only arms on + the main thread — criterion 1 in :mod:`sieval.core.utils.offload`. + """ + from math_verify import parse, verify + + # math_verify.verify takes the gold answer first. + return bool(verify(parse(gold), parse(pred))) diff --git a/sieval/tasks/aime_2024_0shot_gen.py b/sieval/tasks/aime_2024_0shot_gen.py index 591d03e6..fe630d79 100644 --- a/sieval/tasks/aime_2024_0shot_gen.py +++ b/sieval/tasks/aime_2024_0shot_gen.py @@ -19,8 +19,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import AIME2024DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="aime_2024_0shot_gen", @@ -86,8 +89,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -98,10 +99,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/aime_2025_0shot_gen.py b/sieval/tasks/aime_2025_0shot_gen.py index 504131d1..383f076a 100644 --- a/sieval/tasks/aime_2025_0shot_gen.py +++ b/sieval/tasks/aime_2025_0shot_gen.py @@ -19,8 +19,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import AIME2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="aime_2025_0shot_gen", @@ -86,8 +89,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -98,10 +99,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/aime_2026_0shot_gen.py b/sieval/tasks/aime_2026_0shot_gen.py index 68f87b0e..64f66817 100644 --- a/sieval/tasks/aime_2026_0shot_gen.py +++ b/sieval/tasks/aime_2026_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import AIME2026DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="aime_2026_0shot_gen", @@ -104,8 +107,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -116,10 +117,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/apex_2025_0shot_gen.py b/sieval/tasks/apex_2025_0shot_gen.py index cb526892..db9e6ced 100644 --- a/sieval/tasks/apex_2025_0shot_gen.py +++ b/sieval/tasks/apex_2025_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import Apex2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="apex_2025_0shot_gen", @@ -123,8 +126,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -135,10 +136,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/apex_shortlist_2025_0shot_gen.py b/sieval/tasks/apex_shortlist_2025_0shot_gen.py index 6aa64238..05d77e8e 100644 --- a/sieval/tasks/apex_shortlist_2025_0shot_gen.py +++ b/sieval/tasks/apex_shortlist_2025_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import ApexShortlist2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="apex_shortlist_2025_0shot_gen", @@ -130,8 +133,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -142,10 +143,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/brumo_2025_0shot_gen.py b/sieval/tasks/brumo_2025_0shot_gen.py index 565c021b..72cb262b 100644 --- a/sieval/tasks/brumo_2025_0shot_gen.py +++ b/sieval/tasks/brumo_2025_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import BRUMO2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="brumo_2025_0shot_gen", @@ -120,8 +123,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -132,10 +133,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/cmimc_2025_0shot_gen.py b/sieval/tasks/cmimc_2025_0shot_gen.py index 0370bd26..2578c8d7 100644 --- a/sieval/tasks/cmimc_2025_0shot_gen.py +++ b/sieval/tasks/cmimc_2025_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import CMIMC2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="cmimc_2025_0shot_gen", @@ -121,8 +124,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -133,10 +134,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/gsm8k_0shot_gen.py b/sieval/tasks/gsm8k_0shot_gen.py index c02a281b..ba09c2b3 100644 --- a/sieval/tasks/gsm8k_0shot_gen.py +++ b/sieval/tasks/gsm8k_0shot_gen.py @@ -46,6 +46,8 @@ from typing import override +from loguru import logger + from sieval.core.models import ModelOutput from sieval.core.tasks import ( EvalMode, @@ -60,6 +62,7 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import GSM8KDatasetSample # Verbatim from run_subset_parallel.py::markup_question (language="en", @@ -141,7 +144,31 @@ async def feedback(self, post, ctx): gold = _gold_answer(ctx.raw_sample["answer"]) # `or ""` restores exactly what the grader saw pre-migration. prediction = post["rollouts"][0].get("prediction") or "" - correct = is_correct({"prediction": prediction, "answer": gold}) + # `math_equal` runs `parse_latex` + `simplify`: ~11 ms typical, 1.7 s + # worst case — measured on *reference* data, and `simplify` on arbitrary + # model output has no ceiling. Reached with `timeout=False`, so nothing + # else bounds it: criterion 2 in `core/utils/offload.py`. + try: + correct = await run_cpu_bound( + is_correct, + {"prediction": prediction, "answer": gold}, + timeout=GRADE_TIMEOUT, + ) + except TimeoutError: + # An answer that cannot be graded is a wrong answer, not a failed + # run — the contract every sibling math grader keeps. Letting this + # propagate would land the sample in `fails` instead, which reads as + # an infrastructure failure and is one of the signals a run is + # promoted on. The accuracy is identical either way (`report` counts + # fails in the denominator), so the only thing at stake is whether + # the number means what it says. + logger.warning( + "Grading sample {} exceeded {}s and was scored wrong; the " + "prediction is likely a shape `simplify` cannot bound.", + ctx.sample_id, + GRADE_TIMEOUT, + ) + correct = False return True, build_judgement_record(gold, [build_rollout_judgement(0, correct)]) @override diff --git a/sieval/tasks/hendrycks_math_kshot_base_gen.py b/sieval/tasks/hendrycks_math_kshot_base_gen.py index ab9ddb06..88e5a5f8 100644 --- a/sieval/tasks/hendrycks_math_kshot_base_gen.py +++ b/sieval/tasks/hendrycks_math_kshot_base_gen.py @@ -29,6 +29,8 @@ from typing import override +from loguru import logger + from sieval.community.deepseek_math import ( STOP_WORDS, eval_math, @@ -50,6 +52,7 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import HendrycksMathDatasetSample N_SHOT = 4 @@ -126,7 +129,33 @@ async def feedback(self, post, ctx): ctx.raw_sample["problem"], ctx.raw_sample["solution"], "cot" ) prediction = post["rollouts"][0].get("prediction") or "" - correct = bool(eval_math({"prediction": prediction, "answer": reference})) + # `math_equal` runs `parse_latex` + `simplify`: ~11 ms typical, 1.7 s + # worst case — measured on *reference* data, and `simplify` on arbitrary + # model output has no ceiling. Reached with `timeout=False`, so nothing + # else bounds it: criterion 2 in `core/utils/offload.py`. + try: + correct = bool( + await run_cpu_bound( + eval_math, + {"prediction": prediction, "answer": reference}, + timeout=GRADE_TIMEOUT, + ) + ) + except TimeoutError: + # An answer that cannot be graded is a wrong answer, not a failed + # run — the contract every sibling math grader keeps. Letting this + # propagate would land the sample in `fails` instead, which reads as + # an infrastructure failure and is one of the signals a run is + # promoted on. The accuracy is identical either way (`report` counts + # fails in the denominator), so the only thing at stake is whether + # the number means what it says. + logger.warning( + "Grading sample {} exceeded {}s and was scored wrong; the " + "prediction is likely a shape `simplify` cannot bound.", + ctx.sample_id, + GRADE_TIMEOUT, + ) + correct = False return True, build_judgement_record( reference, [build_rollout_judgement(0, correct)] ) diff --git a/sieval/tasks/hmmt_feb_2025_0shot_gen.py b/sieval/tasks/hmmt_feb_2025_0shot_gen.py index 4383cae8..bc79dc10 100644 --- a/sieval/tasks/hmmt_feb_2025_0shot_gen.py +++ b/sieval/tasks/hmmt_feb_2025_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import HMMTFeb2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="hmmt_feb_2025_0shot_gen", @@ -116,8 +119,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -128,10 +129,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/hmmt_feb_2026_0shot_gen.py b/sieval/tasks/hmmt_feb_2026_0shot_gen.py index 5ddc6d79..48ce9de0 100644 --- a/sieval/tasks/hmmt_feb_2026_0shot_gen.py +++ b/sieval/tasks/hmmt_feb_2026_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import HMMTFeb2026DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="hmmt_feb_2026_0shot_gen", @@ -104,8 +107,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -116,10 +117,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/hmmt_nov_2025_0shot_gen.py b/sieval/tasks/hmmt_nov_2025_0shot_gen.py index 1e6bb399..0d818ce3 100644 --- a/sieval/tasks/hmmt_nov_2025_0shot_gen.py +++ b/sieval/tasks/hmmt_nov_2025_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import HMMTNov2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="hmmt_nov_2025_0shot_gen", @@ -118,8 +121,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -130,10 +131,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/imo_answer_bench_0shot_gen.py b/sieval/tasks/imo_answer_bench_0shot_gen.py index 9c17a527..24917288 100644 --- a/sieval/tasks/imo_answer_bench_0shot_gen.py +++ b/sieval/tasks/imo_answer_bench_0shot_gen.py @@ -42,6 +42,7 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import IMOAnswerBenchDatasetSample # IMO-Bench AnswerBench is an agentic harness whose only instruction is @@ -184,7 +185,11 @@ async def feedback(self, post, ctx): # Verbatim upstream grader (math_verify); symmetric $-wrapping like # the HMMT sibling so full expressions parse, gold first. math_verify # handles commutativity / factoring / set-equality — no bespoke logic. - correct = verify_math_answer(f"${gold}$", f"${pred}$") + # In a worker process, like every other math-verify grader: its + # timeouts are signal-based and it raises off the main thread. + correct = await run_cpu_bound( + verify_math_answer, f"${gold}$", f"${pred}$", timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/math_500_0shot_gen.py b/sieval/tasks/math_500_0shot_gen.py index 622eb900..5946e962 100644 --- a/sieval/tasks/math_500_0shot_gen.py +++ b/sieval/tasks/math_500_0shot_gen.py @@ -19,8 +19,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import MATH500DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="math_500_0shot_gen", @@ -92,8 +95,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -104,10 +105,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/scicode_0shot_gen.py b/sieval/tasks/scicode_0shot_gen.py index c374abcb..5947acb6 100644 --- a/sieval/tasks/scicode_0shot_gen.py +++ b/sieval/tasks/scicode_0shot_gen.py @@ -32,12 +32,12 @@ AI-Generated Code - Claude Opus 4.8 (1M context) (Anthropic) """ -import asyncio import os import time from typing import Literal, TypedDict, override import httpx +from anyio.to_thread import run_sync from loguru import logger from sieval.community.scicode import ( @@ -403,7 +403,12 @@ def read_targets() -> dict[str, str]: for sc, _code, cases in pending } - targets_by_step = await asyncio.to_thread(read_targets) + # `anyio.to_thread`, not `asyncio.to_thread`: the latter uses the loop's + # own executor and so escapes anyio's CapacityLimiter, putting these + # reads outside the session's thread budget. Shares the default limiter + # with the loader and the deployer (grading has its own — see + # `core/utils/offload.py`). + targets_by_step = await run_sync(read_targets) programs: list[StepProgram] = [] for sc, code, cases in pending: diff --git a/sieval/tasks/smt_2025_0shot_gen.py b/sieval/tasks/smt_2025_0shot_gen.py index ad8f4e44..7f4f88e6 100644 --- a/sieval/tasks/smt_2025_0shot_gen.py +++ b/sieval/tasks/smt_2025_0shot_gen.py @@ -22,8 +22,11 @@ build_rollout_judgement, sieval_task, ) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound from sieval.datasets import SMT2025DatasetSample +from ._math_verify import verify_answer + @sieval_task( name="smt_2025_0shot_gen", @@ -118,8 +121,6 @@ async def postprocess(self, inf, ctx): @override async def feedback(self, post, ctx): - from math_verify import parse, verify - rollouts = [] ground_truth = ctx.raw_sample["answer"] for rollout in post["rollouts"]: @@ -130,10 +131,9 @@ async def feedback(self, post, ctx): pred_with_env = f"${pred}$" ref_with_env = f"${ground_truth}$" try: - parsed_pred = parse(pred_with_env) - parsed_ref = parse(ref_with_env) - # math_verify.verify expects the gold answer as the first arg. - correct = verify(parsed_ref, parsed_pred) + correct = await run_cpu_bound( + verify_answer, ref_with_env, pred_with_env, timeout=GRADE_TIMEOUT + ) except Exception as e: logger.warning("Feedback failed for sample {}: {}", ctx.sample_id, e) correct = False diff --git a/sieval/tasks/ugmathbench_0shot_gen_fixed.py b/sieval/tasks/ugmathbench_0shot_gen_fixed.py new file mode 100644 index 00000000..a8b1b7b9 --- /dev/null +++ b/sieval/tasks/ugmathbench_0shot_gen_fixed.py @@ -0,0 +1,575 @@ +""" +UGMathBench 0-shot generative task, corrected — effective accuracy and the gap. + +The ``_fixed`` variant: this grades slots that upstream's judge cannot win, so +it is deliberately *not* a reproduction of the published numbers. The +unqualified name ``ugmathbench_0shot_gen`` is reserved for a faithful port and +will stay vacant — upstream's grader is GPL-3.0 and cannot ship in an +Apache-2.0 distribution, so no faithful port is possible here at all. + +The divergence from upstream is measured rather than asserted, two independent +ways — against upstream's judge on a full live run, and on replayed references. +Both are below, with the live one carrying the weight. + +One sample is one *(problem, version)* pair, so a full run issues three +inferences per problem, one per randomized version. That is what the benchmark's +headline metric needs: + +* **AAcc** — average accuracy over every version. +* **EAcc** — effective accuracy: the share of problems answered correctly in + *all* three versions. The headline (``score``). +* **Delta** — the reasoning gap, ``AAcc - EAcc``. A model that reasons rather + than recognizes drives this toward zero; the paper reports double-digit gaps + for every model it evaluated. +* **CAcc** — the share of problems answered correctly in at least one version. + Upstream reports it as the optimistic bound bracketing EAcc. + +Grading is per answer *slot*: a problem states how many ``[ANS]`` placeholders +it has and what type each one takes, and a sample counts as correct only when +every slot is. The per-slot rules live in +:mod:`sieval.community.ugmathbench`, which also explains why the grader is an +independent implementation rather than a port of the GPL-licensed reference. + +Two pinned rows are upstream-corrupt (empty answer sequence, problem text +replaced by an error message); they are prompted and graded like any other +sample, which scores them 0 exactly as the reference harness does. + +**Status: stable, on one full live run.** Qwen3-30B-A3B, thinking on, +temperature 0.6 / top_p 0.95 / top_k 20, one rollout per version; all 15,183 +versions over all 5,061 problems; sglang tp2xdp4 on 8xH100, 75.7M output tokens. + + EAcc 38.49 (``score``) · AAcc 45.59 · CAcc 53.53 · Delta 7.10 + fails 0 · incomplete_problems 0 · extracted=False 32/15,183 (0.21%) + +**Read 38.49 as protocol-faithful, not as this model's mathematical ability.** +The two are separated by a *format tax* of roughly 33 EAcc points, and the tax +is the benchmark's rather than this port's: only the last ``\\boxed{}`` counts +and a slot-count mismatch scores every slot wrong, while Qwen3 ends multi-part +problems with ``\\boxed{a}, \\boxed{b}, \\boxed{c}`` instead of +``\\boxed{a, b, c}``. Measured off the same stored responses, single-answer rows +mismatch 0.30% of the time against multi-answer rows' 86.31% — same model, same +subjects, so this is formatting and not difficulty. Upstream's ``judge_rule.py`` +has the identical rule, so reproducing it is the point. Repairing extraction on +those stored responses lifts EAcc to 74.43, which is where a model scoring 72.5 +on AIME 2026 in this harness belongs. + +**Evidence that the grader itself is right.** Upstream's ``judge_rule.py`` is an +independent implementation of the same spec, so it can be run as a local +*instrument* over the same 15,183 stored responses (GPL-3.0 restricts +distribution, not use; nothing is vendored). Against it this task agrees on +95.51% of samples, and where the two differ it is **591 to 91 in this task's +favour**. The 591 is the direction that matters, since a too-lenient grader +would show up there: a sampled eyeball found 12 of 12 genuinely correct +(``2\\sqrt{2t+9}`` against ``sqrt(2*4*t+36)``, ``4^20`` against ``1.09951E+12``, +``\\ln(2)/2`` against ``0.346573590279973``). The residual misses are +91/15,183 = 0.60% of samples, with named non-systematic causes: LaTeX interval +notation (``\\cup``, ``\\infty``), absolute-value bars, a ``y = `` prefix on an +``EX`` answer. Upstream's own verifier scores EAcc 35.55 on these responses, +i.e. *below* this task. + +**How the promotion criteria resolved.** The gate was never "matches upstream" — +that bar is unreachable by construction here and would pin this task to +``experimental`` forever. It was evidence that *this* grader is right: + +1. ``extracted=False`` rate low — **met**, 0.21%, with truncation at 0.19%, so + the box-or-nothing rule is not what costs this model points; +2. ``fails`` 0 across the run — **met**; +3. EAcc in a plausible band next to sibling math benchmarks on the same model — + **explained rather than met.** 38.49 sits far from the same model's 72.5 on + AIME 2026, but this criterion exists to catch an *unexplained* anomaly (a + mis-wired prompt, a mis-joined gold, a broken extractor). This gap is + attributed to the format tax above and the attribution is checked two + independent ways — the single- against multi-answer mismatch split, and + tracking upstream's own verifier to within 3 points. A harness that were + actually broken would not track the reference implementation that closely; +4. a sampled false-negative rate on wrong verdicts — **met**, via the + independent-instrument audit above. It is the criterion that matters, and it + tests "this grader is correct" directly instead of by proxy. + +**What criterion (4) caught the first time, and why the earlier evidence +missed it.** On the first run it *failed* at 34.9%: of 1,634 wrong slots where +extraction and the reference agreed on slot count, 570 were the grader's error +rather than the model's, entirely in the free-form types (EX 59.6%, NV 25.4%) +and at exactly 0% in every structured one. The cause was in +:func:`~sieval.community.ugmathbench.math_equal`: ``math_verify.parse`` routes +everything through a LaTeX reader, so the dataset's plain-sympy gold was mangled +(``7*sin(pi*x/5)+1`` read as ``7*s*i*n*(i*p*x)/5 + 1``, ``sin`` as s·i·n) and a +gold naming any function could only match by exact string equality. Fixing it +moved EAcc 34.46 -> 38.49 with **716 verdicts wrong-to-right and 0 +right-to-wrong**. + +The zero matters more than the +4.03, and so does the shape of the miss: the +reference-replay figure below could not see this defect at all, because +replaying a gold as its own answer short-circuits on +``_squash(pred) == _squash(gold)`` and never reaches the symbolic path. **A +self-replay canary exercises the fast path and is silent about exactly the +comparison logic it appears to certify.** Worth remembering beyond this task. + +**The divergence from upstream, measured on replayed references.** Replaying +every pinned reference back as a boxed answer through both graders, upstream +accepts its own reference on 14,616 of 15,183 rows (96.27%) and this task on +15,160 (99.85%). The 552 rows that disagree span 192 of 5,061 problems: an EAcc +**ceiling** difference of 3.79 pp, about five times the 0.70 pp binomial +standard error, and 548 of the 552 are rows upstream cannot win at all — so the +gap is repair, not drift. :mod:`sieval.community.ugmathbench` enumerates each +divergence and how the figure was obtained. It is a ceiling on the *grader*, +realized only on a problem a model would otherwise answer correctly in all three +versions, and — per the paragraph above — it is the weaker of the two +measurements. The live head-to-head is the one to trust. + +**Caveat on Delta.** At temperature 0.6 the reasoning gap is mostly the sampler. +A control on Geometry with ``n=3`` gives Delta 8.70 across the three randomized +*versions* but 7.25 across three *rollouts of one fixed version* — only 1.45 pp +is version sensitivity, 83% is sampling noise. Upstream generates greedily, +where this does not arise; read Delta only against the sampling settings that +produced it. + +Budget note: a full run is 15,183 inferences, and grading a wrong answer costs +roughly 25 ms of sympy per sample (a correct one is effectively free, since it +short-circuits on string equality). That work runs in a worker process +(:func:`~sieval.core.utils.offload.run_cpu_bound`) rather than on the event loop +the rest of the session shares, but it is still CPU the run has to spend. +Feedback is therefore worth a few minutes on a whole-benchmark run, and a +subject subset is a reasonable smoke test — pass +``datasets..args.subjects``. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from collections import defaultdict +from typing import override + +from loguru import logger + +from sieval.community.ugmathbench import ( + VERSIONS, + build_prompt, + extract_predictions, + judge_answers, +) +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.utils.offload import GRADE_TIMEOUT, run_cpu_bound +from sieval.datasets import UGMathBenchDatasetSample + +#: Relative tolerance for numeric answers. Matches the reference evaluator's +#: CLI default (``eval_rule.py --precision``), not the stricter 1e-8 its +#: ``Judger`` class defaults to. +DEFAULT_PRECISION = 1e-3 + + +@sieval_task( + name="ugmathbench_0shot_gen_fixed", + display_name="UGMathBench (0-shot, generative, corrected)", + description="Undergraduate math, 3 randomized versions per problem; EAcc + gap.", + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "open-ended"), + deps_group="math", + model_type="chat", + reference_impl=ReferenceImpl( + source="UGMathBench", + url="https://github.com/YangLabHKUST/UGMathBench/blob/df47bfa639bfb89bdb0220036a7b2f216e72b0b3/eval_rule.py", + notes=( + "Prompt and metric definitions mirror upstream's `raw` template and " + "`eval_file` (aacc / eacc / cacc / Delta / RE). The grader does NOT, " + "and cannot: upstream's `judge_rule.py` is GPL-3.0 and would not ship " + "in an Apache-2.0 distribution, so answer comparison is an " + "independent math-verify-based implementation of the same 10 answer " + "types. The `_fixed` variant therefore does not reproduce the paper's " + "numbers, and the unqualified name stays vacant rather than reserved. " + "MEASURED DIVERGENCE: upstream's judge was run as a local instrument " + "(GPL-3.0 restricts distribution, not use; nothing is vendored) over " + "all 15,183 pinned rows, each row's own reference replayed back as a " + "boxed answer. Upstream accepts its own reference on 14,616 rows " + "(96.27%), this task on 15,160 (99.85%); the 552 rows that disagree " + "span 192 of 5,061 problems — an EAcc ceiling difference of 3.79 pp, " + "about 5x the 0.70 pp binomial standard error. 548 of the 552 are rows " + "upstream rejects its OWN reference on, so the gap is repair, not " + "drift; only 4 go the other way (a UOL reference whose top-level " + "commas sit outside any bracket). Dominant cause: upstream normalizes " + "the prediction and the reference by different passes — its " + "extraction-time normalize_answer rewrites sqrt(x) into sqrt{(}x) on " + "the prediction only, which a plain-sympy reference never survives. " + "Contributing: a TF slot whose reference is not a boolean (9 of 1665; " + "upstream asserts it is and swallows the AssertionError into a False, " + "so no answer wins), a zero-valued numeric reference, the declared " + "answer type deciding the rule where upstream's `is_equal` retries " + "every method until one accepts, and three deliberate differences in " + "where commas are split (9 rows): `<` and `>` are the relational " + "operators here, not brackets, so counting them as brackets — which " + "upstream does — swallows the comma after them and the row comes out " + "a slot short (6 rows); the bracket depth clamps at zero instead of " + "going negative, where upstream loses every remaining slot after one " + "unmatched closer (3 rows); and `{}` counts as a bracket, which costs " + "0 rows on the references but keeps a comma inside a LaTeX group in " + "the model's answer from splitting a slot in two. All enumerated in " + "sieval/community/ugmathbench.py. The figure bounds the GRADER, not a " + "model's score: it is a replay of stored references, so it says " + "nothing about extraction on real model prose. LIVE HEAD-TO-HEAD " + "(the stronger measurement, and the one that promoted this task): " + "over a full 15,183-version run, upstream's judge re-graded the same " + "stored responses and agreed on 95.51% of samples, disagreeing 591 " + "to 91 in this task's favour; upstream scores EAcc 35.55 where this " + "task scores 38.49, and the residual misses are 0.60% of samples " + "(LaTeX interval notation, absolute-value bars, a `y = ` prefix on " + "an EX answer). Note that the replay figure could NOT see the " + "largest defect the live run found — replaying a gold as its own " + "answer short-circuits on string equality and never reaches the " + "symbolic path. GUARDS: since the parsed text is model output, three " + "shapes are refused rather than evaluated — the parse namespace has " + "its builtins removed; an answer containing a quote is refused " + "outright, because a quoted string handed to any callable (eval, " + "sympify, S, N, or any name at all, since auto_symbol makes unknown " + "names callable) is re-sympified with sympy's own default namespace " + "and gets the builtins back; and an answer requiring unbounded " + "arithmetic (a power tower) is screened out by an unevaluated " + "pre-parse. All three grade the answer wrong, and none is reachable " + "by any pinned reference — the largest exponent is three digits and " + "not one of the 42,064 gold slots contains a quote. " + "SAMPLING: upstream generates " + "greedily, one sample per version (temperature 0, max_tokens 2048), " + "and this task issues one rollout per version to match; set " + "temperature via the model config. Upstream also offers a " + "model-as-judge variant (eval_marj.py) which it now recommends over " + "the rule-based path — not implemented here. METRICS: `relative_delta` " + "is upstream's RE scaled by 100 — upstream prints the bare ratio " + "(0.1667) where this reports 16.67; do not compare the two directly." + ), + ), + # Promoted on the first live run (Qwen3-30B-A3B, all 15,183 versions, + # fails 0). (1) extracted=False 0.21%, (2) 0 fails, (4) audited against + # upstream's judge as an independent instrument: 95.51% agreement, + # disagreement 591-to-91 in this grader's favour, 12/12 of the risky + # direction genuinely correct, residual misses 0.60% of samples. The defect + # that run exposed -- math_verify.parse LaTeX-parsing the dataset's + # plain-sympy gold, `sin` -> s*i*n -- is fixed, worth EAcc 34.46 -> 38.49 + # with 0 regressions. (3) is explained rather than met: the remaining gap + # to sibling benchmarks is the BENCHMARK's last-box rule, reproduced + # faithfully, and this task scores above upstream's own verifier on + # identical responses. The module docstring records all of it. + status="stable", +) +class UGMathBenchZeroShotGenFixedTask( + Task[ + UGMathBenchDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + precision: float = DEFAULT_PRECISION, + ): + super().__init__(dataset=dataset, model=model, name=name) + if precision <= 0: + raise ValueError( + f"precision must be > 0 (got {precision}); it is the relative " + "tolerance for numeric answers." + ) + self._precision = precision + + @override + async def preprocess(self, raw, ctx): + prompt = build_prompt( + raw["subject"], + raw["problem"], + len(raw["answer"]), + raw["answer_type"], + raw["options"], + ) + return build_prompt_record( + [{"role": "user", "content": prompt}], + reference=raw["answer"], + # The version and its problem are what report() groups on; kept here + # too so a prompt row identifies its sibling versions on its own. + extra={ + "problem_id": raw["id"], + "version": raw["version"], + "subject": raw["subject"], + "answer_type": raw["answer_type"], + }, + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"]) + + @override + async def postprocess(self, inf, ctx): + # One prediction per rollout, itself the list of per-slot answers. + predictions: list = [extract_predictions(text) for text in inf.texts] + return build_prediction_record(predictions) + + @override + async def feedback(self, post, ctx): + raw = ctx.raw_sample + if raw is None: + # Nothing to compare against; the reference is genuinely unknown + # rather than a procedure, so the verdict is wrong-by-default. + # + # The grouping keys still have to survive, and the prompt record + # carries them. Without that, report() cannot tell which problem + # this version belonged to and drops it from the effective-accuracy + # denominator, while the wrong verdict stays in AAcc's -- so EAcc is + # computed over the survivors and biased *upward*. That is the same + # failure `_identify` guards for failed samples; a wrong-by-default + # verdict has to hold its problem's place just as a failure does. + problem_id, subject = _identify(ctx) + return True, build_judgement_record( + None, + [ + build_rollout_judgement(rollout["index"], False) + for rollout in post["rollouts"] + ], + extra={"problem_id": problem_id, "subject": subject} + if problem_id is not None + else None, + ) + + golds = raw["answer"] + rollouts = [] + for rollout in post["rollouts"]: + # Grading is synchronous sympy — ~23 ms for a wrong answer, and every + # runner in the session shares one event loop, so doing it here would + # stall every other task too. `run_cpu_bound` moves it to a worker + # process; a process rather than a thread because math-verify's + # timeouts are signal-based and it refuses to run threaded at all. + try: + per_slot = await run_cpu_bound( + judge_answers, + rollout.get("prediction"), + golds, + raw["answer_type"], + raw["options"], + self._precision, + timeout=GRADE_TIMEOUT, + ) + except TimeoutError: + # Same contract as the rest of the grader: an answer that cannot + # be graded is a wrong answer, not a failed run. Loud, because a + # timeout here means an input the in-module guards did not catch. + logger.warning( + "Grading sample {} exceeded {}s and was scored wrong; the " + "prediction is likely a shape the parser guards miss.", + ctx.sample_id, + GRADE_TIMEOUT, + ) + per_slot = [False] * len(golds) + n_correct = sum(per_slot) + rollouts.append( + build_rollout_judgement( + rollout["index"], + bool(per_slot) and all(per_slot), + metrics={ + # Slot-level credit, so a near-miss on a 20-blank table + # is distinguishable from a blank answer. The headline + # verdict stays all-or-nothing, as upstream grades. + "answer_accuracy": n_correct / len(per_slot) + if per_slot + else 0.0 + }, + extra={"per_answer": per_slot, "n_answers": len(per_slot)}, + ) + ) + return True, build_judgement_record( + golds, + rollouts, + # Aggregation raw material: report() reads these instead of + # raw_sample, which a persisted context is not required to carry. + extra={ + "problem_id": raw["id"], + "version": raw["version"], + "subject": raw["subject"], + }, + ) + + @override + async def report(self, finals, fails): + by_problem: dict[str, list[bool]] = defaultdict(list) + by_subject: dict[str, dict[str, list[bool]]] = defaultdict( + lambda: defaultdict(list) + ) + n_correct = 0 + + unattributed_finals = 0 + for final in finals: + judgement = final.feedback_result + extra = judgement.get("extra", {}) + problem_id = extra.get("problem_id") + subject = extra.get("subject") + if problem_id is None: + # Same recovery as the failed-sample loop below. A judged + # version that cannot name its problem would otherwise leave + # `by_problem` while its verdict stayed in AAcc's denominator, + # which biases EAcc *upward* — silently, and in the direction + # that flatters the run. + problem_id, subject = _identify(final) + # UGMathBench asks one answer per version, so the verdict is the + # first rollout's. A model configured for n > 1 does not turn this + # into pass@n -- that would inflate every version-level accuracy the + # effective-accuracy metric is built from. + verdicts = judgement["rollouts"] + correct = bool(verdicts) and verdicts[0]["correct"] + n_correct += int(correct) + if problem_id is None: + unattributed_finals += 1 + continue + by_problem[problem_id].append(correct) + by_subject[subject or "unknown"][problem_id].append(correct) + + # A failed version still belongs to a problem, and that problem still + # owes three correct answers. Registering it here keeps it in the + # effective-accuracy denominator: without this, a problem whose three + # versions all failed would vanish from `by_problem` entirely while its + # three failures stayed in AAcc's denominator, so EAcc would be computed + # over the survivors and silently biased *upward* — the opposite + # direction from the partial-failure case below, and with no warning. + unattributed_fails = 0 + for failed in fails: + problem_id, subject = _identify(failed) + if problem_id is None: + unattributed_fails += 1 + continue + by_problem.setdefault(problem_id, []) + by_subject[subject or "unknown"].setdefault(problem_id, []) + + # A failed sample is an unanswered version, so it counts against the + # average — same convention as the pass@1 math tasks. + n_versions = len(finals) + len(fails) + aacc = n_correct * 100 / n_versions if n_versions else 0.0 + + unattributed = unattributed_fails + unattributed_finals + if unattributed: + logger.warning( + "{} sample(s) ({} failed, {} judged) carry neither a raw sample " + "nor a prompt record, so the problem they belong to could not be " + "kept in the effective-accuracy denominator; EAcc is an upper " + "bound by up to that many problems.", + unattributed, + unattributed_fails, + unattributed_finals, + ) + + incomplete = sum( + 1 for verdicts in by_problem.values() if len(verdicts) != VERSIONS + ) + if incomplete: + logger.warning( + "{}/{} problem(s) were judged on fewer than {} versions (failed or " + "sliced samples) and cannot count as effective-accuracy hits; " + "EAcc is a lower bound for this run.", + incomplete, + len(by_problem), + VERSIONS, + ) + + eacc = _effective_accuracy(by_problem) + + # EAcc counts problems correct in *every* version; AAcc counts correct + # versions. A problem cannot be correct in all three without those three + # being correct, so EAcc <= AAcc holds for any run — and the way to break + # it is for a problem to leave EAcc's denominator while its versions stay + # in AAcc's. That is exactly what an unattributed sample does, so the + # invariant is the cheapest detector for the whole class. A wrong number + # that still looks plausible is the worst thing an eval can emit; say so + # rather than let it be read as a score. + if eacc > aacc + 1e-9: + logger.error( + "EAcc ({:.2f}) exceeds AAcc ({:.2f}), which is impossible: " + "{} problem(s) are in AAcc's denominator but not EAcc's. Treat " + "this run's EAcc as invalid rather than optimistic.", + eacc, + aacc, + unattributed, + ) + + metrics: dict[str, float] = { + "score": eacc, + "fails": float(len(fails)), + "eacc": eacc, + "aacc": aacc, + "cacc": _covered_accuracy(by_problem), + # Reasoning gap, in accuracy points; `relative_delta` expresses it as + # a percentage of EAcc (upstream's RE). + "delta": aacc - eacc, + "relative_delta": (aacc - eacc) * 100 / eacc if eacc else 0.0, + "n_problems": float(len(by_problem)), + "n_versions_judged": float(len(finals)), + "incomplete_problems": float(incomplete), + # Non-zero means EAcc's denominator is short by this many samples, + # so the figure is an upper bound rather than the usual lower one. + # Split by origin: a failed sample never reached `feedback`, a + # judged one did and still could not name its problem. + "unattributed_fails": float(unattributed_fails), + "unattributed_finals": float(unattributed_finals), + } + for subject, problems in sorted(by_subject.items()): + metrics[f"eacc_{subject.lower()}"] = _effective_accuracy(problems) + return metrics + + +def _identify(ctx) -> tuple[str | None, str | None]: + """The problem and subject a sample belongs to, without a judgement. + + Two callers, one reason. A failed sample never reaches ``feedback``; a + sample whose ``raw_sample`` is gone reaches it but has no reference to + record. Either way the grouping keys the judgement normally carries are + unavailable — and either way the sample still has to hold its place in the + effective-accuracy denominator, because leaving it out biases EAcc upward. + + Two sources, in order: ``raw_sample``, which survives ``to_failed`` (a + dataclass ``replace``), and the prompt record, which carries the same keys + for a context persisted without its raw sample. Both absent means the sample + was lost before either existed, which is what ``unattributed_*`` counts. + """ + raw = ctx.raw_sample + if raw is not None and raw.get("id") is not None: + return raw["id"], raw.get("subject") + pre = ctx.preprocess_result + if pre is not None: + extra = pre.get("extra") or {} + return extra.get("problem_id"), extra.get("subject") + return None, None + + +def _effective_accuracy(by_problem: dict[str, list[bool]]) -> float: + """Share of problems correct in *every* one of their randomized versions. + + A problem judged on fewer versions than the benchmark defines cannot be + confirmed correct across all of them, so it never counts as a hit. + """ + if not by_problem: + return 0.0 + hits = sum( + 1 + for verdicts in by_problem.values() + if len(verdicts) == VERSIONS and all(verdicts) + ) + return hits * 100 / len(by_problem) + + +def _covered_accuracy(by_problem: dict[str, list[bool]]) -> float: + """Share of problems correct in at least one version — EAcc's upper bracket.""" + if not by_problem: + return 0.0 + hits = sum(1 for verdicts in by_problem.values() if any(verdicts)) + return hits * 100 / len(by_problem) diff --git a/tests/unit/community/test_ugmathbench.py b/tests/unit/community/test_ugmathbench.py new file mode 100644 index 00000000..a66dd5ec --- /dev/null +++ b/tests/unit/community/test_ugmathbench.py @@ -0,0 +1,501 @@ +"""Unit tests for UGMathBench prompting, extraction, and per-type grading. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest + +from sieval.community.ugmathbench import ( + SUBJECTS, + VERSIONS, + build_prompt, + describe_answer_type, + extract_answer, + extract_predictions, + judge_answer, + judge_answers, + math_equal, + split_answers, +) + +# --- prompt ---------------------------------------------------------------- + + +def test_single_answer_prompt_uses_singular_wording(): + prompt = build_prompt("Algebra", "Solve $x+1=2$. [ANS]", 1, ["NV"]) + assert "This problem involves only one placeholders [ANS]" in prompt + assert "The answer type is a numerical value without units." in prompt + assert 'end your response with: "The final answer is \\boxed{ANSWER}"' in prompt + assert "Problem:\nSolve $x+1=2$. [ANS]" in prompt + + +def test_multi_answer_prompt_lists_every_type_in_order(): + prompt = build_prompt( + "Trigonometry", "a) [ANS] b) [ANS]", 2, ["NV", "TF"], [[], []] + ) + assert "This problem involves 2 placeholders [ANS]" in prompt + assert ( + "Their answer types are, in order, a numerical value without units, " + "either True or False." in prompt + ) + assert 'end your response with: "The final answers are \\boxed{ANSWER}"' in prompt + + +def test_multiple_choice_description_embeds_the_option_list(): + # Upstream interpolates the Python list repr; the model sees the brackets. + assert describe_answer_type("MCS", ["A", "B"]) == ( + "one option of a multiple choice question with options ['A', 'B']" + ) + + +def test_unknown_answer_type_is_rejected(): + with pytest.raises(KeyError, match="unknown UGMathBench answer type"): + describe_answer_type("XX") + + +def test_answer_count_drives_the_wording_not_the_declared_type_count(): + # A handful of pinned rows declare fewer types than answers; upstream takes + # the count from the answers and describes only the declared types. + prompt = build_prompt("Linear_algebra", "p", 5, ["NV", "NV"]) + assert "This problem involves 5 placeholders [ANS]" in prompt + assert prompt.count("a numerical value without units") == 2 + + +def test_missing_option_entries_are_padded_not_fatal(): + prompt = build_prompt("Algebra", "p", 2, ["MCS", "MCS"], [["A", "B"]]) + assert "options ['A', 'B']" in prompt + assert "options []" in prompt + + +def test_benchmark_shape_constants(): + assert len(SUBJECTS) == 16 + assert VERSIONS == 3 + + +# --- extraction ------------------------------------------------------------ + + +def test_extracts_the_last_box(): + response = "First \\boxed{1}. On reflection, \\boxed{2}." + assert extract_answer(response) == "2" + + +def test_boxed_extraction_is_brace_balanced(): + assert extract_answer("So \\boxed{\\frac{1}{2}}") == "\\frac{1}{2}" + + +def test_falls_back_to_an_explicit_answer_handoff(): + assert extract_answer("blah blah. The final answer is 42.") == "42" + + +def test_no_answer_at_all_is_none_not_a_guess(): + # Strict extraction: a response full of numbers but no box and no hand-off + # must not be scored on the last number it happened to mention. + assert extract_answer("We compute 17, then 3, and stop.") is None + assert extract_predictions("We compute 17, then 3, and stop.") is None + + +def test_normalization_strips_decoration_but_not_content(): + assert extract_answer("\\boxed{\\left(\\frac{1}{2}\\right)}") == "(\\frac{1}{2})" + assert extract_answer("\\boxed{\\text{none}}") == "none" + + +def test_split_keeps_bracketed_commas_together(): + assert split_answers("1, (2, 3), [4, 5]") == ["1", "(2, 3)", "[4, 5]"] + + +def test_split_keeps_latex_group_commas_together(): + assert split_answers("\\frac{a,b}{c}, 2") == ["\\frac{a,b}{c}", "2"] + + +def test_split_folds_latex_set_delimiters(): + assert split_answers("\\{1, 2\\}, 3") == ["(1, 2)", "3"] + + +def test_split_treats_angle_brackets_as_operators_not_grouping(): + # Upstream counts `<` and `>` as brackets. Here they are the relational + # operators the dataset actually uses -- a slot whose whole answer is `<` -- + # so counting them swallows the following comma and the row comes out short + # by a slot, which grades every slot in it wrong however good the answer. + assert split_answers("<, 55000") == ["<", "55000"] + assert split_answers("10/[5^{2*n+1}], <, monotone decreasing") == [ + "10/[5^{2*n+1}]", + "<", + "monotone decreasing", + ] + + +def test_split_does_not_let_an_unmatched_closer_swallow_the_rest(): + # Upstream's depth goes negative here, and below zero no later comma splits + # -- so one stray `>` costs every remaining slot in the row, not just its + # own. `Arithmetic_0071` is exactly this shape. + assert split_answers("12+20/4, >, (12+20)/4") == ["12+20/4", ">", "(12+20)/4"] + assert split_answers(") , a, b") == [")", "a", "b"] + + +def test_split_still_groups_latex_angle_delimiters(): + # Dropping `<`/`>` costs no real grouping: the inner-product form is + # `\langle ... \rangle`, which is folded to parentheses before the scan. + # The fold is textual, so `\langle `'s trailing space rides along -- what + # matters is that the comma inside stays inside. + assert split_answers("\\langle 1, 2\\rangle, 3") == ["( 1, 2)", "3"] + assert split_answers("\\langle1,2\\rangle, 3") == ["(1,2)", "3"] + + +# --- per-type grading ------------------------------------------------------ + + +def test_numerical_value_accepts_latex_equivalents(): + assert judge_answer("\\frac{\\sqrt{3}}{3}", "1/sqrt(3)", "NV") + assert not judge_answer("2", "1/sqrt(3)", "NV") + + +def test_numerical_value_uses_relative_tolerance(): + assert judge_answer("1000.5", "1000.0", "NV", precision=1e-2) + assert not judge_answer("1000.5", "1000.0", "NV", precision=1e-6) + + +def test_numerical_value_normalizes_a_percent_marked_reference(): + # The dataset stores a few "numerical value without units" answers with a + # percent sign; a plain-number prediction still matches. + assert judge_answer("0.66", "0.66%", "NV") + + +def test_expression_equivalence_is_symbolic(): + assert judge_answer("x^3+2x^2+6", "x^3+2*x^2+6", "EX") + assert not judge_answer("x^3+2x^2+7", "x^3+2*x^2+6", "EX") + + +def test_equation_equivalence(): + assert judge_answer("T = 1.02x + 10", "T = 1.02*x+10", "EQ") + + +def test_interval_equivalence(): + assert judge_answer("(-\\infty, 5)", "(-infinity,5)", "INT") + assert not judge_answer("(-\\infty, 5]", "(-infinity,5)", "INT") + + +def test_true_false_accepts_the_datasets_yn_spelling(): + assert judge_answer("True", "Y", "TF") + assert judge_answer("yes", "Y", "TF") + assert judge_answer("False", "N", "TF") + assert not judge_answer("True", "N", "TF") + # Not a boolean at all -> wrong, never silently true. + assert not judge_answer("maybe", "Y", "TF") + + +def test_single_choice_letter_forms(): + options = ["A", "B", "C", "D"] + assert judge_answer("C", "C", "MCS", options) + assert judge_answer("c", "C", "MCS", options) + assert judge_answer("C. the third one", "C", "MCS", options) + assert not judge_answer("D", "C", "MCS", options) + + +def test_single_choice_supports_non_letter_option_labels(): + assert judge_answer("Q(X)", "Q(X)", "MCS", ["P(X)", "Q(X)"]) + + +def test_multiple_choice_ignores_order_but_not_membership(): + options = ["A", "B", "C", "D", "E"] + assert judge_answer("DCA", "ACD", "MCM", options) + assert not judge_answer("ACDE", "ACD", "MCM", options) + assert not judge_answer("AC", "ACD", "MCM", options) + + +def test_multiple_choice_supports_non_letter_option_labels(): + # Option letters are matched a character at a time, so word-labelled choices + # yield no letters at all and the slot would be unwinnable without the + # whole-answer comparison the single-choice rule already does. + options = ["even", "odd", "neither"] + assert judge_answer("even, odd", "even, odd", "MCM", options) + assert not judge_answer("even", "even, odd", "MCM", options) + + +def test_open_ended_is_text_not_mathematics(): + assert judge_answer("None", "none", "OE") + assert not judge_answer("0", "none", "OE") + + +def test_ordered_list_respects_order(): + assert judge_answer("(1, 2, 3, 6)", "(1, 2, 3, 6)", "OL") + assert not judge_answer("(2, 1, 3, 6)", "(1, 2, 3, 6)", "OL") + + +def test_unordered_list_ignores_order(): + assert judge_answer("(x3, e)", "(e, x3)", "UOL") + assert not judge_answer("(x3, x4)", "(e, x3)", "UOL") + + +def test_unordered_list_matches_multiset_not_set(): + assert not judge_answer("(1, 1)", "(1, 2)", "UOL") + + +def test_list_elements_are_not_read_as_booleans(): + # Upstream converts booleans only for a slot the dataset typed TF — + # `norm_ans_str` gates `norm_str2bool` on `ans_type == "TF"` — and leaves + # list elements alone under a standing `TODO: deal with OL with boolean`. + # No OL/UOL reference on the pinned revision holds a boolean: all 1,244 are + # values or parameter names, so there is nothing here to win by coercing. + assert not judge_answer("(True, 1)", "(Y, 1)", "OL") + + +@pytest.mark.parametrize( + ("pred", "gold", "kind"), + [ + ("(x, t)", "(x, y)", "OL"), + ("(1, f)", "(1, n)", "OL"), + ("(y, 2)", "(t, 2)", "UOL"), + ("(s, t)", "(s, y)", "UOL"), + ], +) +def test_parameter_names_are_not_truth_values(pred, gold, kind): + # `t`, `y`, `f` and `n` are the parameter names this dataset actually uses — + # 13 OL/UOL references on the pinned revision carry a bare `t` or `y`. Read + # as booleans they collapse into each other and a wrong answer scores right, + # which is the one direction a grader must never fail in. + assert not judge_answer(pred, gold, kind) + + +def test_math_equal_survives_unparseable_input(): + # Malformed LaTeX must grade wrong, not raise into the runner. + assert not math_equal("\\frac{{{", "1") + + +# --- sample-level grading -------------------------------------------------- + + +def test_every_slot_must_be_right(): + golds = ["1", "2"] + types = ["NV", "NV"] + assert judge_answers(["1", "2"], golds, types) == [True, True] + assert judge_answers(["1", "3"], golds, types) == [True, False] + + +def test_slot_count_mismatch_grades_everything_wrong(): + assert judge_answers(["1"], ["1", "2"], ["NV", "NV"]) == [False, False] + assert judge_answers(["1", "2", "3"], ["1", "2"], ["NV", "NV"]) == [False, False] + + +def test_missing_extraction_grades_everything_wrong(): + assert judge_answers(None, ["1", "2"], ["NV", "NV"]) == [False, False] + + +def test_undeclared_trailing_types_are_still_graded(): + # 3 answers, 2 declared types: the trailing slot falls through to the + # mathematical rule instead of being dropped. + assert judge_answers(["1", "2", "3"], ["1", "2", "3"], ["NV", "NV"]) == [ + True, + True, + True, + ] + assert judge_answers(["1", "2", "9"], ["1", "2", "3"], ["NV", "NV"]) == [ + True, + True, + False, + ] + + +def test_end_to_end_boxed_response(): + response = ( + "Working through it... The final answers are " + "\\boxed{-\\sqrt{3}, -1, \\sqrt{3}, \\frac{2\\sqrt{3}}{3}, \\sqrt{2}}" + ) + golds = ["-sqrt(3)", "-1", "sqrt(3)", "2/sqrt(3)", "sqrt(2)"] + predictions = extract_predictions(response) + assert predictions is not None and len(predictions) == 5 + assert all(judge_answers(predictions, golds, ["NV"] * 5)) + + +def test_a_reference_with_no_answers_is_never_correct(): + # Two pinned rows are upstream-corrupt: the problem text is an error message + # and the answer sequence is empty. `all([])` is True, so an empty verdict + # list must not read as a correct sample. + assert judge_answers(["1"], [], []) == [] + assert judge_answers(None, [], []) == [] + + +# --- gold parsed as sympy source, not as LaTeX ----------------------------- +# UGMathBench stores gold answers in a plain-sympy dialect while models answer +# in LaTeX. `math_verify.parse` reads everything as LaTeX, where an unescaped +# `sin` is the product s*i*n and `pi` is p*i, so `7*sin(pi*x/5)+1` came back as +# `7*s*i*n*(i*p*x)/5 + 1` and could only ever match by exact string equality. +# The first live run measured the cost at 34.9% of wrong slots in the +# slot-aligned bucket, entirely in the free-form answer types. + + +@pytest.mark.parametrize( + ("pred", "gold"), + [ + # the canonical shape: LaTeX function names against sympy ones + ("7 \\sin(\\frac{\\pi}{5}x) + 1", "7*sin(pi*x/5)+1"), + ("a^x \\ln a", "ln(a)*a^x"), + ("7z - \\cos z + 11", "7*z-cos(z)+11"), + # equal only after simplification, which no normalizer reaches + ("3\\cos(2\\sqrt{35}t)", "3*cos(sqrt(980/7)*t)"), + ("\\dfrac{8}{5 - 4\\cos(q^2)}", "2*8/(2 + 1*8 - 1*8*cos(q^2))"), + ("-162\\pi", "-pi*2*9^2"), + # the dataset's square brackets are grouping, and x^0 is a literal 1 + ("e^{-8x} \\cos(9x)", "x^0*e^(-8*x)*cos(9*x)"), + ("\\frac{18.02}{0.013} ( e^{0.013t} - 1 )", "1386.15*[e^{0.013*t}-1]"), + # implicit multiplication on the prediction side + ("5 - 5c", "-5*c+5*1"), + # same line, different parameterization + ("\\frac{23 - x}{54}", "0.333333+(-0.0185185)*(x-5)"), + # overflows to infinity at a large probe -- the ladder has to stay + # small enough to leave usable points, or a correct exponential answer + # is marked wrong for want of evidence + ("4 e^{\\cosh(4x)} \\sinh(4x)", "2.71828182845905^{cosh(4*x)} * sinh(4*x) * 4"), + ], +) +def test_latex_prediction_matches_plain_sympy_gold(pred, gold): + assert math_equal(pred, gold) + + +@pytest.mark.parametrize( + ("pred", "gold"), + [ + # rounded more coarsely than the benchmark's own relative tolerance: + # still wrong, and substitution must not rescue it + ("2.3", "2.2892"), + ("1.97", "1.97423402868049"), + ("0.16", "0.160257708151404"), + ("25/37", "0.657894736842105"), + # plainly different + ("x^2", "x^3"), + ("\\sin(x)", "cos(x)"), + ("10\\sqrt{2}", "5*[1+sqrt(2)]"), + ("162", "54"), + # same shape, different variable: not the same answer + ("x+1", "t+1"), + # exponentials that differ only in rate: the small probe ladder must + # still separate them + ("e^{x}", "e^{2x}"), + ("\\frac{1}{x}", "\\frac{1}{x^2}"), + ], +) +def test_substitution_does_not_credit_wrong_answers(pred, gold): + assert not math_equal(pred, gold) + + +def test_substitution_reaches_list_elements_too(): + # OL/UOL compare element-wise through `math_equal`, so the fix has to show + # up there as well -- that is where the multi-answer rows live. + assert judge_answer("(0, 2\\cos x \\sin x)", "(0,2*cos(x)*sin(x))", "OL") + assert judge_answer( + "(e^{-8x} \\cos(9x), x e^{-8x})", + "(x^1*e^(-8*x), x^0*e^(-8*x)*cos(9*x))", + "UOL", + ) + + +def test_structured_types_are_untouched_by_the_substitution_pass(): + # TF / MCS / MCM / OE never route through `math_equal`, and the live-run + # audit found 0 false negatives in all of them. Pin that they stay strict. + assert not judge_answer("True", "False", "TF") + assert not judge_answer("B", "A", "MCS", ["A", "B", "C"]) + assert not judge_answer("AB", "AC", "MCM", ["A", "B", "C"]) + assert not judge_answer("prime", "composite", "OE") + + +def test_substitution_verdicts_are_deterministic(): + # The probe ladder is fixed rather than a seeded RNG on purpose: a grader + # has to return the same verdict for the same pair regardless of how many + # comparisons ran before it, and in every process. + pairs = [("7 \\sin(\\frac{\\pi}{5}x) + 1", "7*sin(pi*x/5)+1"), ("x^2", "x^3")] + first = [math_equal(p, g) for p, g in pairs] + for _ in range(3): + assert [math_equal(p, g) for p, g in pairs] == first + assert first == [True, False] + + +# --- untrusted-input guards ------------------------------------------------ + + +def test_parsing_a_prediction_cannot_reach_the_interpreter(tmp_path): + # `parse_expr` evaluates what it parses, and the string reaching it is model + # output -- on the ordinary path, since every wrong free-form answer falls + # through to the substitution pass. Its default global namespace is built by + # `exec("from sympy import *", ...)`, which also injects `__builtins__`. + marker = tmp_path / "executed" + payload = f"__import__('os').system('touch {marker}')" + assert judge_answers([payload], ["42"], ["EX"]) == [False] + assert not marker.exists() + + payload = f"open('{marker}', 'w')" + assert judge_answers([payload], ["42"], ["EX"]) == [False] + assert not marker.exists() + + +#: Every way a call can hand a *string* back to sympy, which re-sympifies it +#: with sympy's own default namespace -- builtins included. Clearing +#: `__builtins__` for the top-level parse does not reach into that nested one, +#: so each of these executes despite the sanitized namespace. `eval` is the +#: shape that survives the sanitizing most surprisingly: it is not a sympy name, +#: so `auto_symbol` rewrites it to `Function('eval')`, and calling a sympy +#: Function on a `str` sympifies the argument. +_NESTED_SYMPIFY_CARRIERS = ["eval", "sympify", "S", "N", "Function('f')"] + + +@pytest.mark.parametrize("carrier", _NESTED_SYMPIFY_CARRIERS) +def test_no_call_shape_can_smuggle_a_string_back_into_the_interpreter( + tmp_path, carrier +): + # The namespace restriction covers one parse. These carriers start another + # one, and refusing the callee by name cannot work -- `auto_symbol` turns + # every unknown name into a callable, so the allowlist would have to be of + # *all* names. What is refused instead is the quote. + marker = tmp_path / f"executed-{carrier[:4]}" + payload = f"{carrier}(\"__import__('os').system('touch {marker}')\")" + assert judge_answers([payload], ["42"], ["EX"]) == [False] + assert not marker.exists(), f"{carrier}(...) reached the interpreter" + + +def test_a_quote_free_call_is_still_parsed_normally(): + # The guard refuses quotes, not calls -- a nested sympify with no string + # literal to read gets a sympy object and does nothing (`chr(112)` stays + # symbolic). Refusing calls outright would drop every `sin(pi*x/5)` gold. + assert judge_answer("sin(pi/6)", "1/2", "EX") is True + assert judge_answer("eval(chr(112))", "42", "EX") is False + + +def test_refusing_a_quoted_prediction_costs_only_the_sympy_reading(): + # A refused prediction is not a refused sample: `_parse_sympy_source` is one + # of three readings, and the LaTeX and literal-equality paths still run. A + # correct answer that happens to carry a quote must still grade correct. + assert judge_answer("42'", "42'", "EX") is True + + +def test_a_power_tower_grades_wrong_instead_of_hanging(): + # `^` is rewritten to `**` before parsing and sympy exponentiates eagerly, + # so `9^9^9^9` asks for a 370-million-digit integer. Grading has to reject + # it, not compute it: `feedback()` offloads to a worker process, so one + # such sample would hold a worker for the rest of the run. + assert judge_answer("9^9^9^9", "x+1", "EX") is False + assert judge_answer("2^{2^{100}}", "x+1", "EX") is False + + +def test_the_exponent_cap_leaves_real_answers_alone(): + # Only the tower shape and absurd exponents are refused. A large-but-sane + # power, and a left-nested one, still compare normally. + assert math_equal("2^100", "2**100") + assert math_equal("(x^2)^3", "x**6") + assert math_equal("4^20", "1.09951162778E+12") + + +def test_a_mangled_numeric_gold_still_reaches_the_substitution_pass(): + # The LaTeX reader turns `2**100` into `2`. Returning that comparison's + # verdict made the substitution pass unreachable for every pair the mangling + # happens to turn into a *number*, which is the exact defect the pass exists + # to repair -- so a correct `2^100` was graded against 2 and lost. + assert math_equal("2^100", "2**100") + + +def test_single_answer_branch_describes_only_the_first_declared_type(): + # Upstream's single-answer branch reads `answer_type[0]` alone, while its + # multi-answer branch joins every declared type. Every pinned row agrees + # (one answer means one type), so this pins the shape rather than a + # coincidence of the current data cut. + prompt = build_prompt("Algebra", "p", 1, ["NV", "EX"]) + assert "The answer type is a numerical value without units." in prompt + assert "an expression" not in prompt diff --git a/tests/unit/core/utils/test_offload.py b/tests/unit/core/utils/test_offload.py new file mode 100644 index 00000000..08e3f4b8 --- /dev/null +++ b/tests/unit/core/utils/test_offload.py @@ -0,0 +1,483 @@ +"""Unit tests for the CPU-bound offload pool. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import os +import time + +import pytest + +from sieval.core.utils import offload + + +@pytest.fixture +def pool_spy(monkeypatch): + """Capture the kwargs the pool is constructed with. + + Asserting on the construction rather than on the executor's private + attributes: `_mp_context` / `_max_workers` are CPython internals, and a test + that reads them is one stdlib refactor away from breaking. + """ + captured: dict = {} + + class _Spy: + def __init__(self, **kwargs): + captured.update(kwargs) + + def shutdown(self, **kwargs): + pass + + monkeypatch.setattr(offload, "ProcessPoolExecutor", _Spy) + return captured + + +@pytest.fixture +def warnings_sink(): + """Collect loguru WARNINGs — this module logs through loguru, not stdlib.""" + from loguru import logger + + collected: list[str] = [] + sink_id = logger.add(collected.append, format="{message}", level="WARNING") + try: + yield collected + finally: + logger.remove(sink_id) + + +@pytest.fixture(autouse=True) +def _reset_pool(): + """Each test starts from a cold module, and leaves no pool behind.""" + offload.shutdown() + offload._pool_failed = False + yield + offload.shutdown() + offload._pool_failed = False + + +# Module-level so the worker can import it by name; a closure would not pickle. +def _double(value: int) -> int: + return value * 2 + + +def _boom() -> None: + raise ValueError("worker raised") + + +def _sleep_forever() -> None: + time.sleep(30) + + +def _sleep(seconds: float) -> float: + time.sleep(seconds) + return seconds + + +@pytest.mark.anyio +async def test_runs_the_function_and_returns_its_value(): + assert await offload.run_cpu_bound(_double, 21) == 42 + + +@pytest.mark.anyio +async def test_worker_exceptions_propagate_to_the_caller(): + # A grader relies on its own try/except; swallowing here would turn a real + # error into a silent wrong verdict. + with pytest.raises(ValueError, match="worker raised"): + await offload.run_cpu_bound(_boom) + + +#: mutmut instruments the module under test, and the trampoline it injects +#: imports `mutmut.__main__` on its first hit. In a *spawned* worker that module +#: is not yet in `sys.modules`, so the import re-executes it, and its top-level +#: `set_start_method('fork')` raises `RuntimeError: context has already been set` +#: — spawn fixes the start method before user code runs. The worker dies, the +#: pool reports itself broken, and `run_cpu_bound` correctly falls back to +#: running inline. Every other test still passes through that fallback (inline +#: returns the same answer), which is why only the timeout assertion notices. +_UNDER_MUTMUT = "MUTANT_UNDER_TEST" in os.environ + + +@pytest.mark.anyio +@pytest.mark.skipif( + _UNDER_MUTMUT, + reason="mutmut cannot instrument a spawned worker; the pool degrades to " + "inline, which by design cannot time out", +) +async def test_timeout_raises_rather_than_returning_a_wrong_answer(): + with pytest.raises(TimeoutError): + await offload.run_cpu_bound(_sleep_forever, timeout=0.5) + + +@pytest.mark.anyio +async def test_runs_inline_when_the_pool_is_disabled(monkeypatch): + # The documented escape hatch for a sandbox that cannot spawn. Behaviour + # must stay correct, only slower. + monkeypatch.setenv(offload._ENV_WORKERS, "0") + offload.shutdown() + offload._pool_failed = False + assert await offload.run_cpu_bound(_double, 5) == 10 + assert offload._get_pool() is None + + +@pytest.mark.anyio +async def test_falls_back_inline_when_the_pool_cannot_start(monkeypatch): + def _explode(*_args, **_kwargs): + raise OSError("no processes here") + + monkeypatch.setattr(offload, "ProcessPoolExecutor", _explode) + assert await offload.run_cpu_bound(_double, 8) == 16 + + +@pytest.mark.anyio +async def test_a_broken_pool_degrades_instead_of_failing_the_run(monkeypatch): + from concurrent.futures import BrokenExecutor + + class _BrokenPool: + def submit(self, *_args, **_kwargs): + raise BrokenExecutor("worker died") + + def shutdown(self, **_kwargs): + pass + + monkeypatch.setattr(offload, "_pool", _BrokenPool()) + assert await offload.run_cpu_bound(_double, 3) == 6 + # And it does not keep retrying into the broken pool. + assert offload._pool_failed is True + + +@pytest.mark.anyio +async def test_a_pool_that_dies_mid_run_is_not_retried_per_sample(monkeypatch): + # The sibling above pins the *flag*, which is not the same thing: `_get_pool` + # returns `_pool` whenever it is set, so setting the flag alone stops the + # pool being rebuilt and never stops the dead one being handed back. Every + # later sample then pays another failed `submit` before falling back — which + # is not what the module's own warning ("for the rest of this run") says. + from concurrent.futures import BrokenExecutor + + attempts = {"n": 0} + + class _DyingPool: + def submit(self, *_args, **_kwargs): + attempts["n"] += 1 + raise BrokenExecutor("worker died") + + def shutdown(self, **_kwargs): + pass + + monkeypatch.setattr(offload, "_pool", _DyingPool()) + for value in range(4): + assert await offload.run_cpu_bound(_double, value) == value * 2 + assert attempts["n"] == 1, "the dead pool must be dropped, not re-submitted to" + assert offload._pool is None + assert offload._get_pool() is None + + +def test_a_failed_limiter_does_not_leave_a_live_pool_behind(monkeypatch): + # The limiter is what makes `timeout` mean "one grade" rather than "grade + # plus however long the queue in front of it is". A pool that outlived a + # failed limiter would still be handed out (the guard returns `_pool` + # whenever it is set) and would run against anyio's shared 40-token default, + # silently undoing that bound — the failure mode being a timeout that fires + # on a healthy sample, which reads as a bad model answer. + shutdowns = {"n": 0} + + class _Pool: + def __init__(self, **_kwargs): + pass + + def shutdown(self, **_kwargs): + shutdowns["n"] += 1 + + def _no_limiter(*_args, **_kwargs): + raise RuntimeError("no async backend here") + + monkeypatch.setattr(offload, "ProcessPoolExecutor", _Pool) + monkeypatch.setattr(offload.anyio, "CapacityLimiter", _no_limiter) + + assert offload._get_pool() is None + assert offload._pool is None + assert offload._limiter is None + assert offload._pool_failed is True + assert shutdowns["n"] == 1, "the half-built pool must not be leaked" + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "failure", + [ + RuntimeError("cannot schedule new futures after shutdown"), + OSError(12, "Cannot allocate memory"), + PermissionError(1, "Operation not permitted"), + MemoryError(), + ], + ids=["shutting-down", "enomem", "clone-blocked", "oom"], +) +async def test_any_submit_failure_degrades_rather_than_failing_the_run( + monkeypatch, failure +): + # The module's contract is "degrades rather than fails", and a submit can + # fail for more reasons than the pool being broken: a worker that cannot be + # started surfaces as OSError (ENOMEM, RLIMIT_NPROC) or PermissionError (a + # seccomp-blocked `clone`), neither of which is a BrokenExecutor. + # + # Letting one escape is not a loud failure: callers wrap grading in a broad + # `except` and score the sample wrong, so a whole run silently reports zero. + class _FailingPool: + def submit(self, *_args, **_kwargs): + raise failure + + def shutdown(self, **_kwargs): + pass + + monkeypatch.setattr(offload, "_pool", _FailingPool()) + assert await offload.run_cpu_bound(_double, 3) == 6 + assert offload._pool_failed is True + + +def test_the_pool_is_spawned_never_forked(pool_spy): + # The load-bearing choice in this module: the parent is an async process with + # live worker threads, and forking one can inherit a held lock and deadlock + # the child. A mutation to "fork" fails no behavioural test — it just + # occasionally hangs a run — so it is asserted at construction. + offload._get_pool() + assert pool_spy["mp_context"].get_start_method() == "spawn" + + +def test_the_pool_is_built_once_and_reused(monkeypatch): + # Rebuilding per call would pay spawn + sympy import on every grade, which is + # the cost this module exists to avoid. + monkeypatch.setenv(offload._ENV_WORKERS, "2") + first = offload._get_pool() + assert first is not None + assert offload._get_pool() is first + + +def test_the_pool_is_sized_by_the_worker_count(monkeypatch, pool_spy): + monkeypatch.setenv(offload._ENV_WORKERS, "3") + offload._get_pool() + assert pool_spy["max_workers"] == 3 + + +def test_a_disabled_pool_is_not_reconsidered(monkeypatch): + # Once disabled it must stay disabled for the run, or every sample pays the + # decision again. + monkeypatch.setenv(offload._ENV_WORKERS, "0") + assert offload._get_pool() is None + assert offload._pool_failed is True + # Even with the env var flipped back, the run does not silently change mode. + monkeypatch.setenv(offload._ENV_WORKERS, "4") + assert offload._get_pool() is None + + +def test_a_failed_pool_start_is_not_retried_per_sample(monkeypatch): + starts = {"n": 0} + + def _explode(*_args, **_kwargs): + starts["n"] += 1 + raise OSError("no processes here") + + monkeypatch.setattr(offload, "ProcessPoolExecutor", _explode) + assert offload._get_pool() is None + assert offload._get_pool() is None + assert starts["n"] == 1, "a failed start must not be attempted again" + + +def test_worker_count_honours_the_env_var(monkeypatch): + monkeypatch.setenv(offload._ENV_WORKERS, "3") + assert offload._worker_count() == 3 + + +def test_worker_count_ignores_a_non_integer(monkeypatch, warnings_sink): + monkeypatch.setenv(offload._ENV_WORKERS, "many") + assert offload._worker_count() >= 1 + # Silently falling back would hide a typo'd override for the whole run, so + # the warning has to quote what was actually set. + text = " ".join(warnings_sink) + assert offload._ENV_WORKERS in text + assert "many" in text + + +def test_worker_count_clamps_a_negative_request_to_disabled(monkeypatch): + # Negative is nonsense, and 0 is the documented "run inline" value, so it + # floors there rather than becoming a huge pool via a sign error. + monkeypatch.setenv(offload._ENV_WORKERS, "-5") + assert offload._worker_count() == 0 + + +def test_worker_count_caps_the_default_on_a_big_machine(monkeypatch): + # Grading is CPU-bound but the pool is shared across a whole session; more + # than 8 workers buys nothing and costs one interpreter each. + monkeypatch.delenv(offload._ENV_WORKERS, raising=False) + monkeypatch.setattr(offload.os, "cpu_count", lambda: 64) + assert offload._worker_count() == 8 + + +def test_worker_count_keeps_one_worker_on_a_single_core_box(monkeypatch): + # `cpu_count - 1` is 0 here; a pool of 0 workers is not constructible. + monkeypatch.delenv(offload._ENV_WORKERS, raising=False) + monkeypatch.setattr(offload.os, "cpu_count", lambda: 1) + assert offload._worker_count() == 1 + + +def test_worker_count_survives_an_unknown_cpu_count(monkeypatch): + # os.cpu_count() may return None; the fallback must still be constructible. + monkeypatch.delenv(offload._ENV_WORKERS, raising=False) + monkeypatch.setattr(offload.os, "cpu_count", lambda: None) + assert offload._worker_count() == 1 + + +def test_worker_count_leaves_room_for_the_event_loop(monkeypatch): + # One core is deliberately left to the loop that is dispatching the work. + monkeypatch.delenv(offload._ENV_WORKERS, raising=False) + monkeypatch.setattr(offload.os, "cpu_count", lambda: 4) + assert offload._worker_count() == 3 + + +def test_shutdown_is_idempotent(): + offload.shutdown() + offload.shutdown() + + +def test_shutdown_releases_the_pool_without_waiting(monkeypatch): + # Waiting would hang a run on exactly the sample that already misbehaved, + # and leaving `_pool` set would hand out a shut-down executor afterwards. + calls = {} + + class _Pool: + def shutdown(self, **kwargs): + calls.update(kwargs) + + monkeypatch.setattr(offload, "_pool", _Pool()) + offload.shutdown() + + assert calls == {"wait": False, "cancel_futures": True} + assert offload._pool is None + + +def test_a_broken_pool_is_reported_once_not_per_sample(warnings_sink): + # `_mark_unusable` fires on a path taken by every subsequent sample; warning + # each time would bury the run's real output. + offload._pool_failed = False + offload._mark_unusable(RuntimeError("first")) + offload._mark_unusable(RuntimeError("second")) + + warnings = [m for m in warnings_sink if "unusable" in m] + assert len(warnings) == 1 + assert offload._pool_failed is True + # The message is the operator's only signal that grading silently went back + # on the event loop, so it has to name the cause and the consequence. + assert "first" in warnings[0], "the warning must name the exception" + assert "event loop" in warnings[0] + assert "second" not in warnings[0] + + +def test_the_limiter_is_sized_to_the_pool_not_to_the_sample_concurrency(monkeypatch): + # Sized to the pool because it exists to bound the *queue* a caller can sit + # behind, and the queue drains at the worker count. Sizing it to the sample + # concurrency instead would put the backlog back on `timeout`'s clock. + monkeypatch.setenv(offload._ENV_WORKERS, "3") + offload._get_pool() + assert offload._limiter is not None + assert offload._limiter.total_tokens == 3 + offload._QUEUE_SLACK + + +def test_shutdown_drops_the_limiter_with_the_pool(monkeypatch): + # A limiter outliving its pool would size the next pool's admissions to the + # previous pool's worker count. + monkeypatch.setenv(offload._ENV_WORKERS, "2") + offload._get_pool() + assert offload._limiter is not None + offload.shutdown() + assert offload._limiter is None + + +@pytest.mark.anyio +@pytest.mark.skipif( + _UNDER_MUTMUT, + reason="mutmut cannot instrument a spawned worker; the pool degrades to " + "inline, which borrows no thread tokens at all", +) +async def test_grading_does_not_draw_on_the_shared_thread_tokens(monkeypatch): + # anyio's default limiter is one 40-token budget for the whole process, and + # the loader, the deployer and scicode's reads all draw on it. Waiting for a + # worker there would let grading starve unrelated I/O in the same session. + import anyio + import anyio.to_thread + + monkeypatch.setenv(offload._ENV_WORKERS, "2") + default = anyio.to_thread.current_default_thread_limiter() + baseline = default.borrowed_tokens + peak = 0 + + async def _one(): + nonlocal peak + await offload.run_cpu_bound(_sleep, 0.2, timeout=30.0) + peak = max(peak, default.borrowed_tokens - baseline) + + async with anyio.create_task_group() as task_group: + for _ in range(12): + task_group.start_soon(_one) + + assert peak == 0, f"grading borrowed {peak} of the shared thread tokens" + + +@pytest.mark.anyio +@pytest.mark.skipif( + _UNDER_MUTMUT, + reason="mutmut cannot instrument a spawned worker; the pool degrades to " + "inline, which by design cannot time out", +) +async def test_a_backlog_does_not_spend_the_callers_timeout(): + # `future.result(timeout)` counts from when the caller starts waiting, so it + # waits out everything queued ahead of it too. Without admission control a + # perfectly fast grade "times out" purely because the session was busy — + # and the caller cannot tell that apart from a genuinely hung one. + import anyio + + job, jobs, timeout = 0.25, 8, 1.5 + assert job * jobs > timeout, "the backlog must outlast the per-call ceiling" + + os.environ[offload._ENV_WORKERS] = "1" + try: + pool = offload._get_pool() + assert pool is not None + # Warm the workers: ProcessPoolExecutor spawns them on first submit, and + # that one-off cost is charged to whichever caller happens to be first. + await offload.run_cpu_bound(_sleep, 0.0, timeout=60.0) + + results: list[str] = [] + + async def _one(): + try: + await offload.run_cpu_bound(_sleep, job, timeout=timeout) + results.append("ok") + except TimeoutError: + results.append("timeout") + + async with anyio.create_task_group() as task_group: + for _ in range(jobs): + task_group.start_soon(_one) + finally: + os.environ.pop(offload._ENV_WORKERS, None) + + assert results.count("timeout") == 0, ( + f"{results.count('timeout')}/{jobs} jobs of {job}s hit a {timeout}s " + "ceiling; the backlog is being charged to the caller" + ) + + +@pytest.mark.anyio +async def test_math_verify_still_works_in_the_worker(): + """The whole reason this is a process and not a thread. + + In a worker *thread* math-verify raises outright, callers swallow it, and + these verdicts silently flip to False. + """ + from sieval.community.ugmathbench import judge_answers + + for pred, gold, kind in [ + (r"\frac{1}{2}", "0.5", "NV"), + (r"\frac{\pi}{4}", "pi/4", "EX"), + ]: + got = await offload.run_cpu_bound(judge_answers, [pred], [gold], [kind]) + assert got == [True], f"{pred} vs {gold} should still grade correct" diff --git a/tests/unit/datasets/test_ugmathbench.py b/tests/unit/datasets/test_ugmathbench.py new file mode 100644 index 00000000..a73f08c1 --- /dev/null +++ b/tests/unit/datasets/test_ugmathbench.py @@ -0,0 +1,104 @@ +"""Unit tests for the UGMathBench loader: version unpacking and subject selection. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest +from datasets import Dataset as HFDataset + +from sieval.datasets import ugmathbench as module +from sieval.datasets.ugmathbench import UGMathBenchDataset, _unpack_versions + + +def _packed_row(problem_id: str = "Algebra_0001") -> dict: + return { + "id": problem_id, + "subject": "Algebra", + "topic": "Linear equations", + "subtopic": "Solving", + "level": "2", + "keywords": ["algebra"], + "problem_v1": "v1 text", + "answer_v1": ["1"], + "answer_type_v1": ["NV"], + "options_v1": [[]], + "problem_v2": "v2 text", + "answer_v2": ["2"], + "answer_type_v2": ["NV"], + "options_v2": [[]], + "problem_v3": "v3 text", + "answer_v3": ["A"], + "answer_type_v3": ["MCS"], + "options_v3": [["A", "B"]], + } + + +def test_unpacks_one_sample_per_randomized_version(): + samples = _unpack_versions(_packed_row()) + + assert [s["version"] for s in samples] == [1, 2, 3] + assert [s["problem"] for s in samples] == ["v1 text", "v2 text", "v3 text"] + assert [s["answer"] for s in samples] == [["1"], ["2"], ["A"]] + assert samples[2]["answer_type"] == ["MCS"] + assert samples[2]["options"] == [["A", "B"]] + # Shared metadata rides along on every version. + assert {s["id"] for s in samples} == {"Algebra_0001"} + assert {s["subject"] for s in samples} == {"Algebra"} + + +def test_load_is_problem_major_so_slicing_keeps_whole_problems(monkeypatch): + rows = [_packed_row("Algebra_0001"), _packed_row("Algebra_0002")] + monkeypatch.setattr( + module, "load_dataset", lambda *a, **kw: HFDataset.from_list(rows) + ) + + dataset = UGMathBenchDataset("ignored", subjects=["Algebra"]) + test_set = dataset.test_set + + assert test_set is not None + assert len(test_set) == 6 + # A problem's three versions are adjacent, so slice(3) keeps one whole + # problem rather than one version of three different problems. + assert [(r["id"], r["version"]) for r in test_set][:4] == [ + ("Algebra_0001", 1), + ("Algebra_0001", 2), + ("Algebra_0001", 3), + ("Algebra_0002", 1), + ] + + +def test_unknown_subject_is_rejected_with_the_valid_list(monkeypatch): + monkeypatch.setattr( + module, "load_dataset", lambda *a, **kw: HFDataset.from_list([_packed_row()]) + ) + with pytest.raises(ValueError, match="Unknown UGMathBench subject"): + UGMathBenchDataset("ignored", subjects=["Astrology"]) + + +def test_explicitly_empty_subjects_is_rejected_not_read_as_all(monkeypatch): + # `[]` is falsy, so a truthiness check would quietly load all 16 instead. + loaded: list[str] = [] + monkeypatch.setattr( + module, + "load_dataset", + lambda _p, config, **_kw: ( + loaded.append(config) or HFDataset.from_list([_packed_row()]) + ), + ) + with pytest.raises(ValueError, match="`subjects` is empty"): + UGMathBenchDataset("ignored", subjects=[]) + assert loaded == [] + + +def test_all_sixteen_subjects_load_by_default(monkeypatch): + requested: list[str] = [] + + def _fake_load(_path, config, **_kwargs): + requested.append(config) + return HFDataset.from_list([_packed_row()]) + + monkeypatch.setattr(module, "load_dataset", _fake_load) + + UGMathBenchDataset("ignored") + assert requested == list(UGMathBenchDataset.SUBJECTS) + assert len(requested) == 16 diff --git a/tests/unit/scripts/test_check_preflight.py b/tests/unit/scripts/test_check_preflight.py index a8bbe0d1..963d7762 100644 --- a/tests/unit/scripts/test_check_preflight.py +++ b/tests/unit/scripts/test_check_preflight.py @@ -127,13 +127,14 @@ class TestPreflightRunner: """Runner orchestration.""" def test_all_checks_listed(self): - assert len(PreflightRunner.ALL_CHECKS) == 11 + assert len(PreflightRunner.ALL_CHECKS) == 12 assert "check_links" in PreflightRunner.ALL_CHECKS assert "check_examples" in PreflightRunner.ALL_CHECKS assert "check_meta_index_sync" in PreflightRunner.ALL_CHECKS assert "check_version" in PreflightRunner.ALL_CHECKS assert "check_task_shot_knobs" in PreflightRunner.ALL_CHECKS assert "check_record_key_access" in PreflightRunner.ALL_CHECKS + assert "check_mutmut_config" in PreflightRunner.ALL_CHECKS def test_run_all_returns_results(self): runner = PreflightRunner() @@ -1184,17 +1185,44 @@ class TestTaskFileNamingPattern: def test_accepts_valid_suffixes(self, name): assert _TASK_FILE_PATTERN.match(name) is not None + @pytest.mark.parametrize( + "name", + [ + "foo_0shot_gen_fixed.py", + "foo_0shot_base_gen_fixed.py", + "foo_kshot_ppl_fixed_v2.py", + ], + ) + def test_accepts_a_variant_after_the_mode(self, name): + assert _TASK_FILE_PATTERN.match(name) is not None + @pytest.mark.parametrize( "name", [ "foo_clp.py", # missing shot segment - "foo_5shot_clp_extra.py", # trailing junk "foo_5shot_clpx.py", # not a known mode + "foo_0shot_gen_.py", # empty variant + "foo_0shot_gen_Fixed.py", # variant is not lower-case ], ) def test_rejects_malformed(self, name): assert _TASK_FILE_PATTERN.match(name) is None + @pytest.mark.parametrize( + "name", + [ + # The example both CLAUDE.md and rules/tasks.md name as canonical. + "foo_0shot_clp_gen.py", + "foo_0shot_gen_gen.py", + "foo_5shot_clp_ppl.py", + "foo_0shot_gen_base_gen.py", + ], + ) + def test_rejects_a_variant_that_spells_a_mode(self, name): + # Two readings (mode `gen` + variant `ppl`, or mode `ppl` misplaced), so + # the name is rejected rather than settled by alternation precedence. + assert _TASK_FILE_PATTERN.match(name) is None + class TestCheckDatasets: """Integration tests for check_datasets — registry, imports, naming.""" @@ -1225,6 +1253,13 @@ def test_live_repo_index_is_in_sync(self): drift locally before anyone pushes. """ runner = PreflightRunner() + if runner.project_root.name == "mutants": + # mutmut runs the suite from a copy of the tree, where neither side + # of this comparison is the thing it is about: the "committed" index + # is a copy and the registry is importable only by accident. It is + # the last test standing between `mutmut run` and a score, and + # skipping it costs nothing — CI runs this check on the real tree. + pytest.skip("meaningless inside a mutmut copy of the tree") results = runner.check_meta_index_sync() assert len(results) == 1 assert results[0].status == "PASS", results[0].details @@ -2385,3 +2420,69 @@ def test_real_modules_pass(self): """ results = PreflightRunner().check_record_key_access() assert [r.status for r in results] == ["PASS"] + + +class TestCheckMutmutConfig: + """`[tool.mutmut]` must still yield an importable `mutants/` copy.""" + + def _write(self, tmp_path, body: str) -> PreflightRunner: + (tmp_path / "pyproject.toml").write_text(body, encoding="utf-8") + return PreflightRunner(project_root=tmp_path) + + def test_missing_package_root_fails(self, tmp_path): + # The first regression: also_copy lists the subpackages but not + # sieval/__init__.py, so mutants/sieval is a directory rather than a + # package and every mutation run dies during stats collection. + runner = self._write( + tmp_path, + '[tool.mutmut]\npaths_to_mutate = ["sieval/core"]\n' + 'also_copy = ["sieval/tasks", "scripts"]\n', + ) + results = runner.check_mutmut_config() + assert [r.status for r in results] == ["FAIL"] + assert any("sieval/__init__.py" in d for d in results[0].details) + + def test_missing_scripts_fails(self, tmp_path): + # The second, found the same way: `scripts/` is not a package, so + # tests/unit/scripts/ puts it on sys.path by walking up from __file__. + # In the copy that is mutants/scripts, which without this entry does not + # exist -- and the run dies on `No module named 'check_layer_imports'`, + # which reads as a broken test rather than a missing copy path. + runner = self._write( + tmp_path, + '[tool.mutmut]\npaths_to_mutate = ["sieval/core"]\n' + 'also_copy = ["sieval/__init__.py", "sieval/tasks"]\n', + ) + results = runner.check_mutmut_config() + assert [r.status for r in results] == ["FAIL"] + assert any("scripts" in d for d in results[0].details) + + def test_every_required_path_present_passes(self, tmp_path): + runner = self._write( + tmp_path, + '[tool.mutmut]\npaths_to_mutate = ["sieval/core"]\n' + 'also_copy = ["sieval/__init__.py", "scripts", "sieval/tasks"]\n', + ) + assert [r.status for r in runner.check_mutmut_config()] == ["PASS"] + + def test_a_parent_entry_carries_the_package_root(self, tmp_path): + # Copying "sieval" wholesale brings __init__.py with it. + runner = self._write( + tmp_path, + '[tool.mutmut]\npaths_to_mutate = ["sieval"]\nalso_copy = ["scripts"]\n', + ) + assert [r.status for r in runner.check_mutmut_config()] == ["PASS"] + + def test_no_mutmut_section_is_not_a_failure(self, tmp_path): + runner = self._write(tmp_path, "[project]\nname = 'x'\n") + assert [r.status for r in runner.check_mutmut_config()] == ["PASS"] + + def test_missing_pyproject_fails(self, tmp_path): + runner = PreflightRunner(project_root=tmp_path) + assert [r.status for r in runner.check_mutmut_config()] == ["FAIL"] + + def test_the_repo_config_is_registered_and_passes(self): + # Registration is the half that makes it run in CI; without it the + # function is dead code that reports nothing. + assert "check_mutmut_config" in PreflightRunner.ALL_CHECKS + assert [r.status for r in PreflightRunner().check_mutmut_config()] == ["PASS"] diff --git a/tests/unit/tasks/test_gsm8k_0shot_gen.py b/tests/unit/tasks/test_gsm8k_0shot_gen.py index 248ef9cd..6d14c0a8 100644 --- a/tests/unit/tasks/test_gsm8k_0shot_gen.py +++ b/tests/unit/tasks/test_gsm8k_0shot_gen.py @@ -7,6 +7,7 @@ from datasets import Dataset as HFDataset from datasets import DatasetDict as HFDatasetDict +from sieval.community.deepseek_math import is_correct from sieval.core.models import ModelOutput from sieval.core.models.chat_model import ChatModel from sieval.core.tasks import ( @@ -14,7 +15,9 @@ build_judgement_record, build_rollout_judgement, ) +from sieval.core.utils.offload import GRADE_TIMEOUT from sieval.datasets.gsm8k import GSM8KDataset, GSM8KDatasetSample +from sieval.tasks import gsm8k_0shot_gen as module from sieval.tasks.gsm8k_0shot_gen import ( COT_INSTRUCTION, GSM8KZeroShotGenTask, @@ -140,6 +143,61 @@ async def test_feedback_wrong_answer(): assert fb["rollouts"][0]["correct"] is False +@pytest.mark.anyio +async def test_grading_is_bounded_in_a_worker_process(monkeypatch): + """The mechanism, not the verdict — a thread offload scores identically, so + reverting to `anyio.to_thread.run_sync` keeps every other test in this file + passing. Why a process: criterion 2 in `core/utils/offload.py`. + """ + seen: dict[str, object] = {} + + async def _spy(func, *args, timeout=None): + seen.update(func=func, args=args, timeout=timeout) + return func(*args) + + monkeypatch.setattr(module, "run_cpu_bound", _spy) + + task, model = _task("x") + raw = _sample(answer="Work.\n#### 42") + inf = ModelOutput(model=model.meta(), texts=["The answer is $\\boxed{42}$."]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inf) + post = await task.postprocess(inf, ctx) + _, fb = await task.feedback(post, ctx) + + assert seen["func"] is is_correct + assert seen["args"] == ({"prediction": "42", "answer": "42"},) + assert seen["timeout"] == GRADE_TIMEOUT + assert fb["rollouts"][0]["correct"] is True + + +@pytest.mark.anyio +async def test_a_grading_timeout_scores_wrong_rather_than_failing_the_sample( + monkeypatch, +): + # Offloading introduced a failure mode the synchronous call did not have: + # before, a runaway `simplify` blocked: now it raises at GRADE_TIMEOUT. Left + # to propagate, the runner turns it into a failed sample, so a slow grade + # shows up as `fails > 0` -- which reads as infrastructure breakage and is + # one of the signals a run is promoted on. Every sibling math grader scores + # an ungradeable answer wrong instead; this one has to agree. + async def _raise_timeout(_func, *_args, **_kwargs): + raise TimeoutError("grading took too long") + + monkeypatch.setattr(module, "run_cpu_bound", _raise_timeout) + + task, model = _task("x") + raw = _sample(answer="Work.\n#### 42") + inf = ModelOutput(model=model.meta(), texts=["The answer is $\\boxed{42}$."]) + ctx = TaskContext(sample_id=0, raw_sample=raw, infer_result=inf) + post = await task.postprocess(inf, ctx) + + finalize, fb = await task.feedback(post, ctx) + + assert finalize is True + assert fb["rollouts"][0]["correct"] is False + assert fb["reference"] == "42" + + # --- report accuracy + infer injects no decode params --- diff --git a/tests/unit/tasks/test_hendrycks_math_kshot_base_gen.py b/tests/unit/tasks/test_hendrycks_math_kshot_base_gen.py index 63f216d0..d22d0155 100644 --- a/tests/unit/tasks/test_hendrycks_math_kshot_base_gen.py +++ b/tests/unit/tasks/test_hendrycks_math_kshot_base_gen.py @@ -7,6 +7,7 @@ from datasets import Dataset as HFDataset from datasets import DatasetDict as HFDatasetDict +from sieval.community.deepseek_math import eval_math from sieval.core.models import ModelOutput from sieval.core.models.gen_model import GenModel from sieval.core.tasks import ( @@ -15,10 +16,12 @@ build_prediction_record, build_rollout_judgement, ) +from sieval.core.utils.offload import GRADE_TIMEOUT from sieval.datasets.hendrycks_math import ( HendrycksMathDataset, HendrycksMathDatasetSample, ) +from sieval.tasks import hendrycks_math_kshot_base_gen as module from sieval.tasks.hendrycks_math_kshot_base_gen import ( N_SHOT, HendrycksMathFewShotBaseGenTask, @@ -140,6 +143,59 @@ async def test_feedback_scores_against_solution_via_eval_math(): assert wrong_fb["rollouts"][0]["correct"] is False +@pytest.mark.anyio +async def test_grading_is_bounded_in_a_worker_process(monkeypatch): + """The mechanism, not the verdict — a thread offload scores identically, so + reverting to `anyio.to_thread.run_sync` keeps every other test in this file + passing. Why a process: criterion 2 in `core/utils/offload.py`. + """ + seen: dict[str, object] = {} + + async def _spy(func, *args, timeout=None): + seen.update(func=func, args=args, timeout=timeout) + return func(*args) + + monkeypatch.setattr(module, "run_cpu_bound", _spy) + + task, _ = _task() + raw = _sample(solution="Therefore $\\boxed{16}$.") + _, fb = await task.feedback( + build_prediction_record([["16"]]), TaskContext(sample_id=0, raw_sample=raw) + ) + + assert seen["func"] is eval_math + assert seen["args"] == ({"prediction": ["16"], "answer": ["16"]},) + assert seen["timeout"] == GRADE_TIMEOUT + assert fb["rollouts"][0]["correct"] is True + + +@pytest.mark.anyio +async def test_a_grading_timeout_scores_wrong_rather_than_failing_the_sample( + monkeypatch, +): + # Offloading introduced a failure mode the synchronous call did not have: + # before, a runaway `simplify` blocked; now it raises at GRADE_TIMEOUT. Left + # to propagate, the runner turns it into a failed sample, so a slow grade + # shows up as `fails > 0` -- which reads as infrastructure breakage and is + # one of the signals a run is promoted on. Every sibling math grader scores + # an ungradeable answer wrong instead; this one has to agree. + async def _raise_timeout(_func, *_args, **_kwargs): + raise TimeoutError("grading took too long") + + monkeypatch.setattr(module, "run_cpu_bound", _raise_timeout) + + task, _ = _task() + raw = _sample(solution="Therefore $\\boxed{16}$.") + + finalize, fb = await task.feedback( + build_prediction_record([["16"]]), TaskContext(sample_id=0, raw_sample=raw) + ) + + assert finalize is True + assert fb["rollouts"][0]["correct"] is False + assert fb["reference"] == ["16"] + + @pytest.mark.anyio async def test_feedback_percentage_equivalence(): # math_equal's numeric layer treats 50\% as 0.5 (include_percentage), diff --git a/tests/unit/tasks/test_ugmathbench_0shot_gen_fixed.py b/tests/unit/tasks/test_ugmathbench_0shot_gen_fixed.py new file mode 100644 index 00000000..d6f31dda --- /dev/null +++ b/tests/unit/tasks/test_ugmathbench_0shot_gen_fixed.py @@ -0,0 +1,360 @@ +"""Unit tests for the corrected UGMathBench task: stage plumbing and metrics. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import subprocess +import sys + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import ModelMeta, ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + TaskContext, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, +) +from sieval.datasets.ugmathbench import UGMathBenchDataset +from sieval.tasks.ugmathbench_0shot_gen_fixed import UGMathBenchZeroShotGenFixedTask + + +def _sample( + problem_id: str = "Algebra_0001", + version: int = 1, + answer: list[str] | None = None, + answer_type: list[str] | None = None, +) -> dict: + answers = answer if answer is not None else ["4"] + return { + "id": problem_id, + "subject": "Algebra", + "topic": "Linear equations", + "subtopic": "Solving", + "level": "2", + "keywords": ["algebra"], + "version": version, + "problem": "Solve $x+1=5$. [ANS]", + "answer": answers, + "answer_type": answer_type or ["NV"] * len(answers), + "options": [[] for _ in answers], + } + + +def _inferred(*texts: str) -> ModelOutput: + meta: ModelMeta = {"model": "mock-chat", "api_base": None, "default_params": {}} + return ModelOutput(model=meta, texts=list(texts)) + + +def _task(precision: float = 1e-3) -> UGMathBenchZeroShotGenFixedTask: + sample = _sample() + dataset = UGMathBenchDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([sample])}) + ) + model = ChatModel(model="mock-chat", api_key="fake") + return UGMathBenchZeroShotGenFixedTask(dataset, model, precision=precision) + + +def _judged( + problem_id: str, version: int, correct: bool, subject: str = "Algebra" +) -> TaskContext: + return TaskContext( + sample_id=f"{problem_id}-v{version}", + feedback_result=build_judgement_record( + ["1"], + [build_rollout_judgement(0, correct)], + extra={"problem_id": problem_id, "version": version, "subject": subject}, + ), + ).to_final() + + +def _all_versions( + problem_id: str, verdicts: list[bool], subject: str = "Algebra" +) -> list[TaskContext]: + return [ + _judged(problem_id, version, correct, subject) + for version, correct in enumerate(verdicts, start=1) + ] + + +def _failed(problem_id: str, version: int) -> TaskContext: + """A sample that died before feedback but still knows which problem it is.""" + return TaskContext( + sample_id=f"{problem_id}-v{version}", + raw_sample=_sample(problem_id, version), + ).to_failed(None, "error", "boom") + + +def test_precision_must_be_positive(): + with pytest.raises(ValueError, match="precision must be > 0"): + _task(precision=0) + + +@pytest.mark.anyio +async def test_preprocess_builds_the_benchmark_prompt_and_carries_grouping_keys(): + raw = _sample(version=2) + record = await _task().preprocess(raw, TaskContext(sample_id=0, raw_sample=raw)) + + assert record["prompt"][0]["role"] == "user" + content = record["prompt"][0]["content"] + assert "undergraduate-level mathematical problem in Algebra" in content + assert "The final answer is \\boxed{ANSWER}" in content + assert record["reference"] == ["4"] + assert record["extra"]["problem_id"] == "Algebra_0001" + assert record["extra"]["version"] == 2 + + +@pytest.mark.anyio +async def test_postprocess_records_one_prediction_per_answer_slot(): + record = await _task().postprocess( + _inferred("The final answers are \\boxed{1, 2}"), TaskContext(sample_id=0) + ) + assert record["rollouts"][0]["prediction"] == ["1", "2"] + assert record["rollouts"][0]["extracted"] is True + + +@pytest.mark.anyio +async def test_postprocess_marks_an_unboxed_response_as_not_extracted(): + record = await _task().postprocess( + _inferred("I could not solve it."), TaskContext(sample_id=0) + ) + assert record["rollouts"][0]["extracted"] is False + assert record["rollouts"][0].get("prediction") is None + + +@pytest.mark.anyio +async def test_feedback_grades_every_slot_and_records_grouping_keys(): + raw = _sample(answer=["1", "2"], answer_type=["NV", "NV"]) + post = build_prediction_record([["1", "3"]]) + final, judgement = await _task().feedback( + post, TaskContext(sample_id=0, raw_sample=raw) + ) + + assert final is True + assert judgement["n_correct"] == 0 # one slot wrong -> the sample is wrong + rollout = judgement["rollouts"][0] + assert rollout["extra"]["per_answer"] == [True, False] + assert rollout["metrics"]["answer_accuracy"] == 0.5 + assert judgement["extra"]["problem_id"] == "Algebra_0001" + + +@pytest.mark.anyio +async def test_feedback_without_a_raw_sample_is_wrong_not_a_crash(): + final, judgement = await _task().feedback( + build_prediction_record([["1"]]), TaskContext(sample_id=0, raw_sample=None) + ) + assert final is True + assert judgement["n_correct"] == 0 + + +@pytest.mark.anyio +async def test_effective_accuracy_needs_every_version(): + finals = [ + *_all_versions("p1", [True, True, True]), + *_all_versions("p2", [True, True, False]), + ] + report = await _task().report(finals, []) + + assert report["n_problems"] == 2 + assert report["eacc"] == 50.0 # only p1 is correct in all three versions + assert report["aacc"] == pytest.approx(500 / 6) # 5 of 6 versions + assert report["cacc"] == 100.0 # both are right at least once + assert report["delta"] == pytest.approx(report["aacc"] - report["eacc"]) + assert report["relative_delta"] == pytest.approx( + (report["aacc"] - report["eacc"]) * 100 / report["eacc"] + ) + assert report["score"] == report["eacc"] + + +@pytest.mark.anyio +async def test_a_problem_missing_a_version_cannot_be_an_effective_hit(): + finals = _all_versions("p1", [True, True]) # third version never judged + report = await _task().report(finals, []) + + assert report["incomplete_problems"] == 1 + assert report["eacc"] == 0.0 + assert report["cacc"] == 100.0 + + +@pytest.mark.anyio +async def test_failed_samples_count_against_the_average(): + finals = _all_versions("p1", [True, True]) + report = await _task().report(finals, [_failed("p1", 3)]) + + assert report["fails"] == 1.0 + assert report["n_versions_judged"] == 2.0 + assert report["aacc"] == pytest.approx(200 / 3) # 2 correct out of 3 versions + + +@pytest.mark.anyio +async def test_a_wholly_failed_problem_stays_in_the_effective_accuracy_denominator(): + # Every version of p2 failed, so p2 contributes no judgement at all. It must + # still occupy a slot: dropping it would compute EAcc over the survivors and + # report 100.0 for a run that answered half the problems. + finals = _all_versions("p1", [True, True, True]) + report = await _task().report(finals, [_failed("p2", v) for v in (1, 2, 3)]) + + assert report["n_problems"] == 2.0 + assert report["eacc"] == 50.0 + assert report["cacc"] == 50.0 + assert report["incomplete_problems"] == 1.0 + assert report["unattributed_fails"] == 0.0 + + +@pytest.mark.anyio +async def test_a_failed_sample_is_identified_from_its_prompt_record(): + # Persisted contexts are not required to carry raw_sample; the prompt record + # carries the same grouping keys, so identity survives either way. + raw = _sample("p2", 1) + ctx = await _task().preprocess(raw, TaskContext(sample_id="p2-v1", raw_sample=raw)) + orphan = TaskContext(sample_id="p2-v1", preprocess_result=ctx).to_failed( + None, "error", "boom" + ) + report = await _task().report(_all_versions("p1", [True, True, True]), [orphan]) + + assert report["n_problems"] == 2.0 + assert report["unattributed_fails"] == 0.0 + + +@pytest.mark.anyio +async def test_a_fail_with_no_identity_is_counted_not_silently_dropped(): + orphan = TaskContext(sample_id="?").to_failed(None, "error", "boom") + report = await _task().report(_all_versions("p1", [True, True, True]), [orphan]) + + assert report["unattributed_fails"] == 1.0 + assert report["n_problems"] == 1.0 # nothing to attribute it to + + +@pytest.mark.anyio +async def test_extra_rollouts_do_not_become_pass_at_n(): + # A model configured with n > 1 must not turn a version into "any rollout + # was right" -- that would inflate every accuracy built on top of it. + ctx = TaskContext( + sample_id=0, + feedback_result=build_judgement_record( + ["1"], + [build_rollout_judgement(0, False), build_rollout_judgement(1, True)], + extra={"problem_id": "p1", "version": 1, "subject": "Algebra"}, + ), + ).to_final() + report = await _task().report([ctx], []) + assert report["aacc"] == 0.0 + + +@pytest.mark.anyio +async def test_per_subject_effective_accuracy_is_reported(): + finals = [ + *_all_versions("a1", [True, True, True], subject="Algebra"), + *_all_versions("g1", [False, True, True], subject="Geometry"), + ] + report = await _task().report(finals, []) + + assert report["eacc_algebra"] == 100.0 + assert report["eacc_geometry"] == 0.0 + + +@pytest.mark.anyio +async def test_empty_run_reports_the_same_keys(): + report = await _task().report([], []) + for key in ("score", "eacc", "aacc", "cacc", "delta", "relative_delta", "fails"): + assert key in report + assert report["score"] == 0.0 + + +def test_import_does_not_pull_math_verify(): + code = ( + "import sys\n" + "import sieval.tasks.ugmathbench_0shot_gen_fixed\n" + "assert 'math_verify' not in sys.modules, " + "'math_verify must be lazy-imported'\n" + ) + # Fresh interpreter so pytest's already-loaded modules don't mask the check. + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stderr + + +def _judged_without_extra(problem_id: str, version: int, correct: bool) -> TaskContext: + """A judged version whose judgement lost its grouping keys. + + What ``feedback()`` emits when ``raw_sample`` is gone: a wrong-by-default + verdict. The prompt record still names the problem. + """ + return TaskContext( + sample_id=f"{problem_id}-v{version}", + preprocess_result=build_prompt_record( + [{"role": "user", "content": "q"}], + reference=["1"], + extra={ + "problem_id": problem_id, + "version": version, + "subject": "Algebra", + }, + ), + feedback_result=build_judgement_record( + ["1"], [build_rollout_judgement(0, correct)] + ), + ).to_final() + + +@pytest.mark.anyio +async def test_a_judged_version_without_grouping_keys_is_recovered_from_the_prompt(): + # Otherwise the problem leaves EAcc's denominator while its three wrong + # verdicts stay in AAcc's, so EAcc is computed over the survivors — biased + # *upward*, in the direction that flatters the run, and silently. + good = _all_versions("p1", [True, True, True]) + lost = [_judged_without_extra("p2", version, False) for version in (1, 2, 3)] + + report = await _task().report(good + lost, []) + + assert report["n_problems"] == 2 + assert report["eacc"] == 50.0 # not 100.0 + assert report["aacc"] == 50.0 + assert report["unattributed_finals"] == 0.0 + + +@pytest.mark.anyio +async def test_feedback_without_a_raw_sample_still_names_its_problem(): + ctx = TaskContext( + sample_id="p9-v2", + raw_sample=None, + preprocess_result=build_prompt_record( + [{"role": "user", "content": "q"}], + reference=["1"], + extra={"problem_id": "p9", "version": 2, "subject": "Algebra"}, + ), + ) + + _, judgement = await _task().feedback(build_prediction_record([["1"]]), ctx) + + assert judgement["extra"]["problem_id"] == "p9" + assert judgement["rollouts"][0]["correct"] is False + + +@pytest.mark.anyio +async def test_an_unrecoverable_version_is_counted_rather_than_dropped(): + # Nothing left to recover from, so EAcc really is an upper bound here. The + # point is that the run says so instead of reporting a clean number. + good = _all_versions("p1", [True, True, True]) + lost = [ + TaskContext( + sample_id=f"p2-v{version}", + feedback_result=build_judgement_record( + ["1"], [build_rollout_judgement(0, False)] + ), + ).to_final() + for version in (1, 2, 3) + ] + + report = await _task().report(good + lost, []) + + assert report["unattributed_finals"] == 3.0 + assert report["eacc"] > report["aacc"] # the invariant this makes visible