diff --git a/sieval/community/complex_constraints.py b/sieval/community/complex_constraints.py new file mode 100644 index 00000000..745f8a24 --- /dev/null +++ b/sieval/community/complex_constraints.py @@ -0,0 +1,178 @@ +"""ComplexConstraints rubric-grading assets: judge prompt, verdict parsing, metrics. + +ComplexConstraints (Mehta et al., 2026, arXiv:2606.09118) is a 75-prompt +instruction-following benchmark. Each prompt ships 10-40 *atomic* rubric criteria +(1,559 in total) describing what a correct response must satisfy; criteria are +graded by rubric -- human or LLM-as-a-judge -- never by exact match. + +Upstream publishes **no evaluation code and no judge prompt**. The paper names +GPT-5-mini as the per-criterion judge and defines the metrics, but the template, +its decoding settings, and the call structure are all unstated, and the dataset +card adds nothing. So ``GRADER_TEMPLATE`` and :func:`parse_verdicts` below are +**authored by this port**, not reproduced from upstream -- which is why +``sieval.tasks.complex_constraints_0shot_gen`` ships ``status="experimental"``. +Contrast ``sieval.community.aa_lcr``, whose templates at least come from the +upstream dataset card verbatim. + +Two published metrics, both computed by :func:`aggregate_metrics`: + +* **task pass rate** -- the fraction of prompts whose response satisfies *every* + criterion. This is what the paper's public 75-prompt leaderboard reports + (its Table 1, snapshot 2026-06-03), so it is the port's headline. +* **mean per-criterion pass rate** -- "the fraction of rubric criteria satisfied, + averaged across tasks" (its Table 3 caption), i.e. a **macro** average over + prompts. Criteria counts vary 10-40, so the pooled (**micro**) rate is a + genuinely different number; both are reported and named, and the macro one is + the published one. + +Grading is **one judge call per rollout**, covering all of that prompt's criteria, +with the verdicts emitted as an indexed list. Upstream never says whether it +grades one criterion per call; batching keeps a rollout's whole verdict set in a +single persisted ``ModelOutput`` -- which is also what the runner's grader-spend +accounting expects, since it reads exactly one output per rollout -- and the +indexing makes misalignment detectable: an index the judge never emits is +recorded as unparsed rather than silently shifting its neighbours' verdicts. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import re +from collections.abc import Sequence + +#: One rubric line as the judge sees it. 1-based, matching the verdict indices +#: the judge is asked to emit. +CRITERION_TEMPLATE = "{number}. {criterion}" + +#: Rubric-grading prompt. Authored by this port (upstream publishes none). +#: The verdict block is requested *last* so a reasoning judge puts it after its +#: deliberation, which is what makes "last verdict per index wins" correct. +GRADER_TEMPLATE = """You are grading one model RESPONSE against a rubric of atomic criteria. + +Judge each criterion independently, against the RESPONSE alone. A criterion is satisfied +only if the RESPONSE clearly meets it. If the response only partially meets a criterion, or +gives you nothing to check it against, that criterion is NOT satisfied. Grade exactly what +each criterion asks for -- do not reward or penalise anything else about the response. + +BEGIN PROMPT +{prompt} +END PROMPT + +BEGIN RESPONSE +{response} +END RESPONSE + +BEGIN CRITERIA +{criteria} +END CRITERIA + +Grade all {n_criteria} criteria. End your reply with one verdict per criterion, in order, +one per line, in exactly this format and nothing after it: + +1: +2: +... +{n_criteria}: +""" + + +def format_criteria(criteria: Sequence[str]) -> str: + """Render *criteria* as the 1-based numbered block the judge grades.""" + return "\n".join( + CRITERION_TEMPLATE.format(number=i + 1, criterion=criterion) + for i, criterion in enumerate(criteria) + ) + + +def build_grader_prompt(prompt: str, response: str, criteria: Sequence[str]) -> str: + """Assemble the rubric-grading prompt for one response. + + The original *prompt* is included because criteria are written against it + ("the response should schedule ... 15th-21st December 2025"): many are + uncheckable from the response alone. + """ + return GRADER_TEMPLATE.format( + prompt=prompt, + response=response, + criteria=format_criteria(criteria), + n_criteria=len(criteria), + ) + + +# A verdict line: leading list/emphasis punctuation ("- ", "* ", "**"), an +# optional "criterion" word, the 1-based index, a separator, more optional +# emphasis, then the verdict. Anchored to line starts so prose that merely +# mentions a number cannot register as a verdict. +_VERDICT_RE = re.compile( + r"^[^\w\n]*(?:criterion\s*)?(\d{1,3})\s*[:.)\-]\s*[^\w\n]*(PASS|FAIL)\b", + re.IGNORECASE | re.MULTILINE, +) + + +def parse_verdicts(reply: str, n_criteria: int) -> list[bool | None]: + """Map a judge reply to one verdict per criterion, in criterion order. + + Returns a list of length *n_criteria*: ``True`` (satisfied), ``False`` (not + satisfied), or ``None`` for a criterion the judge never returned a readable + verdict for. ``None`` is deliberately distinct from ``False`` -- the caller + scores it as not-satisfied (an unreadable verdict must not inflate a score) + but records the count separately, so judge format drift stays visible + instead of masquerading as a model that failed the rubric. + + The **last** verdict for an index wins: the judge is asked to put the verdict + block at the end, so a reasoning judge's earlier tentative pass over the + criteria must not override its final answer. Indices outside ``1..n_criteria`` + are ignored rather than clamped -- a hallucinated "41: PASS" is not evidence + about criterion 41 of a 40-criterion rubric. + """ + verdicts: list[bool | None] = [None] * n_criteria + for index_text, verdict in _VERDICT_RE.findall(reply): + index = int(index_text) + if 1 <= index <= n_criteria: + verdicts[index - 1] = verdict.upper() == "PASS" + return verdicts + + +def aggregate_metrics(units: Sequence[tuple[int, int]]) -> dict[str, float]: + """Aggregate ``(n_satisfied, n_criteria)`` pairs into the published metrics. + + One *unit* is one graded rollout, plus one stand-in per attempt that never + produced a gradeable response (contributing ``(0, n_criteria)``) so the rates + span the full requested set rather than only the successfully-graded subset. + + Returns rates in ``[0, 1]``: + + * ``task_pass_rate`` -- units satisfying every criterion. The leaderboard's + metric, and the port's headline. + * ``criterion_pass_rate_macro`` -- per-unit satisfied fraction, averaged over + units. The paper's "mean per-criterion pass rate". + * ``criterion_pass_rate_micro`` -- criteria satisfied pooled over all units. + Differs from the macro rate because criteria counts vary 10-40 per prompt. + + A unit with ``n_criteria == 0`` (a failure whose rubric size could not be + recovered) counts as a task failure at rate 0 and adds nothing to the pooled + denominator -- so it can only ever drag the score down, never flatter it. + """ + total = len(units) + if total == 0: + return { + "task_pass_rate": 0.0, + "criterion_pass_rate_macro": 0.0, + "criterion_pass_rate_micro": 0.0, + } + + pooled_criteria = sum(count for _, count in units) + return { + "task_pass_rate": sum( + 1 for satisfied, count in units if count > 0 and satisfied == count + ) + / total, + "criterion_pass_rate_macro": sum( + satisfied / count if count else 0.0 for satisfied, count in units + ) + / total, + "criterion_pass_rate_micro": ( + sum(satisfied for satisfied, _ in units) / pooled_criteria + if pooled_criteria + else 0.0 + ), + } diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index e5cd3cbe..6e467a86 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -53,6 +53,10 @@ from .cmmlu import ( CMMLUDataset, CMMLUDatasetSample, ) +from .complex_constraints import ( + ComplexConstraintsDataset, + ComplexConstraintsDatasetSample, +) from .drop import ( DROPDataset, DROPDatasetSample, @@ -189,6 +193,8 @@ __all__ = [ "CMIMC2025DatasetSample", "CMMLUDataset", "CMMLUDatasetSample", + "ComplexConstraintsDataset", + "ComplexConstraintsDatasetSample", "DROPDataset", "DROPDatasetSample", "GPQADiamondDataset", diff --git a/sieval/datasets/complex_constraints.py b/sieval/datasets/complex_constraints.py new file mode 100644 index 00000000..af7160f2 --- /dev/null +++ b/sieval/datasets/complex_constraints.py @@ -0,0 +1,116 @@ +"""ComplexConstraints dataset loader (Surge AI). + +ComplexConstraints (Mehta et al., 2026, arXiv:2606.09118) is a 75-prompt +multi-constraint instruction-following benchmark (``CIF-001``-``CIF-075``). Each +row is one realistic prompt plus 10-40 atomic rubric criteria (1,559 in total) +describing what a correct response must satisfy. + +The Hub repo ships a single wide CSV: five item columns and 40 sparse +``criterion_{i}`` columns, of which a row uses the first 10-40. This loader +collapses those 40 columns into one ``criteria`` list and drops them. That is a +**reshape, not a rename for uniformity**: a 40-key ``TypedDict`` of mostly-absent +columns is unusable as a sample type, and every consumer wants the list. The +other five columns keep their upstream names, and no dtype cast is applied -- +the pinned revision already ships all 45 columns as strings. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import os +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +# Pin the Hub revision for reproducibility (current `main` at integration time). +COMPLEX_CONSTRAINTS_REVISION = "e9625c6f635f42b72cb85a04c2be64746f945126" + +#: The repo's one data file. The dataset card's ``configs.data_files.path`` spells +#: it ``ComplexConstraints_Benchmark_Set.csv``, which does not exist -- so +#: ``load_dataset("surgeai/ComplexConstraints")`` cannot resolve the file at all. +#: Reading the staged snapshot by its real name sidesteps the card's typo. +CSV_FILENAME = "ComplexConstraints_benchmark_set.csv" + +#: Widest criterion column upstream ships; a row fills the first 10-40. +MAX_CRITERIA = 40 + +_CRITERION_COLUMNS = tuple(f"criterion_{i}" for i in range(1, MAX_CRITERIA + 1)) + + +class ComplexConstraintsDatasetSample(TypedDict): + benchmark_id: str + prompt: str + use_case: str + instruction_type: str + prompt_style: str + criteria: list[str] + + +def _collapse_criteria(row: dict) -> dict: + """Gather a row's non-empty ``criterion_{i}`` cells into one ordered list. + + Every non-empty cell is kept, rather than stopping at the first empty one. + On the pinned revision the filled cells are a contiguous prefix (verified: + 0 of 75 rows have a gap), so the two readings agree there -- but stopping + early would silently drop criteria if a later revision ever left a hole, + and a dropped criterion inflates the score. + """ + criteria = [ + text + for column in _CRITERION_COLUMNS + if (value := row[column]) is not None and (text := str(value).strip()) + ] + return {"criteria": criteria} + + +@sieval_dataset( + name="complex_constraints", + display_name="ComplexConstraints", + description="75 multi-constraint prompts with 1,559 rubric criteria (Surge AI).", + source=f"hf:surgeai/ComplexConstraints@{COMPLEX_CONSTRAINTS_REVISION}", + categories=(Category(Level1Category.LANGUAGE, "InstructionFollowing"),), + tags=("english", "instruction-following", "open-ended"), + license="CC-BY-4.0", +) +class ComplexConstraintsDataset(Dataset[ComplexConstraintsDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + csv_path = ( + os.path.join(name_or_path, CSV_FILENAME) + if os.path.isdir(name_or_path) + else name_or_path + ) + dataset = load_dataset("csv", data_files={"test": csv_path}, **kwargs) + dataset = ensure_dataset_dict(dataset) + split = dataset["test"] + if len(split) == 0: + raise ValueError( + f"ComplexConstraints produced an empty 'test' split from " + f"{csv_path!r}; check that the dataset has been downloaded via " + "`sieval dataset download complex_constraints`." + ) + + missing = [c for c in _CRITERION_COLUMNS if c not in split.column_names] + if missing: + raise ValueError( + f"ComplexConstraints is missing criterion column(s) {missing} in " + f"{csv_path!r}; the loader expects the wide format of revision " + f"{COMPLEX_CONSTRAINTS_REVISION} (criterion_1..criterion_" + f"{MAX_CRITERIA})." + ) + + return HFDatasetDict( + { + "test": split.map( + _collapse_criteria, remove_columns=list(_CRITERION_COLUMNS) + ) + } + ) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index d77ab81d..cad03cb9 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -284,6 +284,28 @@ "d6e7b716d8ac694f38969a6c0407437d1fded799.zip": "sha256:154593336d5074d793ed990222876b83490b0aed97638a62618d1fe2da7c2cac" } }, + { + "name": "complex_constraints", + "display_name": "ComplexConstraints", + "description": "75 multi-constraint prompts with 1,559 rubric criteria (Surge AI).", + "source": [ + "hf:surgeai/ComplexConstraints@e9625c6f635f42b72cb85a04c2be64746f945126" + ], + "categories": [ + { + "level1": "Language", + "level2": "InstructionFollowing" + } + ], + "tags": [ + "english", + "instruction-following", + "open-ended" + ], + "deps_group": null, + "license": "CC-BY-4.0", + "checksums": {} + }, { "name": "drop", "display_name": "DROP", @@ -1208,6 +1230,27 @@ }, "status": "stable" }, + { + "name": "complex_constraints_0shot_gen", + "display_name": "ComplexConstraints (0-shot, generative)", + "description": "Multi-constraint instruction following; rubric graded by an LLM judge.", + "dataset": "complex_constraints", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "instruction-following", + "open-ended" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "complex-constraints", + "url": "https://arxiv.org/abs/2606.09118", + "notes": "Generative port of ComplexConstraints (Surge AI, arXiv:2606.09118) — 75 multi-constraint prompts (CIF-001..CIF-075) with 10-40 atomic rubric criteria each (1,559 total), graded by rubric rather than exact match. NO UPSTREAM EVAL CODE AND NO UPSTREAM JUDGE PROMPT: the paper names GPT-5-mini as the per-criterion judge and defines the metrics, but the template, decoding settings and call structure are unstated, and the dataset card (https://huggingface.co/datasets/surgeai/ComplexConstraints/blob/e9625c6f635f42b72cb85a04c2be64746f945126/README.md) adds nothing — so the rubric prompt and verdict parsing are AUTHORED BY THIS PORT (sieval.community.complex_constraints), hence status=experimental. METRICS: headline = task pass rate (response satisfies EVERY criterion), the metric the paper's public leaderboard reports (Table 1). Also reported: criterion_pass_rate_macro (per-prompt satisfied fraction averaged over prompts — the paper's 'mean per-criterion pass rate', Table 3 caption) and criterion_pass_rate_micro (pooled over all criteria); the two differ because criteria counts vary 10-40 per prompt. GRADING: one judge call per rollout covering all of that prompt's criteria as an indexed PASS/FAIL list (upstream's call structure is unstated); an unreadable per-criterion verdict scores not-satisfied but is counted as n_unparsed so judge format drift stays visible; empty/whitespace responses satisfy zero criteria without invoking the judge (grader_output absent there, no call made). REPRODUCIBILITY: scores depend on the grader endpoint's model version (not pinnable like a Hub revision) — pin the grader model + temperature=0; per-criterion verdicts and the judge's full ModelOutput (extra.grader_output) are persisted per rollout, the reply being the only evidence of a verdict a re-grade need not reproduce. REPEATS: the leaderboard states no repeat count, so the port defaults to n=1; `n` is a task arg (tasks..args.n), NOT a model arg — infer forwards it call-time and call-time wins. NOT YET VALIDATED against the Table 1 leaderboard (snapshot 2026-06-03: Gemini 3.1 Pro 40.4, GPT-5.5 38.7, Claude Opus 4.8 34.9 task pass %)." + }, + "status": "experimental" + }, { "name": "drop_kshot_gen", "display_name": "DROP (few-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 2136c931..2894d178 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -46,6 +46,9 @@ from .cmimc_2025_0shot_gen import ( from .cmmlu_kshot_clp import ( CMMLUFewShotClpTask, ) +from .complex_constraints_0shot_gen import ( + ComplexConstraintsZeroShotGenTask, +) from .drop_kshot_gen import ( DROPFewShotGenTask, ) @@ -168,6 +171,7 @@ __all__ = [ "CEvalFewShotCLPTask", "CMIMC2025ZeroShotGenTask", "CMMLUFewShotClpTask", + "ComplexConstraintsZeroShotGenTask", "DROPFewShotGenTask", "GPQADiamondZeroShotGenTask", "GSM8KFewShotBaseGenTask", diff --git a/sieval/tasks/complex_constraints_0shot_gen.py b/sieval/tasks/complex_constraints_0shot_gen.py new file mode 100644 index 00000000..c9a90830 --- /dev/null +++ b/sieval/tasks/complex_constraints_0shot_gen.py @@ -0,0 +1,343 @@ +"""ComplexConstraints — 0-shot generative, LLM-judge graded against a rubric. + +Generative port of ComplexConstraints (Mehta et al., 2026, arXiv:2606.09118): the +model answers one realistic multi-constraint instruction, and a separate **LLM +judge** grades the free-form response against that prompt's 10-40 atomic rubric +criteria, one PASS/FAIL verdict each. The headline metric is the **task pass +rate** — the fraction of prompts whose response satisfies *every* criterion — +which is what the paper's public 75-prompt leaderboard reports (its Table 1). +The paper's other metric, the mean per-criterion pass rate, is reported +alongside it in both the macro (published) and pooled-micro readings. + +Upstream ships **no evaluation code and no judge prompt** — the paper names +GPT-5-mini as the judge and defines the metrics, nothing more — so the rubric +prompt and verdict parsing are authored by this port (see +``sieval.community.complex_constraints``). Scores are therefore not comparable to +the leaderboard at the precision a vendored grader would give, which is why this +task ships ``status="experimental"``. + +The judge is supplied via the ``grader`` task arg (a model-config dict, or a +pre-built Model, on its own ``api_base``/``api_key``). As with sieval's other +LLM-graded tasks, correctness depends on a grader model whose version sieval +cannot pin the way it pins a Hub revision, so for reproducibility pin the grader +model and set ``temperature: 0``; each rollout's per-criterion verdicts and the +judge's whole ``ModelOutput`` (``extra.grader_output``: reply, reasoning, usage, +finish reasons, model id) are persisted — see :meth:`feedback`. + +Deviations / by-design behavior worth knowing: + +* **One judge call per rollout**, grading all of that prompt's criteria as an + indexed list, rather than one call per criterion. Upstream never states its + call structure. Batching keeps a rollout's whole verdict set in the single + ``ModelOutput`` the runner's grader-spend accounting expects, and the indexing + makes misalignment detectable instead of silently shifting verdicts. +* A criterion the judge returns no readable verdict for is scored **not + satisfied** — an unreadable verdict must never inflate a score — but counted + separately as ``n_unparsed`` (per rollout, and pooled in the report), so judge + format drift stays distinguishable from a model that failed the rubric. +* An empty/whitespace response is scored as satisfying **zero** criteria + **without** invoking the judge; ``extra.grader_output`` is absent on that path + because no call was made, and the matching prediction rollout's + ``extracted: false`` identifies it independently. +* Pipeline failures (exhausted retries) count as task failures satisfying zero + criteria, weighted by ``n``, so all three rates span the full requested set. + +Reproduction decoding: ``n`` (repeats) is a **task arg** — set it in +``tasks..args.n``. The paper's leaderboard does not state a repeat count, +so the port defaults to ``n=1``; ``infer`` forwards ``n`` as a call-time kwarg to +``agenerate``, and call-time wins over model config, so setting ``n`` on the +model is silently overridden by the task default. Comparison target is the +paper's Table 1 leaderboard (snapshot 2026-06-03; Gemini 3.1 Pro 40.4, GPT-5.5 +38.7, Claude Opus 4.8 34.9 task pass %). + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from collections.abc import Mapping +from typing import override + +from sieval.community.complex_constraints import ( + aggregate_metrics, + build_grader_prompt, + parse_verdicts, +) +from sieval.core.models import ChatModel, Model, ModelOutput +from sieval.core.tasks import ( + GRADER_OUTPUT_KEY, + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + RolloutJudgement, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.utils.serialization import obj_to_dict +from sieval.datasets import ComplexConstraintsDatasetSample + + +@sieval_task( + name="complex_constraints_0shot_gen", + display_name="ComplexConstraints (0-shot, generative)", + description=( + "Multi-constraint instruction following; rubric graded by an LLM judge." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "instruction-following", "open-ended"), + model_type="chat", + status="experimental", + reference_impl=ReferenceImpl( + source="complex-constraints", + url="https://arxiv.org/abs/2606.09118", + notes=( + "Generative port of ComplexConstraints (Surge AI, arXiv:2606.09118) " + "— 75 multi-constraint prompts (CIF-001..CIF-075) with 10-40 atomic " + "rubric criteria each (1,559 total), graded by rubric rather than " + "exact match. NO UPSTREAM EVAL CODE AND NO UPSTREAM JUDGE PROMPT: " + "the paper names GPT-5-mini as the per-criterion judge and defines " + "the metrics, but the template, decoding settings and call structure " + "are unstated, and the dataset card " + "(https://huggingface.co/datasets/surgeai/ComplexConstraints/blob/" + "e9625c6f635f42b72cb85a04c2be64746f945126/README.md) adds nothing — " + "so the rubric prompt and verdict parsing are AUTHORED BY THIS PORT " + "(sieval.community.complex_constraints), hence status=experimental. " + "METRICS: headline = task pass rate (response satisfies EVERY " + "criterion), the metric the paper's public leaderboard reports " + "(Table 1). Also reported: criterion_pass_rate_macro (per-prompt " + "satisfied fraction averaged over prompts — the paper's 'mean " + "per-criterion pass rate', Table 3 caption) and " + "criterion_pass_rate_micro (pooled over all criteria); the two " + "differ because criteria counts vary 10-40 per prompt. GRADING: one " + "judge call per rollout covering all of that prompt's criteria as an " + "indexed PASS/FAIL list (upstream's call structure is unstated); an " + "unreadable per-criterion verdict scores not-satisfied but is " + "counted as n_unparsed so judge format drift stays visible; " + "empty/whitespace responses satisfy zero criteria without invoking " + "the judge (grader_output absent there, no call made). " + "REPRODUCIBILITY: scores depend on the grader endpoint's model " + "version (not pinnable like a Hub revision) — pin the grader model + " + "temperature=0; per-criterion verdicts and the judge's full " + "ModelOutput (extra.grader_output) are persisted per rollout, the " + "reply being the only evidence of a verdict a re-grade need not " + "reproduce. REPEATS: the leaderboard states no repeat count, so the " + "port defaults to n=1; `n` is a task arg (tasks..args.n), NOT " + "a model arg — infer forwards it call-time and call-time wins. " + "NOT YET VALIDATED against the Table 1 leaderboard (snapshot " + "2026-06-03: Gemini 3.1 Pro 40.4, GPT-5.5 38.7, Claude Opus 4.8 " + "34.9 task pass %)." + ), + ), +) +class ComplexConstraintsZeroShotGenTask( + Task[ + ComplexConstraintsDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + grader: Mapping | Model | None = None, + n: int = 1, + ): + super().__init__(dataset=dataset, model=model, name=name) + self._n = n + self._grader = self._build_grader(grader) + + @staticmethod + def _build_grader(grader: Mapping | Model | None) -> Model: + """Resolve the ``grader`` task arg into a Model. + + Accepts a pre-built Model (used by tests / advanced configs) or a + model-config mapping (the YAML path, e.g. + ``{model: gpt-5-mini, api_base: ..., temperature: 0}``). Grading is + mandatory — the rubric is natural language, so there is no deterministic + fallback — and ``None`` raises. + """ + if isinstance(grader, Model): + return grader + if isinstance(grader, Mapping): + return ChatModel(**grader) + raise ValueError( + "ComplexConstraints requires an LLM grader. Pass `grader:` in the " + "task args — a model-config dict such as " + "{model: gpt-5-mini, api_base: ..., api_key: ..., temperature: 0}." + ) + + @override + async def preprocess(self, raw, ctx): + # No `reference`: the ground truth is a *rubric* (a procedure), not a + # value. It goes to `extra` instead, once per sample — the judgement's + # per-criterion verdicts are index-aligned to this list. + return build_prompt_record( + [{"role": "user", "content": raw["prompt"]}], + extra={ + "benchmark_id": raw["benchmark_id"], + "criteria": list(raw["criteria"]), + "use_case": raw["use_case"], + "instruction_type": raw["instruction_type"], + "prompt_style": raw["prompt_style"], + }, + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + # Open-ended task: the response *is* the answer, so no extraction step. + # Normalizing a blank to None keeps `extracted` a real signal AND is + # exactly the empty-response condition feedback() short-circuits on -- + # one notion of "no answer", spelled once. + return build_prediction_record( + [text if text.strip() else None for text in inf.texts] + ) + + @override + async def feedback(self, post, ctx): + """Grade every rollout against the full rubric in one judge call. + + The judge is a model, so its output is persisted the way any model output + is: ``extra["grader_output"]`` is its whole ``ModelOutput`` flattened to a + plain dict (``add_type=False``, so the judgement record stays uniformly + plain-dict). Nothing is hand-picked, so no field is silently dropped and + the reply survives — the only durable evidence of a verdict set that a + re-grade need not reproduce, and the only way to tell judge format drift + (``n_unparsed``) from a response that genuinely failed the rubric. + + ``extra["criterion_verdicts"]`` is one ``True``/``False``/``None`` per + criterion, index-aligned to the prompt record's ``criteria`` — task + logic on top of the raw reply, which is why it is a separate key. It is + absent entirely on the empty-response path, where no judge call is made. + """ + raw = ctx.raw_sample + criteria = list(raw["criteria"]) + n_criteria = len(criteria) + + rollouts: list[RolloutJudgement] = [] + for rollout in post["rollouts"]: + predicted = rollout.get("prediction") + if predicted is None: + # An empty response satisfies nothing, and asking the judge to + # confirm that costs a call per rollout and invites a spurious + # PASS on a criterion phrased as a prohibition ("should not + # mention X"). No call, so no grader output to record -- its + # ABSENCE is the durable signal here. + rollouts.append( + build_rollout_judgement( + rollout["index"], + False, + score=0.0, + metrics={"task_pass": False, "criterion_pass_rate": 0.0}, + extra={ + "n_criteria": n_criteria, + "n_satisfied": 0, + "n_unparsed": 0, + }, + ) + ) + continue + + out = await self._grader.agenerate( + build_grader_prompt(raw["prompt"], predicted, criteria) + ) + reply = out.texts[0] if out.texts else "" + verdicts = parse_verdicts(reply, n_criteria) + n_satisfied = sum(1 for verdict in verdicts if verdict) + n_unparsed = sum(1 for verdict in verdicts if verdict is None) + # Both published readings are co-equal metrics, so both go in + # `metrics`; the headline merely points at task_pass. Derived from + # the mapping, not recomputed, so the two cannot drift. + metrics: dict[str, bool | float] = { + "task_pass": n_criteria > 0 and n_satisfied == n_criteria, + "criterion_pass_rate": n_satisfied / n_criteria if n_criteria else 0.0, + } + rollouts.append( + build_rollout_judgement( + rollout["index"], + bool(metrics["task_pass"]), + score=float(metrics["criterion_pass_rate"]), + metrics=metrics, + extra={ + "criterion_verdicts": verdicts, + "n_criteria": n_criteria, + "n_satisfied": n_satisfied, + "n_unparsed": n_unparsed, + GRADER_OUTPUT_KEY: obj_to_dict(out, add_type=False), + }, + ) + ) + + # Sample-level partial credit: the mean criterion pass rate across + # rollouts. Genuine partial credit rather than a mirror of + # n_correct/n_rollouts, which already records the task-pass side. + score = ( + sum(float(r["metrics"]["criterion_pass_rate"]) for r in rollouts) + / len(rollouts) + if rollouts + else 0.0 + ) + return True, build_judgement_record( + # The rubric is a procedure, not a value; `extra` describes it and + # the criterion texts live once on the prompt record. + None, + rollouts, + score=score, + extra={ + "benchmark_id": raw["benchmark_id"], + "n_criteria": n_criteria, + "use_case": raw["use_case"], + "instruction_type": raw["instruction_type"], + "prompt_style": raw["prompt_style"], + }, + ) + + @override + async def report(self, finals, fails): + graded = [ + rollout + for f in finals + for rollout in (f.feedback_result or {}).get("rollouts", []) + ] + # Pooled from raw per-rollout counts rather than averaged from the + # per-rollout rates -- the two differ when prompts carry different + # criteria counts, which is exactly the macro/micro split below. + units = [(r["extra"]["n_satisfied"], r["extra"]["n_criteria"]) for r in graded] + + # Pipeline failures (exhausted retries) never produced a gradeable + # response; each failed sample stands in for its n requested attempts, + # satisfying zero of its criteria, so all three rates span the full + # requested set rather than only the successfully-graded subset. The + # rubric size comes from the raw sample when the context still carries + # one; without it the attempt still counts as a task failure but adds + # nothing to the pooled denominator. + for f in fails: + n_criteria = len(f.raw_sample["criteria"]) if f.raw_sample else 0 + units.extend([(0, n_criteria)] * self._n) + + m = aggregate_metrics(units) + return { + "score": m["task_pass_rate"] * 100, + "task_pass_rate": m["task_pass_rate"] * 100, + "criterion_pass_rate_macro": m["criterion_pass_rate_macro"] * 100, + "criterion_pass_rate_micro": m["criterion_pass_rate_micro"] * 100, + "n_graded": len(graded), + "n_criteria_graded": sum(r["extra"]["n_criteria"] for r in graded), + # Judge format drift, kept out of the rates it would otherwise be + # invisible inside: these criteria scored not-satisfied. + "n_unparsed": sum(r["extra"]["n_unparsed"] for r in graded), + "fails": len(fails), + } diff --git a/tests/unit/community/test_complex_constraints.py b/tests/unit/community/test_complex_constraints.py new file mode 100644 index 00000000..4d0a0d0b --- /dev/null +++ b/tests/unit/community/test_complex_constraints.py @@ -0,0 +1,126 @@ +"""Unit tests for the ComplexConstraints rubric-grading assets. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest + +from sieval.community.complex_constraints import ( + aggregate_metrics, + build_grader_prompt, + format_criteria, + parse_verdicts, +) + +# --- prompt assembly --- + + +def test_format_criteria_numbers_from_one(): + assert format_criteria(["alpha", "beta"]) == "1. alpha\n2. beta" + + +def test_build_grader_prompt_carries_prompt_response_and_count(): + prompt = build_grader_prompt("do the thing", "here you go", ["a", "b", "c"]) + # The original prompt must reach the judge: most criteria are written + # against it and are uncheckable from the response alone. + assert "do the thing" in prompt + assert "here you go" in prompt + assert "1. a\n2. b\n3. c" in prompt + # The count is interpolated in both places the template asks for it, so a + # judge told to grade "3 criteria" cannot be handed a different rubric size. + assert "Grade all 3 criteria" in prompt + assert "3: " in prompt + + +# --- verdict parsing --- + + +def test_parse_verdicts_reads_indexed_block(): + reply = "1: PASS\n2: FAIL\n3: PASS" + assert parse_verdicts(reply, 3) == [True, False, True] + + +@pytest.mark.parametrize( + "line", + [ + "1: PASS", + "1. PASS", + "1) PASS", + "1 - PASS", + "- 1: PASS", + "* **1: PASS**", + "**1:** PASS", + "Criterion 1: pass", + ], +) +def test_parse_verdicts_tolerates_judge_formatting(line: str): + # Judges wrap verdicts in list bullets and markdown emphasis; a parser that + # only accepts the bare form silently reports every criterion unparsed. + assert parse_verdicts(line, 1) == [True] + + +def test_parse_verdicts_last_verdict_per_index_wins(): + # A reasoning judge revises: the block it is asked to end with is the answer. + reply = "Working through it: 1: PASS, but on reflection no.\n\n1: FAIL\n2: PASS" + assert parse_verdicts(reply, 2) == [False, True] + + +def test_parse_verdicts_missing_index_is_none_not_false(): + # None and False must stay distinct -- the caller scores both as + # not-satisfied but counts None separately as judge format drift. + assert parse_verdicts("1: PASS\n3: PASS", 3) == [True, None, True] + + +def test_parse_verdicts_empty_reply_is_all_none(): + assert parse_verdicts("", 2) == [None, None] + + +def test_parse_verdicts_ignores_out_of_range_indices(): + # A hallucinated "3: PASS" is not evidence about a 2-criterion rubric, and + # must not be clamped onto criterion 2. + assert parse_verdicts("1: PASS\n2: FAIL\n3: PASS\n0: PASS", 2) == [True, False] + + +def test_parse_verdicts_ignores_mid_sentence_prose(): + # Anchored to line starts: a sentence mentioning a number and the word PASS + # is not a verdict line. + assert parse_verdicts("The response would need 2 more sections to PASS.", 2) == [ + None, + None, + ] + + +# --- aggregation --- + + +def test_aggregate_metrics_empty_is_zero(): + m = aggregate_metrics([]) + assert m["task_pass_rate"] == 0.0 + assert m["criterion_pass_rate_macro"] == 0.0 + assert m["criterion_pass_rate_micro"] == 0.0 + + +def test_aggregate_metrics_task_pass_requires_every_criterion(): + # 10/10 passes the task; 9/10 does not, despite a 0.9 criterion rate. + m = aggregate_metrics([(10, 10), (9, 10)]) + assert m["task_pass_rate"] == pytest.approx(0.5) + assert m["criterion_pass_rate_macro"] == pytest.approx(0.95) + + +def test_aggregate_metrics_macro_and_micro_diverge_on_uneven_rubrics(): + # 5/10 and 30/40: macro averages the two rates (0.5, 0.75) => 0.625; + # micro pools 35/50 => 0.70. Reporting one number for both would be wrong + # for whichever reading the reader assumed. + m = aggregate_metrics([(5, 10), (30, 40)]) + assert m["criterion_pass_rate_macro"] == pytest.approx(0.625) + assert m["criterion_pass_rate_micro"] == pytest.approx(0.70) + + +def test_aggregate_metrics_zero_criteria_unit_is_a_failure_not_a_pass(): + # A failure whose rubric size could not be recovered enters as (0, 0). It + # must dilute the rates, never satisfy "every criterion" vacuously. + m = aggregate_metrics([(10, 10), (0, 0)]) + assert m["task_pass_rate"] == pytest.approx(0.5) + assert m["criterion_pass_rate_macro"] == pytest.approx(0.5) + # It adds nothing to the pooled denominator, so micro stays 10/10. + assert m["criterion_pass_rate_micro"] == pytest.approx(1.0) diff --git a/tests/unit/datasets/test_complex_constraints.py b/tests/unit/datasets/test_complex_constraints.py new file mode 100644 index 00000000..e874b33e --- /dev/null +++ b/tests/unit/datasets/test_complex_constraints.py @@ -0,0 +1,113 @@ +"""Unit tests for the ComplexConstraints dataset wrapper. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from unittest.mock import patch + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +import sieval.datasets.complex_constraints as cc_module +from sieval.core.datasets.meta import get_dataset_meta +from sieval.datasets.complex_constraints import ( + COMPLEX_CONSTRAINTS_REVISION, + CSV_FILENAME, + MAX_CRITERIA, + ComplexConstraintsDataset, +) + + +def _row(criteria: list[str], benchmark_id: str = "CIF-001") -> dict: + """One wide-format CSV row: 5 item columns + 40 sparse criterion columns.""" + padded = list(criteria) + [None] * (MAX_CRITERIA - len(criteria)) + return { + "benchmark_id": benchmark_id, + "prompt": "Write a rota.", + "use_case": "Logistics, Scheduling & Event Planning", + "instruction_type": "Negative", + "prompt_style": "Context prompting", + **{f"criterion_{i + 1}": value for i, value in enumerate(padded)}, + } + + +def _hf_dict(rows: list[dict] | None = None) -> HFDatasetDict: + rows = rows if rows is not None else [_row(["alpha", "beta"])] + return HFDatasetDict({"test": HFDataset.from_list(rows)}) + + +def _load(hf_dict: HFDatasetDict, path: str = "/staged/complex_constraints"): + dataset = ComplexConstraintsDataset(_hf_dict=_hf_dict()) + with ( + patch.object(cc_module, "load_dataset", return_value=hf_dict) as mock_load, + patch("os.path.isdir", return_value=True), + ): + return dataset.load(path), mock_load + + +def test_source_pins_hf_revision(): + meta_source = get_dataset_meta(ComplexConstraintsDataset).source + assert meta_source == ( + f"hf:surgeai/ComplexConstraints@{COMPLEX_CONSTRAINTS_REVISION}", + ) + + +def test_load_reads_the_repos_real_csv_filename(): + loaded, mock_load = _load(_hf_dict()) + assert mock_load.call_args.args[0] == "csv" + data_files = mock_load.call_args.kwargs["data_files"] + # The dataset card's configs entry spells this file with different casing + # and does not exist; loading that name would fail outright. + assert data_files["test"].endswith(f"/complex_constraints/{CSV_FILENAME}") + assert CSV_FILENAME == "ComplexConstraints_benchmark_set.csv" + assert len(loaded["test"]) == 1 + + +def test_load_collapses_criterion_columns_into_a_list(): + loaded, _ = _load(_hf_dict()) + split = loaded["test"] + assert split[0]["criteria"] == ["alpha", "beta"] + # The 40 sparse source columns are gone -- leaving them would make the + # sample type a 40-key mostly-absent dict. + assert not [c for c in split.column_names if c.startswith("criterion_")] + # The five item columns keep their upstream names. + assert split[0]["benchmark_id"] == "CIF-001" + assert split[0]["prompt_style"] == "Context prompting" + + +def test_load_keeps_every_non_empty_criterion_across_a_gap(): + # Upstream's filled cells are a contiguous prefix, but stopping at the first + # empty cell would silently drop criteria if that ever changed -- and a + # dropped criterion inflates the score. Every non-empty cell is kept. + row = _row(["alpha", "beta"]) + row["criterion_2"] = None + row["criterion_5"] = "epsilon" + loaded, _ = _load(_hf_dict([row])) + assert loaded["test"][0]["criteria"] == ["alpha", "epsilon"] + + +def test_load_drops_whitespace_only_cells(): + row = _row(["alpha", " ", "gamma"]) + loaded, _ = _load(_hf_dict([row])) + assert loaded["test"][0]["criteria"] == ["alpha", "gamma"] + + +def test_empty_test_split_raises(): + empty = HFDatasetDict({"test": HFDataset.from_list([])}) + dataset = ComplexConstraintsDataset(_hf_dict=_hf_dict()) + with ( + patch.object(cc_module, "load_dataset", return_value=empty), + patch("os.path.isdir", return_value=False), + pytest.raises(ValueError, match="empty 'test' split"), + ): + dataset.load(f"/staged/complex_constraints/{CSV_FILENAME}") + + +def test_missing_criterion_columns_raise(): + # A shape change upstream must fail loudly, not yield empty rubrics that + # would score every response as a vacuous task pass. + narrow = _row(["alpha"]) + del narrow[f"criterion_{MAX_CRITERIA}"] + with pytest.raises(ValueError, match=r"missing criterion column\(s\)"): + _load(_hf_dict([narrow])) diff --git a/tests/unit/tasks/test_complex_constraints_0shot_gen.py b/tests/unit/tasks/test_complex_constraints_0shot_gen.py new file mode 100644 index 00000000..e0a391bf --- /dev/null +++ b/tests/unit/tasks/test_complex_constraints_0shot_gen.py @@ -0,0 +1,372 @@ +"""Unit tests for the ComplexConstraints 0-shot generative task. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + TaskContext, + build_judgement_record, + build_prediction_record, + build_rollout_judgement, +) +from sieval.datasets.complex_constraints import ( + ComplexConstraintsDataset, + ComplexConstraintsDatasetSample, +) +from sieval.tasks.complex_constraints_0shot_gen import ( + ComplexConstraintsZeroShotGenTask, +) + + +class _ScriptedChatModel(ChatModel): + """ChatModel returning a fixed reply, recording every prompt it was sent.""" + + def __init__(self, reply: str, model: str = "mock"): + super().__init__(model=model, api_key="fake") + self._reply = reply + self.prompts: list[object] = [] + self.last_kwargs: dict[str, object] = {} + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + self.prompts.append(prompt) + self.last_kwargs = dict(kwargs) + return ModelOutput( + model=self.meta(), texts=[self._reply], finish_reasons=["stop"] + ) + + async def _alogprobs_impl( + self, prompt, *, max_tokens=1, logprobs=5, echo=True, temperature=0.0, **kwargs + ) -> ModelOutput: + _ = (prompt, max_tokens, logprobs, echo, temperature, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _sample(n_criteria: int = 3) -> ComplexConstraintsDatasetSample: + return { + "benchmark_id": "CIF-001", + "prompt": "Write a rota for the week.", + "use_case": "Logistics, Scheduling & Event Planning", + "instruction_type": "Negative", + "prompt_style": "Context prompting", + "criteria": [ + f"The response should satisfy rule {i}." for i in range(n_criteria) + ], + } + + +def _task( + answer_reply: str = "here is the rota", + grader_reply: str = "1: PASS\n2: PASS\n3: PASS", + n: int = 1, +): + sample = _sample() + dataset = ComplexConstraintsDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(sample)])}) + ) + model = _ScriptedChatModel(reply=answer_reply, model="candidate") + grader = _ScriptedChatModel(reply=grader_reply, model="gpt-5-mini") + task = ComplexConstraintsZeroShotGenTask(dataset, model, grader=grader, n=n) + return task, model, grader + + +# --- grader is mandatory; the rubric is natural language, no fallback exists --- + + +def test_build_grader_requires_config(): + with pytest.raises(ValueError, match="requires an LLM grader"): + ComplexConstraintsZeroShotGenTask._build_grader(None) + + +def test_build_grader_accepts_mapping_and_model(): + built = ComplexConstraintsZeroShotGenTask._build_grader( + {"model": "gpt-5-mini", "api_key": "fake"} + ) + assert isinstance(built, ChatModel) + existing = _ScriptedChatModel(reply="1: PASS") + assert ComplexConstraintsZeroShotGenTask._build_grader(existing) is existing + + +# --- preprocess: the rubric is a procedure, so it goes to extra, not reference --- + + +@pytest.mark.anyio +async def test_preprocess_puts_rubric_in_extra_not_reference(): + task, _, _ = _task() + sample = _sample() + pre = await task.preprocess(sample, TaskContext(sample_id=0, raw_sample=sample)) + + assert pre["prompt"] == [{"role": "user", "content": "Write a rota for the week."}] + # A rubric is a procedure, not a value -- `reference` must stay absent so a + # reader cannot mistake the criteria list for a gold answer. + assert "reference" not in pre + assert pre["extra"]["criteria"] == sample["criteria"] + assert pre["extra"]["benchmark_id"] == "CIF-001" + assert pre["extra"]["instruction_type"] == "Negative" + + +@pytest.mark.anyio +async def test_infer_forwards_n(): + task, model, _ = _task(n=3) + await task.infer( + {"prompt": [{"role": "user", "content": "q"}]}, TaskContext(sample_id=0) + ) + assert model.last_kwargs.get("n") == 3 + + +@pytest.mark.anyio +@pytest.mark.parametrize("blank", ["", " ", "\n\n"]) +async def test_postprocess_normalizes_blank_to_none(blank: str): + task, model, _ = _task() + out = ModelOutput(model=model.meta(), texts=[blank]) + post = await task.postprocess(out, TaskContext(sample_id=0)) + assert post["rollouts"][0]["extracted"] is False + + +# --- feedback: one judge call per rollout, verdicts index-aligned --- + + +@pytest.mark.anyio +async def test_feedback_all_criteria_pass_is_a_task_pass(): + task, _, grader = _task(grader_reply="1: PASS\n2: PASS\n3: PASS") + sample = _sample() + ctx = TaskContext(sample_id=0, raw_sample=sample) + finalize, judgement = await task.feedback(build_prediction_record(["answer"]), ctx) + + assert finalize is True + fb = judgement["rollouts"][0] + assert fb["correct"] is True + assert fb["metrics"]["task_pass"] is True + assert fb["metrics"]["criterion_pass_rate"] == pytest.approx(1.0) + assert fb["extra"]["criterion_verdicts"] == [True, True, True] + assert fb["extra"]["n_satisfied"] == 3 + assert fb["extra"]["n_criteria"] == 3 + # One call, carrying the rubric and the response together. + assert len(grader.prompts) == 1 + assert "1. The response should satisfy rule 0." in grader.prompts[0] + assert "answer" in grader.prompts[0] + + +@pytest.mark.anyio +async def test_feedback_one_failed_criterion_sinks_the_task_pass(): + # The headline is all-or-nothing: 2/3 criteria is a task failure that still + # carries 0.667 partial credit. Collapsing the two would lose the paper's + # distinction between its two metrics. + task, _, _ = _task(grader_reply="1: PASS\n2: FAIL\n3: PASS") + ctx = TaskContext(sample_id=0, raw_sample=_sample()) + _, judgement = await task.feedback(build_prediction_record(["answer"]), ctx) + + fb = judgement["rollouts"][0] + assert fb["correct"] is False + assert fb["metrics"]["task_pass"] is False + assert fb["score"] == pytest.approx(2 / 3) + assert fb["extra"]["criterion_verdicts"] == [True, False, True] + + +@pytest.mark.anyio +async def test_feedback_unparsed_verdict_scores_unsatisfied_but_is_counted(): + # An unreadable verdict must not inflate the score, and must stay + # distinguishable from a criterion the judge actually failed. + task, _, _ = _task(grader_reply="1: PASS\n3: PASS") + ctx = TaskContext(sample_id=0, raw_sample=_sample()) + _, judgement = await task.feedback(build_prediction_record(["answer"]), ctx) + + fb = judgement["rollouts"][0] + assert fb["extra"]["criterion_verdicts"] == [True, None, True] + assert fb["extra"]["n_satisfied"] == 2 + assert fb["extra"]["n_unparsed"] == 1 + assert fb["correct"] is False + + +@pytest.mark.anyio +async def test_feedback_persists_the_whole_judge_output(): + reply = "Let me check each one.\n\n1: PASS\n2: PASS\n3: PASS" + task, _, _ = _task(grader_reply=reply) + ctx = TaskContext(sample_id=0, raw_sample=_sample()) + _, judgement = await task.feedback(build_prediction_record(["answer"]), ctx) + + grader_output = judgement["rollouts"][0]["extra"]["grader_output"] + # The whole ModelOutput, not hand-picked fields: the reply is the only + # durable evidence of a verdict set a re-grade need not reproduce. + assert grader_output["texts"] == [reply] + assert grader_output["finish_reasons"] == ["stop"] + assert grader_output["model"]["model"] == "gpt-5-mini" + + +@pytest.mark.anyio +async def test_feedback_empty_response_scores_zero_without_calling_the_judge(): + # A judge that would pass everything cannot rescue an empty response. + task, _, grader = _task(grader_reply="1: PASS\n2: PASS\n3: PASS") + ctx = TaskContext(sample_id=0, raw_sample=_sample()) + post = build_prediction_record([None]) + _, judgement = await task.feedback(post, ctx) + + fb = judgement["rollouts"][0] + assert fb["correct"] is False + assert fb["extra"]["n_satisfied"] == 0 + assert fb["extra"]["n_criteria"] == 3 + assert grader.prompts == [] + # No call, so no grader output and no verdict list -- absence is the signal. + assert "grader_output" not in fb["extra"] + assert "criterion_verdicts" not in fb["extra"] + + +@pytest.mark.anyio +async def test_feedback_grades_each_rollout_and_records_sample_level_score(): + task, _, grader = _task(grader_reply="1: PASS\n2: FAIL\n3: PASS") + ctx = TaskContext(sample_id=0, raw_sample=_sample()) + _, judgement = await task.feedback( + build_prediction_record(["first", "second"]), ctx + ) + + assert judgement["n_rollouts"] == 2 + assert judgement["n_correct"] == 0 + assert len(grader.prompts) == 2 + # Sample-level score is the mean criterion pass rate, genuine partial credit + # rather than a mirror of n_correct/n_rollouts (which is 0 here). + assert judgement["score"] == pytest.approx(2 / 3) + # The rubric is a procedure, so there is no reference *value*: the builder + # keeps the key None in memory and serialization drops it, leaving the row + # without a gold answer a reader could mistake the criteria for. + assert judgement["reference"] is None + assert judgement["extra"]["n_criteria"] == 3 + assert judgement["extra"]["benchmark_id"] == "CIF-001" + + +@pytest.mark.anyio +async def test_feedback_short_circuit_does_not_inherit_a_prior_verdict(): + # With n>1 the rollouts share one loop; an ungraded attempt must not be + # attributed the graded attempt's verdicts. + task, _, _ = _task(grader_reply="1: PASS\n2: PASS\n3: PASS") + ctx = TaskContext(sample_id=0, raw_sample=_sample()) + _, judgement = await task.feedback(build_prediction_record(["answer", None]), ctx) + + graded, skipped = judgement["rollouts"] + assert graded["extra"]["criterion_verdicts"] == [True, True, True] + assert "criterion_verdicts" not in skipped["extra"] + assert "grader_output" not in skipped["extra"] + assert skipped["extra"]["n_satisfied"] == 0 + + +# --- report: the three published rates over graded + failed samples --- + + +def _final( + sample_id: int, satisfied: int, n_criteria: int, n_unparsed: int = 0 +) -> TaskContext: + return TaskContext( + sample_id=sample_id, + feedback_result=build_judgement_record( + None, + [ + build_rollout_judgement( + 0, + satisfied == n_criteria, + score=satisfied / n_criteria, + metrics={ + "task_pass": satisfied == n_criteria, + "criterion_pass_rate": satisfied / n_criteria, + }, + extra={ + "n_criteria": n_criteria, + "n_satisfied": satisfied, + "n_unparsed": n_unparsed, + }, + ) + ], + ), + ) + + +@pytest.mark.anyio +async def test_report_headline_is_the_task_pass_rate(): + task, _, _ = _task() + finals = [_final(0, 10, 10), _final(1, 9, 10), _final(2, 40, 40), _final(3, 0, 10)] + report = await task.report(finals, fails=[]) + + # 2 of 4 responses satisfied every criterion. + assert report["task_pass_rate"] == pytest.approx(50.0) + assert report["score"] == report["task_pass_rate"] + assert report["n_graded"] == 4 + assert report["n_criteria_graded"] == 70 + assert report["n_unparsed"] == 0 + assert report["fails"] == 0 + + +@pytest.mark.anyio +async def test_report_macro_and_micro_criterion_rates_both_reported(): + task, _, _ = _task() + # 5/10 and 30/40: macro = mean(0.5, 0.75) = 0.625; micro = 35/50 = 0.70. + finals = [_final(0, 5, 10), _final(1, 30, 40)] + report = await task.report(finals, fails=[]) + + assert report["criterion_pass_rate_macro"] == pytest.approx(62.5) + assert report["criterion_pass_rate_micro"] == pytest.approx(70.0) + + +@pytest.mark.anyio +async def test_report_counts_fails_as_zero_criteria_satisfied(): + # Failed samples must dilute all three rates (full-set metric). Excluding + # them would report 100% here. + task, _, _ = _task() + finals = [_final(0, 10, 10)] + fails = [TaskContext(sample_id=1, raw_sample=_sample(n_criteria=10))] + report = await task.report(finals, fails) + + assert report["task_pass_rate"] == pytest.approx(50.0) + assert report["criterion_pass_rate_macro"] == pytest.approx(50.0) + # The failed sample's rubric size is recovered from its raw sample, so it + # reaches the pooled denominator too: 10 / 20. + assert report["criterion_pass_rate_micro"] == pytest.approx(50.0) + assert report["fails"] == 1 + + +@pytest.mark.anyio +async def test_report_fails_weighted_by_n(): + task, _, _ = _task(n=2) + finals = [_final(0, 10, 10)] + fails = [TaskContext(sample_id=1, raw_sample=_sample(n_criteria=10))] + report = await task.report(finals, fails) + + # 1 passing rollout + n*1 = 2 failed attempts => 3 units, 33.3%. + # An unweighted count would give 50%. + assert report["task_pass_rate"] == pytest.approx(100 / 3) + + +@pytest.mark.anyio +async def test_report_fail_without_raw_sample_still_counts_as_a_failure(): + # A context that never got a raw sample has no known rubric size; it must + # still dilute the task-pass and macro rates rather than vanish. + task, _, _ = _task() + finals = [_final(0, 10, 10)] + report = await task.report(finals, fails=[TaskContext(sample_id=1)]) + + assert report["task_pass_rate"] == pytest.approx(50.0) + assert report["criterion_pass_rate_macro"] == pytest.approx(50.0) + # Unknown rubric size adds nothing to the pooled denominator. + assert report["criterion_pass_rate_micro"] == pytest.approx(100.0) + + +@pytest.mark.anyio +async def test_report_surfaces_unparsed_verdicts(): + # Judge format drift must be visible in the report, not buried inside the + # rates it silently depresses. + task, _, _ = _task() + report = await task.report([_final(0, 8, 10, n_unparsed=2)], fails=[]) + assert report["n_unparsed"] == 2 + # The two unreadable verdicts are already inside the 8/10 -- surfacing them + # separately is what lets a reader tell drift from a real rubric failure. + assert report["criterion_pass_rate_micro"] == pytest.approx(80.0) + + +@pytest.mark.anyio +async def test_report_empty_is_zero(): + task, _, _ = _task() + report = await task.report([], fails=[]) + assert report["score"] == 0.0 + assert report["n_graded"] == 0