diff --git a/sieval/community/advanced_if.py b/sieval/community/advanced_if.py new file mode 100644 index 00000000..eb70e36b --- /dev/null +++ b/sieval/community/advanced_if.py @@ -0,0 +1,388 @@ +# Upstream: https://github.com/facebookresearch/AdvancedIF/blob/f9d30137c4139d4d9af260ae28108b5afae828c0/judge.py +# +# The judge prompts are deliberately NOT vendored. Every file in the upstream +# repository is CC-BY-NC-4.0, which cannot be redistributed inside sieval's +# Apache-2.0 tree. The operator stages their own checkout and points +# SIEVAL_ADVANCED_IF_SRC at it; this module loads the prompts from there at +# eval time and contributes only sieval-authored scoring code. That mirrors how +# the benchmark data is handled -- also CC-BY-NC-4.0, also never vendored -- so +# running AdvancedIF requires accepting the upstream terms either way. +"""AdvancedIF rubric-judge assets and scoring kernel. + +AdvancedIF (Meta, arXiv:2511.10507) scores a response against expert-written +rubrics: a grader LLM answers every rubric question yes/no and declares whether +the response satisfied all of them. This module holds + +* :func:`load_judge_prompts` -- the upstream prompt templates, read from the + operator's own checkout (see the note above) and digest-checked against the + pinned commit; +* :func:`compose_judge_prompt` -- prompt assembly, including the conversation + rendering and the user/system-turn extraction upstream performs; +* :func:`parse_judgement` -- grader reply -> rubric verdicts; +* the counting helpers the two published rates are pooled from. + +Two published rates, and they do **not** share a denominator -- upstream +computes them in different places and they disagree whenever the grader emits a +number of answers that differs from the rubric count: + +* per-sample ``rubric_level_pass_rate`` (``judge._calc_rubric_level_pass_rate``) + divides in-range passes by ``len(rubrics)``, the count the *data* carries, and + skips answer keys that index past it; +* the aggregate ``micro_pass_rate`` (``processor._calculate_stats``) pools over + every key the *grader* emitted, with no range check at all. + +:func:`count_in_range_passes` and :func:`count_all_checks` keep the two separate +so each pooled metric matches its own upstream definition. + +**Upstream defect, reproduced deliberately.** Upstream selects the +system-steerability judge on ``benchmark_name == "if_system_steerability_oss"`` +-- a value the released dataset never contains, since it ships +``system_steerability_v2``. On the public data that judge is therefore +unreachable, and all 507 system-prompt rows are graded by the plain +user-instruction judge against rubrics written for the system prompt. The same +stale spelling makes the CLI's ``--task`` choices match zero rows, while +``processor.process_file``'s own docstring gives ``system_steerability_v2`` as +its example -- so the ``if_*_oss`` literals, not the dataset, are what went +stale. :func:`is_system_steer` keeps upstream's comparison verbatim anyway: +the unqualified task name tracks upstream including its defects, so a run can +be compared against a published number without first asking which routing it +used. Correcting the routing changes scores on a third of the benchmark and so +belongs in a ``_fixed`` variant carrying a measured delta, not here. + +Deviations from upstream @ f9d3013: + +* **Reply parsing.** Upstream guarantees JSON by passing + ``response_format={"type": "json_object"}`` to the OpenAI client. sieval + reaches the grader through the generic ``ChatModel``, and not every endpoint + honours that flag, so :func:`parse_judgement` falls back to extracting the + outermost JSON object (optionally fenced) before giving up. Without it a + grader that fences its JSON would score zero everywhere -- a harness artifact, + not a property of the model under test. +* **Non-string rubric answers** are stringified rather than raising. Upstream + calls ``.lower()`` directly, so a non-string answer aborts the row into its + ``except Exception`` path; stringifying keeps the row gradeable and, for the + schema-conforming string answers upstream expects, is identical. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import hashlib +import importlib.util +import json +import os +import re +import sys +from dataclasses import dataclass +from functools import cache +from pathlib import Path + +UPSTREAM_COMMIT = "f9d30137c4139d4d9af260ae28108b5afae828c0" +"""Upstream revision the prompts and scoring rules are pinned to.""" + +UPSTREAM_JUDGE_SHA256 = ( + "415164e9c3cb1e267321fa0561a8d61b81b3ac134a7d018764ae53a6e5a84955" +) +"""sha256 of ``judge.py`` at :data:`UPSTREAM_COMMIT`.""" + +SRC_ENV_VAR = "SIEVAL_ADVANCED_IF_SRC" +"""Environment variable pointing at the operator's upstream checkout.""" + +# Upstream's literal, verbatim. The released dataset spells the same aspect +# `system_steerability_v2`, so this never matches it -- see the module docstring. +SYSTEM_STEER_BENCHMARK = "if_system_steerability_oss" + +RELEASED_SYSTEM_STEER_BENCHMARK = "system_steerability_v2" +"""What the released dataset calls the aspect :data:`SYSTEM_STEER_BENCHMARK` misses.""" + +_JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL) + +_MISSING_SOURCE_HINT = ( + f"AdvancedIF's judge prompts are CC-BY-NC-4.0 and are not redistributed " + f"with sieval. Clone the upstream harness and point {SRC_ENV_VAR} at it:\n" + f" git clone https://github.com/facebookresearch/AdvancedIF\n" + f" git -C AdvancedIF checkout {UPSTREAM_COMMIT}\n" + f" export {SRC_ENV_VAR}=$PWD/AdvancedIF" +) + + +@dataclass(frozen=True) +class JudgePrompts: + """The three prompt templates upstream ``judge.py`` defines.""" + + judge_prompt: str + system_steer_judge_prompt: str + steer_few_shot_examples: str + + +@dataclass(frozen=True) +class Judgement: + """A parsed grader verdict. + + Attributes: + rubrics_check: Answer keyed by ``question_``, as the grader emitted + it -- keys are neither renumbered nor range-filtered here, because + the two pooled rates disagree about which of them count. + satisfied_all: The grader's all-rubrics-passed declaration. + """ + + rubrics_check: dict[str, str] + satisfied_all: bool + + +def is_system_steer(benchmark_name: str) -> bool: + """Whether *benchmark_name* routes to the system-steerability judge. + + Upstream's comparison, kept verbatim -- which means this returns ``False`` + for every row of the released dataset, including all 507 system-prompt + ones. That is upstream's behaviour, not an oversight here; see the module + docstring. + """ + return benchmark_name == SYSTEM_STEER_BENCHMARK + + +def _judge_source_path() -> Path: + raw = os.environ.get(SRC_ENV_VAR, "").strip() + if not raw: + raise RuntimeError(f"{SRC_ENV_VAR} is not set.\n\n{_MISSING_SOURCE_HINT}") + root = Path(raw).expanduser() + path = root / "judge.py" if root.is_dir() else root + if not path.is_file(): + raise RuntimeError( + f"{SRC_ENV_VAR}={raw} does not contain judge.py (looked at {path})." + f"\n\n{_MISSING_SOURCE_HINT}" + ) + return path + + +@cache +def load_judge_prompts() -> JudgePrompts: + """Load the upstream prompt templates from the operator's checkout. + + The file is digest-checked against :data:`UPSTREAM_JUDGE_SHA256` before it + is imported, so a score is always attributable to a known prompt revision. + A mismatch is fatal by design -- these prompts *are* the benchmark, and + silently grading against a drifted revision would make runs incomparable. + Recovery is to check out :data:`UPSTREAM_COMMIT`. + """ + path = _judge_source_path() + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != UPSTREAM_JUDGE_SHA256: + raise RuntimeError( + f"{path} does not match the pinned AdvancedIF revision.\n" + f" expected sha256 {UPSTREAM_JUDGE_SHA256} (commit {UPSTREAM_COMMIT})\n" + f" found sha256 {digest}\n" + f"Check out the pinned commit:\n" + f" git -C {path.parent} checkout {UPSTREAM_COMMIT}" + ) + + spec = importlib.util.spec_from_file_location("sieval._advanced_if_upstream", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load AdvancedIF judge module from {path}.") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + return JudgePrompts( + judge_prompt=module.JUDGE_PROMPT, + system_steer_judge_prompt=module.SYSTEM_STEER_JUDGE_PROMPT, + steer_few_shot_examples=module.STEER_FEW_SHOT_EXAMPLES, + ) + + +def parse_conversation(conversation_history: str | list) -> list: + """Decode the dataset's ``conversation_history`` into role/content dicts. + + Only the two keys the judge prompt reads are kept, so the record persisted + downstream is the message list actually sent to the model. The return type + is left bare because this list becomes a record's ``prompt`` (a + ``JSONValue``), and ``list`` is invariant. + """ + messages = ( + json.loads(conversation_history) + if isinstance(conversation_history, str) + else conversation_history + ) + if not isinstance(messages, list): + raise ValueError( + f"conversation_history must decode to a list, got {type(messages).__name__}" + ) + return [{"role": str(m["role"]), "content": str(m["content"])} for m in messages] + + +def parse_rubrics(prompt_metadata: str | dict) -> list[str]: + """Extract the rubric list from the dataset's ``prompt_metadata``. + + ``rubrics`` is itself sometimes a JSON-encoded string rather than a list, + which is why upstream decodes it a second time. + """ + metadata = ( + json.loads(prompt_metadata) + if isinstance(prompt_metadata, str) + else prompt_metadata + ) + if "rubrics" not in metadata: + raise ValueError("Rubrics not found in prompt_metadata") + rubrics = metadata["rubrics"] + if isinstance(rubrics, str): + rubrics = json.loads(rubrics) + return [str(rubric) for rubric in rubrics] + + +def format_conversation_history(messages: list[dict]) -> str: + """Render prior turns as upstream's ``role [turn]: content`` block. + + The final message is dropped: the dataset's ``conversation_history`` ends on + the user prompt being answered, and that turn is passed to the prompt + separately. The turn counter advances on assistant messages, so a + user/assistant pair shares one number. + """ + formatted = [] + turn = 1 + for message in messages[:-1]: + formatted.append(f"{message['role']} [{turn}]: {message['content']}") + if message["role"] == "assistant": + turn += 1 + return "\n".join(formatted) + + +def last_user_turn(messages: list[dict]) -> str: + """The most recent user message, or ``""`` when there is none.""" + for message in reversed(messages): + if message["role"] == "user": + return message["content"] + return "" + + +def system_prompt_of(messages: list[dict]) -> str: + """The leading system message, or ``""`` when the turn list has none.""" + if messages and messages[0]["role"] == "system": + return messages[0]["content"] + return "" + + +def compose_judge_prompt( + benchmark_name: str, + messages: list[dict], + response_text: str, + rubrics: list[str], +) -> str: + """Assemble the grader prompt for one sample. + + Routing between the user-instruction and system-steerability judges follows + :func:`is_system_steer`; the rubric block is JSON with upstream's + ``indent=4``, which the grader's ``question_`` keys are positional over. + """ + prompts = load_judge_prompts() + rubrics_text = json.dumps(rubrics, indent=4) + + if is_system_steer(benchmark_name): + return prompts.system_steer_judge_prompt.format( + few_shot_examples=prompts.steer_few_shot_examples, + system_prompt=system_prompt_of(messages), + user_prompt_last_turn=last_user_turn(messages), + response_text=response_text, + rubrics_text=rubrics_text, + ) + return prompts.judge_prompt.format( + full_conversation=format_conversation_history(messages), + user_prompt_last_turn=last_user_turn(messages), + response_text=response_text, + rubrics_text=rubrics_text, + ) + + +def _loads_json_object(reply: str) -> dict | None: + try: + parsed = json.loads(reply) + except (json.JSONDecodeError, ValueError): + return None + return parsed if isinstance(parsed, dict) else None + + +def parse_judgement(reply: str) -> Judgement | None: + """Parse a grader reply, or ``None`` when it yields no JSON object. + + ``None`` is the analogue of upstream's ``JudgeResult(success=False)``: the + sample counts against the overall pass rate but contributes no rubrics to + the pooled micro rate. + """ + parsed = _loads_json_object(reply) + if parsed is None: + # Fenced or prose-wrapped JSON -- see the parsing deviation above. + match = _JSON_OBJECT_RE.search(reply) + parsed = _loads_json_object(match.group(0)) if match else None + if parsed is None: + return None + + raw_checks = parsed.get("rubrics_check", {}) + if not isinstance(raw_checks, dict): + raw_checks = {} + # Upstream defaults a missing declaration to "NO" and compares + # case-insensitively -- its own few-shot examples answer "Yes"/"No", not + # "YES"/"NO", so a case-sensitive check would fail every passing sample. + declared = parsed.get("SATISFIED_ALL_REQUIREMENTS", "NO") + return Judgement( + rubrics_check={str(k): str(v) for k, v in raw_checks.items()}, + satisfied_all=str(declared).strip().lower() == "yes", + ) + + +def _is_pass(answer: str) -> bool: + # Upstream's substring test, not equality: rubric answers routinely carry a + # justification ("The intro is four sentences. No"). + return "yes" in answer.lower() + + +def count_in_range_passes(rubrics_check: dict[str, str], rubrics: list[str]) -> int: + """Passes among answers whose ``question_`` indexes a real rubric. + + Numerator of the per-sample ``rubric_level_pass_rate``; keys that are + unparseable or index past the rubric list are skipped, as upstream does. + """ + passes = 0 + for key, answer in rubrics_check.items(): + try: + index = int(key.split("_")[1]) - 1 + except (IndexError, ValueError): + continue + if index >= len(rubrics): + continue + if _is_pass(answer): + passes += 1 + return passes + + +def rubric_level_pass_rate(rubrics_check: dict[str, str], rubrics: list[str]) -> float: + """Per-sample rubric pass rate, over the rubric count the data carries.""" + return count_in_range_passes(rubrics_check, rubrics) / max(len(rubrics), 1) + + +def count_all_checks(rubrics_check: dict[str, str]) -> tuple[int, int]: + """``(answers emitted, answers passed)`` with no range filtering. + + The pooled ``micro_pass_rate`` counts exactly these, so a grader that + answers fewer questions than there are rubrics shrinks its own denominator. + """ + return ( + len(rubrics_check), + sum(1 for answer in rubrics_check.values() if _is_pass(answer)), + ) + + +def aggregate_metrics(verdicts: list[dict]) -> dict[str, float]: + """Pool per-rollout verdicts into the two published rates. + + Each entry carries ``satisfied_all``, ``n_checks`` and ``n_checks_passed`` + (a rollout the grader failed to produce a verdict for contributes only to + the denominator of the pass rate, matching upstream). + """ + total = len(verdicts) + passed = sum(1 for v in verdicts if v["satisfied_all"]) + checks = sum(v["n_checks"] for v in verdicts) + checks_passed = sum(v["n_checks_passed"] for v in verdicts) + return { + "overall_pass_rate": passed / total * 100 if total else 0.0, + "micro_pass_rate": checks_passed / checks * 100 if checks else 0.0, + "n_samples": float(total), + "n_rubric_checks": float(checks), + } diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index e5cd3cbe..042019fe 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -5,6 +5,10 @@ from .aa_lcr import ( AALCRDataset, AALCRDatasetSample, ) +from .advanced_if import ( + AdvancedIFDataset, + AdvancedIFDatasetSample, +) from .aime_2024 import ( AIME2024Dataset, AIME2024DatasetSample, @@ -175,6 +179,8 @@ __all__ = [ "ARCChallengeDatasetSample", "ARCEasyDataset", "ARCEasyDatasetSample", + "AdvancedIFDataset", + "AdvancedIFDatasetSample", "Apex2025Dataset", "Apex2025DatasetSample", "ApexShortlist2025Dataset", diff --git a/sieval/datasets/advanced_if.py b/sieval/datasets/advanced_if.py new file mode 100644 index 00000000..f29826d9 --- /dev/null +++ b/sieval/datasets/advanced_if.py @@ -0,0 +1,67 @@ +"""AdvancedIF dataset loader. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import apply_eval_split, ensure_dataset_dict + +ADVANCED_IF_REVISION = "e20cba9b94b59c027dfab00b29244e8bc42e4ab4" + + +class AdvancedIFDatasetSample(TypedDict): + """One AdvancedIF prompt. + + Upstream ships all three columns as strings, including the two that hold + JSON; they are decoded in the task rather than here, so the persisted prompt + record keeps the bytes the grader prompt was actually built from. No cast + needed -- the pinned revision already ships these as strings. + + Attributes: + conversation_history: JSON list of ``{"role", "content"}`` turns, ending + on the user prompt to answer (the assistant turn under test is + absent by construction). A system-steerability row leads with a + ``system`` turn. + benchmark_name: Which of the three aspects the row belongs to -- + ``complex_if_single_turn_v5`` (402), ``system_steerability_v2`` + (507) or ``carried_context_multi_turn_eval_v5`` (736). + prompt_metadata: JSON object whose ``rubrics`` key holds the + expert-written yes/no checks (itself sometimes JSON-encoded again). + """ + + conversation_history: str + benchmark_name: str + prompt_metadata: str + + +@sieval_dataset( + name="advanced_if", + display_name="AdvancedIF", + description=( + "Expert-written prompts with human-curated rubrics for advanced " + "instruction following." + ), + source=f"hf:facebook/AdvancedIF@{ADVANCED_IF_REVISION}", + categories=(Category(Level1Category.LANGUAGE, "InstructionFollowing"),), + tags=("english", "open-ended"), + # Non-commercial. The judge prompts carry the same terms and are likewise + # not redistributed -- see sieval.community.advanced_if. + license="CC-BY-NC-4.0", +) +class AdvancedIFDataset(Dataset[AdvancedIFDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + # AdvancedIF ships its 1,645 rows in a single "train" split (the card + # calls the same rows "test"); mirror it to "test" for the runner. + dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs)) + return apply_eval_split(dataset, "train") diff --git a/sieval/meta/index.json b/sieval/meta/index.json index d77ab81d..dabc17f3 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -24,6 +24,27 @@ "license": "apache-2.0", "checksums": {} }, + { + "name": "advanced_if", + "display_name": "AdvancedIF", + "description": "Expert-written prompts with human-curated rubrics for advanced instruction following.", + "source": [ + "hf:facebook/AdvancedIF@e20cba9b94b59c027dfab00b29244e8bc42e4ab4" + ], + "categories": [ + { + "level1": "Language", + "level2": "InstructionFollowing" + } + ], + "tags": [ + "english", + "open-ended" + ], + "deps_group": null, + "license": "CC-BY-NC-4.0", + "checksums": {} + }, { "name": "aime_2024", "display_name": "AIME 2024", @@ -917,6 +938,26 @@ }, "status": "experimental" }, + { + "name": "advanced_if_0shot_gen", + "display_name": "AdvancedIF (0-shot, generative)", + "description": "Instruction following graded against expert-written rubrics by an LLM judge.", + "dataset": "advanced_if", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "open-ended" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "facebookresearch/AdvancedIF", + "url": "https://github.com/facebookresearch/AdvancedIF/blob/f9d30137c4139d4d9af260ae28108b5afae828c0/judge.py", + "notes": "Port of AdvancedIF (Meta, arXiv:2511.10507), 1,645 prompts across complex_if_single_turn_v5 (402), system_steerability_v2 (507) and carried_context_multi_turn_eval_v5 (736). Headline score is the overall pass rate (share of samples the grader marked SATISFIED_ALL_REQUIREMENTS=yes), the number the paper reports; micro_pass_rate is the co-published rubric-level rate. LICENSING: upstream ships judge.py under CC-BY-NC-4.0, incompatible with sieval's Apache-2.0 tree, so the prompts are NOT vendored -- the operator stages a checkout and points SIEVAL_ADVANCED_IF_SRC at it, digest-checked against commit f9d30137c4139d4d9af260ae28108b5afae828c0. The dataset is CC-BY-NC-4.0 too, so running this benchmark accepts those terms either way. Loading from the operator's own checkout also makes the prompts byte-exact by construction rather than by review. UPSTREAM DEFECT (reproduced, not corrected): upstream routes to the system-steerability judge on benchmark_name == 'if_system_steerability_oss', a value the released dataset never contains (it ships 'system_steerability_v2'), so all 507 system-prompt rows are graded by the plain user-instruction judge and the CLI's --task choices match zero rows; processor.process_file's own docstring uses the released spelling, so the if_*_oss literals are what went stale. This port keeps upstream's comparison verbatim so the unqualified name tracks upstream. Correcting the routing moves scores on a third of the benchmark and belongs in a _fixed variant carrying a measured delta. Grader is a REAL LLM (upstream: o3-mini-2025-01-31, temperature 0, max_completion_tokens=32768, response_format=json_object) supplied via the `grader` task arg; pin it, as its version is not pinnable the way a Hub revision is. The grader's full ModelOutput and per-rubric answers are persisted under the judgement record's `extra`. VALIDATION: none -- no published number has been reproduced with this port, and the paper's own figures come from Meta's internal pipeline rather than the released CLI." + }, + "status": "experimental" + }, { "name": "aime_2024_0shot_gen", "display_name": "AIME 2024 (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 2136c931..f4fc2acc 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -4,6 +4,9 @@ from .aa_lcr_0shot_gen import ( AALCRZeroShotGenTask, ) +from .advanced_if_0shot_gen import ( + AdvancedIFZeroShotGenTask, +) from .aime_2024_0shot_gen import ( AIME2024ZeroShotGenTask, ) @@ -161,6 +164,7 @@ __all__ = [ "ARCChallengeFewShotPplTask", "ARCEasyFewShotClpTask", "ARCEasyFewShotPplTask", + "AdvancedIFZeroShotGenTask", "Apex2025ZeroShotGenTask", "ApexShortlist2025ZeroShotGenTask", "BRUMO2025ZeroShotGenTask", diff --git a/sieval/tasks/advanced_if_0shot_gen.py b/sieval/tasks/advanced_if_0shot_gen.py new file mode 100644 index 00000000..a889aff3 --- /dev/null +++ b/sieval/tasks/advanced_if_0shot_gen.py @@ -0,0 +1,315 @@ +"""AdvancedIF — 0-shot generative, rubric-graded by an LLM judge. + +AdvancedIF (Meta, Hu et al., 2025, arXiv:2511.10507) probes instruction +following past the verifiable-constraint regime IFEval and IFBench cover: every +prompt is expert-written and paired with a human-curated rubric of yes/no +checks, and the three aspects it spans -- complex single-turn instructions +(6+ per prompt), instructions carried across a multi-turn dialog, and system- +prompt steerability -- are graded by an LLM judge rather than by checkers. + +The model answers the conversation's final user turn; a separate **grader** +model then answers every rubric question and declares whether the response +satisfied all of them. Headline metric is the overall pass rate: the share of +samples where the grader answered yes to that declaration. + +The grader is supplied via the ``grader`` task arg (a model-config dict, or a +pre-built Model, on its own ``api_base``/``api_key``); upstream's judge is +o3-mini-2025-01-31 at temperature 0 with ``max_completion_tokens=32768`` and +``response_format={"type": "json_object"}``. Set all of those in the grader's +model config, not here -- sieval does not force ``response_format`` on the +request because endpoints that reject the field would fail the whole run, and +the reply parser tolerates fenced JSON either way. + +Upstream's judge routing is reproduced with its defect intact: the +system-steerability judge is selected on a ``benchmark_name`` the released +dataset never contains, so every row -- including the 507 system-prompt ones -- +is graded by the user-instruction judge. The unqualified task name tracks +upstream, defects included; see :mod:`sieval.community.advanced_if`. + +Running this task needs an upstream checkout: the judge prompts are CC-BY-NC-4.0 +and are not redistributed with sieval, so point ``SIEVAL_ADVANCED_IF_SRC`` at +your own clone (see :mod:`sieval.community.advanced_if`, which digest-checks it +against the pinned commit). The benchmark data carries the same terms. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from collections.abc import Mapping +from typing import override + +from sieval.community.advanced_if import ( + aggregate_metrics, + compose_judge_prompt, + count_all_checks, + count_in_range_passes, + parse_conversation, + parse_judgement, + parse_rubrics, +) +from sieval.core.models import ChatModel, Model, ModelOutput +from sieval.core.tasks import ( + GRADER_OUTPUT_KEY, + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + RolloutJudgement, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.utils.serialization import obj_to_dict +from sieval.datasets import AdvancedIFDatasetSample + + +@sieval_task( + name="advanced_if_0shot_gen", + display_name="AdvancedIF (0-shot, generative)", + description=( + "Instruction following graded against expert-written rubrics by an LLM judge." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "open-ended"), + model_type="chat", + reference_impl=ReferenceImpl( + source="facebookresearch/AdvancedIF", + url=( + "https://github.com/facebookresearch/AdvancedIF/blob/" + "f9d30137c4139d4d9af260ae28108b5afae828c0/judge.py" + ), + notes=( + "Port of AdvancedIF (Meta, arXiv:2511.10507), 1,645 prompts across " + "complex_if_single_turn_v5 (402), system_steerability_v2 (507) and " + "carried_context_multi_turn_eval_v5 (736). Headline score is the " + "overall pass rate (share of samples the grader marked " + "SATISFIED_ALL_REQUIREMENTS=yes), the number the paper reports; " + "micro_pass_rate is the co-published rubric-level rate. " + "LICENSING: upstream ships judge.py under CC-BY-NC-4.0, " + "incompatible with sieval's Apache-2.0 tree, so the prompts are " + "NOT vendored -- the operator stages a checkout and points " + "SIEVAL_ADVANCED_IF_SRC at it, digest-checked against commit " + "f9d30137c4139d4d9af260ae28108b5afae828c0. The dataset is " + "CC-BY-NC-4.0 too, so running this benchmark accepts those terms " + "either way. Loading from the operator's own checkout also makes " + "the prompts byte-exact by construction rather than by review. " + "UPSTREAM DEFECT (reproduced, not corrected): upstream routes to " + "the system-steerability judge on benchmark_name == " + "'if_system_steerability_oss', a value the released dataset never " + "contains (it ships 'system_steerability_v2'), so all 507 " + "system-prompt rows are graded by the plain user-instruction judge " + "and the CLI's --task choices match zero rows; " + "processor.process_file's own docstring uses the released " + "spelling, so the if_*_oss literals are what went stale. This port " + "keeps upstream's comparison verbatim so the unqualified name " + "tracks upstream. Correcting the routing moves scores on a third " + "of the benchmark and belongs in a _fixed variant carrying a " + "measured delta. " + "Grader is a REAL LLM (upstream: o3-mini-2025-01-31, temperature " + "0, max_completion_tokens=32768, response_format=json_object) " + "supplied via the `grader` task arg; pin it, as its version is not " + "pinnable the way a Hub revision is. The grader's full ModelOutput " + "and per-rubric answers are persisted under the judgement record's " + "`extra`. " + "VALIDATION: none -- no published number has been reproduced with " + "this port, and the paper's own figures come from Meta's internal " + "pipeline rather than the released CLI." + ), + ), + # Faithful port of upstream's routing and scoring kernel, but no published + # number has been reproduced with it: faithful port, unverified reproduction. + status="experimental", +) +class AdvancedIFZeroShotGenTask( + Task[ + AdvancedIFDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + grader: Mapping | Model | None = None, + n: int = 1, + ): + super().__init__(dataset=dataset, model=model, name=name) + self._n = n + self._grader = self._build_grader(grader) + + @staticmethod + def _build_grader(grader: Mapping | Model | None) -> Model: + """Resolve the ``grader`` task arg into a Model. + + Accepts a pre-built Model (tests / advanced configs) or a model-config + mapping (the YAML path). Rubric grading is the only scorer AdvancedIF + has -- there is no deterministic fallback -- so ``None`` raises. + """ + if isinstance(grader, Model): + return grader + if isinstance(grader, Mapping): + return ChatModel(**grader) + raise ValueError( + "AdvancedIF requires an LLM grader. Pass `grader:` in the task " + "args — a model-config dict such as {model: o3-mini, api_base: " + "..., api_key: ..., temperature: 0}." + ) + + @override + async def preprocess(self, raw, ctx): + messages = parse_conversation(raw["conversation_history"]) + # No `reference`: the ground truth is a rubric (a procedure), not a + # value. The rubric itself rides in `extra` so a prompt row is readable + # on its own and feedback() need not re-decode the raw sample. + return build_prompt_record( + messages, + extra={ + "benchmark_name": raw["benchmark_name"], + "rubrics": parse_rubrics(raw["prompt_metadata"]), + }, + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + # Open-ended: the response *is* the answer, so no extraction step. A + # blank response normalizes to None so `extracted` stays a real signal; + # the grader still sees "" and will fail its rubrics. + return build_prediction_record( + [text if text.strip() else None for text in inf.texts] + ) + + @override + async def feedback(self, post, ctx): + """Grade every rollout against its rubrics, recording the judge's reply. + + The grader is a model, so its output is persisted the way any model + output is: ``extra["grader_output"]`` is the full ``ModelOutput`` + flattened to a plain dict. Nothing is hand-picked, so no field is + silently dropped. That matters more here than for a short-answer + autorater: the verdict is a whole rubric-by-rubric JSON blob, a + re-grade need not reproduce it, and an unparseable reply is scored the + same as a failing one -- only the raw text separates a grader that + broke format from a response that genuinely missed every rubric. + + Per-rubric answers and the two raw counts the pooled rates need live in + the rollout's ``extra``: the published micro rate pools over the + grader's own answer keys, which a per-sample rate cannot reconstruct. + """ + prompt_extra = ctx.preprocess_result["extra"] + benchmark_name = prompt_extra["benchmark_name"] + rubrics = prompt_extra["rubrics"] + messages = ctx.preprocess_result["prompt"] + + rollouts: list[RolloutJudgement] = [] + for rollout in post["rollouts"]: + response = rollout.get("prediction") or "" + out = await self._grader.agenerate( + compose_judge_prompt(benchmark_name, messages, response, rubrics) + ) + reply = out.texts[0] if out.texts else "" + judgement = parse_judgement(reply) + + if judgement is None: + # Upstream's failed-row path: counts against the pass rate, + # contributes no rubrics to the pooled micro rate. + checks: dict[str, str] = {} + satisfied_all = False + n_checks, n_checks_passed = 0, 0 + pass_rate = 0.0 + else: + checks = judgement.rubrics_check + satisfied_all = judgement.satisfied_all + n_checks, n_checks_passed = count_all_checks(checks) + pass_rate = count_in_range_passes(checks, rubrics) / max( + len(rubrics), 1 + ) + + rollouts.append( + build_rollout_judgement( + rollout["index"], + satisfied_all, + score=pass_rate, + metrics={ + "satisfied_all": satisfied_all, + "rubric_level_pass_rate": pass_rate, + }, + extra={ + "judge_parsed": judgement is not None, + "rubrics_check": checks, + "n_checks": n_checks, + "n_checks_passed": n_checks_passed, + GRADER_OUTPUT_KEY: obj_to_dict(out, add_type=False), + }, + ) + ) + + mean_pass_rate = sum(r["score"] for r in rollouts) / len(rollouts) + return True, build_judgement_record( + # Rubric grading is a procedure, not a value to compare against. + None, + rollouts, + score=mean_pass_rate, + extra={"benchmark_name": benchmark_name, "n_rubrics": len(rubrics)}, + ) + + @override + async def report(self, finals, fails): + """Pool the two published rates overall and per aspect. + + Reads the persisted verdicts rather than ``raw_sample``, so the report + survives a resume. Pipeline failures (exhausted retries) never produced + a gradeable answer; each failed sample's requested rollouts count as + non-passes so the headline spans the full requested set, matching + upstream's denominator (which likewise includes rows its judge failed). + """ + by_benchmark: dict[str, list[dict]] = {} + verdicts: list[dict] = [] + for final in finals: + judgement = final.feedback_result or {} + benchmark_name = judgement.get("extra", {}).get("benchmark_name", "unknown") + for rollout in judgement.get("rollouts", []): + extra = rollout.get("extra", {}) + verdict = { + "satisfied_all": rollout["correct"], + "n_checks": extra.get("n_checks", 0), + "n_checks_passed": extra.get("n_checks_passed", 0), + } + verdicts.append(verdict) + by_benchmark.setdefault(benchmark_name, []).append(verdict) + + n_graded = len(verdicts) + # A failed sample has no verdict to attribute to an aspect, so it lands + # in the overall rates only; the per-aspect rates below cover graded + # rollouts, and `fails` reports the shortfall. + failed = [{"satisfied_all": False, "n_checks": 0, "n_checks_passed": 0}] * ( + self._n * len(fails) + ) + + overall = aggregate_metrics(verdicts + failed) + results: dict[str, float] = { + "score": overall["overall_pass_rate"], + "overall_pass_rate": overall["overall_pass_rate"], + "micro_pass_rate": overall["micro_pass_rate"], + "n_rubric_checks": overall["n_rubric_checks"], + "n_graded": float(n_graded), + "fails": len(fails), + } + for benchmark_name, group in sorted(by_benchmark.items()): + aspect = aggregate_metrics(group) + results[f"{benchmark_name}_pass_rate"] = aspect["overall_pass_rate"] + results[f"{benchmark_name}_micro_pass_rate"] = aspect["micro_pass_rate"] + results[f"{benchmark_name}_n_graded"] = aspect["n_samples"] + return results diff --git a/tests/unit/community/test_advanced_if.py b/tests/unit/community/test_advanced_if.py new file mode 100644 index 00000000..f99e2f54 --- /dev/null +++ b/tests/unit/community/test_advanced_if.py @@ -0,0 +1,334 @@ +"""Unit tests for the AdvancedIF judge assets and scoring kernel. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json + +import pytest + +from sieval.community import advanced_if +from sieval.community.advanced_if import ( + RELEASED_SYSTEM_STEER_BENCHMARK, + SRC_ENV_VAR, + SYSTEM_STEER_BENCHMARK, + UPSTREAM_JUDGE_SHA256, + JudgePrompts, + aggregate_metrics, + compose_judge_prompt, + count_all_checks, + count_in_range_passes, + format_conversation_history, + is_system_steer, + last_user_turn, + load_judge_prompts, + parse_conversation, + parse_judgement, + parse_rubrics, + rubric_level_pass_rate, + system_prompt_of, +) + +# --- dataset field decoding --- + + +def test_parse_conversation_from_json_string_keeps_role_and_content(): + raw = json.dumps( + [ + {"role": "user", "content": "hi", "extra_field": "dropped"}, + {"role": "assistant", "content": "hello"}, + ] + ) + assert parse_conversation(raw) == [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] + + +def test_parse_conversation_accepts_decoded_list(): + assert parse_conversation([{"role": "user", "content": "hi"}]) == [ + {"role": "user", "content": "hi"} + ] + + +def test_parse_conversation_rejects_non_list(): + with pytest.raises(ValueError, match="must decode to a list"): + parse_conversation(json.dumps({"role": "user"})) + + +def test_parse_rubrics_handles_double_encoded_rubrics(): + """Upstream decodes ``rubrics`` twice because it is sometimes a JSON string.""" + as_list = json.dumps({"rubrics": ["a?", "b?"]}) + as_string = json.dumps({"rubrics": json.dumps(["a?", "b?"])}) + assert parse_rubrics(as_list) == ["a?", "b?"] + assert parse_rubrics(as_string) == ["a?", "b?"] + assert parse_rubrics({"rubrics": ["a?"]}) == ["a?"] + + +def test_parse_rubrics_requires_the_key(): + with pytest.raises(ValueError, match="Rubrics not found"): + parse_rubrics(json.dumps({"something_else": []})) + + +# --- conversation rendering (feeds the judge prompt verbatim) --- + + +def test_format_conversation_history_drops_last_turn_and_numbers_by_assistant(): + messages = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + ] + # The trailing user turn is excluded -- it is passed to the prompt + # separately -- and the turn counter advances only past an assistant reply. + assert format_conversation_history(messages) == "user [1]: u1\nassistant [1]: a1" + + +def test_last_user_turn_and_system_prompt(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + ] + assert last_user_turn(messages) == "u2" + assert system_prompt_of(messages) == "sys" + assert system_prompt_of(messages[1:]) == "" + assert last_user_turn([{"role": "assistant", "content": "a"}]) == "" + + +# --- judge routing: upstream's defect, reproduced on purpose --- + + +def test_system_steer_routing_never_fires_on_released_data(): + """Pin the reproduced defect so a "cleanup" cannot silently change scores. + + Upstream compares against ``if_system_steerability_oss``; the released + dataset ships ``system_steerability_v2``, so every row -- including all 507 + system-prompt ones -- goes to the user-instruction judge. + """ + assert is_system_steer(SYSTEM_STEER_BENCHMARK) + assert not is_system_steer(RELEASED_SYSTEM_STEER_BENCHMARK) + assert not is_system_steer("complex_if_single_turn_v5") + assert not is_system_steer("carried_context_multi_turn_eval_v5") + + +# --- grader reply parsing --- + + +def _reply(checks: dict, satisfied: str) -> str: + return json.dumps( + {"rubrics_check": checks, "SATISFIED_ALL_REQUIREMENTS": satisfied} + ) + + +def test_parse_judgement_reads_checks_and_declaration(): + judgement = parse_judgement(_reply({"question_1": "Yes"}, "YES")) + assert judgement is not None + assert judgement.rubrics_check == {"question_1": "Yes"} + assert judgement.satisfied_all + + +@pytest.mark.parametrize("declared", ["Yes", "yes", "YES", " yes "]) +def test_parse_judgement_declaration_is_case_insensitive(declared): + """Upstream's own few-shot examples answer "Yes"/"No", not "YES"/"NO".""" + judgement = parse_judgement(_reply({}, declared)) + assert judgement is not None + assert judgement.satisfied_all + + +def test_parse_judgement_defaults_missing_declaration_to_not_satisfied(): + judgement = parse_judgement(json.dumps({"rubrics_check": {"question_1": "Yes"}})) + assert judgement is not None + assert not judgement.satisfied_all + + +def test_parse_judgement_recovers_fenced_json(): + """sieval cannot force response_format=json_object on every endpoint.""" + fenced = f"Here you go:\n```json\n{_reply({'question_1': 'Yes'}, 'Yes')}\n```" + judgement = parse_judgement(fenced) + assert judgement is not None + assert judgement.rubrics_check == {"question_1": "Yes"} + + +def test_parse_judgement_returns_none_without_json(): + assert parse_judgement("I could not evaluate this.") is None + assert parse_judgement("") is None + + +def test_parse_judgement_stringifies_non_string_answers(): + judgement = parse_judgement(_reply({"question_1": ["Yes"]}, "No")) + assert judgement is not None + assert judgement.rubrics_check == {"question_1": "['Yes']"} + + +# --- counting: the two rates do not share a denominator --- + + +def test_count_in_range_passes_skips_out_of_range_and_malformed_keys(): + checks = { + "question_1": "Yes", + "question_2": "No", + "question_3": "Yes", # indexes past a 2-rubric sample + "notaquestion": "Yes", # unparseable key + "question_x": "Yes", # unparseable index + } + assert count_in_range_passes(checks, ["r1", "r2"]) == 1 + + +def test_count_in_range_passes_matches_justified_answers_by_substring(): + """Rubric answers routinely carry a justification before the verdict.""" + checks = {"question_1": "The intro is four sentences. No", "question_2": "Yes"} + assert count_in_range_passes(checks, ["r1", "r2"]) == 1 + + +def test_rubric_level_pass_rate_divides_by_the_data_rubric_count(): + checks = {"question_1": "Yes"} + # Two rubrics in the data, one answered -> 0.5, not 1.0. + assert rubric_level_pass_rate(checks, ["r1", "r2"]) == 0.5 + assert rubric_level_pass_rate({}, []) == 0.0 + + +def test_count_all_checks_ignores_the_rubric_count(): + """The pooled micro rate counts grader-emitted keys, with no range filter.""" + checks = {"question_1": "Yes", "question_9": "Yes", "question_2": "No"} + assert count_all_checks(checks) == (3, 2) + + +def test_the_two_denominators_disagree_when_the_grader_under_answers(): + checks = {"question_1": "Yes"} + rubrics = ["r1", "r2", "r3", "r4"] + # Per-sample rate is over the data's 4 rubrics ... + assert rubric_level_pass_rate(checks, rubrics) == 0.25 + # ... while the pooled micro rate is over the single key the grader emitted. + assert count_all_checks(checks) == (1, 1) + + +# --- aggregation --- + + +def _verdict(satisfied: bool, n_checks: int, n_passed: int) -> dict: + return { + "satisfied_all": satisfied, + "n_checks": n_checks, + "n_checks_passed": n_passed, + } + + +def test_aggregate_metrics_pools_both_rates(): + metrics = aggregate_metrics( + [_verdict(True, 4, 4), _verdict(False, 4, 2), _verdict(False, 2, 0)] + ) + assert metrics["overall_pass_rate"] == pytest.approx(100 / 3) + assert metrics["micro_pass_rate"] == pytest.approx(600 / 10) + assert metrics["n_samples"] == 3.0 + assert metrics["n_rubric_checks"] == 10.0 + + +def test_aggregate_metrics_counts_ungradeable_rollouts_against_the_pass_rate_only(): + """Upstream's failed-row path: in the pass-rate denominator, out of micro.""" + metrics = aggregate_metrics([_verdict(True, 2, 2), _verdict(False, 0, 0)]) + assert metrics["overall_pass_rate"] == pytest.approx(50.0) + # The failed row contributes no rubrics, so micro stays 100%. + assert metrics["micro_pass_rate"] == pytest.approx(100.0) + + +def test_aggregate_metrics_handles_an_empty_set(): + metrics = aggregate_metrics([]) + assert metrics["overall_pass_rate"] == 0.0 + assert metrics["micro_pass_rate"] == 0.0 + + +# --- loading the upstream prompts (never vendored: CC-BY-NC-4.0) --- + + +def test_load_judge_prompts_requires_the_env_var(monkeypatch): + monkeypatch.delenv(SRC_ENV_VAR, raising=False) + load_judge_prompts.cache_clear() + with pytest.raises(RuntimeError, match=SRC_ENV_VAR): + load_judge_prompts() + + +def test_load_judge_prompts_reports_a_missing_file(monkeypatch, tmp_path): + monkeypatch.setenv(SRC_ENV_VAR, str(tmp_path)) + load_judge_prompts.cache_clear() + with pytest.raises(RuntimeError, match="does not contain judge.py"): + load_judge_prompts() + + +def test_load_judge_prompts_rejects_a_drifted_revision(monkeypatch, tmp_path): + """The prompts are the benchmark; grading against a drifted copy is fatal.""" + (tmp_path / "judge.py").write_text("JUDGE_PROMPT = 'not upstream'\n") + monkeypatch.setenv(SRC_ENV_VAR, str(tmp_path)) + load_judge_prompts.cache_clear() + with pytest.raises(RuntimeError, match=UPSTREAM_JUDGE_SHA256): + load_judge_prompts() + + +# --- prompt assembly (upstream prompts stubbed: they are not redistributable) --- + + +@pytest.fixture +def stub_prompts(monkeypatch): + prompts = JudgePrompts( + judge_prompt=( + "IF|{full_conversation}|{user_prompt_last_turn}|" + "{response_text}|{rubrics_text}" + ), + system_steer_judge_prompt=( + "STEER|{few_shot_examples}|{system_prompt}|" + "{user_prompt_last_turn}|{response_text}|{rubrics_text}" + ), + steer_few_shot_examples="SHOTS", + ) + monkeypatch.setattr(advanced_if, "load_judge_prompts", lambda: prompts) + return prompts + + +@pytest.mark.usefixtures("stub_prompts") +def test_compose_judge_prompt_fills_the_if_judge_slots(): + messages = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + ] + composed = compose_judge_prompt( + "complex_if_single_turn_v5", messages, "the answer", ["r1"] + ) + kind, conversation, last_turn, response, rubrics_text = composed.split("|") + assert kind == "IF" + assert conversation == "user [1]: u1\nassistant [1]: a1" + assert last_turn == "u2" + assert response == "the answer" + # Upstream renders the rubric block with indent=4. + assert rubrics_text == json.dumps(["r1"], indent=4) + + +@pytest.mark.usefixtures("stub_prompts") +def test_compose_judge_prompt_uses_the_steer_judge_for_upstreams_literal(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u1"}, + ] + composed = compose_judge_prompt( + SYSTEM_STEER_BENCHMARK, messages, "the answer", ["r1"] + ) + kind, shots, system_prompt, last_turn, response, _ = composed.split("|") + assert kind == "STEER" + assert shots == "SHOTS" + assert system_prompt == "sys" + assert last_turn == "u1" + assert response == "the answer" + + +@pytest.mark.usefixtures("stub_prompts") +def test_released_system_steer_rows_compose_the_if_prompt(): + """The reproduced routing defect, seen end to end at the prompt level.""" + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u1"}, + ] + composed = compose_judge_prompt( + RELEASED_SYSTEM_STEER_BENCHMARK, messages, "answer", ["r1"] + ) + assert composed.startswith("IF|") diff --git a/tests/unit/tasks/test_advanced_if_0shot_gen.py b/tests/unit/tasks/test_advanced_if_0shot_gen.py new file mode 100644 index 00000000..bdc326c0 --- /dev/null +++ b/tests/unit/tasks/test_advanced_if_0shot_gen.py @@ -0,0 +1,324 @@ +"""Unit tests for the AdvancedIF 0-shot generative task. + +The upstream judge prompts are CC-BY-NC-4.0 and are never redistributed, so +``compose_judge_prompt`` is stubbed here; :mod:`tests.unit.community.test_advanced_if` +covers the real assembly against stand-in templates. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + GRADER_OUTPUT_KEY, + TaskContext, + build_judgement_record, + build_rollout_judgement, + iter_grader_outputs, +) +from sieval.datasets.advanced_if import AdvancedIFDataset, AdvancedIFDatasetSample +from sieval.tasks import advanced_if_0shot_gen +from sieval.tasks.advanced_if_0shot_gen import AdvancedIFZeroShotGenTask + +COMPLEX = "complex_if_single_turn_v5" +STEERABILITY = "system_steerability_v2" + + +class _ScriptedChatModel(ChatModel): + """ChatModel returning a fixed reply, recording the last prompt it saw.""" + + def __init__(self, reply: str, model: str = "mock"): + super().__init__(model=model, api_key="fake") + self._reply = reply + self.last_prompt = None + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + self.last_prompt = prompt + return ModelOutput( + model=self.meta(), + texts=[self._reply], + usage={"input_tokens": 40, "output_tokens": 3, "total_tokens": 43}, + ) + + async def _alogprobs_impl( + self, prompt, *, max_tokens=1, logprobs=5, echo=True, temperature=0.0, **kwargs + ) -> ModelOutput: + _ = (prompt, max_tokens, logprobs, echo, temperature, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _sample( + benchmark_name: str = COMPLEX, + rubrics: tuple[str, ...] = ("Is it two paragraphs?", "Are there two metaphors?"), +) -> AdvancedIFDatasetSample: + return { + "conversation_history": json.dumps( + [ + {"role": "user", "content": "Write a story."}, + {"role": "assistant", "content": "Once upon a time."}, + {"role": "user", "content": "Now make it rhyme."}, + ] + ), + "benchmark_name": benchmark_name, + "prompt_metadata": json.dumps({"rubrics": list(rubrics)}), + } + + +def _judge_reply(checks: dict, satisfied: str) -> str: + return json.dumps( + {"rubrics_check": checks, "SATISFIED_ALL_REQUIREMENTS": satisfied} + ) + + +def _task(answer: str = "A rhyming story.", grader_reply: str = "{}"): + sample = _sample() + dataset = AdvancedIFDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(sample)])}) + ) + model = _ScriptedChatModel(reply=answer, model="candidate") + grader = _ScriptedChatModel(reply=grader_reply, model="o3-mini") + return AdvancedIFZeroShotGenTask(dataset, model, grader=grader), grader + + +async def _run_to_feedback(task, sample): + """Drive preprocess -> infer -> postprocess -> feedback for one sample.""" + ctx = TaskContext(sample_id=0, raw_sample=sample) + pre = await task.preprocess(sample, ctx) + ctx = TaskContext(sample_id=0, raw_sample=sample, preprocess_result=pre) + inferred = await task.infer(pre, ctx) + post = await task.postprocess(inferred, ctx) + return await task.feedback(post, ctx) + + +@pytest.fixture(autouse=True) +def stub_compose(monkeypatch): + """Stand in for the non-redistributable judge prompt assembly.""" + monkeypatch.setattr( + advanced_if_0shot_gen, + "compose_judge_prompt", + lambda benchmark_name, messages, response, rubrics: ( + f"JUDGE[{benchmark_name}] resp={response} rubrics={len(rubrics)}" + ), + ) + + +# --- grader is mandatory; rubric grading is the only scorer AdvancedIF has --- + + +def test_build_grader_requires_config(): + with pytest.raises(ValueError, match="requires an LLM grader"): + AdvancedIFZeroShotGenTask._build_grader(None) + + +def test_build_grader_accepts_mapping_and_model(): + built = AdvancedIFZeroShotGenTask._build_grader( + {"model": "o3-mini", "api_key": "fake"} + ) + assert isinstance(built, ChatModel) + existing = _ScriptedChatModel(reply="{}") + assert AdvancedIFZeroShotGenTask._build_grader(existing) is existing + + +# --- preprocess --- + + +@pytest.mark.anyio +async def test_preprocess_sends_the_conversation_as_messages(): + task, _ = _task() + sample = _sample() + record = await task.preprocess(sample, TaskContext(sample_id=0, raw_sample=sample)) + assert record["prompt"] == [ + {"role": "user", "content": "Write a story."}, + {"role": "assistant", "content": "Once upon a time."}, + {"role": "user", "content": "Now make it rhyme."}, + ] + + +@pytest.mark.anyio +async def test_preprocess_omits_reference_and_carries_the_rubric(): + """The ground truth is a rubric -- a procedure, not a value to compare.""" + task, _ = _task() + sample = _sample() + record = await task.preprocess(sample, TaskContext(sample_id=0, raw_sample=sample)) + assert "reference" not in record + assert record["extra"]["benchmark_name"] == COMPLEX + assert record["extra"]["rubrics"] == [ + "Is it two paragraphs?", + "Are there two metaphors?", + ] + + +# --- postprocess --- + + +@pytest.mark.anyio +async def test_postprocess_normalizes_a_blank_response_to_none(): + task, _ = _task(answer=" ") + ctx = TaskContext(sample_id=0) + inferred = ModelOutput(model=task.model.meta(), texts=[" "]) + record = await task.postprocess(inferred, ctx) + assert record["rollouts"][0].get("prediction") is None + assert record["rollouts"][0]["extracted"] is False + + +# --- feedback --- + + +@pytest.mark.anyio +async def test_feedback_scores_a_fully_satisfied_response(): + task, grader = _task( + grader_reply=_judge_reply({"question_1": "Yes", "question_2": "Yes"}, "Yes") + ) + ok, judgement = await _run_to_feedback(task, _sample()) + + assert ok is True + rollout = judgement["rollouts"][0] + assert rollout["correct"] is True + assert rollout["score"] == 1.0 + assert rollout["metrics"] == { + "satisfied_all": True, + "rubric_level_pass_rate": 1.0, + } + assert judgement["reference"] is None + assert judgement["extra"] == {"benchmark_name": COMPLEX, "n_rubrics": 2} + # The grader saw the composed judge prompt, not the raw conversation. + assert grader.last_prompt.startswith(f"JUDGE[{COMPLEX}]") + + +@pytest.mark.anyio +async def test_feedback_records_partial_credit_and_raw_counts(): + task, _ = _task( + grader_reply=_judge_reply({"question_1": "Yes", "question_2": "No"}, "No") + ) + _, judgement = await _run_to_feedback(task, _sample()) + + rollout = judgement["rollouts"][0] + assert rollout["correct"] is False + assert rollout["score"] == 0.5 + # Raw counts, because a per-sample rate cannot reconstruct a pooled one. + assert rollout["extra"]["n_checks"] == 2 + assert rollout["extra"]["n_checks_passed"] == 1 + assert rollout["extra"]["judge_parsed"] is True + assert rollout["extra"]["rubrics_check"] == { + "question_1": "Yes", + "question_2": "No", + } + + +@pytest.mark.anyio +async def test_feedback_persists_the_whole_grader_output(): + task, _ = _task(grader_reply=_judge_reply({"question_1": "Yes"}, "No")) + _, judgement = await _run_to_feedback(task, _sample()) + + grader_output = judgement["rollouts"][0]["extra"][GRADER_OUTPUT_KEY] + assert grader_output["texts"] == [_judge_reply({"question_1": "Yes"}, "No")] + assert grader_output["usage"]["total_tokens"] == 43 + + +@pytest.mark.anyio +async def test_grader_spend_reaches_the_profiler(): + """One batched grader call per rollout, stored as a mapping -- not a list. + + ``iter_grader_outputs`` skips a list, so fanning the rubric out into one + judge call per criterion would make the grader's tokens vanish from + profile.json. The whole rubric goes in a single indexed call instead. + """ + task, _ = _task(grader_reply=_judge_reply({"question_1": "Yes"}, "Yes")) + _, judgement = await _run_to_feedback(task, _sample()) + + outputs = iter_grader_outputs(judgement) + assert len(outputs) == 1 + assert outputs[0]["usage"]["total_tokens"] == 43 + + +@pytest.mark.anyio +async def test_feedback_treats_an_unparseable_reply_as_a_failed_row(): + """Upstream's failed-row path: no pass, and no rubrics into the micro rate.""" + task, _ = _task(grader_reply="the judge rambled without emitting JSON") + _, judgement = await _run_to_feedback(task, _sample()) + + rollout = judgement["rollouts"][0] + assert rollout["correct"] is False + assert rollout["score"] == 0.0 + assert rollout["extra"]["judge_parsed"] is False + assert rollout["extra"]["n_checks"] == 0 + assert rollout["extra"]["n_checks_passed"] == 0 + # The reply is still on disk -- the only evidence of what the grader did. + assert rollout["extra"][GRADER_OUTPUT_KEY]["texts"] == [ + "the judge rambled without emitting JSON" + ] + + +# --- report --- + + +def _final(benchmark_name: str, satisfied: bool, n_checks: int, n_passed: int): + judgement = build_judgement_record( + None, + [ + build_rollout_judgement( + 0, + satisfied, + score=n_passed / n_checks if n_checks else 0.0, + extra={"n_checks": n_checks, "n_checks_passed": n_passed}, + ) + ], + extra={"benchmark_name": benchmark_name, "n_rubrics": n_checks}, + ) + return TaskContext(sample_id=0, feedback_result=judgement) + + +@pytest.mark.anyio +async def test_report_pools_both_published_rates(): + task, _ = _task() + finals = [ + _final(COMPLEX, True, 4, 4), + _final(COMPLEX, False, 4, 2), + ] + report = await task.report(finals, []) + + assert report["score"] == pytest.approx(50.0) + assert report["overall_pass_rate"] == pytest.approx(50.0) + assert report["micro_pass_rate"] == pytest.approx(75.0) + assert report["n_graded"] == 2.0 + assert report["n_rubric_checks"] == 8.0 + assert report["fails"] == 0 + + +@pytest.mark.anyio +async def test_report_breaks_down_by_aspect(): + task, _ = _task() + finals = [ + _final(COMPLEX, True, 2, 2), + _final(STEERABILITY, False, 2, 1), + _final(STEERABILITY, False, 2, 0), + ] + report = await task.report(finals, []) + + assert report[f"{COMPLEX}_pass_rate"] == pytest.approx(100.0) + assert report[f"{COMPLEX}_n_graded"] == 1.0 + assert report[f"{STEERABILITY}_pass_rate"] == pytest.approx(0.0) + assert report[f"{STEERABILITY}_micro_pass_rate"] == pytest.approx(25.0) + assert report[f"{STEERABILITY}_n_graded"] == 2.0 + + +@pytest.mark.anyio +async def test_report_counts_pipeline_failures_as_non_passes(): + """A sample that never produced an answer still spans the requested set.""" + task, _ = _task() + finals = [_final(COMPLEX, True, 2, 2)] + report = await task.report(finals, [TaskContext(sample_id=1)]) + + assert report["overall_pass_rate"] == pytest.approx(50.0) + # A failure contributes no rubrics, so the micro rate is unaffected. + assert report["micro_pass_rate"] == pytest.approx(100.0) + assert report["n_graded"] == 1.0 + assert report["fails"] == 1 + # It has no aspect to attribute to, so the breakdown covers graded rollouts. + assert report[f"{COMPLEX}_n_graded"] == 1.0