From 53df4f673e53aafbe5246aeb4ec25c1f7c48f24b Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 7 Aug 2026 16:17:45 +0800 Subject: [PATCH] feat(tasks): add AdvancedIF rubric-graded instruction-following benchmark AdvancedIF (Meta, arXiv:2511.10507) is 1,645 expert-written prompts paired with human-curated rubrics, spanning complex single-turn (402), system steerability (507) and carried-context multi-turn (736) instruction following. A grader LLM answers every rubric question and declares whether the response satisfied all of them; the headline score is the overall pass rate, with the co-published micro rubric rate and a per-aspect breakdown. The judge prompts are NOT vendored. Upstream ships every file under CC-BY-NC-4.0, which cannot be redistributed inside this Apache-2.0 tree, so the operator stages their own checkout and points SIEVAL_ADVANCED_IF_SRC at it; the loader sha256-pins judge.py to commit f9d3013 and refuses a drifted revision. The benchmark data carries the same terms and is likewise only referenced, so running AdvancedIF accepts them either way. Loading the prompts from the operator's checkout also makes them byte-exact by construction rather than by review. Upstream's judge routing is reproduced with its defect intact: it selects the system-steerability judge on benchmark_name == "if_system_steerability _oss", a value the released dataset never contains (it ships "system_steerability_v2"), so all 507 system-prompt rows are graded by the user-instruction judge and the CLI's --task choices match zero rows. processor.process_file's own docstring uses the released spelling, so the if_*_oss literals are what went stale. The unqualified task name tracks upstream including defects; correcting the routing moves scores on a third of the benchmark and belongs in a _fixed variant carrying a measured delta. A test pins the behaviour so a "cleanup" cannot silently change scores. The two published rates deliberately do not share a denominator: the per-sample rubric rate divides by the rubric count the data carries, while the pooled micro rate counts every answer key the grader emitted. Both are kept, each matching its own upstream definition, with the raw counts a pooled rate cannot reconstruct persisted per rollout. Deviations, both documented in sieval/community/advanced_if.py: the reply parser falls back to extracting fenced JSON, because sieval cannot force response_format=json_object on an arbitrary grader endpoint the way upstream's OpenAI client does; and non-string rubric answers are stringified rather than aborting the row. The whole rubric goes to the grader in one indexed call per rollout, so its ModelOutput is stored as a single mapping and its spend reaches profile.json (iter_grader_outputs skips a list). status="experimental": faithful port, no published number reproduced -- the paper's figures come from Meta's internal pipeline, not the released CLI. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/community/advanced_if.py | 388 ++++++++++++++++++ sieval/datasets/__init__.pyi | 6 + sieval/datasets/advanced_if.py | 67 +++ sieval/meta/index.json | 41 ++ sieval/tasks/__init__.pyi | 4 + sieval/tasks/advanced_if_0shot_gen.py | 315 ++++++++++++++ tests/unit/community/test_advanced_if.py | 334 +++++++++++++++ .../unit/tasks/test_advanced_if_0shot_gen.py | 324 +++++++++++++++ 8 files changed, 1479 insertions(+) create mode 100644 sieval/community/advanced_if.py create mode 100644 sieval/datasets/advanced_if.py create mode 100644 sieval/tasks/advanced_if_0shot_gen.py create mode 100644 tests/unit/community/test_advanced_if.py create mode 100644 tests/unit/tasks/test_advanced_if_0shot_gen.py diff --git a/sieval/community/advanced_if.py b/sieval/community/advanced_if.py new file mode 100644 index 00000000..eb70e36b --- /dev/null +++ b/sieval/community/advanced_if.py @@ -0,0 +1,388 @@ +# Upstream: https://github.com/facebookresearch/AdvancedIF/blob/f9d30137c4139d4d9af260ae28108b5afae828c0/judge.py +# +# The judge prompts are deliberately NOT vendored. Every file in the upstream +# repository is CC-BY-NC-4.0, which cannot be redistributed inside sieval's +# Apache-2.0 tree. The operator stages their own checkout and points +# SIEVAL_ADVANCED_IF_SRC at it; this module loads the prompts from there at +# eval time and contributes only sieval-authored scoring code. That mirrors how +# the benchmark data is handled -- also CC-BY-NC-4.0, also never vendored -- so +# running AdvancedIF requires accepting the upstream terms either way. +"""AdvancedIF rubric-judge assets and scoring kernel. + +AdvancedIF (Meta, arXiv:2511.10507) scores a response against expert-written +rubrics: a grader LLM answers every rubric question yes/no and declares whether +the response satisfied all of them. This module holds + +* :func:`load_judge_prompts` -- the upstream prompt templates, read from the + operator's own checkout (see the note above) and digest-checked against the + pinned commit; +* :func:`compose_judge_prompt` -- prompt assembly, including the conversation + rendering and the user/system-turn extraction upstream performs; +* :func:`parse_judgement` -- grader reply -> rubric verdicts; +* the counting helpers the two published rates are pooled from. + +Two published rates, and they do **not** share a denominator -- upstream +computes them in different places and they disagree whenever the grader emits a +number of answers that differs from the rubric count: + +* per-sample ``rubric_level_pass_rate`` (``judge._calc_rubric_level_pass_rate``) + divides in-range passes by ``len(rubrics)``, the count the *data* carries, and + skips answer keys that index past it; +* the aggregate ``micro_pass_rate`` (``processor._calculate_stats``) pools over + every key the *grader* emitted, with no range check at all. + +:func:`count_in_range_passes` and :func:`count_all_checks` keep the two separate +so each pooled metric matches its own upstream definition. + +**Upstream defect, reproduced deliberately.** Upstream selects the +system-steerability judge on ``benchmark_name == "if_system_steerability_oss"`` +-- a value the released dataset never contains, since it ships +``system_steerability_v2``. On the public data that judge is therefore +unreachable, and all 507 system-prompt rows are graded by the plain +user-instruction judge against rubrics written for the system prompt. The same +stale spelling makes the CLI's ``--task`` choices match zero rows, while +``processor.process_file``'s own docstring gives ``system_steerability_v2`` as +its example -- so the ``if_*_oss`` literals, not the dataset, are what went +stale. :func:`is_system_steer` keeps upstream's comparison verbatim anyway: +the unqualified task name tracks upstream including its defects, so a run can +be compared against a published number without first asking which routing it +used. Correcting the routing changes scores on a third of the benchmark and so +belongs in a ``_fixed`` variant carrying a measured delta, not here. + +Deviations from upstream @ f9d3013: + +* **Reply parsing.** Upstream guarantees JSON by passing + ``response_format={"type": "json_object"}`` to the OpenAI client. sieval + reaches the grader through the generic ``ChatModel``, and not every endpoint + honours that flag, so :func:`parse_judgement` falls back to extracting the + outermost JSON object (optionally fenced) before giving up. Without it a + grader that fences its JSON would score zero everywhere -- a harness artifact, + not a property of the model under test. +* **Non-string rubric answers** are stringified rather than raising. Upstream + calls ``.lower()`` directly, so a non-string answer aborts the row into its + ``except Exception`` path; stringifying keeps the row gradeable and, for the + schema-conforming string answers upstream expects, is identical. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import hashlib +import importlib.util +import json +import os +import re +import sys +from dataclasses import dataclass +from functools import cache +from pathlib import Path + +UPSTREAM_COMMIT = "f9d30137c4139d4d9af260ae28108b5afae828c0" +"""Upstream revision the prompts and scoring rules are pinned to.""" + +UPSTREAM_JUDGE_SHA256 = ( + "415164e9c3cb1e267321fa0561a8d61b81b3ac134a7d018764ae53a6e5a84955" +) +"""sha256 of ``judge.py`` at :data:`UPSTREAM_COMMIT`.""" + +SRC_ENV_VAR = "SIEVAL_ADVANCED_IF_SRC" +"""Environment variable pointing at the operator's upstream checkout.""" + +# Upstream's literal, verbatim. The released dataset spells the same aspect +# `system_steerability_v2`, so this never matches it -- see the module docstring. +SYSTEM_STEER_BENCHMARK = "if_system_steerability_oss" + +RELEASED_SYSTEM_STEER_BENCHMARK = "system_steerability_v2" +"""What the released dataset calls the aspect :data:`SYSTEM_STEER_BENCHMARK` misses.""" + +_JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL) + +_MISSING_SOURCE_HINT = ( + f"AdvancedIF's judge prompts are CC-BY-NC-4.0 and are not redistributed " + f"with sieval. Clone the upstream harness and point {SRC_ENV_VAR} at it:\n" + f" git clone https://github.com/facebookresearch/AdvancedIF\n" + f" git -C AdvancedIF checkout {UPSTREAM_COMMIT}\n" + f" export {SRC_ENV_VAR}=$PWD/AdvancedIF" +) + + +@dataclass(frozen=True) +class JudgePrompts: + """The three prompt templates upstream ``judge.py`` defines.""" + + judge_prompt: str + system_steer_judge_prompt: str + steer_few_shot_examples: str + + +@dataclass(frozen=True) +class Judgement: + """A parsed grader verdict. + + Attributes: + rubrics_check: Answer keyed by ``question_``, as the grader emitted + it -- keys are neither renumbered nor range-filtered here, because + the two pooled rates disagree about which of them count. + satisfied_all: The grader's all-rubrics-passed declaration. + """ + + rubrics_check: dict[str, str] + satisfied_all: bool + + +def is_system_steer(benchmark_name: str) -> bool: + """Whether *benchmark_name* routes to the system-steerability judge. + + Upstream's comparison, kept verbatim -- which means this returns ``False`` + for every row of the released dataset, including all 507 system-prompt + ones. That is upstream's behaviour, not an oversight here; see the module + docstring. + """ + return benchmark_name == SYSTEM_STEER_BENCHMARK + + +def _judge_source_path() -> Path: + raw = os.environ.get(SRC_ENV_VAR, "").strip() + if not raw: + raise RuntimeError(f"{SRC_ENV_VAR} is not set.\n\n{_MISSING_SOURCE_HINT}") + root = Path(raw).expanduser() + path = root / "judge.py" if root.is_dir() else root + if not path.is_file(): + raise RuntimeError( + f"{SRC_ENV_VAR}={raw} does not contain judge.py (looked at {path})." + f"\n\n{_MISSING_SOURCE_HINT}" + ) + return path + + +@cache +def load_judge_prompts() -> JudgePrompts: + """Load the upstream prompt templates from the operator's checkout. + + The file is digest-checked against :data:`UPSTREAM_JUDGE_SHA256` before it + is imported, so a score is always attributable to a known prompt revision. + A mismatch is fatal by design -- these prompts *are* the benchmark, and + silently grading against a drifted revision would make runs incomparable. + Recovery is to check out :data:`UPSTREAM_COMMIT`. + """ + path = _judge_source_path() + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != UPSTREAM_JUDGE_SHA256: + raise RuntimeError( + f"{path} does not match the pinned AdvancedIF revision.\n" + f" expected sha256 {UPSTREAM_JUDGE_SHA256} (commit {UPSTREAM_COMMIT})\n" + f" found sha256 {digest}\n" + f"Check out the pinned commit:\n" + f" git -C {path.parent} checkout {UPSTREAM_COMMIT}" + ) + + spec = importlib.util.spec_from_file_location("sieval._advanced_if_upstream", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load AdvancedIF judge module from {path}.") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + return JudgePrompts( + judge_prompt=module.JUDGE_PROMPT, + system_steer_judge_prompt=module.SYSTEM_STEER_JUDGE_PROMPT, + steer_few_shot_examples=module.STEER_FEW_SHOT_EXAMPLES, + ) + + +def parse_conversation(conversation_history: str | list) -> list: + """Decode the dataset's ``conversation_history`` into role/content dicts. + + Only the two keys the judge prompt reads are kept, so the record persisted + downstream is the message list actually sent to the model. The return type + is left bare because this list becomes a record's ``prompt`` (a + ``JSONValue``), and ``list`` is invariant. + """ + messages = ( + json.loads(conversation_history) + if isinstance(conversation_history, str) + else conversation_history + ) + if not isinstance(messages, list): + raise ValueError( + f"conversation_history must decode to a list, got {type(messages).__name__}" + ) + return [{"role": str(m["role"]), "content": str(m["content"])} for m in messages] + + +def parse_rubrics(prompt_metadata: str | dict) -> list[str]: + """Extract the rubric list from the dataset's ``prompt_metadata``. + + ``rubrics`` is itself sometimes a JSON-encoded string rather than a list, + which is why upstream decodes it a second time. + """ + metadata = ( + json.loads(prompt_metadata) + if isinstance(prompt_metadata, str) + else prompt_metadata + ) + if "rubrics" not in metadata: + raise ValueError("Rubrics not found in prompt_metadata") + rubrics = metadata["rubrics"] + if isinstance(rubrics, str): + rubrics = json.loads(rubrics) + return [str(rubric) for rubric in rubrics] + + +def format_conversation_history(messages: list[dict]) -> str: + """Render prior turns as upstream's ``role [turn]: content`` block. + + The final message is dropped: the dataset's ``conversation_history`` ends on + the user prompt being answered, and that turn is passed to the prompt + separately. The turn counter advances on assistant messages, so a + user/assistant pair shares one number. + """ + formatted = [] + turn = 1 + for message in messages[:-1]: + formatted.append(f"{message['role']} [{turn}]: {message['content']}") + if message["role"] == "assistant": + turn += 1 + return "\n".join(formatted) + + +def last_user_turn(messages: list[dict]) -> str: + """The most recent user message, or ``""`` when there is none.""" + for message in reversed(messages): + if message["role"] == "user": + return message["content"] + return "" + + +def system_prompt_of(messages: list[dict]) -> str: + """The leading system message, or ``""`` when the turn list has none.""" + if messages and messages[0]["role"] == "system": + return messages[0]["content"] + return "" + + +def compose_judge_prompt( + benchmark_name: str, + messages: list[dict], + response_text: str, + rubrics: list[str], +) -> str: + """Assemble the grader prompt for one sample. + + Routing between the user-instruction and system-steerability judges follows + :func:`is_system_steer`; the rubric block is JSON with upstream's + ``indent=4``, which the grader's ``question_`` keys are positional over. + """ + prompts = load_judge_prompts() + rubrics_text = json.dumps(rubrics, indent=4) + + if is_system_steer(benchmark_name): + return prompts.system_steer_judge_prompt.format( + few_shot_examples=prompts.steer_few_shot_examples, + system_prompt=system_prompt_of(messages), + user_prompt_last_turn=last_user_turn(messages), + response_text=response_text, + rubrics_text=rubrics_text, + ) + return prompts.judge_prompt.format( + full_conversation=format_conversation_history(messages), + user_prompt_last_turn=last_user_turn(messages), + response_text=response_text, + rubrics_text=rubrics_text, + ) + + +def _loads_json_object(reply: str) -> dict | None: + try: + parsed = json.loads(reply) + except (json.JSONDecodeError, ValueError): + return None + return parsed if isinstance(parsed, dict) else None + + +def parse_judgement(reply: str) -> Judgement | None: + """Parse a grader reply, or ``None`` when it yields no JSON object. + + ``None`` is the analogue of upstream's ``JudgeResult(success=False)``: the + sample counts against the overall pass rate but contributes no rubrics to + the pooled micro rate. + """ + parsed = _loads_json_object(reply) + if parsed is None: + # Fenced or prose-wrapped JSON -- see the parsing deviation above. + match = _JSON_OBJECT_RE.search(reply) + parsed = _loads_json_object(match.group(0)) if match else None + if parsed is None: + return None + + raw_checks = parsed.get("rubrics_check", {}) + if not isinstance(raw_checks, dict): + raw_checks = {} + # Upstream defaults a missing declaration to "NO" and compares + # case-insensitively -- its own few-shot examples answer "Yes"/"No", not + # "YES"/"NO", so a case-sensitive check would fail every passing sample. + declared = parsed.get("SATISFIED_ALL_REQUIREMENTS", "NO") + return Judgement( + rubrics_check={str(k): str(v) for k, v in raw_checks.items()}, + satisfied_all=str(declared).strip().lower() == "yes", + ) + + +def _is_pass(answer: str) -> bool: + # Upstream's substring test, not equality: rubric answers routinely carry a + # justification ("The intro is four sentences. No"). + return "yes" in answer.lower() + + +def count_in_range_passes(rubrics_check: dict[str, str], rubrics: list[str]) -> int: + """Passes among answers whose ``question_`` indexes a real rubric. + + Numerator of the per-sample ``rubric_level_pass_rate``; keys that are + unparseable or index past the rubric list are skipped, as upstream does. + """ + passes = 0 + for key, answer in rubrics_check.items(): + try: + index = int(key.split("_")[1]) - 1 + except (IndexError, ValueError): + continue + if index >= len(rubrics): + continue + if _is_pass(answer): + passes += 1 + return passes + + +def rubric_level_pass_rate(rubrics_check: dict[str, str], rubrics: list[str]) -> float: + """Per-sample rubric pass rate, over the rubric count the data carries.""" + return count_in_range_passes(rubrics_check, rubrics) / max(len(rubrics), 1) + + +def count_all_checks(rubrics_check: dict[str, str]) -> tuple[int, int]: + """``(answers emitted, answers passed)`` with no range filtering. + + The pooled ``micro_pass_rate`` counts exactly these, so a grader that + answers fewer questions than there are rubrics shrinks its own denominator. + """ + return ( + len(rubrics_check), + sum(1 for answer in rubrics_check.values() if _is_pass(answer)), + ) + + +def aggregate_metrics(verdicts: list[dict]) -> dict[str, float]: + """Pool per-rollout verdicts into the two published rates. + + Each entry carries ``satisfied_all``, ``n_checks`` and ``n_checks_passed`` + (a rollout the grader failed to produce a verdict for contributes only to + the denominator of the pass rate, matching upstream). + """ + total = len(verdicts) + passed = sum(1 for v in verdicts if v["satisfied_all"]) + checks = sum(v["n_checks"] for v in verdicts) + checks_passed = sum(v["n_checks_passed"] for v in verdicts) + return { + "overall_pass_rate": passed / total * 100 if total else 0.0, + "micro_pass_rate": checks_passed / checks * 100 if checks else 0.0, + "n_samples": float(total), + "n_rubric_checks": float(checks), + } diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index e5cd3cbe..042019fe 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -5,6 +5,10 @@ from .aa_lcr import ( AALCRDataset, AALCRDatasetSample, ) +from .advanced_if import ( + AdvancedIFDataset, + AdvancedIFDatasetSample, +) from .aime_2024 import ( AIME2024Dataset, AIME2024DatasetSample, @@ -175,6 +179,8 @@ __all__ = [ "ARCChallengeDatasetSample", "ARCEasyDataset", "ARCEasyDatasetSample", + "AdvancedIFDataset", + "AdvancedIFDatasetSample", "Apex2025Dataset", "Apex2025DatasetSample", "ApexShortlist2025Dataset", diff --git a/sieval/datasets/advanced_if.py b/sieval/datasets/advanced_if.py new file mode 100644 index 00000000..f29826d9 --- /dev/null +++ b/sieval/datasets/advanced_if.py @@ -0,0 +1,67 @@ +"""AdvancedIF dataset loader. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import apply_eval_split, ensure_dataset_dict + +ADVANCED_IF_REVISION = "e20cba9b94b59c027dfab00b29244e8bc42e4ab4" + + +class AdvancedIFDatasetSample(TypedDict): + """One AdvancedIF prompt. + + Upstream ships all three columns as strings, including the two that hold + JSON; they are decoded in the task rather than here, so the persisted prompt + record keeps the bytes the grader prompt was actually built from. No cast + needed -- the pinned revision already ships these as strings. + + Attributes: + conversation_history: JSON list of ``{"role", "content"}`` turns, ending + on the user prompt to answer (the assistant turn under test is + absent by construction). A system-steerability row leads with a + ``system`` turn. + benchmark_name: Which of the three aspects the row belongs to -- + ``complex_if_single_turn_v5`` (402), ``system_steerability_v2`` + (507) or ``carried_context_multi_turn_eval_v5`` (736). + prompt_metadata: JSON object whose ``rubrics`` key holds the + expert-written yes/no checks (itself sometimes JSON-encoded again). + """ + + conversation_history: str + benchmark_name: str + prompt_metadata: str + + +@sieval_dataset( + name="advanced_if", + display_name="AdvancedIF", + description=( + "Expert-written prompts with human-curated rubrics for advanced " + "instruction following." + ), + source=f"hf:facebook/AdvancedIF@{ADVANCED_IF_REVISION}", + categories=(Category(Level1Category.LANGUAGE, "InstructionFollowing"),), + tags=("english", "open-ended"), + # Non-commercial. The judge prompts carry the same terms and are likewise + # not redistributed -- see sieval.community.advanced_if. + license="CC-BY-NC-4.0", +) +class AdvancedIFDataset(Dataset[AdvancedIFDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + # AdvancedIF ships its 1,645 rows in a single "train" split (the card + # calls the same rows "test"); mirror it to "test" for the runner. + dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs)) + return apply_eval_split(dataset, "train") diff --git a/sieval/meta/index.json b/sieval/meta/index.json index d77ab81d..dabc17f3 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -24,6 +24,27 @@ "license": "apache-2.0", "checksums": {} }, + { + "name": "advanced_if", + "display_name": "AdvancedIF", + "description": "Expert-written prompts with human-curated rubrics for advanced instruction following.", + "source": [ + "hf:facebook/AdvancedIF@e20cba9b94b59c027dfab00b29244e8bc42e4ab4" + ], + "categories": [ + { + "level1": "Language", + "level2": "InstructionFollowing" + } + ], + "tags": [ + "english", + "open-ended" + ], + "deps_group": null, + "license": "CC-BY-NC-4.0", + "checksums": {} + }, { "name": "aime_2024", "display_name": "AIME 2024", @@ -917,6 +938,26 @@ }, "status": "experimental" }, + { + "name": "advanced_if_0shot_gen", + "display_name": "AdvancedIF (0-shot, generative)", + "description": "Instruction following graded against expert-written rubrics by an LLM judge.", + "dataset": "advanced_if", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "open-ended" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "facebookresearch/AdvancedIF", + "url": "https://github.com/facebookresearch/AdvancedIF/blob/f9d30137c4139d4d9af260ae28108b5afae828c0/judge.py", + "notes": "Port of AdvancedIF (Meta, arXiv:2511.10507), 1,645 prompts across complex_if_single_turn_v5 (402), system_steerability_v2 (507) and carried_context_multi_turn_eval_v5 (736). Headline score is the overall pass rate (share of samples the grader marked SATISFIED_ALL_REQUIREMENTS=yes), the number the paper reports; micro_pass_rate is the co-published rubric-level rate. LICENSING: upstream ships judge.py under CC-BY-NC-4.0, incompatible with sieval's Apache-2.0 tree, so the prompts are NOT vendored -- the operator stages a checkout and points SIEVAL_ADVANCED_IF_SRC at it, digest-checked against commit f9d30137c4139d4d9af260ae28108b5afae828c0. The dataset is CC-BY-NC-4.0 too, so running this benchmark accepts those terms either way. Loading from the operator's own checkout also makes the prompts byte-exact by construction rather than by review. UPSTREAM DEFECT (reproduced, not corrected): upstream routes to the system-steerability judge on benchmark_name == 'if_system_steerability_oss', a value the released dataset never contains (it ships 'system_steerability_v2'), so all 507 system-prompt rows are graded by the plain user-instruction judge and the CLI's --task choices match zero rows; processor.process_file's own docstring uses the released spelling, so the if_*_oss literals are what went stale. This port keeps upstream's comparison verbatim so the unqualified name tracks upstream. Correcting the routing moves scores on a third of the benchmark and belongs in a _fixed variant carrying a measured delta. Grader is a REAL LLM (upstream: o3-mini-2025-01-31, temperature 0, max_completion_tokens=32768, response_format=json_object) supplied via the `grader` task arg; pin it, as its version is not pinnable the way a Hub revision is. The grader's full ModelOutput and per-rubric answers are persisted under the judgement record's `extra`. VALIDATION: none -- no published number has been reproduced with this port, and the paper's own figures come from Meta's internal pipeline rather than the released CLI." + }, + "status": "experimental" + }, { "name": "aime_2024_0shot_gen", "display_name": "AIME 2024 (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 2136c931..f4fc2acc 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -4,6 +4,9 @@ from .aa_lcr_0shot_gen import ( AALCRZeroShotGenTask, ) +from .advanced_if_0shot_gen import ( + AdvancedIFZeroShotGenTask, +) from .aime_2024_0shot_gen import ( AIME2024ZeroShotGenTask, ) @@ -161,6 +164,7 @@ __all__ = [ "ARCChallengeFewShotPplTask", "ARCEasyFewShotClpTask", "ARCEasyFewShotPplTask", + "AdvancedIFZeroShotGenTask", "Apex2025ZeroShotGenTask", "ApexShortlist2025ZeroShotGenTask", "BRUMO2025ZeroShotGenTask", diff --git a/sieval/tasks/advanced_if_0shot_gen.py b/sieval/tasks/advanced_if_0shot_gen.py new file mode 100644 index 00000000..a889aff3 --- /dev/null +++ b/sieval/tasks/advanced_if_0shot_gen.py @@ -0,0 +1,315 @@ +"""AdvancedIF — 0-shot generative, rubric-graded by an LLM judge. + +AdvancedIF (Meta, Hu et al., 2025, arXiv:2511.10507) probes instruction +following past the verifiable-constraint regime IFEval and IFBench cover: every +prompt is expert-written and paired with a human-curated rubric of yes/no +checks, and the three aspects it spans -- complex single-turn instructions +(6+ per prompt), instructions carried across a multi-turn dialog, and system- +prompt steerability -- are graded by an LLM judge rather than by checkers. + +The model answers the conversation's final user turn; a separate **grader** +model then answers every rubric question and declares whether the response +satisfied all of them. Headline metric is the overall pass rate: the share of +samples where the grader answered yes to that declaration. + +The grader is supplied via the ``grader`` task arg (a model-config dict, or a +pre-built Model, on its own ``api_base``/``api_key``); upstream's judge is +o3-mini-2025-01-31 at temperature 0 with ``max_completion_tokens=32768`` and +``response_format={"type": "json_object"}``. Set all of those in the grader's +model config, not here -- sieval does not force ``response_format`` on the +request because endpoints that reject the field would fail the whole run, and +the reply parser tolerates fenced JSON either way. + +Upstream's judge routing is reproduced with its defect intact: the +system-steerability judge is selected on a ``benchmark_name`` the released +dataset never contains, so every row -- including the 507 system-prompt ones -- +is graded by the user-instruction judge. The unqualified task name tracks +upstream, defects included; see :mod:`sieval.community.advanced_if`. + +Running this task needs an upstream checkout: the judge prompts are CC-BY-NC-4.0 +and are not redistributed with sieval, so point ``SIEVAL_ADVANCED_IF_SRC`` at +your own clone (see :mod:`sieval.community.advanced_if`, which digest-checks it +against the pinned commit). The benchmark data carries the same terms. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from collections.abc import Mapping +from typing import override + +from sieval.community.advanced_if import ( + aggregate_metrics, + compose_judge_prompt, + count_all_checks, + count_in_range_passes, + parse_conversation, + parse_judgement, + parse_rubrics, +) +from sieval.core.models import ChatModel, Model, ModelOutput +from sieval.core.tasks import ( + GRADER_OUTPUT_KEY, + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + RolloutJudgement, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.utils.serialization import obj_to_dict +from sieval.datasets import AdvancedIFDatasetSample + + +@sieval_task( + name="advanced_if_0shot_gen", + display_name="AdvancedIF (0-shot, generative)", + description=( + "Instruction following graded against expert-written rubrics by an LLM judge." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "open-ended"), + model_type="chat", + reference_impl=ReferenceImpl( + source="facebookresearch/AdvancedIF", + url=( + "https://github.com/facebookresearch/AdvancedIF/blob/" + "f9d30137c4139d4d9af260ae28108b5afae828c0/judge.py" + ), + notes=( + "Port of AdvancedIF (Meta, arXiv:2511.10507), 1,645 prompts across " + "complex_if_single_turn_v5 (402), system_steerability_v2 (507) and " + "carried_context_multi_turn_eval_v5 (736). Headline score is the " + "overall pass rate (share of samples the grader marked " + "SATISFIED_ALL_REQUIREMENTS=yes), the number the paper reports; " + "micro_pass_rate is the co-published rubric-level rate. " + "LICENSING: upstream ships judge.py under CC-BY-NC-4.0, " + "incompatible with sieval's Apache-2.0 tree, so the prompts are " + "NOT vendored -- the operator stages a checkout and points " + "SIEVAL_ADVANCED_IF_SRC at it, digest-checked against commit " + "f9d30137c4139d4d9af260ae28108b5afae828c0. The dataset is " + "CC-BY-NC-4.0 too, so running this benchmark accepts those terms " + "either way. Loading from the operator's own checkout also makes " + "the prompts byte-exact by construction rather than by review. " + "UPSTREAM DEFECT (reproduced, not corrected): upstream routes to " + "the system-steerability judge on benchmark_name == " + "'if_system_steerability_oss', a value the released dataset never " + "contains (it ships 'system_steerability_v2'), so all 507 " + "system-prompt rows are graded by the plain user-instruction judge " + "and the CLI's --task choices match zero rows; " + "processor.process_file's own docstring uses the released " + "spelling, so the if_*_oss literals are what went stale. This port " + "keeps upstream's comparison verbatim so the unqualified name " + "tracks upstream. Correcting the routing moves scores on a third " + "of the benchmark and belongs in a _fixed variant carrying a " + "measured delta. " + "Grader is a REAL LLM (upstream: o3-mini-2025-01-31, temperature " + "0, max_completion_tokens=32768, response_format=json_object) " + "supplied via the `grader` task arg; pin it, as its version is not " + "pinnable the way a Hub revision is. The grader's full ModelOutput " + "and per-rubric answers are persisted under the judgement record's " + "`extra`. " + "VALIDATION: none -- no published number has been reproduced with " + "this port, and the paper's own figures come from Meta's internal " + "pipeline rather than the released CLI." + ), + ), + # Faithful port of upstream's routing and scoring kernel, but no published + # number has been reproduced with it: faithful port, unverified reproduction. + status="experimental", +) +class AdvancedIFZeroShotGenTask( + Task[ + AdvancedIFDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + grader: Mapping | Model | None = None, + n: int = 1, + ): + super().__init__(dataset=dataset, model=model, name=name) + self._n = n + self._grader = self._build_grader(grader) + + @staticmethod + def _build_grader(grader: Mapping | Model | None) -> Model: + """Resolve the ``grader`` task arg into a Model. + + Accepts a pre-built Model (tests / advanced configs) or a model-config + mapping (the YAML path). Rubric grading is the only scorer AdvancedIF + has -- there is no deterministic fallback -- so ``None`` raises. + """ + if isinstance(grader, Model): + return grader + if isinstance(grader, Mapping): + return ChatModel(**grader) + raise ValueError( + "AdvancedIF requires an LLM grader. Pass `grader:` in the task " + "args — a model-config dict such as {model: o3-mini, api_base: " + "..., api_key: ..., temperature: 0}." + ) + + @override + async def preprocess(self, raw, ctx): + messages = parse_conversation(raw["conversation_history"]) + # No `reference`: the ground truth is a rubric (a procedure), not a + # value. The rubric itself rides in `extra` so a prompt row is readable + # on its own and feedback() need not re-decode the raw sample. + return build_prompt_record( + messages, + extra={ + "benchmark_name": raw["benchmark_name"], + "rubrics": parse_rubrics(raw["prompt_metadata"]), + }, + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + # Open-ended: the response *is* the answer, so no extraction step. A + # blank response normalizes to None so `extracted` stays a real signal; + # the grader still sees "" and will fail its rubrics. + return build_prediction_record( + [text if text.strip() else None for text in inf.texts] + ) + + @override + async def feedback(self, post, ctx): + """Grade every rollout against its rubrics, recording the judge's reply. + + The grader is a model, so its output is persisted the way any model + output is: ``extra["grader_output"]`` is the full ``ModelOutput`` + flattened to a plain dict. Nothing is hand-picked, so no field is + silently dropped. That matters more here than for a short-answer + autorater: the verdict is a whole rubric-by-rubric JSON blob, a + re-grade need not reproduce it, and an unparseable reply is scored the + same as a failing one -- only the raw text separates a grader that + broke format from a response that genuinely missed every rubric. + + Per-rubric answers and the two raw counts the pooled rates need live in + the rollout's ``extra``: the published micro rate pools over the + grader's own answer keys, which a per-sample rate cannot reconstruct. + """ + prompt_extra = ctx.preprocess_result["extra"] + benchmark_name = prompt_extra["benchmark_name"] + rubrics = prompt_extra["rubrics"] + messages = ctx.preprocess_result["prompt"] + + rollouts: list[RolloutJudgement] = [] + for rollout in post["rollouts"]: + response = rollout.get("prediction") or "" + out = await self._grader.agenerate( + compose_judge_prompt(benchmark_name, messages, response, rubrics) + ) + reply = out.texts[0] if out.texts else "" + judgement = parse_judgement(reply) + + if judgement is None: + # Upstream's failed-row path: counts against the pass rate, + # contributes no rubrics to the pooled micro rate. + checks: dict[str, str] = {} + satisfied_all = False + n_checks, n_checks_passed = 0, 0 + pass_rate = 0.0 + else: + checks = judgement.rubrics_check + satisfied_all = judgement.satisfied_all + n_checks, n_checks_passed = count_all_checks(checks) + pass_rate = count_in_range_passes(checks, rubrics) / max( + len(rubrics), 1 + ) + + rollouts.append( + build_rollout_judgement( + rollout["index"], + satisfied_all, + score=pass_rate, + metrics={ + "satisfied_all": satisfied_all, + "rubric_level_pass_rate": pass_rate, + }, + extra={ + "judge_parsed": judgement is not None, + "rubrics_check": checks, + "n_checks": n_checks, + "n_checks_passed": n_checks_passed, + GRADER_OUTPUT_KEY: obj_to_dict(out, add_type=False), + }, + ) + ) + + mean_pass_rate = sum(r["score"] for r in rollouts) / len(rollouts) + return True, build_judgement_record( + # Rubric grading is a procedure, not a value to compare against. + None, + rollouts, + score=mean_pass_rate, + extra={"benchmark_name": benchmark_name, "n_rubrics": len(rubrics)}, + ) + + @override + async def report(self, finals, fails): + """Pool the two published rates overall and per aspect. + + Reads the persisted verdicts rather than ``raw_sample``, so the report + survives a resume. Pipeline failures (exhausted retries) never produced + a gradeable answer; each failed sample's requested rollouts count as + non-passes so the headline spans the full requested set, matching + upstream's denominator (which likewise includes rows its judge failed). + """ + by_benchmark: dict[str, list[dict]] = {} + verdicts: list[dict] = [] + for final in finals: + judgement = final.feedback_result or {} + benchmark_name = judgement.get("extra", {}).get("benchmark_name", "unknown") + for rollout in judgement.get("rollouts", []): + extra = rollout.get("extra", {}) + verdict = { + "satisfied_all": rollout["correct"], + "n_checks": extra.get("n_checks", 0), + "n_checks_passed": extra.get("n_checks_passed", 0), + } + verdicts.append(verdict) + by_benchmark.setdefault(benchmark_name, []).append(verdict) + + n_graded = len(verdicts) + # A failed sample has no verdict to attribute to an aspect, so it lands + # in the overall rates only; the per-aspect rates below cover graded + # rollouts, and `fails` reports the shortfall. + failed = [{"satisfied_all": False, "n_checks": 0, "n_checks_passed": 0}] * ( + self._n * len(fails) + ) + + overall = aggregate_metrics(verdicts + failed) + results: dict[str, float] = { + "score": overall["overall_pass_rate"], + "overall_pass_rate": overall["overall_pass_rate"], + "micro_pass_rate": overall["micro_pass_rate"], + "n_rubric_checks": overall["n_rubric_checks"], + "n_graded": float(n_graded), + "fails": len(fails), + } + for benchmark_name, group in sorted(by_benchmark.items()): + aspect = aggregate_metrics(group) + results[f"{benchmark_name}_pass_rate"] = aspect["overall_pass_rate"] + results[f"{benchmark_name}_micro_pass_rate"] = aspect["micro_pass_rate"] + results[f"{benchmark_name}_n_graded"] = aspect["n_samples"] + return results diff --git a/tests/unit/community/test_advanced_if.py b/tests/unit/community/test_advanced_if.py new file mode 100644 index 00000000..f99e2f54 --- /dev/null +++ b/tests/unit/community/test_advanced_if.py @@ -0,0 +1,334 @@ +"""Unit tests for the AdvancedIF judge assets and scoring kernel. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json + +import pytest + +from sieval.community import advanced_if +from sieval.community.advanced_if import ( + RELEASED_SYSTEM_STEER_BENCHMARK, + SRC_ENV_VAR, + SYSTEM_STEER_BENCHMARK, + UPSTREAM_JUDGE_SHA256, + JudgePrompts, + aggregate_metrics, + compose_judge_prompt, + count_all_checks, + count_in_range_passes, + format_conversation_history, + is_system_steer, + last_user_turn, + load_judge_prompts, + parse_conversation, + parse_judgement, + parse_rubrics, + rubric_level_pass_rate, + system_prompt_of, +) + +# --- dataset field decoding --- + + +def test_parse_conversation_from_json_string_keeps_role_and_content(): + raw = json.dumps( + [ + {"role": "user", "content": "hi", "extra_field": "dropped"}, + {"role": "assistant", "content": "hello"}, + ] + ) + assert parse_conversation(raw) == [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] + + +def test_parse_conversation_accepts_decoded_list(): + assert parse_conversation([{"role": "user", "content": "hi"}]) == [ + {"role": "user", "content": "hi"} + ] + + +def test_parse_conversation_rejects_non_list(): + with pytest.raises(ValueError, match="must decode to a list"): + parse_conversation(json.dumps({"role": "user"})) + + +def test_parse_rubrics_handles_double_encoded_rubrics(): + """Upstream decodes ``rubrics`` twice because it is sometimes a JSON string.""" + as_list = json.dumps({"rubrics": ["a?", "b?"]}) + as_string = json.dumps({"rubrics": json.dumps(["a?", "b?"])}) + assert parse_rubrics(as_list) == ["a?", "b?"] + assert parse_rubrics(as_string) == ["a?", "b?"] + assert parse_rubrics({"rubrics": ["a?"]}) == ["a?"] + + +def test_parse_rubrics_requires_the_key(): + with pytest.raises(ValueError, match="Rubrics not found"): + parse_rubrics(json.dumps({"something_else": []})) + + +# --- conversation rendering (feeds the judge prompt verbatim) --- + + +def test_format_conversation_history_drops_last_turn_and_numbers_by_assistant(): + messages = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + ] + # The trailing user turn is excluded -- it is passed to the prompt + # separately -- and the turn counter advances only past an assistant reply. + assert format_conversation_history(messages) == "user [1]: u1\nassistant [1]: a1" + + +def test_last_user_turn_and_system_prompt(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + ] + assert last_user_turn(messages) == "u2" + assert system_prompt_of(messages) == "sys" + assert system_prompt_of(messages[1:]) == "" + assert last_user_turn([{"role": "assistant", "content": "a"}]) == "" + + +# --- judge routing: upstream's defect, reproduced on purpose --- + + +def test_system_steer_routing_never_fires_on_released_data(): + """Pin the reproduced defect so a "cleanup" cannot silently change scores. + + Upstream compares against ``if_system_steerability_oss``; the released + dataset ships ``system_steerability_v2``, so every row -- including all 507 + system-prompt ones -- goes to the user-instruction judge. + """ + assert is_system_steer(SYSTEM_STEER_BENCHMARK) + assert not is_system_steer(RELEASED_SYSTEM_STEER_BENCHMARK) + assert not is_system_steer("complex_if_single_turn_v5") + assert not is_system_steer("carried_context_multi_turn_eval_v5") + + +# --- grader reply parsing --- + + +def _reply(checks: dict, satisfied: str) -> str: + return json.dumps( + {"rubrics_check": checks, "SATISFIED_ALL_REQUIREMENTS": satisfied} + ) + + +def test_parse_judgement_reads_checks_and_declaration(): + judgement = parse_judgement(_reply({"question_1": "Yes"}, "YES")) + assert judgement is not None + assert judgement.rubrics_check == {"question_1": "Yes"} + assert judgement.satisfied_all + + +@pytest.mark.parametrize("declared", ["Yes", "yes", "YES", " yes "]) +def test_parse_judgement_declaration_is_case_insensitive(declared): + """Upstream's own few-shot examples answer "Yes"/"No", not "YES"/"NO".""" + judgement = parse_judgement(_reply({}, declared)) + assert judgement is not None + assert judgement.satisfied_all + + +def test_parse_judgement_defaults_missing_declaration_to_not_satisfied(): + judgement = parse_judgement(json.dumps({"rubrics_check": {"question_1": "Yes"}})) + assert judgement is not None + assert not judgement.satisfied_all + + +def test_parse_judgement_recovers_fenced_json(): + """sieval cannot force response_format=json_object on every endpoint.""" + fenced = f"Here you go:\n```json\n{_reply({'question_1': 'Yes'}, 'Yes')}\n```" + judgement = parse_judgement(fenced) + assert judgement is not None + assert judgement.rubrics_check == {"question_1": "Yes"} + + +def test_parse_judgement_returns_none_without_json(): + assert parse_judgement("I could not evaluate this.") is None + assert parse_judgement("") is None + + +def test_parse_judgement_stringifies_non_string_answers(): + judgement = parse_judgement(_reply({"question_1": ["Yes"]}, "No")) + assert judgement is not None + assert judgement.rubrics_check == {"question_1": "['Yes']"} + + +# --- counting: the two rates do not share a denominator --- + + +def test_count_in_range_passes_skips_out_of_range_and_malformed_keys(): + checks = { + "question_1": "Yes", + "question_2": "No", + "question_3": "Yes", # indexes past a 2-rubric sample + "notaquestion": "Yes", # unparseable key + "question_x": "Yes", # unparseable index + } + assert count_in_range_passes(checks, ["r1", "r2"]) == 1 + + +def test_count_in_range_passes_matches_justified_answers_by_substring(): + """Rubric answers routinely carry a justification before the verdict.""" + checks = {"question_1": "The intro is four sentences. No", "question_2": "Yes"} + assert count_in_range_passes(checks, ["r1", "r2"]) == 1 + + +def test_rubric_level_pass_rate_divides_by_the_data_rubric_count(): + checks = {"question_1": "Yes"} + # Two rubrics in the data, one answered -> 0.5, not 1.0. + assert rubric_level_pass_rate(checks, ["r1", "r2"]) == 0.5 + assert rubric_level_pass_rate({}, []) == 0.0 + + +def test_count_all_checks_ignores_the_rubric_count(): + """The pooled micro rate counts grader-emitted keys, with no range filter.""" + checks = {"question_1": "Yes", "question_9": "Yes", "question_2": "No"} + assert count_all_checks(checks) == (3, 2) + + +def test_the_two_denominators_disagree_when_the_grader_under_answers(): + checks = {"question_1": "Yes"} + rubrics = ["r1", "r2", "r3", "r4"] + # Per-sample rate is over the data's 4 rubrics ... + assert rubric_level_pass_rate(checks, rubrics) == 0.25 + # ... while the pooled micro rate is over the single key the grader emitted. + assert count_all_checks(checks) == (1, 1) + + +# --- aggregation --- + + +def _verdict(satisfied: bool, n_checks: int, n_passed: int) -> dict: + return { + "satisfied_all": satisfied, + "n_checks": n_checks, + "n_checks_passed": n_passed, + } + + +def test_aggregate_metrics_pools_both_rates(): + metrics = aggregate_metrics( + [_verdict(True, 4, 4), _verdict(False, 4, 2), _verdict(False, 2, 0)] + ) + assert metrics["overall_pass_rate"] == pytest.approx(100 / 3) + assert metrics["micro_pass_rate"] == pytest.approx(600 / 10) + assert metrics["n_samples"] == 3.0 + assert metrics["n_rubric_checks"] == 10.0 + + +def test_aggregate_metrics_counts_ungradeable_rollouts_against_the_pass_rate_only(): + """Upstream's failed-row path: in the pass-rate denominator, out of micro.""" + metrics = aggregate_metrics([_verdict(True, 2, 2), _verdict(False, 0, 0)]) + assert metrics["overall_pass_rate"] == pytest.approx(50.0) + # The failed row contributes no rubrics, so micro stays 100%. + assert metrics["micro_pass_rate"] == pytest.approx(100.0) + + +def test_aggregate_metrics_handles_an_empty_set(): + metrics = aggregate_metrics([]) + assert metrics["overall_pass_rate"] == 0.0 + assert metrics["micro_pass_rate"] == 0.0 + + +# --- loading the upstream prompts (never vendored: CC-BY-NC-4.0) --- + + +def test_load_judge_prompts_requires_the_env_var(monkeypatch): + monkeypatch.delenv(SRC_ENV_VAR, raising=False) + load_judge_prompts.cache_clear() + with pytest.raises(RuntimeError, match=SRC_ENV_VAR): + load_judge_prompts() + + +def test_load_judge_prompts_reports_a_missing_file(monkeypatch, tmp_path): + monkeypatch.setenv(SRC_ENV_VAR, str(tmp_path)) + load_judge_prompts.cache_clear() + with pytest.raises(RuntimeError, match="does not contain judge.py"): + load_judge_prompts() + + +def test_load_judge_prompts_rejects_a_drifted_revision(monkeypatch, tmp_path): + """The prompts are the benchmark; grading against a drifted copy is fatal.""" + (tmp_path / "judge.py").write_text("JUDGE_PROMPT = 'not upstream'\n") + monkeypatch.setenv(SRC_ENV_VAR, str(tmp_path)) + load_judge_prompts.cache_clear() + with pytest.raises(RuntimeError, match=UPSTREAM_JUDGE_SHA256): + load_judge_prompts() + + +# --- prompt assembly (upstream prompts stubbed: they are not redistributable) --- + + +@pytest.fixture +def stub_prompts(monkeypatch): + prompts = JudgePrompts( + judge_prompt=( + "IF|{full_conversation}|{user_prompt_last_turn}|" + "{response_text}|{rubrics_text}" + ), + system_steer_judge_prompt=( + "STEER|{few_shot_examples}|{system_prompt}|" + "{user_prompt_last_turn}|{response_text}|{rubrics_text}" + ), + steer_few_shot_examples="SHOTS", + ) + monkeypatch.setattr(advanced_if, "load_judge_prompts", lambda: prompts) + return prompts + + +@pytest.mark.usefixtures("stub_prompts") +def test_compose_judge_prompt_fills_the_if_judge_slots(): + messages = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + ] + composed = compose_judge_prompt( + "complex_if_single_turn_v5", messages, "the answer", ["r1"] + ) + kind, conversation, last_turn, response, rubrics_text = composed.split("|") + assert kind == "IF" + assert conversation == "user [1]: u1\nassistant [1]: a1" + assert last_turn == "u2" + assert response == "the answer" + # Upstream renders the rubric block with indent=4. + assert rubrics_text == json.dumps(["r1"], indent=4) + + +@pytest.mark.usefixtures("stub_prompts") +def test_compose_judge_prompt_uses_the_steer_judge_for_upstreams_literal(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u1"}, + ] + composed = compose_judge_prompt( + SYSTEM_STEER_BENCHMARK, messages, "the answer", ["r1"] + ) + kind, shots, system_prompt, last_turn, response, _ = composed.split("|") + assert kind == "STEER" + assert shots == "SHOTS" + assert system_prompt == "sys" + assert last_turn == "u1" + assert response == "the answer" + + +@pytest.mark.usefixtures("stub_prompts") +def test_released_system_steer_rows_compose_the_if_prompt(): + """The reproduced routing defect, seen end to end at the prompt level.""" + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u1"}, + ] + composed = compose_judge_prompt( + RELEASED_SYSTEM_STEER_BENCHMARK, messages, "answer", ["r1"] + ) + assert composed.startswith("IF|") diff --git a/tests/unit/tasks/test_advanced_if_0shot_gen.py b/tests/unit/tasks/test_advanced_if_0shot_gen.py new file mode 100644 index 00000000..bdc326c0 --- /dev/null +++ b/tests/unit/tasks/test_advanced_if_0shot_gen.py @@ -0,0 +1,324 @@ +"""Unit tests for the AdvancedIF 0-shot generative task. + +The upstream judge prompts are CC-BY-NC-4.0 and are never redistributed, so +``compose_judge_prompt`` is stubbed here; :mod:`tests.unit.community.test_advanced_if` +covers the real assembly against stand-in templates. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import json + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import ModelOutput +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + GRADER_OUTPUT_KEY, + TaskContext, + build_judgement_record, + build_rollout_judgement, + iter_grader_outputs, +) +from sieval.datasets.advanced_if import AdvancedIFDataset, AdvancedIFDatasetSample +from sieval.tasks import advanced_if_0shot_gen +from sieval.tasks.advanced_if_0shot_gen import AdvancedIFZeroShotGenTask + +COMPLEX = "complex_if_single_turn_v5" +STEERABILITY = "system_steerability_v2" + + +class _ScriptedChatModel(ChatModel): + """ChatModel returning a fixed reply, recording the last prompt it saw.""" + + def __init__(self, reply: str, model: str = "mock"): + super().__init__(model=model, api_key="fake") + self._reply = reply + self.last_prompt = None + + async def _agenerate_impl(self, prompt, **kwargs) -> ModelOutput: + self.last_prompt = prompt + return ModelOutput( + model=self.meta(), + texts=[self._reply], + usage={"input_tokens": 40, "output_tokens": 3, "total_tokens": 43}, + ) + + async def _alogprobs_impl( + self, prompt, *, max_tokens=1, logprobs=5, echo=True, temperature=0.0, **kwargs + ) -> ModelOutput: + _ = (prompt, max_tokens, logprobs, echo, temperature, kwargs) + return ModelOutput(model=self.meta(), texts=[""]) + + +def _sample( + benchmark_name: str = COMPLEX, + rubrics: tuple[str, ...] = ("Is it two paragraphs?", "Are there two metaphors?"), +) -> AdvancedIFDatasetSample: + return { + "conversation_history": json.dumps( + [ + {"role": "user", "content": "Write a story."}, + {"role": "assistant", "content": "Once upon a time."}, + {"role": "user", "content": "Now make it rhyme."}, + ] + ), + "benchmark_name": benchmark_name, + "prompt_metadata": json.dumps({"rubrics": list(rubrics)}), + } + + +def _judge_reply(checks: dict, satisfied: str) -> str: + return json.dumps( + {"rubrics_check": checks, "SATISFIED_ALL_REQUIREMENTS": satisfied} + ) + + +def _task(answer: str = "A rhyming story.", grader_reply: str = "{}"): + sample = _sample() + dataset = AdvancedIFDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([dict(sample)])}) + ) + model = _ScriptedChatModel(reply=answer, model="candidate") + grader = _ScriptedChatModel(reply=grader_reply, model="o3-mini") + return AdvancedIFZeroShotGenTask(dataset, model, grader=grader), grader + + +async def _run_to_feedback(task, sample): + """Drive preprocess -> infer -> postprocess -> feedback for one sample.""" + ctx = TaskContext(sample_id=0, raw_sample=sample) + pre = await task.preprocess(sample, ctx) + ctx = TaskContext(sample_id=0, raw_sample=sample, preprocess_result=pre) + inferred = await task.infer(pre, ctx) + post = await task.postprocess(inferred, ctx) + return await task.feedback(post, ctx) + + +@pytest.fixture(autouse=True) +def stub_compose(monkeypatch): + """Stand in for the non-redistributable judge prompt assembly.""" + monkeypatch.setattr( + advanced_if_0shot_gen, + "compose_judge_prompt", + lambda benchmark_name, messages, response, rubrics: ( + f"JUDGE[{benchmark_name}] resp={response} rubrics={len(rubrics)}" + ), + ) + + +# --- grader is mandatory; rubric grading is the only scorer AdvancedIF has --- + + +def test_build_grader_requires_config(): + with pytest.raises(ValueError, match="requires an LLM grader"): + AdvancedIFZeroShotGenTask._build_grader(None) + + +def test_build_grader_accepts_mapping_and_model(): + built = AdvancedIFZeroShotGenTask._build_grader( + {"model": "o3-mini", "api_key": "fake"} + ) + assert isinstance(built, ChatModel) + existing = _ScriptedChatModel(reply="{}") + assert AdvancedIFZeroShotGenTask._build_grader(existing) is existing + + +# --- preprocess --- + + +@pytest.mark.anyio +async def test_preprocess_sends_the_conversation_as_messages(): + task, _ = _task() + sample = _sample() + record = await task.preprocess(sample, TaskContext(sample_id=0, raw_sample=sample)) + assert record["prompt"] == [ + {"role": "user", "content": "Write a story."}, + {"role": "assistant", "content": "Once upon a time."}, + {"role": "user", "content": "Now make it rhyme."}, + ] + + +@pytest.mark.anyio +async def test_preprocess_omits_reference_and_carries_the_rubric(): + """The ground truth is a rubric -- a procedure, not a value to compare.""" + task, _ = _task() + sample = _sample() + record = await task.preprocess(sample, TaskContext(sample_id=0, raw_sample=sample)) + assert "reference" not in record + assert record["extra"]["benchmark_name"] == COMPLEX + assert record["extra"]["rubrics"] == [ + "Is it two paragraphs?", + "Are there two metaphors?", + ] + + +# --- postprocess --- + + +@pytest.mark.anyio +async def test_postprocess_normalizes_a_blank_response_to_none(): + task, _ = _task(answer=" ") + ctx = TaskContext(sample_id=0) + inferred = ModelOutput(model=task.model.meta(), texts=[" "]) + record = await task.postprocess(inferred, ctx) + assert record["rollouts"][0].get("prediction") is None + assert record["rollouts"][0]["extracted"] is False + + +# --- feedback --- + + +@pytest.mark.anyio +async def test_feedback_scores_a_fully_satisfied_response(): + task, grader = _task( + grader_reply=_judge_reply({"question_1": "Yes", "question_2": "Yes"}, "Yes") + ) + ok, judgement = await _run_to_feedback(task, _sample()) + + assert ok is True + rollout = judgement["rollouts"][0] + assert rollout["correct"] is True + assert rollout["score"] == 1.0 + assert rollout["metrics"] == { + "satisfied_all": True, + "rubric_level_pass_rate": 1.0, + } + assert judgement["reference"] is None + assert judgement["extra"] == {"benchmark_name": COMPLEX, "n_rubrics": 2} + # The grader saw the composed judge prompt, not the raw conversation. + assert grader.last_prompt.startswith(f"JUDGE[{COMPLEX}]") + + +@pytest.mark.anyio +async def test_feedback_records_partial_credit_and_raw_counts(): + task, _ = _task( + grader_reply=_judge_reply({"question_1": "Yes", "question_2": "No"}, "No") + ) + _, judgement = await _run_to_feedback(task, _sample()) + + rollout = judgement["rollouts"][0] + assert rollout["correct"] is False + assert rollout["score"] == 0.5 + # Raw counts, because a per-sample rate cannot reconstruct a pooled one. + assert rollout["extra"]["n_checks"] == 2 + assert rollout["extra"]["n_checks_passed"] == 1 + assert rollout["extra"]["judge_parsed"] is True + assert rollout["extra"]["rubrics_check"] == { + "question_1": "Yes", + "question_2": "No", + } + + +@pytest.mark.anyio +async def test_feedback_persists_the_whole_grader_output(): + task, _ = _task(grader_reply=_judge_reply({"question_1": "Yes"}, "No")) + _, judgement = await _run_to_feedback(task, _sample()) + + grader_output = judgement["rollouts"][0]["extra"][GRADER_OUTPUT_KEY] + assert grader_output["texts"] == [_judge_reply({"question_1": "Yes"}, "No")] + assert grader_output["usage"]["total_tokens"] == 43 + + +@pytest.mark.anyio +async def test_grader_spend_reaches_the_profiler(): + """One batched grader call per rollout, stored as a mapping -- not a list. + + ``iter_grader_outputs`` skips a list, so fanning the rubric out into one + judge call per criterion would make the grader's tokens vanish from + profile.json. The whole rubric goes in a single indexed call instead. + """ + task, _ = _task(grader_reply=_judge_reply({"question_1": "Yes"}, "Yes")) + _, judgement = await _run_to_feedback(task, _sample()) + + outputs = iter_grader_outputs(judgement) + assert len(outputs) == 1 + assert outputs[0]["usage"]["total_tokens"] == 43 + + +@pytest.mark.anyio +async def test_feedback_treats_an_unparseable_reply_as_a_failed_row(): + """Upstream's failed-row path: no pass, and no rubrics into the micro rate.""" + task, _ = _task(grader_reply="the judge rambled without emitting JSON") + _, judgement = await _run_to_feedback(task, _sample()) + + rollout = judgement["rollouts"][0] + assert rollout["correct"] is False + assert rollout["score"] == 0.0 + assert rollout["extra"]["judge_parsed"] is False + assert rollout["extra"]["n_checks"] == 0 + assert rollout["extra"]["n_checks_passed"] == 0 + # The reply is still on disk -- the only evidence of what the grader did. + assert rollout["extra"][GRADER_OUTPUT_KEY]["texts"] == [ + "the judge rambled without emitting JSON" + ] + + +# --- report --- + + +def _final(benchmark_name: str, satisfied: bool, n_checks: int, n_passed: int): + judgement = build_judgement_record( + None, + [ + build_rollout_judgement( + 0, + satisfied, + score=n_passed / n_checks if n_checks else 0.0, + extra={"n_checks": n_checks, "n_checks_passed": n_passed}, + ) + ], + extra={"benchmark_name": benchmark_name, "n_rubrics": n_checks}, + ) + return TaskContext(sample_id=0, feedback_result=judgement) + + +@pytest.mark.anyio +async def test_report_pools_both_published_rates(): + task, _ = _task() + finals = [ + _final(COMPLEX, True, 4, 4), + _final(COMPLEX, False, 4, 2), + ] + report = await task.report(finals, []) + + assert report["score"] == pytest.approx(50.0) + assert report["overall_pass_rate"] == pytest.approx(50.0) + assert report["micro_pass_rate"] == pytest.approx(75.0) + assert report["n_graded"] == 2.0 + assert report["n_rubric_checks"] == 8.0 + assert report["fails"] == 0 + + +@pytest.mark.anyio +async def test_report_breaks_down_by_aspect(): + task, _ = _task() + finals = [ + _final(COMPLEX, True, 2, 2), + _final(STEERABILITY, False, 2, 1), + _final(STEERABILITY, False, 2, 0), + ] + report = await task.report(finals, []) + + assert report[f"{COMPLEX}_pass_rate"] == pytest.approx(100.0) + assert report[f"{COMPLEX}_n_graded"] == 1.0 + assert report[f"{STEERABILITY}_pass_rate"] == pytest.approx(0.0) + assert report[f"{STEERABILITY}_micro_pass_rate"] == pytest.approx(25.0) + assert report[f"{STEERABILITY}_n_graded"] == 2.0 + + +@pytest.mark.anyio +async def test_report_counts_pipeline_failures_as_non_passes(): + """A sample that never produced an answer still spans the requested set.""" + task, _ = _task() + finals = [_final(COMPLEX, True, 2, 2)] + report = await task.report(finals, [TaskContext(sample_id=1)]) + + assert report["overall_pass_rate"] == pytest.approx(50.0) + # A failure contributes no rubrics, so the micro rate is unaffected. + assert report["micro_pass_rate"] == pytest.approx(100.0) + assert report["n_graded"] == 1.0 + assert report["fails"] == 1 + # It has no aspect to attribute to, so the breakdown covers graded rollouts. + assert report[f"{COMPLEX}_n_graded"] == 1.0