feat(tasks): add AdvancedIF rubric-graded instruction-following benchmark

sieval/datasets/__init__.pyi

@@ -5,6 +5,10 @@ from .aa_lcr import (
     AALCRDataset,
     AALCRDatasetSample,
 )
+from .advanced_if import (
+    AdvancedIFDataset,
+    AdvancedIFDatasetSample,
+)
 from .aime_2024 import (
     AIME2024Dataset,
     AIME2024DatasetSample,
@@ -175,6 +179,8 @@ __all__ = [
     "ARCChallengeDatasetSample",
     "ARCEasyDataset",
     "ARCEasyDatasetSample",
+    "AdvancedIFDataset",
+    "AdvancedIFDatasetSample",
     "Apex2025Dataset",
     "Apex2025DatasetSample",
     "ApexShortlist2025Dataset",

sieval/datasets/advanced_if.py

@@ -0,0 +1,67 @@
+"""AdvancedIF dataset loader.
+
+AI-Generated Code - Claude Opus 5 (Anthropic)
+"""
+
+from typing import TypedDict, override
+
+from datasets import DatasetDict as HFDatasetDict
+from datasets import load_dataset
+
+from sieval.core.datasets import (
+    Category,
+    Dataset,
+    Level1Category,
+    sieval_dataset,
+)
+from sieval.core.utils.hf import apply_eval_split, ensure_dataset_dict
+
+ADVANCED_IF_REVISION = "e20cba9b94b59c027dfab00b29244e8bc42e4ab4"
+
+
+class AdvancedIFDatasetSample(TypedDict):
+    """One AdvancedIF prompt.
+
+    Upstream ships all three columns as strings, including the two that hold
+    JSON; they are decoded in the task rather than here, so the persisted prompt
+    record keeps the bytes the grader prompt was actually built from. No cast
+    needed -- the pinned revision already ships these as strings.
+
+    Attributes:
+        conversation_history: JSON list of ``{"role", "content"}`` turns, ending
+            on the user prompt to answer (the assistant turn under test is
+            absent by construction). A system-steerability row leads with a
+            ``system`` turn.
+        benchmark_name: Which of the three aspects the row belongs to --
+            ``complex_if_single_turn_v5`` (402), ``system_steerability_v2``
+            (507) or ``carried_context_multi_turn_eval_v5`` (736).
+        prompt_metadata: JSON object whose ``rubrics`` key holds the
+            expert-written yes/no checks (itself sometimes JSON-encoded again).
+    """
+
+    conversation_history: str
+    benchmark_name: str
+    prompt_metadata: str
+
+
+@sieval_dataset(
+    name="advanced_if",
+    display_name="AdvancedIF",
+    description=(
+        "Expert-written prompts with human-curated rubrics for advanced "
+        "instruction following."
+    ),
+    source=f"hf:facebook/AdvancedIF@{ADVANCED_IF_REVISION}",
+    categories=(Category(Level1Category.LANGUAGE, "InstructionFollowing"),),
+    tags=("english", "open-ended"),
+    # Non-commercial. The judge prompts carry the same terms and are likewise
+    # not redistributed -- see sieval.community.advanced_if.
+    license="CC-BY-NC-4.0",
+)
+class AdvancedIFDataset(Dataset[AdvancedIFDatasetSample]):
+    @override
+    def load(self, name_or_path: str, **kwargs) -> HFDatasetDict:
+        # AdvancedIF ships its 1,645 rows in a single "train" split (the card
+        # calls the same rows "test"); mirror it to "test" for the runner.
+        dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs))
+        return apply_eval_split(dataset, "train")

sieval/meta/index.json

@@ -24,6 +24,27 @@
       "license": "apache-2.0",
       "checksums": {}
     },
+    {
+      "name": "advanced_if",
+      "display_name": "AdvancedIF",
+      "description": "Expert-written prompts with human-curated rubrics for advanced instruction following.",
+      "source": [
+        "hf:facebook/AdvancedIF@e20cba9b94b59c027dfab00b29244e8bc42e4ab4"
+      ],
+      "categories": [
+        {
+          "level1": "Language",
+          "level2": "InstructionFollowing"
+        }
+      ],
+      "tags": [
+        "english",
+        "open-ended"
+      ],
+      "deps_group": null,
+      "license": "CC-BY-NC-4.0",
+      "checksums": {}
+    },
     {
       "name": "aime_2024",
       "display_name": "AIME 2024",
@@ -917,6 +938,26 @@
       },
       "status": "experimental"
     },
+    {
+      "name": "advanced_if_0shot_gen",
+      "display_name": "AdvancedIF (0-shot, generative)",
+      "description": "Instruction following graded against expert-written rubrics by an LLM judge.",
+      "dataset": "advanced_if",
+      "eval_mode": "gen",
+      "n_shot": 0,
+      "tags": [
+        "english",
+        "open-ended"
+      ],
+      "deps_group": null,
+      "model_type": "chat",
+      "reference_impl": {
+        "source": "facebookresearch/AdvancedIF",
+        "url": "https://github.com/facebookresearch/AdvancedIF/blob/f9d30137c4139d4d9af260ae28108b5afae828c0/judge.py",
+        "notes": "Port of AdvancedIF (Meta, arXiv:2511.10507), 1,645 prompts across complex_if_single_turn_v5 (402), system_steerability_v2 (507) and carried_context_multi_turn_eval_v5 (736). Headline score is the overall pass rate (share of samples the grader marked SATISFIED_ALL_REQUIREMENTS=yes), the number the paper reports; micro_pass_rate is the co-published rubric-level rate. LICENSING: upstream ships judge.py under CC-BY-NC-4.0, incompatible with sieval's Apache-2.0 tree, so the prompts are NOT vendored -- the operator stages a checkout and points SIEVAL_ADVANCED_IF_SRC at it, digest-checked against commit f9d30137c4139d4d9af260ae28108b5afae828c0. The dataset is CC-BY-NC-4.0 too, so running this benchmark accepts those terms either way. Loading from the operator's own checkout also makes the prompts byte-exact by construction rather than by review. UPSTREAM DEFECT (reproduced, not corrected): upstream routes to the system-steerability judge on benchmark_name == 'if_system_steerability_oss', a value the released dataset never contains (it ships 'system_steerability_v2'), so all 507 system-prompt rows are graded by the plain user-instruction judge and the CLI's --task choices match zero rows; processor.process_file's own docstring uses the released spelling, so the if_*_oss literals are what went stale. This port keeps upstream's comparison verbatim so the unqualified name tracks upstream. Correcting the routing moves scores on a third of the benchmark and belongs in a _fixed variant carrying a measured delta. Grader is a REAL LLM (upstream: o3-mini-2025-01-31, temperature 0, max_completion_tokens=32768, response_format=json_object) supplied via the `grader` task arg; pin it, as its version is not pinnable the way a Hub revision is. The grader's full ModelOutput and per-rubric answers are persisted under the judgement record's `extra`. VALIDATION: none -- no published number has been reproduced with this port, and the paper's own figures come from Meta's internal pipeline rather than the released CLI."
+      },
+      "status": "experimental"
+    },
     {
       "name": "aime_2024_0shot_gen",
       "display_name": "AIME 2024 (0-shot, generative)",

sieval/tasks/__init__.pyi

@@ -4,6 +4,9 @@
 from .aa_lcr_0shot_gen import (
     AALCRZeroShotGenTask,
 )
+from .advanced_if_0shot_gen import (
+    AdvancedIFZeroShotGenTask,
+)
 from .aime_2024_0shot_gen import (
     AIME2024ZeroShotGenTask,
 )
@@ -161,6 +164,7 @@ __all__ = [
     "ARCChallengeFewShotPplTask",
     "ARCEasyFewShotClpTask",
     "ARCEasyFewShotPplTask",
+    "AdvancedIFZeroShotGenTask",
     "Apex2025ZeroShotGenTask",
     "ApexShortlist2025ZeroShotGenTask",
     "BRUMO2025ZeroShotGenTask",