feat(tasks): add ComplexConstraints (dataset + 0-shot rubric-judged task)

sieval/community/complex_constraints.py

@@ -0,0 +1,178 @@
+"""ComplexConstraints rubric-grading assets: judge prompt, verdict parsing, metrics.
+
+ComplexConstraints (Mehta et al., 2026, arXiv:2606.09118) is a 75-prompt
+instruction-following benchmark. Each prompt ships 10-40 *atomic* rubric criteria
+(1,559 in total) describing what a correct response must satisfy; criteria are
+graded by rubric -- human or LLM-as-a-judge -- never by exact match.
+
+Upstream publishes **no evaluation code and no judge prompt**. The paper names
+GPT-5-mini as the per-criterion judge and defines the metrics, but the template,
+its decoding settings, and the call structure are all unstated, and the dataset
+card adds nothing. So ``GRADER_TEMPLATE`` and :func:`parse_verdicts` below are
+**authored by this port**, not reproduced from upstream -- which is why
+``sieval.tasks.complex_constraints_0shot_gen`` ships ``status="experimental"``.
+Contrast ``sieval.community.aa_lcr``, whose templates at least come from the
+upstream dataset card verbatim.
+
+Two published metrics, both computed by :func:`aggregate_metrics`:
+
+* **task pass rate** -- the fraction of prompts whose response satisfies *every*
+  criterion. This is what the paper's public 75-prompt leaderboard reports
+  (its Table 1, snapshot 2026-06-03), so it is the port's headline.
+* **mean per-criterion pass rate** -- "the fraction of rubric criteria satisfied,
+  averaged across tasks" (its Table 3 caption), i.e. a **macro** average over
+  prompts. Criteria counts vary 10-40, so the pooled (**micro**) rate is a
+  genuinely different number; both are reported and named, and the macro one is
+  the published one.
+
+Grading is **one judge call per rollout**, covering all of that prompt's criteria,
+with the verdicts emitted as an indexed list. Upstream never says whether it
+grades one criterion per call; batching keeps a rollout's whole verdict set in a
+single persisted ``ModelOutput`` -- which is also what the runner's grader-spend
+accounting expects, since it reads exactly one output per rollout -- and the
+indexing makes misalignment detectable: an index the judge never emits is
+recorded as unparsed rather than silently shifting its neighbours' verdicts.
+
+AI-Generated Code - Claude Opus 5 (1M context) (Anthropic)
+"""
+
+import re
+from collections.abc import Sequence
+
+#: One rubric line as the judge sees it. 1-based, matching the verdict indices
+#: the judge is asked to emit.
+CRITERION_TEMPLATE = "{number}. {criterion}"
+
+#: Rubric-grading prompt. Authored by this port (upstream publishes none).
+#: The verdict block is requested *last* so a reasoning judge puts it after its
+#: deliberation, which is what makes "last verdict per index wins" correct.
+GRADER_TEMPLATE = """You are grading one model RESPONSE against a rubric of atomic criteria.
+
+Judge each criterion independently, against the RESPONSE alone. A criterion is satisfied
+only if the RESPONSE clearly meets it. If the response only partially meets a criterion, or
+gives you nothing to check it against, that criterion is NOT satisfied. Grade exactly what
+each criterion asks for -- do not reward or penalise anything else about the response.
+
+BEGIN PROMPT
+{prompt}
+END PROMPT
+
+BEGIN RESPONSE
+{response}
+END RESPONSE
+
+BEGIN CRITERIA
+{criteria}
+END CRITERIA
+
+Grade all {n_criteria} criteria. End your reply with one verdict per criterion, in order,
+one per line, in exactly this format and nothing after it:
+
+1: <PASS|FAIL>
+2: <PASS|FAIL>
+...
+{n_criteria}: <PASS|FAIL>
+"""
+
+
+def format_criteria(criteria: Sequence[str]) -> str:
+    """Render *criteria* as the 1-based numbered block the judge grades."""
+    return "\n".join(
+        CRITERION_TEMPLATE.format(number=i + 1, criterion=criterion)
+        for i, criterion in enumerate(criteria)
+    )
+
+
+def build_grader_prompt(prompt: str, response: str, criteria: Sequence[str]) -> str:
+    """Assemble the rubric-grading prompt for one response.
+
+    The original *prompt* is included because criteria are written against it
+    ("the response should schedule ... 15th-21st December 2025"): many are
+    uncheckable from the response alone.
+    """
+    return GRADER_TEMPLATE.format(
+        prompt=prompt,
+        response=response,
+        criteria=format_criteria(criteria),
+        n_criteria=len(criteria),
+    )
+
+
+# A verdict line: leading list/emphasis punctuation ("- ", "* ", "**"), an
+# optional "criterion" word, the 1-based index, a separator, more optional
+# emphasis, then the verdict. Anchored to line starts so prose that merely
+# mentions a number cannot register as a verdict.
+_VERDICT_RE = re.compile(
+    r"^[^\w\n]*(?:criterion\s*)?(\d{1,3})\s*[:.)\-]\s*[^\w\n]*(PASS|FAIL)\b",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+
+def parse_verdicts(reply: str, n_criteria: int) -> list[bool | None]:
+    """Map a judge reply to one verdict per criterion, in criterion order.
+
+    Returns a list of length *n_criteria*: ``True`` (satisfied), ``False`` (not
+    satisfied), or ``None`` for a criterion the judge never returned a readable
+    verdict for. ``None`` is deliberately distinct from ``False`` -- the caller
+    scores it as not-satisfied (an unreadable verdict must not inflate a score)
+    but records the count separately, so judge format drift stays visible
+    instead of masquerading as a model that failed the rubric.
+
+    The **last** verdict for an index wins: the judge is asked to put the verdict
+    block at the end, so a reasoning judge's earlier tentative pass over the
+    criteria must not override its final answer. Indices outside ``1..n_criteria``
+    are ignored rather than clamped -- a hallucinated "41: PASS" is not evidence
+    about criterion 41 of a 40-criterion rubric.
+    """
+    verdicts: list[bool | None] = [None] * n_criteria
+    for index_text, verdict in _VERDICT_RE.findall(reply):
+        index = int(index_text)
+        if 1 <= index <= n_criteria:
+            verdicts[index - 1] = verdict.upper() == "PASS"
+    return verdicts
+
+
+def aggregate_metrics(units: Sequence[tuple[int, int]]) -> dict[str, float]:
+    """Aggregate ``(n_satisfied, n_criteria)`` pairs into the published metrics.
+
+    One *unit* is one graded rollout, plus one stand-in per attempt that never
+    produced a gradeable response (contributing ``(0, n_criteria)``) so the rates
+    span the full requested set rather than only the successfully-graded subset.
+
+    Returns rates in ``[0, 1]``:
+
+    * ``task_pass_rate`` -- units satisfying every criterion. The leaderboard's
+      metric, and the port's headline.
+    * ``criterion_pass_rate_macro`` -- per-unit satisfied fraction, averaged over
+      units. The paper's "mean per-criterion pass rate".
+    * ``criterion_pass_rate_micro`` -- criteria satisfied pooled over all units.
+      Differs from the macro rate because criteria counts vary 10-40 per prompt.
+
+    A unit with ``n_criteria == 0`` (a failure whose rubric size could not be
+    recovered) counts as a task failure at rate 0 and adds nothing to the pooled
+    denominator -- so it can only ever drag the score down, never flatter it.
+    """
+    total = len(units)
+    if total == 0:
+        return {
+            "task_pass_rate": 0.0,
+            "criterion_pass_rate_macro": 0.0,
+            "criterion_pass_rate_micro": 0.0,
+        }
+
+    pooled_criteria = sum(count for _, count in units)
+    return {
+        "task_pass_rate": sum(
+            1 for satisfied, count in units if count > 0 and satisfied == count
+        )
+        / total,
+        "criterion_pass_rate_macro": sum(
+            satisfied / count if count else 0.0 for satisfied, count in units
+        )
+        / total,
+        "criterion_pass_rate_micro": (
+            sum(satisfied for satisfied, _ in units) / pooled_criteria
+            if pooled_criteria
+            else 0.0
+        ),
+    }

sieval/datasets/__init__.pyi

@@ -53,6 +53,10 @@ from .cmmlu import (
     CMMLUDataset,
     CMMLUDatasetSample,
 )
+from .complex_constraints import (
+    ComplexConstraintsDataset,
+    ComplexConstraintsDatasetSample,
+)
 from .drop import (
     DROPDataset,
     DROPDatasetSample,
@@ -189,6 +193,8 @@ __all__ = [
     "CMIMC2025DatasetSample",
     "CMMLUDataset",
     "CMMLUDatasetSample",
+    "ComplexConstraintsDataset",
+    "ComplexConstraintsDatasetSample",
     "DROPDataset",
     "DROPDatasetSample",
     "GPQADiamondDataset",

sieval/datasets/complex_constraints.py

@@ -0,0 +1,116 @@
+"""ComplexConstraints dataset loader (Surge AI).
+
+ComplexConstraints (Mehta et al., 2026, arXiv:2606.09118) is a 75-prompt
+multi-constraint instruction-following benchmark (``CIF-001``-``CIF-075``). Each
+row is one realistic prompt plus 10-40 atomic rubric criteria (1,559 in total)
+describing what a correct response must satisfy.
+
+The Hub repo ships a single wide CSV: five item columns and 40 sparse
+``criterion_{i}`` columns, of which a row uses the first 10-40. This loader
+collapses those 40 columns into one ``criteria`` list and drops them. That is a
+**reshape, not a rename for uniformity**: a 40-key ``TypedDict`` of mostly-absent
+columns is unusable as a sample type, and every consumer wants the list. The
+other five columns keep their upstream names, and no dtype cast is applied --
+the pinned revision already ships all 45 columns as strings.
+
+AI-Generated Code - Claude Opus 5 (1M context) (Anthropic)
+"""
+
+import os
+from typing import TypedDict, override
+
+from datasets import DatasetDict as HFDatasetDict
+from datasets import load_dataset
+
+from sieval.core.datasets import (
+    Category,
+    Dataset,
+    Level1Category,
+    sieval_dataset,
+)
+from sieval.core.utils.hf import ensure_dataset_dict
+
+# Pin the Hub revision for reproducibility (current `main` at integration time).
+COMPLEX_CONSTRAINTS_REVISION = "e9625c6f635f42b72cb85a04c2be64746f945126"
+
+#: The repo's one data file. The dataset card's ``configs.data_files.path`` spells
+#: it ``ComplexConstraints_Benchmark_Set.csv``, which does not exist -- so
+#: ``load_dataset("surgeai/ComplexConstraints")`` cannot resolve the file at all.
+#: Reading the staged snapshot by its real name sidesteps the card's typo.
+CSV_FILENAME = "ComplexConstraints_benchmark_set.csv"
+
+#: Widest criterion column upstream ships; a row fills the first 10-40.
+MAX_CRITERIA = 40
+
+_CRITERION_COLUMNS = tuple(f"criterion_{i}" for i in range(1, MAX_CRITERIA + 1))
+
+
+class ComplexConstraintsDatasetSample(TypedDict):
+    benchmark_id: str
+    prompt: str
+    use_case: str
+    instruction_type: str
+    prompt_style: str
+    criteria: list[str]
+
+
+def _collapse_criteria(row: dict) -> dict:
+    """Gather a row's non-empty ``criterion_{i}`` cells into one ordered list.
+
+    Every non-empty cell is kept, rather than stopping at the first empty one.
+    On the pinned revision the filled cells are a contiguous prefix (verified:
+    0 of 75 rows have a gap), so the two readings agree there -- but stopping
+    early would silently drop criteria if a later revision ever left a hole,
+    and a dropped criterion inflates the score.
+    """
+    criteria = [
+        text
+        for column in _CRITERION_COLUMNS
+        if (value := row[column]) is not None and (text := str(value).strip())
+    ]
+    return {"criteria": criteria}
+
+
+@sieval_dataset(
+    name="complex_constraints",
+    display_name="ComplexConstraints",
+    description="75 multi-constraint prompts with 1,559 rubric criteria (Surge AI).",
+    source=f"hf:surgeai/ComplexConstraints@{COMPLEX_CONSTRAINTS_REVISION}",
+    categories=(Category(Level1Category.LANGUAGE, "InstructionFollowing"),),
+    tags=("english", "instruction-following", "open-ended"),
+    license="CC-BY-4.0",
+)
+class ComplexConstraintsDataset(Dataset[ComplexConstraintsDatasetSample]):
+    @override
+    def load(self, name_or_path: str, **kwargs) -> HFDatasetDict:
+        csv_path = (
+            os.path.join(name_or_path, CSV_FILENAME)
+            if os.path.isdir(name_or_path)
+            else name_or_path
+        )
+        dataset = load_dataset("csv", data_files={"test": csv_path}, **kwargs)
+        dataset = ensure_dataset_dict(dataset)
+        split = dataset["test"]
+        if len(split) == 0:
+            raise ValueError(
+                f"ComplexConstraints produced an empty 'test' split from "
+                f"{csv_path!r}; check that the dataset has been downloaded via "
+                "`sieval dataset download complex_constraints`."
+            )
+
+        missing = [c for c in _CRITERION_COLUMNS if c not in split.column_names]
+        if missing:
+            raise ValueError(
+                f"ComplexConstraints is missing criterion column(s) {missing} in "
+                f"{csv_path!r}; the loader expects the wide format of revision "
+                f"{COMPLEX_CONSTRAINTS_REVISION} (criterion_1..criterion_"
+                f"{MAX_CRITERIA})."
+            )
+
+        return HFDatasetDict(
+            {
+                "test": split.map(
+                    _collapse_criteria, remove_columns=list(_CRITERION_COLUMNS)
+                )
+            }
+        )

sieval/meta/index.json

@@ -284,6 +284,28 @@
         "d6e7b716d8ac694f38969a6c0407437d1fded799.zip": "sha256:154593336d5074d793ed990222876b83490b0aed97638a62618d1fe2da7c2cac"
       }
     },
+    {
+      "name": "complex_constraints",
+      "display_name": "ComplexConstraints",
+      "description": "75 multi-constraint prompts with 1,559 rubric criteria (Surge AI).",
+      "source": [
+        "hf:surgeai/ComplexConstraints@e9625c6f635f42b72cb85a04c2be64746f945126"
+      ],
+      "categories": [
+        {
+          "level1": "Language",
+          "level2": "InstructionFollowing"
+        }
+      ],
+      "tags": [
+        "english",
+        "instruction-following",
+        "open-ended"
+      ],
+      "deps_group": null,
+      "license": "CC-BY-4.0",
+      "checksums": {}
+    },
     {
       "name": "drop",
       "display_name": "DROP",
@@ -1208,6 +1230,27 @@
       },
       "status": "stable"
     },
+    {
+      "name": "complex_constraints_0shot_gen",
+      "display_name": "ComplexConstraints (0-shot, generative)",
+      "description": "Multi-constraint instruction following; rubric graded by an LLM judge.",
+      "dataset": "complex_constraints",
+      "eval_mode": "gen",
+      "n_shot": 0,
+      "tags": [
+        "english",
+        "instruction-following",
+        "open-ended"
+      ],
+      "deps_group": null,
+      "model_type": "chat",
+      "reference_impl": {
+        "source": "complex-constraints",
+        "url": "https://arxiv.org/abs/2606.09118",
+        "notes": "Generative port of ComplexConstraints (Surge AI, arXiv:2606.09118) — 75 multi-constraint prompts (CIF-001..CIF-075) with 10-40 atomic rubric criteria each (1,559 total), graded by rubric rather than exact match. NO UPSTREAM EVAL CODE AND NO UPSTREAM JUDGE PROMPT: the paper names GPT-5-mini as the per-criterion judge and defines the metrics, but the template, decoding settings and call structure are unstated, and the dataset card (https://huggingface.co/datasets/surgeai/ComplexConstraints/blob/e9625c6f635f42b72cb85a04c2be64746f945126/README.md) adds nothing — so the rubric prompt and verdict parsing are AUTHORED BY THIS PORT (sieval.community.complex_constraints), hence status=experimental. METRICS: headline = task pass rate (response satisfies EVERY criterion), the metric the paper's public leaderboard reports (Table 1). Also reported: criterion_pass_rate_macro (per-prompt satisfied fraction averaged over prompts — the paper's 'mean per-criterion pass rate', Table 3 caption) and criterion_pass_rate_micro (pooled over all criteria); the two differ because criteria counts vary 10-40 per prompt. GRADING: one judge call per rollout covering all of that prompt's criteria as an indexed PASS/FAIL list (upstream's call structure is unstated); an unreadable per-criterion verdict scores not-satisfied but is counted as n_unparsed so judge format drift stays visible; empty/whitespace responses satisfy zero criteria without invoking the judge (grader_output absent there, no call made). REPRODUCIBILITY: scores depend on the grader endpoint's model version (not pinnable like a Hub revision) — pin the grader model + temperature=0; per-criterion verdicts and the judge's full ModelOutput (extra.grader_output) are persisted per rollout, the reply being the only evidence of a verdict a re-grade need not reproduce. REPEATS: the leaderboard states no repeat count, so the port defaults to n=1; `n` is a task arg (tasks.<name>.args.n), NOT a model arg — infer forwards it call-time and call-time wins. NOT YET VALIDATED against the Table 1 leaderboard (snapshot 2026-06-03: Gemini 3.1 Pro 40.4, GPT-5.5 38.7, Claude Opus 4.8 34.9 task pass %)."
+      },
+      "status": "experimental"
+    },
     {
       "name": "drop_kshot_gen",
       "display_name": "DROP (few-shot, generative)",

sieval/tasks/__init__.pyi

@@ -46,6 +46,9 @@ from .cmimc_2025_0shot_gen import (
 from .cmmlu_kshot_clp import (
     CMMLUFewShotClpTask,
 )
+from .complex_constraints_0shot_gen import (
+    ComplexConstraintsZeroShotGenTask,
+)
 from .drop_kshot_gen import (
     DROPFewShotGenTask,
 )
@@ -168,6 +171,7 @@ __all__ = [
     "CEvalFewShotCLPTask",
     "CMIMC2025ZeroShotGenTask",
     "CMMLUFewShotClpTask",
+    "ComplexConstraintsZeroShotGenTask",
     "DROPFewShotGenTask",
     "GPQADiamondZeroShotGenTask",
     "GSM8KFewShotBaseGenTask",