From a17558b396710c0ed48a74801e66ff49c8d210fc Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 7 Aug 2026 16:17:13 +0800 Subject: [PATCH] feat(tasks): add Multi-IF multi-turn multilingual instruction-following MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-IF extends IFEval into three-turn conversations across eight languages (4,501 conversations). Adds the `multi_if` dataset, the `multi_if_0shot_gen` task, and a `multi-if` optional-dependency group. sieval's first multi-turn task: one sample is one whole conversation, and `infer` walks its turns in order, appending the model's own reply before sending the next user turn. It returns `list[ModelOutput]`, which the runner already special-cases, so every turn's tokens reach the profile. Upstream reaches the same conversation by re-running the whole set once per `--steps` value and writing the growing history back to a CSV; one pass makes a conversation a single unit of work and lets one run report all three turns. Upstream's own multilingual fork of the IFEval checkers is vendored in `sieval/community/multi_if/` rather than reusing the google-research sibling. Both carry the same 25 instruction ids, but Multi-IF's are language-aware (langdetect-routed word/sentence counting, CJK and Hindi counters, Thai tokenizers), so they are not interchangeable. The only local adaptation is a lazy pythainlp import. Grading is verified against upstream's own `metrics_gen` on 535 conversations spanning all eight languages and all 56 two-turn rows: 3,098 strict/loose follow-lists and every per-language `overall` agree exactly. `status="experimental"` all the same — Multi-IF publishes paper scores only, with no per-model inference dump to replay, so no published number has been reproduced yet. Two upstream defects are tracked rather than repaired, per the unqualified-name rule; both make upstream's grader unable to reproduce itself: - Two conversations (6 of 13,447 turn-cells) carry kwargs the checker rejects (`letter="#"`; a missing `keyword`), and `build_description` falls back to an unseeded `random.choice`, grading the response against a letter nobody asked for. - `langdetect` is unseeded, and detection picks the counting algorithm behind every length constraint. Other notes: - The 56 conversations without a third turn get no third generation. Upstream sends them a literal "None" prompt and then drops the row when scoring turn 3, so metrics are unchanged and the tokens are not spent. - Constraints accumulate: turn t is graded against turns 1..t's constraints, verified as a prefix-extension on every row. - Instruction-level accuracy is pooled from raw counts, not averaged from per-sample rates, which differ when turns carry different counts. - `kwargs` stay JSON-encoded in the sample; decoding them would make Arrow unify 24 sparse struct fields per constraint. - The dataset license is CC-BY-NC-2.0 (the Hub card's), not the Apache-2.0 of the code repo that ships the evaluator. Co-Authored-By: Claude Opus 5 (1M context) --- pdm.lock | 39 +- pyproject.toml | 14 + sieval/community/multi_if/__init__.py | 32 + sieval/community/multi_if/evaluation_lib.py | 95 + sieval/community/multi_if/ifeval.py | 3491 +++++++++++++++++++ sieval/datasets/__init__.pyi | 6 + sieval/datasets/multi_if.py | 160 + sieval/meta/index.json | 43 + sieval/tasks/__init__.pyi | 4 + sieval/tasks/multi_if_0shot_gen.py | 336 ++ tests/unit/tasks/test_multi_if_0shot_gen.py | 248 ++ 11 files changed, 4456 insertions(+), 12 deletions(-) create mode 100644 sieval/community/multi_if/__init__.py create mode 100644 sieval/community/multi_if/evaluation_lib.py create mode 100644 sieval/community/multi_if/ifeval.py create mode 100644 sieval/datasets/multi_if.py create mode 100644 sieval/tasks/multi_if_0shot_gen.py create mode 100644 tests/unit/tasks/test_multi_if_0shot_gen.py diff --git a/pdm.lock b/pdm.lock index 8a2b9545..a35ef0d2 100644 --- a/pdm.lock +++ b/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "math", "ruler", "ruler-gen", "scicode", "t-eval", "test"] +groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "math", "multi-if", "ruler", "ruler-gen", "scicode", "t-eval", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:3203ae56fa1eac9b9abf195674f7d3f42537f3f202a8c355303ebe14edf74169" +content_hash = "sha256:362b6f111e39525ceb3ae36c9d9498c3e8cafe76ceef454b335080754777a1da" [[metadata.targets]] requires_python = ">=3.12,<3.15" @@ -301,7 +301,7 @@ name = "click" version = "8.3.1" requires_python = ">=3.10" summary = "Composable command line interface toolkit" -groups = ["default", "ifbench", "ifeval", "ruler", "test"] +groups = ["default", "ifbench", "ifeval", "multi-if", "ruler", "test"] dependencies = [ "colorama; platform_system == \"Windows\"", ] @@ -315,7 +315,7 @@ name = "colorama" version = "0.4.6" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Cross-platform colored terminal text." -groups = ["default", "ifbench", "ifeval", "ruler", "t-eval", "test"] +groups = ["default", "ifbench", "ifeval", "multi-if", "ruler", "t-eval", "test"] marker = "platform_system == \"Windows\" or sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, @@ -563,7 +563,7 @@ name = "emoji" version = "2.15.0" requires_python = ">=3.8" summary = "Emoji for Python" -groups = ["ifbench"] +groups = ["ifbench", "multi-if"] files = [ {file = "emoji-2.15.0-py3-none-any.whl", hash = "sha256:205296793d66a89d88af4688fa57fd6496732eb48917a87175a023c8138995eb"}, {file = "emoji-2.15.0.tar.gz", hash = "sha256:eae4ab7d86456a70a00a985125a03263a5eac54cd55e51d7e184b1ed3b6757e4"}, @@ -979,7 +979,7 @@ name = "joblib" version = "1.5.2" requires_python = ">=3.9" summary = "Lightweight pipelining with Python functions" -groups = ["ifbench", "ifeval", "ruler", "t-eval"] +groups = ["ifbench", "ifeval", "multi-if", "ruler", "t-eval"] files = [ {file = "joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241"}, {file = "joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55"}, @@ -989,7 +989,7 @@ files = [ name = "langdetect" version = "1.0.9" summary = "Language detection library ported from Google's language-detection." -groups = ["ifeval"] +groups = ["ifeval", "multi-if"] dependencies = [ "six", ] @@ -1531,7 +1531,7 @@ name = "nltk" version = "3.9.2" requires_python = ">=3.9" summary = "Natural Language Toolkit" -groups = ["ifbench", "ifeval", "ruler"] +groups = ["ifbench", "ifeval", "multi-if", "ruler"] dependencies = [ "click", "joblib", @@ -2353,6 +2353,21 @@ files = [ {file = "pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1"}, ] +[[package]] +name = "pythainlp" +version = "5.3.5" +requires_python = ">=3.9" +summary = "Thai Natural Language Processing library" +groups = ["multi-if"] +dependencies = [ + "importlib-resources; python_version < \"3.11\"", + "tzdata; sys_platform == \"win32\"", +] +files = [ + {file = "pythainlp-5.3.5-py3-none-any.whl", hash = "sha256:147a7a77c5c6d5b387b827ed3b00ee23c3665d242e9d021565cae4c3bca7b2c2"}, + {file = "pythainlp-5.3.5.tar.gz", hash = "sha256:3be53b97e44fdfc55669705b31a2fe96546146d0c1d90f18999c9b04c4e50c83"}, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2457,7 +2472,7 @@ name = "regex" version = "2025.11.3" requires_python = ">=3.9" summary = "Alternative regular expression module, to replace re." -groups = ["ifbench", "ifeval", "math", "ruler", "t-eval"] +groups = ["ifbench", "ifeval", "math", "multi-if", "ruler", "t-eval"] files = [ {file = "regex-2025.11.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41"}, {file = "regex-2025.11.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36"}, @@ -2822,7 +2837,7 @@ name = "six" version = "1.17.0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Python 2 and 3 compatibility utilities" -groups = ["default", "ifeval"] +groups = ["default", "ifeval", "multi-if"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -3040,7 +3055,7 @@ name = "tqdm" version = "4.67.1" requires_python = ">=3.7" summary = "Fast, Extensible Progress Meter" -groups = ["default", "ifbench", "ifeval", "ruler", "t-eval"] +groups = ["default", "ifbench", "ifeval", "multi-if", "ruler", "t-eval"] dependencies = [ "colorama; platform_system == \"Windows\"", ] @@ -3207,7 +3222,7 @@ name = "tzdata" version = "2025.2" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default"] +groups = ["default", "multi-if"] files = [ {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, diff --git a/pyproject.toml b/pyproject.toml index 681de131..f657d063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,20 @@ ifeval = [ "nltk>=3.9.2", "immutabledict>=4.2.2", ] +multi-if = [ + # Multi-IF vendors its own multilingual fork of the IFEval checkers, so this + # group is deliberately not `ifeval`'s: no absl/immutabledict (the fork uses + # stdlib logging and MappingProxyType), and emoji is required rather than + # optional because the CJK word counter behind every Chinese length + # constraint counts emoji as words. + "emoji>=2.15.0", + "langdetect>=1.0.9", + "nltk>=3.9.2", + # Only reached when langdetect reports Thai for a model *response* (Thai is + # not one of the dataset's eight languages). Declared anyway so that path + # cannot die on ImportError mid-run; the import itself is deferred. + "pythainlp>=5.0.0", +] math = [ "latex2sympy2-extended>=1.10.2", # antlr4-11-0 extra pins antlr4-python3-runtime==4.11.0, which sympy 1.14's diff --git a/sieval/community/multi_if/__init__.py b/sieval/community/multi_if/__init__.py new file mode 100644 index 00000000..edb49d05 --- /dev/null +++ b/sieval/community/multi_if/__init__.py @@ -0,0 +1,32 @@ +"""Meta Multi-IF evaluation adaptation. + +Source: https://github.com/facebookresearch/Multi-IF +Revision: 1cdb53ed18499ad729e0766e5d3099dd5344406f (Apache-2.0, archived) + +Multi-IF ships its *own* multilingual fork of Google's IFEval checkers. It +carries the same 25 instruction ids as +``sieval.community.instruction_following_eval``, but routes word counting, +sentence counting and casing through ``langdetect``, so the two are not +interchangeable and both are vendored. + +Local adaptations (each marked "Local adaptation:" at its site): + +- ``ifeval.py``: ``pythainlp`` is imported on first use rather than at module + scope. Thai is not one of the eight languages in the released Multi-IF CSV, + so the Thai branches are unreachable for that data and the dependency stays + optional. +- ``evaluation_lib.py``: only upstream ``metrics.py``'s two per-response + graders are taken; conversation assembly and aggregation live in the task. + +Deliberately *not* adapted: upstream leaves ``langdetect`` unseeded, so grading +is not reproducible run to run (~2-3% of ``detect()`` calls flip on short or +mixed-script text). Seeding it would change the grader, which belongs in a +``_fixed`` task variant with a measured score delta rather than under the +faithful name. The import site carries the full note. + +Infra: scoring needs the NLTK ``punkt`` tokenizer (via +``nltk.data.load("nltk:tokenizers/punkt/english.pickle")``) exactly as the +IFEval sibling does, and does not download it. Offline runs must pre-stage it. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" diff --git a/sieval/community/multi_if/evaluation_lib.py b/sieval/community/multi_if/evaluation_lib.py new file mode 100644 index 00000000..eb5c7ce9 --- /dev/null +++ b/sieval/community/multi_if/evaluation_lib.py @@ -0,0 +1,95 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Vendored from facebookresearch/Multi-IF (Apache-2.0), file `metrics.py`: +# https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/metrics.py +# +# Only the two per-response graders are taken. Upstream's surrounding +# `MultiTurnInstructionFollowingPromptSolution` drives a pandas/scipy pipeline +# that re-reads a CSV per turn and bootstraps confidence intervals; SiEval's +# task owns conversation assembly and aggregation instead, so importing that +# machinery would pull pandas + scipy in for two pure functions. +# +# Local adaptations: +# 1. `import ifeval` -> `from . import ifeval` (upstream is a flat repo). +# 2. The `Dict[str, float]` return annotations are corrected to `dict`: both +# functions return lists, not floats, so upstream's annotation is wrong. +# Otherwise the bodies are byte-identical to upstream. + +from typing import Any + +from . import ifeval + + +def gen_acc_strict(x: dict[str, Any]) -> dict: + # reference: fbcode/gen_ai/github/fair_evals/evals/tasks/finetune/ifeval.py + response = str(x["response"]) + instruction_list = x["instruction_id_list"] + is_following_list = [] + for index, instruction_id in enumerate(instruction_list): + instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + instruction.build_description(**x["kwargs"][index]) + + if response and instruction.check_following(response): + is_following_list.append(True) + else: + is_following_list.append(False) + + return { + "follow_instruction_list": is_following_list, + "instruction_id_list": instruction_list, + } + + +def gen_acc_loose(x: dict[str, Any]) -> dict: + response = str(x["response"]) + r = response.split("\n") + response_remove_first = "\n".join(r[1:]).strip() + response_remove_last = "\n".join(r[:-1]).strip() + response_remove_both = "\n".join(r[1:-1]).strip() + revised_response = response.replace("*", "") + revised_response_remove_first = response_remove_first.replace("*", "") + revised_response_remove_last = response_remove_last.replace("*", "") + revised_response_remove_both = response_remove_both.replace("*", "") + all_responses = [ + response, + revised_response, + response_remove_first, + response_remove_last, + response_remove_both, + revised_response_remove_first, + revised_response_remove_last, + revised_response_remove_both, + ] + instruction_list = x["instruction_id_list"] + is_following_list = [] + for index, instruction_id in enumerate(instruction_list): + instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + instruction.build_description(**x["kwargs"][index]) + + is_following = False + for r in all_responses: # type: ignore + if r.strip() and instruction.check_following(r): # type: ignore + is_following = True + break + + is_following_list.append(is_following) + return { + "follow_instruction_list": is_following_list, + "instruction_id_list": instruction_list, + } diff --git a/sieval/community/multi_if/ifeval.py b/sieval/community/multi_if/ifeval.py new file mode 100644 index 00000000..6ca76da4 --- /dev/null +++ b/sieval/community/multi_if/ifeval.py @@ -0,0 +1,3491 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Vendored from facebookresearch/Multi-IF (Apache-2.0), file `ifeval.py`: +# https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/ifeval.py +# +# This is Multi-IF's own multilingual fork of Google's IFEval checkers -- it +# carries the same 25 instruction ids as +# `sieval/community/instruction_following_eval/`, but the implementations are +# language-aware (langdetect-routed word/sentence counting, CJK and Hindi +# counters, Thai tokenizers). The two are therefore NOT interchangeable, and +# this file is vendored rather than reusing the IFEval sibling. +# +# The only local adaptation is marked "Local adaptation:" inline: pythainlp is +# imported lazily instead of at module scope. The langdetect import carries an +# explanatory note but is NOT changed. Otherwise byte-identical to upstream. + +import collections +import functools +import json +import logging +import random +import re +import string +from types import MappingProxyType +from typing import Dict, Iterable, Optional, Sequence, Union + + +try: + # Upstream does NOT set `langdetect.DetectorFactory.seed`, so `detect()` + # samples from the global `random` state and can return different languages + # for the same text across runs (measured on this build: ~2-3% of calls flip + # on short or mixed-script input). That matters here because detection picks + # the word/sentence counting algorithm for every length constraint, and is + # compared directly by `language:response_language` and the English + # casing checkers. + # + # It is left unseeded on purpose: this task carries the unqualified name, so + # it must measure what upstream measures, defects included. Seeding is a + # grader change, which belongs in a `_fixed` variant together with a + # quantified score delta -- not smuggled in under the faithful name. + # + # In practice the exposure is narrower than the flip rate suggests: the + # observed flips stayed within Latin-script languages, which all route to + # the same counting branch. Only a flip into or out of {th, hi, zh, ja} + # changes the count, and those scripts are not ambiguous to langdetect. + import langdetect +except ImportError: + langdetect = None +try: + import emoji +except ImportError: + emoji = None +import nltk + + +def sent_tokenize_thai(text): + """Thai sentence tokenizer, imported on first use. + + Upstream imports pythainlp at module scope, so every checker pays its + dictionary-loading cost up front. The import is deferred instead. + + This branch is reached when langdetect reports Thai for a *response* -- not + for the dataset row's language, and Thai is not among the released CSV's + eight -- so in practice it fires only if a model answers in Thai script or + detection misfires. Rare, but reachable, which is why pythainlp is still a + declared dependency of the `multi-if` group rather than left to chance. + """ + from pythainlp.tokenize import sent_tokenize + + return sent_tokenize(text) + + +def word_tokenize_thai(text): + """Thai word tokenizer, imported on first use. See sent_tokenize_thai.""" + from pythainlp.tokenize import word_tokenize + + return word_tokenize(text) + + +logger = logging.getLogger() + +WORD_LIST = [ + "western", + "sentence", + "signal", + "dump", + "spot", + "opposite", + "bottom", + "potato", + "administration", + "working", + "welcome", + "morning", + "good", + "agency", + "primary", + "wish", + "responsibility", + "press", + "problem", + "president", + "steal", + "brush", + "read", + "type", + "beat", + "trainer", + "growth", + "lock", + "bone", + "case", + "equal", + "comfortable", + "region", + "replacement", + "performance", + "mate", + "walk", + "medicine", + "film", + "thing", + "rock", + "tap", + "total", + "competition", + "ease", + "south", + "establishment", + "gather", + "parking", + "world", + "plenty", + "breath", + "claim", + "alcohol", + "trade", + "dear", + "highlight", + "street", + "matter", + "decision", + "mess", + "agreement", + "studio", + "coach", + "assist", + "brain", + "wing", + "style", + "private", + "top", + "brown", + "leg", + "buy", + "procedure", + "method", + "speed", + "high", + "company", + "valuable", + "pie", + "analyst", + "session", + "pattern", + "district", + "pleasure", + "dinner", + "swimming", + "joke", + "order", + "plate", + "department", + "motor", + "cell", + "spend", + "cabinet", + "difference", + "power", + "examination", + "engine", + "horse", + "dimension", + "pay", + "toe", + "curve", + "literature", + "bother", + "fire", + "possibility", + "debate", + "activity", + "passage", + "hello", + "cycle", + "background", + "quiet", + "author", + "effect", + "actor", + "page", + "bicycle", + "error", + "throat", + "attack", + "character", + "phone", + "tea", + "increase", + "outcome", + "file", + "specific", + "inspector", + "internal", + "potential", + "staff", + "building", + "employer", + "shoe", + "hand", + "direction", + "garden", + "purchase", + "interview", + "study", + "recognition", + "member", + "spiritual", + "oven", + "sandwich", + "weird", + "passenger", + "particular", + "response", + "reaction", + "size", + "variation", + "a", + "cancel", + "candy", + "exit", + "guest", + "condition", + "fly", + "price", + "weakness", + "convert", + "hotel", + "great", + "mouth", + "mind", + "song", + "sugar", + "suspect", + "telephone", + "ear", + "roof", + "paint", + "refrigerator", + "organization", + "jury", + "reward", + "engineering", + "day", + "possession", + "crew", + "bar", + "road", + "description", + "celebration", + "score", + "mark", + "letter", + "shower", + "suggestion", + "sir", + "luck", + "national", + "progress", + "hall", + "stroke", + "theory", + "offer", + "story", + "tax", + "definition", + "history", + "ride", + "medium", + "opening", + "glass", + "elevator", + "stomach", + "question", + "ability", + "leading", + "village", + "computer", + "city", + "grand", + "confidence", + "candle", + "priest", + "recommendation", + "point", + "necessary", + "body", + "desk", + "secret", + "horror", + "noise", + "culture", + "warning", + "water", + "round", + "diet", + "flower", + "bus", + "tough", + "permission", + "week", + "prompt", + "connection", + "abuse", + "height", + "save", + "corner", + "border", + "stress", + "drive", + "stop", + "rip", + "meal", + "listen", + "confusion", + "girlfriend", + "living", + "relation", + "significance", + "plan", + "creative", + "atmosphere", + "blame", + "invite", + "housing", + "paper", + "drink", + "roll", + "silver", + "drunk", + "age", + "damage", + "smoke", + "environment", + "pack", + "savings", + "influence", + "tourist", + "rain", + "post", + "sign", + "grandmother", + "run", + "profit", + "push", + "clerk", + "final", + "wine", + "swim", + "pause", + "stuff", + "singer", + "funeral", + "average", + "source", + "scene", + "tradition", + "personal", + "snow", + "nobody", + "distance", + "sort", + "sensitive", + "animal", + "major", + "negotiation", + "click", + "mood", + "period", + "arrival", + "expression", + "holiday", + "repeat", + "dust", + "closet", + "gold", + "bad", + "sail", + "combination", + "clothes", + "emphasis", + "duty", + "black", + "step", + "school", + "jump", + "document", + "professional", + "lip", + "chemical", + "front", + "wake", + "while", + "inside", + "watch", + "row", + "subject", + "penalty", + "balance", + "possible", + "adult", + "aside", + "sample", + "appeal", + "wedding", + "depth", + "king", + "award", + "wife", + "blow", + "site", + "camp", + "music", + "safe", + "gift", + "fault", + "guess", + "act", + "shame", + "drama", + "capital", + "exam", + "stupid", + "record", + "sound", + "swing", + "novel", + "minimum", + "ratio", + "machine", + "shape", + "lead", + "operation", + "salary", + "cloud", + "affair", + "hit", + "chapter", + "stage", + "quantity", + "access", + "army", + "chain", + "traffic", + "kick", + "analysis", + "airport", + "time", + "vacation", + "philosophy", + "ball", + "chest", + "thanks", + "place", + "mountain", + "advertising", + "red", + "past", + "rent", + "return", + "tour", + "house", + "construction", + "net", + "native", + "war", + "figure", + "fee", + "spray", + "user", + "dirt", + "shot", + "task", + "stick", + "friend", + "software", + "promotion", + "interaction", + "surround", + "block", + "purpose", + "practice", + "conflict", + "routine", + "requirement", + "bonus", + "hole", + "state", + "junior", + "sweet", + "catch", + "tear", + "fold", + "wall", + "editor", + "life", + "position", + "pound", + "respect", + "bathroom", + "coat", + "script", + "job", + "teach", + "birth", + "view", + "resolve", + "theme", + "employee", + "doubt", + "market", + "education", + "serve", + "recover", + "tone", + "harm", + "miss", + "union", + "understanding", + "cow", + "river", + "association", + "concept", + "training", + "recipe", + "relationship", + "reserve", + "depression", + "proof", + "hair", + "revenue", + "independent", + "lift", + "assignment", + "temporary", + "amount", + "loss", + "edge", + "track", + "check", + "rope", + "estimate", + "pollution", + "stable", + "message", + "delivery", + "perspective", + "mirror", + "assistant", + "representative", + "witness", + "nature", + "judge", + "fruit", + "tip", + "devil", + "town", + "emergency", + "upper", + "drop", + "stay", + "human", + "neck", + "speaker", + "network", + "sing", + "resist", + "league", + "trip", + "signature", + "lawyer", + "importance", + "gas", + "choice", + "engineer", + "success", + "part", + "external", + "worker", + "simple", + "quarter", + "student", + "heart", + "pass", + "spite", + "shift", + "rough", + "lady", + "grass", + "community", + "garage", + "youth", + "standard", + "skirt", + "promise", + "blind", + "television", + "disease", + "commission", + "positive", + "energy", + "calm", + "presence", + "tune", + "basis", + "preference", + "head", + "common", + "cut", + "somewhere", + "presentation", + "current", + "thought", + "revolution", + "effort", + "master", + "implement", + "republic", + "floor", + "principle", + "stranger", + "shoulder", + "grade", + "button", + "tennis", + "police", + "collection", + "account", + "register", + "glove", + "divide", + "professor", + "chair", + "priority", + "combine", + "peace", + "extension", + "maybe", + "evening", + "frame", + "sister", + "wave", + "code", + "application", + "mouse", + "match", + "counter", + "bottle", + "half", + "cheek", + "resolution", + "back", + "knowledge", + "make", + "discussion", + "screw", + "length", + "accident", + "battle", + "dress", + "knee", + "log", + "package", + "it", + "turn", + "hearing", + "newspaper", + "layer", + "wealth", + "profile", + "imagination", + "answer", + "weekend", + "teacher", + "appearance", + "meet", + "bike", + "rise", + "belt", + "crash", + "bowl", + "equivalent", + "support", + "image", + "poem", + "risk", + "excitement", + "remote", + "secretary", + "public", + "produce", + "plane", + "display", + "money", + "sand", + "situation", + "punch", + "customer", + "title", + "shake", + "mortgage", + "option", + "number", + "pop", + "window", + "extent", + "nothing", + "experience", + "opinion", + "departure", + "dance", + "indication", + "boy", + "material", + "band", + "leader", + "sun", + "beautiful", + "muscle", + "farmer", + "variety", + "fat", + "handle", + "director", + "opportunity", + "calendar", + "outside", + "pace", + "bath", + "fish", + "consequence", + "put", + "owner", + "go", + "doctor", + "information", + "share", + "hurt", + "protection", + "career", + "finance", + "force", + "golf", + "garbage", + "aspect", + "kid", + "food", + "boot", + "milk", + "respond", + "objective", + "reality", + "raw", + "ring", + "mall", + "one", + "impact", + "area", + "news", + "international", + "series", + "impress", + "mother", + "shelter", + "strike", + "loan", + "month", + "seat", + "anything", + "entertainment", + "familiar", + "clue", + "year", + "glad", + "supermarket", + "natural", + "god", + "cost", + "conversation", + "tie", + "ruin", + "comfort", + "earth", + "storm", + "percentage", + "assistance", + "budget", + "strength", + "beginning", + "sleep", + "other", + "young", + "unit", + "fill", + "store", + "desire", + "hide", + "value", + "cup", + "maintenance", + "nurse", + "function", + "tower", + "role", + "class", + "camera", + "database", + "panic", + "nation", + "basket", + "ice", + "art", + "spirit", + "chart", + "exchange", + "feedback", + "statement", + "reputation", + "search", + "hunt", + "exercise", + "nasty", + "notice", + "male", + "yard", + "annual", + "collar", + "date", + "platform", + "plant", + "fortune", + "passion", + "friendship", + "spread", + "cancer", + "ticket", + "attitude", + "island", + "active", + "object", + "service", + "buyer", + "bite", + "card", + "face", + "steak", + "proposal", + "patient", + "heat", + "rule", + "resident", + "broad", + "politics", + "west", + "knife", + "expert", + "girl", + "design", + "salt", + "baseball", + "grab", + "inspection", + "cousin", + "couple", + "magazine", + "cook", + "dependent", + "security", + "chicken", + "version", + "currency", + "ladder", + "scheme", + "kitchen", + "employment", + "local", + "attention", + "manager", + "fact", + "cover", + "sad", + "guard", + "relative", + "county", + "rate", + "lunch", + "program", + "initiative", + "gear", + "bridge", + "breast", + "talk", + "dish", + "guarantee", + "beer", + "vehicle", + "reception", + "woman", + "substance", + "copy", + "lecture", + "advantage", + "park", + "cold", + "death", + "mix", + "hold", + "scale", + "tomorrow", + "blood", + "request", + "green", + "cookie", + "church", + "strip", + "forever", + "beyond", + "debt", + "tackle", + "wash", + "following", + "feel", + "maximum", + "sector", + "sea", + "property", + "economics", + "menu", + "bench", + "try", + "language", + "start", + "call", + "solid", + "address", + "income", + "foot", + "senior", + "honey", + "few", + "mixture", + "cash", + "grocery", + "link", + "map", + "form", + "factor", + "pot", + "model", + "writer", + "farm", + "winter", + "skill", + "anywhere", + "birthday", + "policy", + "release", + "husband", + "lab", + "hurry", + "mail", + "equipment", + "sink", + "pair", + "driver", + "consideration", + "leather", + "skin", + "blue", + "boat", + "sale", + "brick", + "two", + "feed", + "square", + "dot", + "rush", + "dream", + "location", + "afternoon", + "manufacturer", + "control", + "occasion", + "trouble", + "introduction", + "advice", + "bet", + "eat", + "kill", + "category", + "manner", + "office", + "estate", + "pride", + "awareness", + "slip", + "crack", + "client", + "nail", + "shoot", + "membership", + "soft", + "anybody", + "web", + "official", + "individual", + "pizza", + "interest", + "bag", + "spell", + "profession", + "queen", + "deal", + "resource", + "ship", + "guy", + "chocolate", + "joint", + "formal", + "upstairs", + "car", + "resort", + "abroad", + "dealer", + "associate", + "finger", + "surgery", + "comment", + "team", + "detail", + "crazy", + "path", + "tale", + "initial", + "arm", + "radio", + "demand", + "single", + "draw", + "yellow", + "contest", + "piece", + "quote", + "pull", + "commercial", + "shirt", + "contribution", + "cream", + "channel", + "suit", + "discipline", + "instruction", + "concert", + "speech", + "low", + "effective", + "hang", + "scratch", + "industry", + "breakfast", + "lay", + "join", + "metal", + "bedroom", + "minute", + "product", + "rest", + "temperature", + "many", + "give", + "argument", + "print", + "purple", + "laugh", + "health", + "credit", + "investment", + "sell", + "setting", + "lesson", + "egg", + "middle", + "marriage", + "level", + "evidence", + "phrase", + "love", + "self", + "benefit", + "guidance", + "affect", + "you", + "dad", + "anxiety", + "special", + "boyfriend", + "test", + "blank", + "payment", + "soup", + "obligation", + "reply", + "smile", + "deep", + "complaint", + "addition", + "review", + "box", + "towel", + "minor", + "fun", + "soil", + "issue", + "cigarette", + "internet", + "gain", + "tell", + "entry", + "spare", + "incident", + "family", + "refuse", + "branch", + "can", + "pen", + "grandfather", + "constant", + "tank", + "uncle", + "climate", + "ground", + "volume", + "communication", + "kind", + "poet", + "child", + "screen", + "mine", + "quit", + "gene", + "lack", + "charity", + "memory", + "tooth", + "fear", + "mention", + "marketing", + "reveal", + "reason", + "court", + "season", + "freedom", + "land", + "sport", + "audience", + "classroom", + "law", + "hook", + "win", + "carry", + "eye", + "smell", + "distribution", + "research", + "country", + "dare", + "hope", + "whereas", + "stretch", + "library", + "if", + "delay", + "college", + "plastic", + "book", + "present", + "use", + "worry", + "champion", + "goal", + "economy", + "march", + "election", + "reflection", + "midnight", + "slide", + "inflation", + "action", + "challenge", + "guitar", + "coast", + "apple", + "campaign", + "field", + "jacket", + "sense", + "way", + "visual", + "remove", + "weather", + "trash", + "cable", + "regret", + "buddy", + "beach", + "historian", + "courage", + "sympathy", + "truck", + "tension", + "permit", + "nose", + "bed", + "son", + "person", + "base", + "meat", + "usual", + "air", + "meeting", + "worth", + "game", + "independence", + "physical", + "brief", + "play", + "raise", + "board", + "she", + "key", + "writing", + "pick", + "command", + "party", + "yesterday", + "spring", + "candidate", + "physics", + "university", + "concern", + "development", + "change", + "string", + "target", + "instance", + "room", + "bitter", + "bird", + "football", + "normal", + "split", + "impression", + "wood", + "long", + "meaning", + "stock", + "cap", + "leadership", + "media", + "ambition", + "fishing", + "essay", + "salad", + "repair", + "today", + "designer", + "night", + "bank", + "drawing", + "inevitable", + "phase", + "vast", + "chip", + "anger", + "switch", + "cry", + "twist", + "personality", + "attempt", + "storage", + "being", + "preparation", + "bat", + "selection", + "white", + "technology", + "contract", + "side", + "section", + "station", + "till", + "structure", + "tongue", + "taste", + "truth", + "difficulty", + "group", + "limit", + "main", + "move", + "feeling", + "light", + "example", + "mission", + "might", + "wait", + "wheel", + "shop", + "host", + "classic", + "alternative", + "cause", + "agent", + "consist", + "table", + "airline", + "text", + "pool", + "craft", + "range", + "fuel", + "tool", + "partner", + "load", + "entrance", + "deposit", + "hate", + "article", + "video", + "summer", + "feature", + "extreme", + "mobile", + "hospital", + "flight", + "fall", + "pension", + "piano", + "fail", + "result", + "rub", + "gap", + "system", + "report", + "suck", + "ordinary", + "wind", + "nerve", + "ask", + "shine", + "note", + "line", + "mom", + "perception", + "brother", + "reference", + "bend", + "charge", + "treat", + "trick", + "term", + "homework", + "bake", + "bid", + "status", + "project", + "strategy", + "orange", + "let", + "enthusiasm", + "parent", + "concentrate", + "device", + "travel", + "poetry", + "business", + "society", + "kiss", + "end", + "vegetable", + "employ", + "schedule", + "hour", + "brave", + "focus", + "process", + "movie", + "illegal", + "general", + "coffee", + "ad", + "highway", + "chemistry", + "psychology", + "hire", + "bell", + "conference", + "relief", + "show", + "neat", + "funny", + "weight", + "quality", + "club", + "daughter", + "zone", + "touch", + "tonight", + "shock", + "burn", + "excuse", + "name", + "survey", + "landscape", + "advance", + "satisfaction", + "bread", + "disaster", + "item", + "hat", + "prior", + "shopping", + "visit", + "east", + "photo", + "home", + "idea", + "father", + "comparison", + "cat", + "pipe", + "winner", + "count", + "lake", + "fight", + "prize", + "foundation", + "dog", + "keep", + "ideal", + "fan", + "struggle", + "peak", + "safety", + "solution", + "hell", + "conclusion", + "population", + "strain", + "alarm", + "measurement", + "second", + "train", + "race", + "due", + "insurance", + "boss", + "tree", + "monitor", + "sick", + "course", + "drag", + "appointment", + "slice", + "still", + "care", + "patience", + "rich", + "escape", + "emotion", + "royal", + "female", + "childhood", + "government", + "picture", + "will", + "sock", + "big", + "gate", + "oil", + "cross", + "pin", + "improvement", + "championship", + "silly", + "help", + "sky", + "pitch", + "man", + "diamond", + "most", + "transition", + "work", + "science", + "committee", + "moment", + "fix", + "teaching", + "dig", + "specialist", + "complex", + "guide", + "people", + "dead", + "voice", + "original", + "break", + "topic", + "data", + "degree", + "reading", + "recording", + "bunch", + "reach", + "judgment", + "lie", + "regular", + "set", + "painting", + "mode", + "list", + "player", + "bear", + "north", + "wonder", + "carpet", + "heavy", + "officer", + "negative", + "clock", + "unique", + "baby", + "pain", + "assumption", + "disk", + "iron", + "bill", + "drawer", + "look", + "double", + "mistake", + "finish", + "future", + "brilliant", + "contact", + "math", + "rice", + "leave", + "restaurant", + "discount", + "sex", + "virus", + "bit", + "trust", + "event", + "wear", + "juice", + "failure", + "bug", + "context", + "mud", + "whole", + "wrap", + "intention", + "draft", + "pressure", + "cake", + "dark", + "explanation", + "space", + "angle", + "word", + "efficiency", + "management", + "habit", + "star", + "chance", + "finding", + "transportation", + "stand", + "criticism", + "flow", + "door", + "injury", + "insect", + "surprise", + "apartment", +] # pylint: disable=line-too-long + +# ISO 639-1 codes to language names. +LANGUAGE_CODES = MappingProxyType( + { + "en": "English", + "es": "Spanish", + "pt": "Portuguese", + "ar": "Arabic", + "hi": "Hindi", + "fr": "French", + "ru": "Russian", + "de": "German", + "ja": "Japanese", + "it": "Italian", + "bn": "Bengali", + "uk": "Ukrainian", + "th": "Thai", + "ur": "Urdu", + "ta": "Tamil", + "te": "Telugu", + "bg": "Bulgarian", + "ko": "Korean", + "pl": "Polish", + "he": "Hebrew", + "fa": "Persian", + "vi": "Vietnamese", + "ne": "Nepali", + "sw": "Swahili", + "kn": "Kannada", + "mr": "Marathi", + "gu": "Gujarati", + "pa": "Punjabi", + "ml": "Malayalam", + "fi": "Finnish", + } +) + +# Chinese characters +_CHINESE_CHARS_PATTERN = r"[\u4E00-\u9FFF\u3400-\u4DBF]" +# Japanese Hiragana & Katakana +_JAPANESE_CHARS_PATTERN = r"[\u3040-\u309f\u30a0-\u30ff]" +# Korean (Hangul Syllables) +_KOREAN_CHARS_PATTERN = r"[\uAC00-\uD7AF]" +_ALPHABETS = "([A-Za-z])" +_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" +_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" +_STARTERS = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" +_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" +_WEBSITES = "[.](com|net|org|io|gov|edu|me)" +_DIGITS = "([0-9])" +_MULTIPLE_DOTS = r"\.{2,}" + + +# Util functions +def split_into_sentences(text): + """Split the text into sentences. + + Args: + text: A string that consists of more than or equal to one sentences. + + Returns: + A list of strings where each string is a sentence. + """ + text = " " + text + " " + text = text.replace("\n", " ") + text = re.sub(_PREFIXES, "\\1", text) + text = re.sub(_WEBSITES, "\\1", text) + text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) + text = re.sub( + _MULTIPLE_DOTS, + lambda match: "" * len(match.group(0)) + "", + text, + ) + if "Ph.D" in text: + text = text.replace("Ph.D.", "PhD") + text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) + text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) + text = re.sub( + _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", + "\\1\\2\\3", + text, + ) + text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) + text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) + text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) + text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) + if "”" in text: + text = text.replace(".”", "”.") + if '"' in text: + text = text.replace('."', '".') + if "!" in text: + text = text.replace('!"', '"!') + if "?" in text: + text = text.replace('?"', '"?') + text = text.replace(".", ".") + text = text.replace("?", "?") + text = text.replace("!", "!") + text = text.replace("", ".") + sentences = text.split("") + sentences = [s.strip() for s in sentences] + if sentences and not sentences[-1]: + sentences = sentences[:-1] + return sentences + + +def count_words(text): + """Counts the number of words.""" + try: + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_words = len(tokens) + except: + print('Failed to count for', text) + return 0 + return num_words + + +def split_chinese_japanese(lines: str) -> Iterable[str]: + """ + Split Chinese and Japanese text into sentences. + From https://stackoverflow.com/questions/27441191/splitting-chinese-document-into-sentences + Special question/exclamation marks were added upon inspection of our raw data + Also supports multiple lines. + """ + for line in lines.splitlines(): + for sent in re.findall( + r"[^!?。\.\!\?\！\？\．\n]+[!?。\.\!\?\！\？\．\n]?", line.strip(), flags=re.U + ): + yield sent + + +def count_words_chinese_japanese(text: str) -> int: + """Counts the number of words for Chinese and Japanese and Korean. + Can be extended to additional languages. + Source: https://stackoverflow.com/questions/49164507/how-to-count-the-number-of-chinese-korean-and-english-words withadditional modifications + Example: + >In: count_words_chinese_japanese('こんにちは、ジェイソンさん、Jason? Nice to meet you☺ ❤') + >Out: 19 + """ + # Non alpha numeric patterns in latin and asian languages. + non_alphanumeric_patterns = ( + r"[\\.\!\?\．\/_,\{\}<>:;$%^&*(+\"\'+——！，。？、`~@#￥……（）：；《）《》“”()\[\]»〔〕\-「」]+" + ) + text = re.sub(non_alphanumeric_patterns, "", text) + if emoji: + emoji_cnt = emoji.emoji_count(text) # count emojis + text = emoji.replace_emoji(text, "") # remove emojis + else: + emoji_cnt = 0 + foreign_chars_patterns = "|".join( + [_CHINESE_CHARS_PATTERN, _JAPANESE_CHARS_PATTERN, _KOREAN_CHARS_PATTERN] + ) + asian_chars = re.findall(foreign_chars_patterns, text) + asian_chars_cnt = len(asian_chars) + non_asian_chars = re.sub(foreign_chars_patterns, " ", text) + non_asian_words_cnt = len(non_asian_chars.split()) + return non_asian_words_cnt + asian_chars_cnt + emoji_cnt + + +@functools.lru_cache(maxsize=None) +def _get_sentence_tokenizer(): + return nltk.data.load("nltk:tokenizers/punkt/english.pickle") + + +def count_sentences(text): + """Count the number of sentences.""" + tokenizer = _get_sentence_tokenizer() + tokenized_sentences = tokenizer.tokenize(text) + return len(tokenized_sentences) + +def count_hindi_num_sentences(text): + sentences = re.split(r'(?<=[।!?])\s*', text) + return len([s for s in sentences if s.strip()]) + +def generate_keywords(num_keywords): + """Randomly generates a few keywords.""" + return random.sample(WORD_LIST, k=num_keywords) + + +"""Library of instructions""" +_InstructionArgsDtype = Optional[Dict[str, Union[int, str, Sequence[str]]]] + +_LANGUAGES = LANGUAGE_CODES + +# The relational operation for comparison. +_COMPARISON_RELATION = ("less than", "at least") + +# The maximum number of sentences. +_MAX_NUM_SENTENCES = 20 + +# The number of placeholders. +_NUM_PLACEHOLDERS = 4 + +# The number of bullet lists. +_NUM_BULLETS = 5 + +# The options of constrained response. +_CONSTRAINED_RESPONSE_OPTIONS = ( + "My answer is yes.", + "My answer is no.", + "My answer is maybe.", +) + +# The options of starter keywords. +_STARTER_OPTIONS = ( + "I would say", + "My answer is", + "I believe", + "In my opinion", + "I think", + "I reckon", + "I feel", + "From my perspective", + "As I see it", + "According to me", + "As far as I'm concerned", + "To my understanding", + "In my view", + "My take on it is", + "As per my perception", +) + +# The options of ending keywords. +# TODO(jeffreyzhou) add more ending options +_ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help with?") + +# The number of highlighted sections. +_NUM_HIGHLIGHTED_SECTIONS = 4 + +# The section spliter. +_SECTION_SPLITER = ("Section", "SECTION") + +# The number of sections. +_NUM_SECTIONS = 5 + +# The number of paragraphs. +_NUM_PARAGRAPHS = 5 + +# The postscript marker. +_POSTSCRIPT_MARKER = ("P.S.", "P.P.S") + +# The number of keywords. +_NUM_KEYWORDS = 2 + +# The occurrences of a single keyword. +_KEYWORD_FREQUENCY = 3 + +# The occurrences of a single letter. +_LETTER_FREQUENCY = 10 + +# The occurrences of words with all capital letters. +_ALL_CAPITAL_WORD_FREQUENCY = 20 + +# The number of words in the response. +_NUM_WORDS_LOWER_LIMIT = 100 +_NUM_WORDS_UPPER_LIMIT = 500 + + +class Instruction: + """An instruction template.""" + + def __init__(self, instruction_id): + self.id = instruction_id + + def build_description(self, **kwargs): + raise NotImplementedError("`build_description` not implemented.") + + def get_instruction_args(self): + raise NotImplementedError("`get_instruction_args` not implemented.") + + def get_instruction_args_keys(self): + raise NotImplementedError("`get_instruction_args_keys` not implemented.") + + def check_following(self, value): + raise NotImplementedError("`check_following` not implemented.") + + +class ResponseLanguageChecker(Instruction): + """Check the language of the entire response.""" + + def build_description(self, *, language=None): + """Build the instruction description. + + Args: + language: A string representing the expected language of the response. The + language has to comply to the 97 types defined in + `langid.py` (https://pypi.org/project/langid/1.1.5/), which follows + ISO 639-1 codes (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes); + for example, `en` for English, `zh` for Chinese, `fr` for French. + + Returns: + A string representing the instruction description. + """ + self._language = language + if self._language is None: + self._language = random.choice(list(_LANGUAGES.keys())) + # TODO(tianjianlu): opens the description generation to more choices. + self._description_pattern = ( + "Your ENTIRE response should be in {language} language, no other " + + "language is allowed." + ) + return self._description_pattern.format(language=_LANGUAGES[self._language]) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"language": self._language} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["language"] + + def check_following(self, value): + """Check if the language of the entire response follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the language of `value` follows instruction; otherwise False. + """ + try: + assert isinstance(value, str) + except: + print('Failed for assertion, got non str type input,', value) + return False + + try: + return langdetect.detect(value) == self._language + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logger.info( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class NumberOfSentences(Instruction): + """Check the number of sentences.""" + + def build_description(self, *, num_sentences=None, relation=None): + """Build the instruction description. + + Args: + num_sentences: An integer specifying the number of sentences as a + threshold. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of sentences < the threshold; + if 'at least', the actual number of sentences >= the threshold. + + Returns: + A string representing the instruction description. + """ + # The number of sentences as a threshold for comparison. + self._num_sentences_threshold = num_sentences + if self._num_sentences_threshold is None or self._num_sentences_threshold < 0: + self._num_sentences_threshold = random.randint(1, _MAX_NUM_SENTENCES) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "Your response should contain {relation} {num_sentences} sentences." + ) + return self._description_pattern.format( + relation=self._comparison_relation, + num_sentences=self._num_sentences_threshold, + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return { + "num_sentences": self._num_sentences_threshold, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "relation"] + + def check_following(self, value): + """Check if the number of sentences follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the instruction. + + Raise: + ValueError if the string in `instruction_args` is not in + [`less_than`, `at_least`]. + """ + try: + lang = langdetect.detect(value) + except: + print("Failed to detect language, got value:", value) + lang = 'en' + if lang == "th": + # print(f"shervin1. lang is {lang}") + # print(value) + # Newline also counts as a new sentence: + num_sentences = sum( + [len(sent_tokenize_thai(line)) for line in value.splitlines()] + ) + # print(f"num sentences: {num_sentences}") + elif lang == 'hi': + num_sentences = count_hindi_num_sentences(value) + elif lang in ["zh", "zh-cn", "zh-tw", "ja"]: + # print(f"shervin2. lang is {lang}") + # print(value) + num_sentences = len(list(split_chinese_japanese(value))) + # print(f"num sentences: {num_sentences}") + else: + # print(f"shervin3: lang is {lang}") + num_sentences = count_sentences(value) + # print(f"num sentences: {num_sentences}") + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_sentences < self._num_sentences_threshold + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_sentences >= self._num_sentences_threshold + + +class PlaceholderChecker(Instruction): + """Check the placeholders in template writing.""" + + def build_description(self, *, num_placeholders=None): + """Build the instruction description. + + Args: + num_placeholders: An integer denoting the minimum number of + placeholders required in the response. + + Returns: + A string representing the instruction description. + """ + self._num_placeholders = num_placeholders + if self._num_placeholders is None or self._num_placeholders < 0: + self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS) + self._description_pattern = ( + "The response must contain at least {num_placeholders} placeholders " + + "represented by square brackets, such as [address]." + ) + return self._description_pattern.format(num_placeholders=self._num_placeholders) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"num_placeholders": self._num_placeholders} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_placeholders"] + + def check_following(self, value): + """Check if the number of placeholders follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the actual number of placeholders in the response is greater than + or equal to `num_placeholders`; otherwise, False. + """ + placeholders = re.findall(r"\[.*?\]", value) + num_placeholders = len(placeholders) + return num_placeholders >= self._num_placeholders + + +class BulletListChecker(Instruction): + """Checks the bullet list in the prompt.""" + + def build_description(self, *, num_bullets=None): + """Build the instruction description. + + Args: + num_bullets: An integer specifying the exact number of bullet lists + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_bullets = num_bullets + if self._num_bullets is None or self._num_bullets < 0: + self._num_bullets = random.randint(1, _NUM_BULLETS) + self._description_pattern = ( + "Your answer must contain exactly {num_bullets} bullet points. " + + "Use the markdown bullet points such as:\n" + + "* This is point 1. \n" + + "* This is point 2" + ) + return self._description_pattern.format(num_bullets=self._num_bullets) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"num_bullets": self._num_bullets} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_bullets"] + + def check_following(self, value): + r"""Check if the number of bullet lists meets the requirement. + + Args: + value: A string representing the response. The response is expected to + contain some bullet lists that start with `\*`. + + Returns: + True if the actual number of bullet lists in the response meets the + requirement. + """ + bullet_lists = re.findall(r"^\s*\*[^\*].*$", value, flags=re.MULTILINE) + bullet_lists_2 = re.findall(r"^\s*-.*$", value, flags=re.MULTILINE) + num_bullet_lists = len(bullet_lists) + len(bullet_lists_2) + return num_bullet_lists == self._num_bullets + + +class ConstrainedResponseChecker(Instruction): + """Checks the constrained response.""" + + def build_description(self): + """Build the instruction description.""" + # A sequence of string(s) representing the options of the expected response. + self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS + self._description_pattern = ( + "Answer with one of the following options: {response_options}" + ) + return self._description_pattern.format( + response_options=self._constrained_responses + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response matches the constrained options. + + Args: + value: A string representing the response. + + Returns: + True if the actual response contains one of the options in the constrained + responses; otherwise False. + """ + value = value.strip() + for constrained_response in self._constrained_responses: + if constrained_response in value: + return True + return False + + +class ConstrainedStartChecker(Instruction): + """Checks the response start.""" + + def build_description(self, *, starter=None): + """Build the instruction description. + + Args: + starter: A string representing the keyward that the response should start + with. + + Returns: + A string representing the instruction description. + """ + self._starter = starter.strip() if isinstance(starter, str) else starter + if self._starter is None: + self._starter = random.choice(_STARTER_OPTIONS) + self._description_pattern = ( + "During the conversation, when it is your turn, " + + "please always start with {starter}" + ) + return self._description_pattern.format(starter=self._starter) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"starter": self._starter} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["starter"] + + def check_following(self, value): + """Checks if the response starts with the constrained keyword or phrase. + + Args: + value: A string representing the response. + + Returns: + True if the response starts with the given phrase or keyword that is + contained in `instruction_args`; otherwise, False. + """ + response_pattern = r"^\s*" + self._starter + r".*$" + response_with_constrained_start = re.search( + response_pattern, value, flags=re.MULTILINE + ) + return True if response_with_constrained_start else False + + +class HighlightSectionChecker(Instruction): + """Checks the highlighted section.""" + + def build_description(self, *, num_highlights=None): + """Build the instruction description. + + Args: + num_highlights: An integer specifying the minimum number of highlighted + sections. + + Returns: + A string representing the instruction description. + """ + self._num_highlights = num_highlights + if self._num_highlights is None or self._num_highlights < 0: + self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS) + + self._description_pattern = ( + "Highlight at least {num_highlights} sections in your answer with " + + "markdown, i.e. *highlighted section*." + ) + + return self._description_pattern.format(num_highlights=self._num_highlights) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"num_highlights": self._num_highlights} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_highlights"] + + def check_following(self, value): + """Checks if the number of highlighted sections meets the requirement. + + Args: + value: a string repesenting the response. The response is expected to + contain highlighted sections in the format of *highlighted*. + + Returns: + True if the actual number of highlighted sections in the format of + *highlighed sections* meets the minimum requirement; otherwise False. + """ + num_highlights = 0 + try: + highlights = re.findall(r"\*[^\n\*]*\*", value) + except: + print('Failed for highlights, got value: ', value) + return False + double_highlights = re.findall(r"\*\*[^\n\*]*\*\*", value) + for highlight in highlights: + if highlight.strip("*").strip(): + num_highlights += 1 + for highlight in double_highlights: + if highlight.removeprefix("**").removesuffix("**").strip(): + num_highlights += 1 + + return num_highlights >= self._num_highlights + + +class SectionChecker(Instruction): + """Checks the sections.""" + + def build_description(self, *, section_spliter=None, num_sections=None): + """Build the instruction description. + + Args: + section_spliter: A string represents the section spliter keyword that + marks a new section, i.e., `Section` or `SECTION`. + num_sections: An integer specifying the number of sections. + + Returns: + A string representing the instruction description. + """ + self._section_spliter = ( + section_spliter.strip() + if isinstance(section_spliter, str) + else section_spliter + ) + if self._section_spliter is None: + self._section_spliter = random.choice(_SECTION_SPLITER) + + self._num_sections = num_sections + if self._num_sections is None or self._num_sections < 0: + self._num_sections = random.randint(1, _NUM_SECTIONS) + + self._description_pattern = ( + "Your response must have {num_sections} sections. Mark the beginning " + + "of each section with {section_spliter} X, such as:\n" + + "{section_spliter} 1\n" + + "[content of section 1]\n" + + "{section_spliter} 2\n" + + "[content of section 2]" + ) + + return self._description_pattern.format( + num_sections=self._num_sections, section_spliter=self._section_spliter + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return { + "section_spliter": self._section_spliter, + "num_sections": self._num_sections, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["section_spliter", "num_sections"] + + def check_following(self, value): + """Checks the response contains multiple sections. + + Args: + value: A string representing the response. The response is expected + to contain multiple sections (number of sections is greater than 1). + A new section starts with `Section 1`, where the number denotes the + section index. + + Returns: + True if the number of sections in the response is greater than or equal to + the minimum number of sections; otherwise, False. + """ + section_splitter_patten = r"\s?" + self._section_spliter + r"\s?\d+\s?" + sections = re.split(section_splitter_patten, value) + num_sections = len(sections) - 1 + return num_sections >= self._num_sections + + +class ParagraphChecker(Instruction): + """Checks the paragraphs.""" + + def build_description(self, *, num_paragraphs=None): + """Build the instruction description. + + Args: + num_paragraphs: An integer specifying the number of paragraphs. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs are separated with the markdown divider: ***" + ) + + return self._description_pattern.format(num_paragraphs=self._num_paragraphs) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"num_paragraphs": self._num_paragraphs} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs"] + + def check_following(self, value): + """Checks the response contains required number of paragraphs. + + Args: + value: A string representing the response. The response may contain + paragraphs that are separated by the markdown divider: `***`. + + Returns: + True if the actual number of paragraphs is the same as required; + otherwise, False. + """ + paragraphs = re.split(r"\s?\*\*\*\s?", value) + num_paragraphs = len(paragraphs) + + for index, paragraph in enumerate(paragraphs): + if not paragraph.strip(): + if index == 0 or index == len(paragraphs) - 1: + num_paragraphs -= 1 + else: + return False + + return num_paragraphs == self._num_paragraphs + + +class PostscriptChecker(Instruction): + """Checks the postscript.""" + + def build_description(self, *, postscript_marker=None): + """Build the instruction description. + + Args: + postscript_marker: A string containing the keyword that marks the start + of the postscript section. + + Returns: + A string representing the instruction description. + """ + self._postscript_marker = ( + postscript_marker.strip() + if isinstance(postscript_marker, str) + else postscript_marker + ) + if self._postscript_marker is None: + self._postscript_marker = random.choice(_POSTSCRIPT_MARKER) + + self._description_pattern = ( + "At the end of your response, please explicitly add a postscript " + + "starting with {postscript}" + ) + + return self._description_pattern.format(postscript=self._postscript_marker) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"postscript_marker": self._postscript_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["postscript_marker"] + + def check_following(self, value): + """Checks if the response follows the postscript format. + + Args: + value: a string representing the response. The response is expected to + contain a postscript section. + + Returns: + True if the response contains a postscript section starting with + the keyword containing in the `instruction_args`; otherwise False. + """ + value = value.lower() + if self._postscript_marker == "P.P.S": + postscript_pattern = r"\s*p\.\s?p\.\s?s.*$" + elif self._postscript_marker == "P.S.": + postscript_pattern = r"\s*p\.\s?s\..*$" + else: + postscript_pattern = r"\s*" + self._postscript_marker.lower() + r".*$" + postscript = re.findall(postscript_pattern, value, flags=re.MULTILINE) + return True if postscript else False + + +class RephraseChecker(Instruction): + """Checks the repharse.""" + + def build_description(self, *, original_message): + """Build the instruction description. + + Args: + original_message: A string representing the original message. The + rephrased response should only change its words/sentences in between + its two asterisks, for example, *change me*. Both original and rephrased + messages should contain the changes in the form of *change me*. + + Returns: + A string representing the instruction description. + """ + if not self.is_change(original_message): + raise ValueError( + f"Message {original_message} does not contain changes " + "in the form of *change me*." + ) + + self._reference_without_change = original_message + self._description = ( + "Rephrasing: Your rephrased response should only" + + "change the words/sentences in between two asterisks" + + "such as *change me*." + ) + return self._description + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"original_message": self._reference_without_change} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_message"] + + def check_following(self, value): + r"""Checks if the rephrasing follows the instruction. + + Args: + value: A string representing the response, which is expected to rephras + the string of `instruction_args`. + + Returns: + True if `value` and `instruction_args` only differ by the words/sentences + in between two asterisks such as *change me*; otherwise, False. + """ + + if not self.is_change(value): + raise ValueError( + f"value {value} does not contain " "changes in the form of *change me*." + ) + + response_without_changes = self.strip_changes(value) + reference_without_changes = self.strip_changes(self._reference_without_change) + + return response_without_changes == reference_without_changes + + def is_change(self, response): + """Check if there is change in the response in the form of *change me*.""" + return re.search(r"\*.*\*", response) + + def strip_changes(self, response): + """Strips off the changes.""" + return re.sub(r"\*.*\*", "", response) + + +class KeywordChecker(Instruction): + """Check the exisitence of certain keywords.""" + + def build_description(self, *, keywords=None): + """Build the instruction description. + + Args: + keywords: A sequence of strings representing the keywords that are + expected in the response. + + Returns: + A string representing the instruction description. + """ + + if not keywords: + self._keywords = generate_keywords(num_keywords=_NUM_KEYWORDS) + else: + self._keywords = keywords + self._keywords = sorted(self._keywords) + + self._description_pattern = "Include keywords {keywords} in the response." + + return self._description_pattern.format(keywords=self._keywords) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"keywords": self._keywords} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keywords"] + + def check_following(self, value): + """Check if the response contain the expected keywords.""" + for keyword in self._keywords: + if not re.search(keyword, value, flags=re.IGNORECASE): + return False + return True + + +class KeywordFrequencyChecker(Instruction): + """Check the keyword frequency.""" + + def build_description(self, *, keyword=None, frequency=None, relation=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + frequency: An integer specifying the number of times `keyword` is expected + to appear in the response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of occurrences < frequency; + if 'at least', the actual number of occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + + self._frequency = frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _KEYWORD_FREQUENCY) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "In your response, the word {keyword} should appear {relation} " + + "{frequency} times." + ) + + return self._description_pattern.format( + keyword=self._keyword, + relation=self._comparison_relation, + frequency=self._frequency, + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return { + "keyword": self._keyword, + "frequency": self._frequency, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "frequency", "relation"] + + def check_following(self, value): + """Checks if the response contain the keyword with required frequency.""" + try: + actual_occurrences = len(re.findall(self._keyword, value, flags=re.IGNORECASE)) + except: + print('Failed to parse for', value) + return False + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return actual_occurrences < self._frequency + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return actual_occurrences >= self._frequency + + +class NumberOfWords(Instruction): + """Checks the number of words.""" + + def build_description(self, *, num_words=None, relation=None): + """Build the instruction description. + + Args: + num_words: An integer specifying the number of words contained in the + response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of words < num_words; + if 'at least', the actual number of words >= num_words. + + Returns: + A string representing the instruction description. + """ + + self._num_words = num_words + if self._num_words is None or self._num_words < 0: + self._num_words = random.randint( + _NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT + ) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = "Answer with {relation} {num_words} words." + + return self._description_pattern.format( + relation=self._comparison_relation, num_words=self._num_words + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"num_words": self._num_words, "relation": self._comparison_relation} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_words", "relation"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + try: + lang = langdetect.detect(value) + except: + print("Failed to detect language, got value:", value) + lang = 'en' + if lang == "th": + # print(f"shervin4. lang is {lang}") + # print(value) + num_words = len(word_tokenize_thai(value)) + # print(f"num words: {num_words}") + elif lang in ["zh", "zh-cn", "zh-tw", "ja"]: + # print(f"shervin5. lang is {lang}") + # print(value) + num_words = count_words_chinese_japanese(value) + # print(f"num words: {num_words}") + else: + # print(f"shervin6. lang is {lang}") + # print(value) + num_words = count_words(value) + # print(f"num words: {num_words}") + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_words < self._num_words + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_words >= self._num_words + + +class JsonFormat(Instruction): + """Check the Json format.""" + + def build_description(self): + self._description_pattern = ( + "Entire output should be wrapped in JSON format. You can use markdown" + " ticks such as ```." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = ( + value.strip() + .removeprefix("```json") + .removeprefix("```Json") + .removeprefix("```JSON") + .removeprefix("```") + .removesuffix("```") + .strip() + ) + try: + json.loads(value) + except ValueError as _: + return False + return True + + +class ParagraphFirstWordCheck(Instruction): + """Check the paragraph and the first word of the nth paragraph.""" + + def build_description( + self, num_paragraphs=None, nth_paragraph=None, first_word=None + ): + r"""Build the instruction description. + + Args: + num_paragraphs: An integer indicating the number of paragraphs expected + in the response. A paragraph is a subset of the string that is + expected to be separated by '\n\n'. + nth_paragraph: An integer indicating the paragraph number that we look at. + Note that n starts from 1. + first_word: A string that represent the first word of the bth paragraph. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._nth_paragraph = nth_paragraph + if ( + self._nth_paragraph is None + or self._nth_paragraph <= 0 + or self._nth_paragraph > self._num_paragraphs + ): + self._nth_paragraph = random.randint(1, self._num_paragraphs + 1) + + self._first_word = first_word + if self._first_word is None: + self._first_word = generate_keywords(num_keywords=1)[0] + self._first_word = self._first_word.lower() + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs and only paragraphs are separated with each other by two " + + "new lines as if it was '\\n\\n' in python. " + + "Paragraph {nth_paragraph} must start with word {first_word}." + ) + + return self._description_pattern.format( + num_paragraphs=self._num_paragraphs, + nth_paragraph=self._nth_paragraph, + first_word=self._first_word, + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return { + "num_paragraphs": self._num_paragraphs, + "nth_paragraph": self._nth_paragraph, + "first_word": self._first_word, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs", "nth_paragraph", "first_word"] + + def check_following(self, value): + """Checks for required number of paragraphs and correct first word. + + Args: + value: a string representing the response. The response may contain + paragraphs that are separated by two new lines and the first word of + the nth paragraph will have to match a specified word. + + Returns: + True if the number of paragraphs is the same as required and the first + word of the specified paragraph is the same as required. Otherwise, false. + """ + + paragraphs = re.split(r"\n\n", value) + num_paragraphs = len(paragraphs) + + for paragraph in paragraphs: + if not paragraph.strip(): + num_paragraphs -= 1 + + # check that index doesn't go out of bounds + if self._nth_paragraph <= num_paragraphs: + paragraph = paragraphs[self._nth_paragraph - 1].strip() + if not paragraph: + return False + else: + return False + + first_word = "" + punctuation = {".", ",", "?", "!", "'", '"'} + + # get first word and remove punctuation + word = paragraph.split()[0].strip() + # TODO(jeffrey): make more complex? + word = word.lstrip("'") + word = word.lstrip('"') + + for letter in word: + if letter in punctuation: + break + first_word += letter.lower() + + return num_paragraphs == self._num_paragraphs and first_word == self._first_word + + +# TODO(jeffrey) add relation - at least/at most? +class KeySentenceChecker(Instruction): + """Check the existence of certain key sentences.""" + + def build_description(self, key_sentences=None, num_sentences=None): + """Build the instruction description. + + Args: + key_sentences: A sequences of strings representing the key sentences that + are expected in the response. + num_sentences: The number of key sentences that are expected to be seen in + the response. + + Returns: + A string representing the instruction description. + """ + + if not key_sentences: + # TODO(jeffrey) make a generate sentences function? wonderwords package + self._key_sentences = set(["For now, this is fine."]) + else: + self._key_sentences = key_sentences + + if not num_sentences: + self._num_sentences = random.randint(1, len(self._key_sentences)) + else: + self._num_sentences = num_sentences + + self._description_pattern = ( + "Include {num_sentences} of the following sentences {key_sentences}" + ) + + return self._description_pattern.format( + num_sentences=self._num_sentences, key_sentences=self._key_sentences + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return { + "num_sentences": self._num_sentences, + "key_sentences": list(self._key_sentences), + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "key_sentences"] + + def check_following(self, value): + """Checks if the response contains the expected key sentences.""" + count = 0 + sentences = split_into_sentences(value) + for sentence in self._key_sentences: + if sentence in sentences: + count += 1 + + return count == self._num_sentences + + +class ForbiddenWords(Instruction): + """Checks that specified words are not used in response.""" + + def build_description(self, forbidden_words=None): + """Build the instruction description. + + Args: + forbidden_words: A sequences of strings respresenting words that are not + allowed in the response. + + Returns: + A string representing the instruction description. + """ + + if not forbidden_words: + self._forbidden_words = generate_keywords(num_keywords=_NUM_KEYWORDS) + else: + self._forbidden_words = list(set(forbidden_words)) + self._forbidden_words = sorted(self._forbidden_words) + self._description_pattern = ( + "Do not include keywords {forbidden_words} in the response." + ) + + return self._description_pattern.format(forbidden_words=self._forbidden_words) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"forbidden_words": self._forbidden_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["forbidden_words"] + + def check_following(self, value): + """Check if the response does not contain the expected keywords.""" + for word in self._forbidden_words: + if re.search(r"\b" + word + r"\b", value, flags=re.IGNORECASE): + return False + return True + + +class RephraseParagraph(Instruction): + """Checks that the paragraph is rephrased.""" + + def build_description(self, *, original_paragraph, low, high): + """Builds the instruction description. + + Args: + original_paragraph: A string presenting the original paragraph. The + rephrases response should have betweeb low-high words in common. + low: An integer presenting the lower bound of similar words. + high: An integer representing the upper bound of similar words. + + Returns: + A string representing the instruction description. + """ + # TODO(jeffrey) make more encompassing + self._original_paragraph = original_paragraph + self._low = low + self._high = high + + self._description = ( + "Rephrase the following paragraph: " + + "{original_paragraph}\nYour response should have " + + "between {low} and {high} of the same words. " + + "Words are the same if and only if all of the " + + "letters, ignoring cases, are the same. For " + + "example, 'run' is the same as 'Run' but different " + + "to 'ran'." + ) + + return self._description.format( + original_paragraph=original_paragraph, low=self._low, high=self._high + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return { + "original_paragraph": self._original_paragraph, + "low": self._low, + "high": self._high, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_paragraph", "low", "high"] + + def check_following(self, value): + val_words = re.findall(r"\w+", value.lower()) + original_words = re.findall(r"\w+", self._original_paragraph.lower()) + similar_words = 0 + + dict_val = collections.Counter(val_words) + dict_original = collections.Counter(original_words) + + for word in dict_original: + similar_words += min(dict_original[word], dict_val[word]) + + return similar_words >= self._low and similar_words <= self._high + + +class TwoResponsesChecker(Instruction): + """Check that two responses were given.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Give two different responses. Responses and only responses should" + " be separated by 6 asterisk symbols: ******." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has two different answers. + + Args: + value: A string representing the response. + + Returns: + True if two responses are detected and false otherwise. + """ + valid_responses = list() + responses = value.split("******") + for index, response in enumerate(responses): + if not response.strip(): + if index != 0 and index != len(responses) - 1: + return False + else: + valid_responses.append(response) + return ( + len(valid_responses) == 2 + and valid_responses[0].strip() != valid_responses[1].strip() + ) + + +class RepeatPromptThenAnswer(Instruction): + """Checks that Prompt is first repeated then answered.""" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + prompt_to_repeat: The prompt that is meant to be repeated. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + self._description_pattern = ( + "First repeat the request word for word without change," + " then give your answer (1. do not say any words or characters" + " before repeating the request; 2. the request you need to repeat" + " does not include this sentence)" + ) + return self._description_pattern + + def get_instruction_args(self): + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + if value.strip().lower().startswith(self._prompt_to_repeat.strip().lower()): + return True + return False + + +class EndChecker(Instruction): + """Checks that the prompt ends with a given phrase.""" + + def build_description(self, *, end_phrase=None): + """Build the instruction description. + + Args: + end_phrase: A string representing the phrase the response should end with. + + Returns: + A string representing the instruction description. + """ + self._end_phrase = ( + end_phrase.strip() if isinstance(end_phrase, str) else end_phrase + ) + if self._end_phrase is None: + self._end_phrase = random.choice(_ENDING_OPTIONS) + self._description_pattern = ( + "Finish your response with this exact phrase {ender}. " + "No other words should follow this phrase." + ) + return self._description_pattern.format(ender=self._end_phrase) + + def get_instruction_args(self): + return {"end_phrase": self._end_phrase} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["end_phrase"] + + def check_following(self, value): + """Checks if the response ends with the expected phrase.""" + value = value.strip().strip('"').lower() + self._end_phrase = self._end_phrase.strip().lower() + return value.endswith(self._end_phrase) + + +class TitleChecker(Instruction): + """Checks the response for a title.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your answer must contain a title, wrapped in double angular brackets," + " such as <>." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a title.""" + pattern = r"<<[^\n]+>>" + re_pattern = re.compile(pattern) + titles = re.findall(re_pattern, value) + + for title in titles: + if title.lstrip("<").rstrip(">").strip(): + return True + return False + + +class LetterFrequencyChecker(Instruction): + """Checks letter frequency.""" + + def build_description(self, *, letter=None, let_frequency=None, let_relation=None): + """Build the instruction description. + + Args: + letter: A string representing a letter that is expected in the response. + let_frequency: An integer specifying the number of times `keyword` is + expected to appear in the response. + let_relation: A string in (`less than`, `at least`), defining the + relational operator for comparison. Two relational comparisons are + supported for now; if 'less than', the actual number of + occurrences < frequency; if 'at least', the actual number of + occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if ( + not letter + or len(letter) > 1 + or ord(letter.lower()) < 97 + or ord(letter.lower()) > 122 + ): + self._letter = random.choice(list(string.ascii_letters)) + else: + self._letter = letter.strip() + self._letter = self._letter.lower() + + self._frequency = let_frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _LETTER_FREQUENCY) + + if let_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif let_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {let_relation} is given." + ) + else: + self._comparison_relation = let_relation + + self._description_pattern = ( + "In your response, the letter {letter} should appear {let_relation}" + " {let_frequency} times." + ) + + return self._description_pattern.format( + letter=self._letter, + let_frequency=self._frequency, + let_relation=self._comparison_relation, + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "letter": self._letter, + "let_frequency": self._frequency, + "let_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["letter", "let_frequency", "let_relation"] + + def check_following(self, value): + """Checks that the response contains the letter at the right frequency.""" + value = value.lower() + letters = collections.Counter(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return letters[self._letter] < self._frequency + else: + return letters[self._letter] >= self._frequency + + +class CapitalLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all capital letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all capital letters." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all capital letters.""" + assert isinstance(value, str) + + try: + return value.isupper() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logger.info( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class LowercaseLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all lowercase letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all lowercase" + " letters. No capital letters are allowed." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all lowercase letters.""" + assert isinstance(value, str) + + try: + return value.islower() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logger.info( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class CommaChecker(Instruction): + """Checks the response for no commas.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "In your entire response, refrain from the use of any commas." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response does not contain commas.""" + return not re.search(r"\,", value) + + +class CapitalWordFrequencyChecker(Instruction): + """Checks frequency of words with all capital letters.""" + + def build_description( + self, + capital_frequency=None, + capital_relation=None, + ): + """Build the instruction description. + + Args: + capital_frequency: An integer that represents the number of words that + should be in all capital letters. + capital_relation: A string that is 'at least' or 'at most' that refers to + the frequency. + + Returns: + A string representing the instruction description. + """ + self._frequency = capital_frequency + if self._frequency is None: + self._frequency = random.randint(1, _ALL_CAPITAL_WORD_FREQUENCY) + + self._comparison_relation = capital_relation + if capital_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif capital_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {capital_relation} is given." + ) + + self._description_pattern = ( + "In your response, words with all capital letters should appear" + " {relation} {frequency} times." + ) + + return self._description_pattern.format( + frequency=self._frequency, relation=self._comparison_relation + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "capital_frequency": self._frequency, + "capital_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["capital_frequency", "capital_relation"] + + def check_following(self, value): + """Checks the frequency of words with all capital letters.""" + # Hyphenated words will count as one word + words = nltk.word_tokenize(value) + capital_words = [word for word in words if word.isupper()] + + capital_words = len(capital_words) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return capital_words < self._frequency + else: + return capital_words >= self._frequency + + +class QuotationChecker(Instruction): + """Checks response is wrapped with double quotation marks.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Wrap your entire response with double quotation marks." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is wrapped with double quotation marks.""" + value = value.strip() + return len(value) > 1 and (value[0] == '"' and value[-1] == '"' or # e.g., English + value[0] == '“' and value[-1] == '”' or # e.g., Chinese + value[0] == '「' and value[-1] == '」') # e.g., Japanese + + +# Define instruction dicts +_KEYWORD = "keywords:" +_LANGUAGE = "language:" +_LENGTH = "length_constraints:" +_CONTENT = "detectable_content:" +_FORMAT = "detectable_format:" +_MULTITURN = "multi-turn:" +_COMBINATION = "combination:" +_STARTEND = "startend:" +_CHANGE_CASES = "change_case:" +_PUNCTUATION = "punctuation:" + +INSTRUCTION_DICT = { + _KEYWORD + "existence": KeywordChecker, + _KEYWORD + "frequency": KeywordFrequencyChecker, + # _KEYWORD + "key_sentences": KeySentenceChecker, + _KEYWORD + "forbidden_words": ForbiddenWords, + _KEYWORD + "letter_frequency": LetterFrequencyChecker, + _LANGUAGE + "response_language": ResponseLanguageChecker, + _LENGTH + "number_sentences": NumberOfSentences, + _LENGTH + "number_paragraphs": ParagraphChecker, + _LENGTH + "number_words": NumberOfWords, + _LENGTH + "nth_paragraph_first_word": ParagraphFirstWordCheck, + _CONTENT + "number_placeholders": PlaceholderChecker, + _CONTENT + "postscript": PostscriptChecker, + _FORMAT + "number_bullet_lists": BulletListChecker, + # _CONTENT + "rephrase_paragraph": RephraseParagraph, + _FORMAT + "constrained_response": ConstrainedResponseChecker, + _FORMAT + "number_highlighted_sections": (HighlightSectionChecker), + _FORMAT + "multiple_sections": SectionChecker, + # _FORMAT + "rephrase": RephraseChecker, + _FORMAT + "json_format": JsonFormat, + _FORMAT + "title": TitleChecker, + # _MULTITURN + "constrained_start": ConstrainedStartChecker, + _COMBINATION + "two_responses": TwoResponsesChecker, + _COMBINATION + "repeat_prompt": RepeatPromptThenAnswer, + _STARTEND + "end_checker": EndChecker, + _CHANGE_CASES + "capital_word_frequency": CapitalWordFrequencyChecker, + _CHANGE_CASES + "english_capital": CapitalLettersEnglishChecker, + _CHANGE_CASES + "english_lowercase": LowercaseLettersEnglishChecker, + _PUNCTUATION + "no_comma": CommaChecker, + _STARTEND + "quotation": QuotationChecker, +} + +INSTRUCTION_LIST = list(INSTRUCTION_DICT.keys()) + [ + _KEYWORD[:-1], + _LANGUAGE[:-1], + _LENGTH[:-1], + _CONTENT[:-1], + _FORMAT[:-1], + _MULTITURN[:-1], + _COMBINATION[:-1], + _STARTEND[:-1], + _CHANGE_CASES[:-1], + _PUNCTUATION[:-1], +] diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index e5cd3cbe..d9633c79 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -129,6 +129,10 @@ from .mmmlu import ( MMMLUDataset, MMMLUDatasetSample, ) +from .multi_if import ( + MultiIFDataset, + MultiIFDatasetSample, +) from .openbookqa import ( OpenBookQADataset, OpenBookQADatasetSample, @@ -227,6 +231,8 @@ __all__ = [ "MMLUProDatasetSample", "MMMLUDataset", "MMMLUDatasetSample", + "MultiIFDataset", + "MultiIFDatasetSample", "OpenBookQADataset", "OpenBookQADatasetSample", "PlatinumBenchDataset", diff --git a/sieval/datasets/multi_if.py b/sieval/datasets/multi_if.py new file mode 100644 index 00000000..57c94328 --- /dev/null +++ b/sieval/datasets/multi_if.py @@ -0,0 +1,160 @@ +"""Multi-IF dataset loader (multi-turn, multilingual instruction following). + +Multi-IF extends IFEval into three-turn conversations across eight languages +(English, French, Hindi, Portuguese, Spanish, Russian, Italian, Chinese), +4,501 conversations in all. Each later turn adds a constraint while keeping the +earlier ones, so a turn is graded against the *accumulated* constraint set -- +verified against the pinned revision: turn ``t``'s ``instruction_id_list`` is a +prefix-extension of turn ``t-1``'s for every row, with no exceptions. + +The Hub repo ships one file, ``multiIF_20241018.csv``, with nine turn columns +(``turn_{1,2,3}_{prompt,instruction_id_list,kwargs}``) plus ``key``, +``language``, and three eval-time placeholders (``turns``, ``responses``, +``turn_index``) that are empty in the release and are dropped here -- upstream +writes them back as it walks the conversation, which is state this loader has no +reason to carry. + +The nine flat columns are reshaped into a ``turns`` list because the turn count +genuinely varies: 56 rows (French 30, English 13, Hindi 13) have no third turn, +and upstream skips those rows when scoring turn 3 rather than padding them. A +list makes that the natural ``len(turns) == 2``. + +Field-by-field, relative to the CSV: + +* ``turn_N_prompt`` is a JSON-encoded chat message. Every one of the 13,447 + turn prompts in the pinned revision is ``{"role": "user", ...}`` with no other + keys, so only ``content`` is kept and the role is re-attached by the task. +* ``turn_N_instruction_id_list`` is a JSON list of strings, decoded here. +* ``turn_N_kwargs`` is a JSON list of *JSON strings* -- double-encoded upstream, + and each inner object holds only the keys its own checker needs (24 distinct + keys across the set). The outer list is decoded; the inner objects are + deliberately left encoded, because decoding them would make Arrow unify 24 + sparse struct fields across every constraint. IFEval hits the same wall and + works around it by stripping ``None``s at use time; keeping the payload opaque + avoids it outright. The task decodes each element immediately before handing + it to ``build_description``. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +from pathlib import Path +from typing import TypedDict, override + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +# Pin the Hub revision for reproducibility (current `main` at integration time). +MULTI_IF_REVISION = "0ab97ce0b45c7f57772e8ba2ac1616f4b00bd3aa" + +_CSV_FILENAME = "multiIF_20241018.csv" +_MAX_TURNS = 3 + + +class MultiIFTurn(TypedDict): + """One user turn and the constraints its response is graded against. + + Attributes: + prompt: The user message text (``content`` of the CSV's JSON message). + instruction_id_list: Constraint ids for this turn, cumulative -- it + includes every earlier turn's constraints as a prefix. + kwargs: Per-constraint arguments, positionally aligned with + ``instruction_id_list`` and still JSON-encoded (see module docstring). + """ + + prompt: str + instruction_id_list: list[str] + kwargs: list[str] + + +class MultiIFDatasetSample(TypedDict): + key: str + language: str + # Three turns, except for the 56 rows that ship only two. + turns: list[MultiIFTurn] + + +@sieval_dataset( + name="multi_if", + display_name="Multi-IF", + description=( + "Multi-IF — 4,501 three-turn, eight-language instruction-following " + "conversations extending IFEval." + ), + source=f"hf:facebook/Multi-IF@{MULTI_IF_REVISION}", + categories=(Category(Level1Category.LANGUAGE, "InstructionFollowing"),), + tags=("multilingual", "multi-turn", "open-ended"), + # The Hub dataset card's own license, which is NOT the Apache-2.0 of the + # facebookresearch/Multi-IF *code* repo that ships the evaluator. + license="CC-BY-NC-2.0", +) +class MultiIFDataset(Dataset[MultiIFDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + # `hf:` stages the repo as a directory; accept only that layout and fail + # loudly rather than probing speculative alternatives. + csv_path = Path(name_or_path) / _CSV_FILENAME + if not csv_path.is_file(): + raise FileNotFoundError( + f"Multi-IF CSV not found at {str(csv_path)!r}. Run " + "'sieval dataset download multi_if' to stage the dataset." + ) + + # `keep_default_na=False` so an absent third turn arrives as "" rather + # than a float nan, matching upstream's own read of this CSV and letting + # one emptiness check cover both spellings. + raw = load_dataset( + "csv", + data_files={"test": str(csv_path)}, + keep_default_na=False, + **kwargs, + ) + rows = [self._build_sample(row) for row in ensure_dataset_dict(raw)["test"]] + + dataset = ensure_dataset_dict( + HFDatasetDict({"test": HFDataset.from_list([{**r} for r in rows])}) + ) + if len(dataset["test"]) == 0: + raise ValueError( + f"Multi-IF produced an empty 'test' split from {str(csv_path)!r}; " + "check that the dataset has been downloaded via " + "'sieval dataset download multi_if'." + ) + return dataset + + def _build_sample(self, row: dict) -> MultiIFDatasetSample: + turns: list[MultiIFTurn] = [] + for index in range(1, _MAX_TURNS + 1): + turn = self._build_turn(row, index) + if turn is None: + # Only the third turn is ever absent in the pinned revision, and + # a conversation cannot resume after a gap -- so stop rather than + # skip, which would silently splice turn 3 onto turn 1. + break + turns.append(turn) + return { + "key": row["key"], + "language": row["language"], + "turns": turns, + } + + def _build_turn(self, row: dict, index: int) -> MultiIFTurn | None: + prompt = row[f"turn_{index}_prompt"] + # Upstream treats both "" and the literal string "None" as "no turn". + if not prompt or prompt == "None": + return None + return { + "prompt": json.loads(prompt)["content"], + "instruction_id_list": json.loads(row[f"turn_{index}_instruction_id_list"]), + # Elements stay JSON-encoded on purpose (see module docstring). + "kwargs": json.loads(row[f"turn_{index}_kwargs"]), + } diff --git a/sieval/meta/index.json b/sieval/meta/index.json index d77ab81d..beaa3ace 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -702,6 +702,28 @@ "license": "MIT", "checksums": {} }, + { + "name": "multi_if", + "display_name": "Multi-IF", + "description": "Multi-IF — 4,501 three-turn, eight-language instruction-following conversations extending IFEval.", + "source": [ + "hf:facebook/Multi-IF@0ab97ce0b45c7f57772e8ba2ac1616f4b00bd3aa" + ], + "categories": [ + { + "level1": "Language", + "level2": "InstructionFollowing" + } + ], + "tags": [ + "multilingual", + "multi-turn", + "open-ended" + ], + "deps_group": null, + "license": "CC-BY-NC-2.0", + "checksums": {} + }, { "name": "openbookqa", "display_name": "OpenBookQA", @@ -1684,6 +1706,27 @@ }, "status": "stable" }, + { + "name": "multi_if_0shot_gen", + "display_name": "Multi-IF (0-shot, generative)", + "description": "Multi-IF — 4,501 three-turn, eight-language instruction-following conversations extending IFEval.", + "dataset": "multi_if", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "multilingual", + "multi-turn", + "open-ended" + ], + "deps_group": "multi-if", + "model_type": "chat", + "reference_impl": { + "source": "facebookresearch/Multi-IF", + "url": "https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/metrics.py", + "notes": "Multi-IF's own multilingual fork of the IFEval checkers is vendored (sieval.community.multi_if); the google-research IFEval sibling is NOT interchangeable with it. Upstream drives one pass per turn (--steps 1 2 3), one sample per turn; this task walks all three in one pass. Upstream reports fractions, this task percentages. Grading matches upstream's metrics_gen exactly — verified on 535 conversations across all 8 languages (3,098 follow-lists, all per-language overalls) — except for two conversations upstream cannot grade reproducibly itself: kwargs it rejects (letter='#'; missing keyword) send build_description to an unseeded random.choice. langdetect is likewise unseeded upstream, and picks the counting algorithm behind every length constraint. Both defects are tracked, not repaired, per the unqualified-name rule; fixing either needs a `_fixed` variant with a measured delta." + }, + "status": "experimental" + }, { "name": "openbookqa_kshot_gen", "display_name": "OpenBookQA (k-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 2136c931..dfb9bdff 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -115,6 +115,9 @@ from .mmlu_pro_0shot_gen import ( from .mmmlu_kshot_clp import ( MMMLUKShotClpTask, ) +from .multi_if_0shot_gen import ( + MultiIFZeroShotGenTask, +) from .openbookqa_kshot_gen import ( OpenBookQAFewShotGenTask, ) @@ -191,6 +194,7 @@ __all__ = [ "MMLUProZeroShotGenTask", "MMLUZeroShotGenTask", "MMMLUKShotClpTask", + "MultiIFZeroShotGenTask", "OpenBookQAFewShotGenTask", "PlatinumGSM8KZeroShotGenTask", "PlatinumMultiArithZeroShotGenTask", diff --git a/sieval/tasks/multi_if_0shot_gen.py b/sieval/tasks/multi_if_0shot_gen.py new file mode 100644 index 00000000..ae2178cc --- /dev/null +++ b/sieval/tasks/multi_if_0shot_gen.py @@ -0,0 +1,336 @@ +"""Multi-IF zero-shot generative task (multi-turn, multilingual IFEval). + +One sample is one whole conversation. ``infer`` walks its turns in order, +appending the model's own reply before sending the next user turn, so turn *t* +is answered with turns *1..t-1* in context -- upstream's ``old_prompt + +old_response + new_prompt``, which it reaches by re-running the whole set once +per ``--steps`` value and writing the growing conversation back to a CSV. Doing +it in one pass makes every turn of a conversation a single unit of work, which +is what lets one run report all three turns. + +Grading is per turn, against that turn's *cumulative* constraint list (turn 3 +carries turns 1 and 2's constraints too), under IFEval's strict and loose +readings. Both are co-equal published metrics, so both are recorded. + +Upstream's headline per turn is a plain mean of four numbers -- strict and loose +x prompt-level and instruction-level -- reported per language and over all of +them (``turn_1_all_languages_overall``, and so on). The paper quotes these as +fractions (o1-preview 0.877 at turn 1, 0.707 at turn 3); this task reports +percentages, matching its IFEval and IFBench siblings. + +Deviations from the official Multi-IF evaluation: + +- The 56 conversations with no third turn get no third generation. Upstream + sends them a literal ``"None"`` prompt and then drops the row when scoring + turn 3, so the metrics are unchanged and the tokens are not spent. +- Bootstrap confidence intervals are not computed. Upstream resamples every + language cell 10,000 times via scipy; nothing in SiEval consumes the interval, + and the per-sample verdicts needed to recompute one are all on disk. +- ``score`` is the mean of the three turns' all-language overalls. Upstream + emits one report per turn and never reduces them to a single number, but a + task needs one headline; every component is in the report. + +Two upstream defects are tracked rather than repaired, because the unqualified +name must measure what upstream measures. Both make the score nondeterministic: + +- Two conversations (6 of 13,447 turn-cells, 0.04%) carry kwargs the checker + rejects -- ``keywords:letter_frequency`` with ``letter="#"``, and + ``keywords:frequency`` with no ``keyword``. On a rejected value + ``build_description`` falls back to ``random.choice`` over the alphabet and + grades the response against a letter nobody asked for, freshly drawn per call. +- ``langdetect`` is unseeded, so detection -- which selects the word- and + sentence-counting algorithm behind every length constraint -- can vary between + runs on short or mixed-script text. + +Verified against upstream's own ``metrics_gen`` on 535 conversations spanning +all eight languages and all 56 two-turn rows: 3,098 strict/loose follow-lists +and every per-language ``overall`` agree exactly, once those two conversations +are set aside. They cannot agree with anything, upstream included. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import json +from typing import override + +from openai.types.chat import ChatCompletionMessageParam + +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.types import JSONValue +from sieval.datasets import MultiIFDatasetSample + +# The two IFEval readings, both published. Neither is subordinate: upstream's +# per-turn headline averages them together rather than picking one. +_GRADES = ("strict", "loose") + +# The pooled cell every published Multi-IF number is quoted against; kept +# distinct from the CSV's own `language` values, which are English names. +_ALL_LANGUAGES = "all_languages" + + +@sieval_task( + name="multi_if_0shot_gen", + display_name="Multi-IF (0-shot, generative)", + description=( + "Multi-IF — 4,501 three-turn, eight-language instruction-following " + "conversations extending IFEval." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("multilingual", "multi-turn", "open-ended"), + deps_group="multi-if", + model_type="chat", + # The grader and aggregation are verified against upstream's own evaluator + # (see the module docstring), but no live run has reproduced a published + # number yet -- Multi-IF publishes paper scores only, not the per-model + # inference dumps that let MathArena and PlatinumBench ports claim `stable` + # by replay. Faithful by construction until a run lands within a stated band. + status="experimental", + reference_impl=ReferenceImpl( + source="facebookresearch/Multi-IF", + url="https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/metrics.py", + notes=( + "Multi-IF's own multilingual fork of the IFEval checkers is vendored " + "(sieval.community.multi_if); the google-research IFEval sibling is " + "NOT interchangeable with it. Upstream drives one pass per turn " + "(--steps 1 2 3), one sample per turn; this task walks all three in " + "one pass. Upstream reports fractions, this task percentages. " + "Grading matches upstream's metrics_gen exactly — verified on 535 " + "conversations across all 8 languages (3,098 follow-lists, all " + "per-language overalls) — except for two conversations upstream " + "cannot grade reproducibly itself: kwargs it rejects " + "(letter='#'; missing keyword) send build_description to an unseeded " + "random.choice. langdetect is likewise unseeded upstream, and picks " + "the counting algorithm behind every length constraint. Both defects " + "are tracked, not repaired, per the unqualified-name rule; fixing " + "either needs a `_fixed` variant with a measured delta." + ), + ), +) +class MultiIFZeroShotGenTask( + Task[ + MultiIFDatasetSample, + PromptRecord, + list[ModelOutput], + PredictionRecord, + JudgementRecord, + dict[str, float], + ] +): + @override + async def preprocess(self, raw, ctx): + turns = raw["turns"] + return build_prompt_record( + # Only the opening turn can be built ahead of inference; the rest + # depend on what the model says. `infer` sends this verbatim and + # grows the conversation from it, so the recorded prompt is provably + # the one the model saw. + [{"role": "user", "content": turns[0]["prompt"]}], + # The "ground truth" is the constraint set each turn must satisfy, + # innermost list ordered to match that turn's kwargs. + reference=[list(turn["instruction_id_list"]) for turn in turns], + extra={ + "key": raw["key"], + "language": raw["language"], + # Says how many prompts this sample really takes, so the single + # recorded message cannot be misread as the whole input. + "n_turns": len(turns), + # The later turns' user text, recorded because it is part of the + # sample's input and is otherwise only in the raw dataset. + "later_turn_prompts": [turn["prompt"] for turn in turns[1:]], + }, + ) + + @override + async def infer(self, pre, ctx): + turns = ctx.raw_sample["turns"] + # Seeded from the record rather than rebuilt, so what was persisted and + # what was sent cannot drift. + messages: list[ChatCompletionMessageParam] = list(pre["prompt"]) + outputs: list[ModelOutput] = [] + + for turn in turns: + if outputs: + previous = outputs[-1] + # An aborted or filtered response has no choices. Feed the empty + # assistant turn through anyway: the conversation continues (as + # upstream's does) and the missing answer scores zero on its own + # turn rather than failing the whole sample. + messages.append( + { + "role": "assistant", + "content": previous.texts[0] if previous.texts else "", + } + ) + messages.append({"role": "user", "content": turn["prompt"]}) + # n=1 per turn: a second sample would fork the conversation, and + # every later turn would have to be answered once per branch. + outputs.append(await self.model.agenerate(messages, n=1)) + + # Returned bare so the runner sums token usage across all turns into the + # stage meta -- it special-cases `list[ModelOutput]` for exactly this. + return outputs + + @override + async def postprocess(self, inf, ctx): + # One rollout per conversation, not one per turn: turns are sequential + # parts of a single answer, while a rollout is one of `n` independent + # samples of the same prompt. Per-turn responses ride inside it. + texts = [output.texts[0] if output.texts else "" for output in inf] + # Annotated because `list` is invariant: an inferred + # `list[dict[str, int | str]]` is not a `list[JSONValue]`. + responses: list[JSONValue] = [ + {"turn": index, "response": text} + for index, text in enumerate(texts, start=1) + ] + # `None` only when no turn produced anything, so `extracted` stays a real + # signal -- a partly-blank conversation is a real answer that scores + # badly. This is also what keeps the empty-postprocess anomaly rule + # meaningful here, since the rules that read `infer` unwrap a single + # ModelOutput and skip a list. + any_text = any(text.strip() for text in texts) + return build_prediction_record([responses if any_text else None]) + + @override + async def feedback(self, post, ctx): + # Graded here rather than in report() so every turn's verdict is on disk + # and inspectable. Both graders are pure per-response. + from sieval.community.multi_if.evaluation_lib import ( + gen_acc_loose, + gen_acc_strict, + ) + + graders = {"strict": gen_acc_strict, "loose": gen_acc_loose} + raw = ctx.raw_sample + turns = raw["turns"] + responses = post["rollouts"][0].get("prediction") or [] + by_turn = {r["turn"]: r["response"] for r in responses} + + metrics: dict[str, bool | float] = {} + detail: dict[str, dict] = {} + for index, turn in enumerate(turns, start=1): + instruction_ids = list(turn["instruction_id_list"]) + payload = { + "response": by_turn.get(index, ""), + "instruction_id_list": instruction_ids, + # Each element is JSON-encoded upstream and stays that way in the + # dataset (see its module docstring); decoded here, immediately + # before `build_description` consumes it. + "kwargs": [json.loads(kwarg) for kwarg in turn["kwargs"]], + } + detail[f"turn_{index}"] = {"instruction_id_list": instruction_ids} + for grade in _GRADES: + followed = list(graders[grade](payload)["follow_instruction_list"]) + metrics[f"turn_{index}_{grade}_follow_all"] = all(followed) + metrics[f"turn_{index}_{grade}_instruction_level"] = ( + sum(followed) / len(followed) if followed else 0.0 + ) + # The raw per-constraint outcomes, which report() pools; a + # per-sample rate cannot reconstruct a pooled one. + detail[f"turn_{index}"][grade] = {"follow_instruction_list": followed} + + # Derived from `metrics`, not recomputed, so the headline cannot disagree + # with the set. `correct` is the strictest reading the benchmark offers: + # every constraint honoured in every turn. + correct = all( + bool(metrics[f"turn_{index}_strict_follow_all"]) + for index in range(1, len(turns) + 1) + ) + score = sum( + float(metrics[f"turn_{index}_strict_instruction_level"]) + for index in range(1, len(turns) + 1) + ) / len(turns) + return True, build_judgement_record( + [list(turn["instruction_id_list"]) for turn in turns], + [build_rollout_judgement(0, correct, score=score, metrics=metrics)], + score=score, + metrics=metrics, + extra={ + "key": raw["key"], + "language": raw["language"], + "n_turns": len(turns), + **detail, + }, + ) + + @override + async def report(self, finals, fails): + judgements = [f.feedback_result for f in finals] + results: dict[str, float] = {"fails": len(fails)} + + # Every language present, plus the pooled cell. Sorted so the report's + # key order does not depend on which sample finished first. + languages = sorted({str(j["extra"]["language"]) for j in judgements}) + turn_overalls: list[float] = [] + + for turn in (1, 2, 3): + # Only conversations that *have* this turn count toward it -- 56 rows + # have no third turn, and upstream skips them rather than scoring + # them zero. + present = [j for j in judgements if j["extra"]["n_turns"] >= turn] + if not present: + continue + results[f"turn_{turn}_prompts_number"] = len(present) + + for language in (_ALL_LANGUAGES, *languages): + cell = ( + present + if language == _ALL_LANGUAGES + else [j for j in present if j["extra"]["language"] == language] + ) + if not cell: + continue + components = [] + for grade in _GRADES: + followed = [ + j["extra"][f"turn_{turn}"][grade]["follow_instruction_list"] + for j in cell + ] + prompt_level = sum(1 for f in followed if all(f)) / len(followed) + # Pooled from raw counts, not averaged from the per-sample + # rates -- the two differ when turns carry different + # constraint counts, and here they always do. + instruction_total = sum(len(f) for f in followed) + instruction_level = ( + sum(sum(f) for f in followed) / instruction_total + if instruction_total + else 0.0 + ) + components += [prompt_level, instruction_level] + if language == _ALL_LANGUAGES: + # The four components are only broken out for the pooled + # cell; per language upstream publishes just `overall`. + results[f"turn_{turn}_{grade}_prompt_level_accuracy"] = ( + prompt_level * 100 + ) + results[f"turn_{turn}_{grade}_instruction_level_accuracy"] = ( + instruction_level * 100 + ) + + # Upstream's per-turn headline: the plain mean of strict and + # loose x prompt-level and instruction-level. + overall = sum(components) / len(components) * 100 + results[f"turn_{turn}_{language}_overall"] = overall + if language == _ALL_LANGUAGES: + turn_overalls.append(overall) + + # Upstream reports each turn separately; a task needs one headline, and + # the benchmark's subject is how following degrades across turns, so the + # mean over turns is the summary that does not privilege one of them. + results["score"] = ( + sum(turn_overalls) / len(turn_overalls) if turn_overalls else 0.0 + ) + return results diff --git a/tests/unit/tasks/test_multi_if_0shot_gen.py b/tests/unit/tasks/test_multi_if_0shot_gen.py new file mode 100644 index 00000000..6602e42b --- /dev/null +++ b/tests/unit/tasks/test_multi_if_0shot_gen.py @@ -0,0 +1,248 @@ +"""Unit tests for the Multi-IF task. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import asyncio +import subprocess +import sys + +import pytest + +from sieval.tasks.multi_if_0shot_gen import MultiIFZeroShotGenTask + + +def test_import_does_not_pull_evaluation_lib(): + # evaluation_lib pulls langdetect/nltk/emoji and the 3.5k-line checker + # fork; registration must not import it. + code = ( + "import sys\n" + "import sieval.tasks.multi_if_0shot_gen\n" + "assert 'sieval.community.multi_if.evaluation_lib' not in sys.modules, " + "'evaluation_lib must be lazy-imported'\n" + ) + # Run in a fresh interpreter so pytest's already-loaded modules + # don't mask the check. + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +def _task() -> MultiIFZeroShotGenTask: + return MultiIFZeroShotGenTask.__new__(MultiIFZeroShotGenTask) + + +class _Ctx: + def __init__(self, raw): + self.raw_sample = raw + + +def _sample(n_turns: int = 3) -> dict: + # Turn t's constraints extend turn t-1's, as they do in the real data. + turns = [ + { + "prompt": "write something in lowercase", + "instruction_id_list": ["change_case:english_lowercase"], + "kwargs": ["{}"], + }, + { + "prompt": "now end with the exact phrase", + "instruction_id_list": [ + "change_case:english_lowercase", + "startend:end_checker", + ], + "kwargs": ["{}", '{"end_phrase": "the end."}'], + }, + { + "prompt": "and use at least three words", + "instruction_id_list": [ + "change_case:english_lowercase", + "startend:end_checker", + "length_constraints:number_words", + ], + "kwargs": [ + "{}", + '{"end_phrase": "the end."}', + '{"relation": "at least", "num_words": 3}', + ], + }, + ] + return {"key": "k:1:en", "language": "English", "turns": turns[:n_turns]} + + +def _judge(responses: list[str], n_turns: int = 3) -> dict: + task = _task() + post = { + "rollouts": [ + { + "index": 0, + "extracted": True, + "prediction": [ + {"turn": i, "response": r} for i, r in enumerate(responses, start=1) + ], + } + ] + } + _final, judgement = asyncio.run(task.feedback(post, _Ctx(_sample(n_turns)))) + return judgement + + +class _StubOutput: + def __init__(self, text): + self.texts = [text] if text is not None else [] + + +class _StubModel: + """Records the conversation it is handed on each call.""" + + def __init__(self, replies): + self._replies = list(replies) + self.seen: list[list[dict]] = [] + + async def agenerate(self, messages, **kwargs): + # Copy: the task keeps mutating the same list across turns. + self.seen.append([dict(m) for m in messages]) + return _StubOutput(self._replies[len(self.seen) - 1]) + + +def _run_infer(replies, n_turns=3): + task = _task() + model = _StubModel(replies) + task._model = model + raw = _sample(n_turns) + ctx = _Ctx(raw) + pre = asyncio.run(task.preprocess(raw, ctx)) + outputs = asyncio.run(task.infer(pre, ctx)) + return model, outputs + + +def test_infer_feeds_each_turn_the_preceding_conversation(): + model, outputs = _run_infer(["reply one", "reply two", "reply three"]) + assert len(outputs) == 3 + assert len(model.seen) == 3 + + # Turn 1: just the opening user message. + assert model.seen[0] == [ + {"role": "user", "content": "write something in lowercase"} + ] + # Turn 2: turn 1 plus the model's own reply plus the new user turn. + assert model.seen[1] == [ + {"role": "user", "content": "write something in lowercase"}, + {"role": "assistant", "content": "reply one"}, + {"role": "user", "content": "now end with the exact phrase"}, + ] + # Turn 3: the whole history, in order, ending on the third user turn. + assert [m["role"] for m in model.seen[2]] == [ + "user", + "assistant", + "user", + "assistant", + "user", + ] + assert model.seen[2][3] == {"role": "assistant", "content": "reply two"} + assert model.seen[2][4]["content"] == "and use at least three words" + + +def test_infer_stops_at_the_last_turn_of_a_two_turn_conversation(): + model, outputs = _run_infer(["reply one", "reply two"], n_turns=2) + assert len(outputs) == 2 + assert len(model.seen) == 2 + + +def test_infer_continues_after_a_choiceless_response(): + # An aborted turn has no choices; the conversation must carry on with an + # empty assistant turn rather than raising. + model, outputs = _run_infer([None, "reply two", "reply three"]) + assert len(outputs) == 3 + assert model.seen[1][1] == {"role": "assistant", "content": ""} + + +def test_preprocess_records_only_the_first_turn(): + task = _task() + raw = _sample() + record = asyncio.run(task.preprocess(raw, _Ctx(raw))) + assert record["prompt"] == [ + {"role": "user", "content": "write something in lowercase"} + ] + assert record["extra"]["n_turns"] == 3 + # The later user turns are recorded even though they are not sent yet. + assert len(record["extra"]["later_turn_prompts"]) == 2 + # Reference is the per-turn cumulative constraint list. + assert [len(ids) for ids in record["reference"]] == [1, 2, 3] + + +def test_feedback_grades_each_turn_against_its_cumulative_constraints(): + # Turn 1 satisfies its only constraint; turn 2 misses the end phrase; + # turn 3 satisfies all three. + judgement = _judge( + [ + "all lowercase here.", + "still lowercase but wrong ending", + "still lowercase and long enough. the end.", + ] + ) + m = judgement["metrics"] + assert m["turn_1_strict_follow_all"] is True + assert m["turn_2_strict_follow_all"] is False + assert m["turn_3_strict_follow_all"] is True + # Turn 2 kept 1 of its 2 constraints. + assert m["turn_2_strict_instruction_level"] == pytest.approx(0.5) + # A single failed turn makes the whole conversation incorrect. + assert judgement["rollouts"][0]["correct"] is False + + +def test_feedback_handles_a_two_turn_conversation(): + judgement = _judge(["all lowercase.", "lowercase. the end."], n_turns=2) + assert judgement["extra"]["n_turns"] == 2 + assert "turn_3" not in judgement["extra"] + assert judgement["rollouts"][0]["correct"] is True + + +def test_missing_turn_response_scores_zero_rather_than_raising(): + # An aborted turn arrives as "": every constraint fails, nothing raises. + judgement = _judge(["", "", ""]) + m = judgement["metrics"] + assert m["turn_1_strict_follow_all"] is False + assert m["turn_1_strict_instruction_level"] == 0.0 + + +def test_postprocess_reports_no_extraction_only_when_every_turn_is_blank(): + task = _task() + + class _Out: + def __init__(self, text): + self.texts = [text] + + blank = asyncio.run(task.postprocess([_Out(""), _Out(" ")], _Ctx(_sample()))) + assert blank["rollouts"][0]["extracted"] is False + + # A partly-blank conversation is still a real answer that scores badly. + partial = asyncio.run(task.postprocess([_Out(""), _Out("hi")], _Ctx(_sample()))) + assert partial["rollouts"][0]["extracted"] is True + assert partial["rollouts"][0]["prediction"][1] == {"turn": 2, "response": "hi"} + + +def test_report_pools_instruction_counts_and_skips_absent_turns(): + class _Final: + def __init__(self, judgement): + self.feedback_result = judgement + + # One three-turn conversation and one two-turn conversation. + three = _judge( + ["all lowercase.", "lowercase. the end.", "lowercase and long. the end."] + ) + two = _judge(["all lowercase.", "lowercase. the end."], n_turns=2) + report = asyncio.run(_task().report([_Final(three), _Final(two)], [])) + + # Turn 3 exists for only one of the two conversations. + assert report["turn_1_prompts_number"] == 2 + assert report["turn_3_prompts_number"] == 1 + # Both conversations pass everything, so every cell is 100. + assert report["turn_1_all_languages_overall"] == pytest.approx(100.0) + assert report["turn_1_English_overall"] == pytest.approx(100.0) + assert report["score"] == pytest.approx(100.0) + assert report["fails"] == 0