feat(tasks): add Multi-IF multi-turn multilingual instruction-following

pyproject.toml

@@ -45,6 +45,20 @@ ifeval = [
   "nltk>=3.9.2",
   "immutabledict>=4.2.2",
 ]
+multi-if = [
+  # Multi-IF vendors its own multilingual fork of the IFEval checkers, so this
+  # group is deliberately not `ifeval`'s: no absl/immutabledict (the fork uses
+  # stdlib logging and MappingProxyType), and emoji is required rather than
+  # optional because the CJK word counter behind every Chinese length
+  # constraint counts emoji as words.
+  "emoji>=2.15.0",
+  "langdetect>=1.0.9",
+  "nltk>=3.9.2",
+  # Only reached when langdetect reports Thai for a model *response* (Thai is
+  # not one of the dataset's eight languages). Declared anyway so that path
+  # cannot die on ImportError mid-run; the import itself is deferred.
+  "pythainlp>=5.0.0",
+]
 math = [
   "latex2sympy2-extended>=1.10.2",
   # antlr4-11-0 extra pins antlr4-python3-runtime==4.11.0, which sympy 1.14's

sieval/community/multi_if/__init__.py

@@ -0,0 +1,32 @@
+"""Meta Multi-IF evaluation adaptation.
+
+Source: https://github.com/facebookresearch/Multi-IF
+Revision: 1cdb53ed18499ad729e0766e5d3099dd5344406f (Apache-2.0, archived)
+
+Multi-IF ships its *own* multilingual fork of Google's IFEval checkers. It
+carries the same 25 instruction ids as
+``sieval.community.instruction_following_eval``, but routes word counting,
+sentence counting and casing through ``langdetect``, so the two are not
+interchangeable and both are vendored.
+
+Local adaptations (each marked "Local adaptation:" at its site):
+
+- ``ifeval.py``: ``pythainlp`` is imported on first use rather than at module
+  scope. Thai is not one of the eight languages in the released Multi-IF CSV,
+  so the Thai branches are unreachable for that data and the dependency stays
+  optional.
+- ``evaluation_lib.py``: only upstream ``metrics.py``'s two per-response
+  graders are taken; conversation assembly and aggregation live in the task.
+
+Deliberately *not* adapted: upstream leaves ``langdetect`` unseeded, so grading
+is not reproducible run to run (~2-3% of ``detect()`` calls flip on short or
+mixed-script text). Seeding it would change the grader, which belongs in a
+``_fixed`` task variant with a measured score delta rather than under the
+faithful name. The import site carries the full note.
+
+Infra: scoring needs the NLTK ``punkt`` tokenizer (via
+``nltk.data.load("nltk:tokenizers/punkt/english.pickle")``) exactly as the
+IFEval sibling does, and does not download it. Offline runs must pre-stage it.
+
+AI-Generated Code - Claude Opus 5 (1M context) (Anthropic)
+"""

sieval/community/multi_if/evaluation_lib.py

@@ -0,0 +1,95 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Vendored from facebookresearch/Multi-IF (Apache-2.0), file `metrics.py`:
+# https://github.com/facebookresearch/Multi-IF/blob/1cdb53ed18499ad729e0766e5d3099dd5344406f/metrics.py
+#
+# Only the two per-response graders are taken. Upstream's surrounding
+# `MultiTurnInstructionFollowingPromptSolution` drives a pandas/scipy pipeline
+# that re-reads a CSV per turn and bootstraps confidence intervals; SiEval's
+# task owns conversation assembly and aggregation instead, so importing that
+# machinery would pull pandas + scipy in for two pure functions.
+#
+# Local adaptations:
+#   1. `import ifeval` -> `from . import ifeval` (upstream is a flat repo).
+#   2. The `Dict[str, float]` return annotations are corrected to `dict`: both
+#      functions return lists, not floats, so upstream's annotation is wrong.
+# Otherwise the bodies are byte-identical to upstream.
+
+from typing import Any
+
+from . import ifeval
+
+
+def gen_acc_strict(x: dict[str, Any]) -> dict:
+    # reference: fbcode/gen_ai/github/fair_evals/evals/tasks/finetune/ifeval.py
+    response = str(x["response"])
+    instruction_list = x["instruction_id_list"]
+    is_following_list = []
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        instruction.build_description(**x["kwargs"][index])
+
+        if response and instruction.check_following(response):
+            is_following_list.append(True)
+        else:
+            is_following_list.append(False)
+
+    return {
+        "follow_instruction_list": is_following_list,
+        "instruction_id_list": instruction_list,
+    }
+
+
+def gen_acc_loose(x: dict[str, Any]) -> dict:
+    response = str(x["response"])
+    r = response.split("\n")
+    response_remove_first = "\n".join(r[1:]).strip()
+    response_remove_last = "\n".join(r[:-1]).strip()
+    response_remove_both = "\n".join(r[1:-1]).strip()
+    revised_response = response.replace("*", "")
+    revised_response_remove_first = response_remove_first.replace("*", "")
+    revised_response_remove_last = response_remove_last.replace("*", "")
+    revised_response_remove_both = response_remove_both.replace("*", "")
+    all_responses = [
+        response,
+        revised_response,
+        response_remove_first,
+        response_remove_last,
+        response_remove_both,
+        revised_response_remove_first,
+        revised_response_remove_last,
+        revised_response_remove_both,
+    ]
+    instruction_list = x["instruction_id_list"]
+    is_following_list = []
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        instruction.build_description(**x["kwargs"][index])
+
+        is_following = False
+        for r in all_responses:  # type: ignore
+            if r.strip() and instruction.check_following(r):  # type: ignore
+                is_following = True
+                break
+
+        is_following_list.append(is_following)
+    return {
+        "follow_instruction_list": is_following_list,
+        "instruction_id_list": instruction_list,
+    }