MultipleChoice評価でfew-shotリーク検査が機能しない問題を修正

flexeval/core/evaluate_multiple_choice.py

@@ -37,7 +37,9 @@ def evaluate_multiple_choice(
                 template_inputs = {**eval_instance.inputs, "choices": eval_instance.choices}
 
                 if few_shot_generator is not None:
-                    few_shot_instances = few_shot_generator(template_inputs)
+                    # Pass the raw inputs (without the injected "choices" key) so that
+                    # the data-leak check can compare them against the sampled instances' inputs.
+                    few_shot_instances = few_shot_generator(eval_instance.inputs)
                     few_shot_item_list: list[dict[str, Any]] = []
                     for few_shot_instance in few_shot_instances:
                         if isinstance(few_shot_instance, MultipleChoiceInstance):

tests/core/test_evaluate_functions.py

@@ -79,6 +79,26 @@ def test_evaluate_multiple_choice(use_few_shot: bool, max_instances: int) -> Non
     assert set(metrics.keys()) == {"accuracy", "byte_norm_accuracy", "macro_f1_score", "micro_f1_score"}
 
 
+def test_evaluate_multiple_choice_detects_few_shot_leak() -> None:
+    # Sampling all instances of the dataset used as both the evaluation dataset and the
+    # few-shot pool guarantees the evaluation instance is always among the sampled shots.
+    few_shot_generator = RandomFewShotGenerator(
+        dataset=DummyMultipleChoiceDataset(),
+        num_shots=len(DummyMultipleChoiceDataset()),
+        num_trials_to_avoid_leak=3,
+    )
+
+    with pytest.raises(ValueError, match="data leak"):
+        evaluate_multiple_choice(
+            language_model=DummyLanguageModel(),
+            eval_dataset=DummyMultipleChoiceDataset(),
+            prompt_template=Jinja2PromptTemplate("{{text}}"),
+            few_shot_generator=few_shot_generator,
+            batch_size=1,
+            max_instances=1,
+        )
+
+
 @pytest.mark.parametrize(
     "max_instances",
     [None, 1],