From 983d01478a97cdc40ac31c76c93e86ded75065b3 Mon Sep 17 00:00:00 2001 From: kiakiraki Date: Sun, 19 Jul 2026 09:40:59 +0900 Subject: [PATCH] =?UTF-8?q?MultipleChoice=E8=A9=95=E4=BE=A1=E3=81=A7few-sh?= =?UTF-8?q?ot=E3=83=AA=E3=83=BC=E3=82=AF=E6=A4=9C=E6=9F=BB=E3=81=8C?= =?UTF-8?q?=E6=A9=9F=E8=83=BD=E3=81=97=E3=81=AA=E3=81=84=E5=95=8F=E9=A1=8C?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit few_shot_generatorに"choices"キーを注入したtemplate_inputsを渡していた ため、サンプル済みインスタンスのinputs（"choices"を含まない）との等価 比較が絶対に成立せず、num_trials_to_avoid_leakによるリーク検査が no-opになっていた。evaluate_generation.pyと同様に未加工の eval_instance.inputsを渡すことで検査が機能するようにする。 Co-Authored-By: Claude Fable 5 --- flexeval/core/evaluate_multiple_choice.py | 4 +++- tests/core/test_evaluate_functions.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/flexeval/core/evaluate_multiple_choice.py b/flexeval/core/evaluate_multiple_choice.py index 5b32c90e..5ac7b08a 100644 --- a/flexeval/core/evaluate_multiple_choice.py +++ b/flexeval/core/evaluate_multiple_choice.py @@ -37,7 +37,9 @@ def evaluate_multiple_choice( template_inputs = {**eval_instance.inputs, "choices": eval_instance.choices} if few_shot_generator is not None: - few_shot_instances = few_shot_generator(template_inputs) + # Pass the raw inputs (without the injected "choices" key) so that + # the data-leak check can compare them against the sampled instances' inputs. + few_shot_instances = few_shot_generator(eval_instance.inputs) few_shot_item_list: list[dict[str, Any]] = [] for few_shot_instance in few_shot_instances: if isinstance(few_shot_instance, MultipleChoiceInstance): diff --git a/tests/core/test_evaluate_functions.py b/tests/core/test_evaluate_functions.py index 350636da..0386e6ee 100644 --- a/tests/core/test_evaluate_functions.py +++ b/tests/core/test_evaluate_functions.py @@ -79,6 +79,26 @@ def test_evaluate_multiple_choice(use_few_shot: bool, max_instances: int) -> Non assert set(metrics.keys()) == {"accuracy", "byte_norm_accuracy", "macro_f1_score", "micro_f1_score"} +def test_evaluate_multiple_choice_detects_few_shot_leak() -> None: + # Sampling all instances of the dataset used as both the evaluation dataset and the + # few-shot pool guarantees the evaluation instance is always among the sampled shots. + few_shot_generator = RandomFewShotGenerator( + dataset=DummyMultipleChoiceDataset(), + num_shots=len(DummyMultipleChoiceDataset()), + num_trials_to_avoid_leak=3, + ) + + with pytest.raises(ValueError, match="data leak"): + evaluate_multiple_choice( + language_model=DummyLanguageModel(), + eval_dataset=DummyMultipleChoiceDataset(), + prompt_template=Jinja2PromptTemplate("{{text}}"), + few_shot_generator=few_shot_generator, + batch_size=1, + max_instances=1, + ) + + @pytest.mark.parametrize( "max_instances", [None, 1],