From c248d787c48d381bf13292822f2bc7b3e6d333ca Mon Sep 17 00:00:00 2001 From: kiakiraki Date: Sun, 19 Jul 2026 10:16:29 +0900 Subject: [PATCH] =?UTF-8?q?LiteLLMChatAPI=E3=81=AEignore=5Fseed=E3=81=8Cse?= =?UTF-8?q?t=5Frandom=5Fseed=E7=B5=8C=E7=94=B1=E3=81=AEseed=E3=82=92?= =?UTF-8?q?=E7=84=A1=E8=A6=96=E3=81=97=E3=81=AA=E3=81=84=E5=95=8F=E9=A1=8C?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- flexeval/core/language_model/litellm_api.py | 4 +++ tests/core/language_model/test_litellm_api.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/flexeval/core/language_model/litellm_api.py b/flexeval/core/language_model/litellm_api.py index 37f8ac55..c55091ce 100644 --- a/flexeval/core/language_model/litellm_api.py +++ b/flexeval/core/language_model/litellm_api.py @@ -75,9 +75,13 @@ def __init__( model_response_object=ModelResponse(), ) self.ignore_seed = ignore_seed + if self.ignore_seed and "seed" in self.default_gen_kwargs: + self.default_gen_kwargs.pop("seed") self.api_call_func = completion def set_random_seed(self, seed: int) -> None: + if self.ignore_seed: + return self.default_gen_kwargs["seed"] = seed def _batch_complete_text( diff --git a/tests/core/language_model/test_litellm_api.py b/tests/core/language_model/test_litellm_api.py index 8786f801..89415733 100644 --- a/tests/core/language_model/test_litellm_api.py +++ b/tests/core/language_model/test_litellm_api.py @@ -115,3 +115,28 @@ def test_if_not_ignore_seed() -> None: def test_set_random_seed(chat_lm: OpenAIChatAPI) -> None: chat_lm.set_random_seed(42) assert chat_lm.default_gen_kwargs["seed"] == 42 + + +@pytest.mark.skipif(not is_openai_enabled(), reason="OpenAI is not installed") +def test_set_random_seed_is_noop_when_ignore_seed() -> None: + chat_lm = LiteLLMChatAPI(MODEL_NAME, ignore_seed=True) + chat_lm.set_random_seed(42) + assert "seed" not in chat_lm.default_gen_kwargs + + +@pytest.mark.skipif(not is_openai_enabled(), reason="OpenAI is not installed") +def test_ignore_seed_removes_seed_from_default_gen_kwargs() -> None: + chat_lm = LiteLLMChatAPI(MODEL_NAME, default_gen_kwargs={"seed": 42, "temperature": 0.0}, ignore_seed=True) + assert "seed" not in chat_lm.default_gen_kwargs + assert chat_lm.default_gen_kwargs["temperature"] == 0.0 + + +@pytest.mark.skipif(not is_openai_enabled(), reason="OpenAI is not installed") +def test_if_ignore_seed_after_set_random_seed() -> None: + chat_lm = LiteLLMChatAPI(MODEL_NAME, ignore_seed=True) + chat_lm.set_random_seed(42) + chat_messages = [{"role": "user", "content": "Hello"}] + with patch.object(OpenAIChatAPI, "_batch_generate_chat_response", return_value=[LMOutput("Hello!")]) as mock_method: + chat_lm.generate_chat_response(chat_messages, temperature=0.7) + # `seed` should not be passed even though set_random_seed() was called + mock_method.assert_called_once_with([chat_messages], [None], temperature=0.7)