From f99f668cabf214d7ad191855049dc6ab3957b55f Mon Sep 17 00:00:00 2001 From: Kotaro-Aono Date: Mon, 13 Jul 2026 18:13:57 +0900 Subject: [PATCH 1/4] =?UTF-8?q?empty=5Fresponse=E3=81=AEfinish=5Freason?= =?UTF-8?q?=E3=82=92error=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flexeval/core/language_model/openai_api.py | 11 ++++++++--- flexeval/core/language_model/openai_batch_api.py | 8 ++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/flexeval/core/language_model/openai_api.py b/flexeval/core/language_model/openai_api.py index 5aa2594f..0a072ff5 100644 --- a/flexeval/core/language_model/openai_api.py +++ b/flexeval/core/language_model/openai_api.py @@ -226,7 +226,7 @@ def _batch_complete_text( LMOutput( text=res.choices[0].message.content, reasoning_text=get_reasoning_text(res.choices[0].message), - finish_reason=res.choices[0].finish_reason, + finish_reason="error" if res is self.empty_response else res.choices[0].finish_reason, ) for res in api_responses ] @@ -246,7 +246,7 @@ def _batch_generate_chat_response( LMOutput( text=res.choices[0].message.content, reasoning_text=get_reasoning_text(res.choices[0].message), - finish_reason=res.choices[0].finish_reason, + finish_reason="error" if res is self.empty_response else res.choices[0].finish_reason, tool_calls=[tool_call.to_dict() for tool_call in res.choices[0].message.tool_calls] if res.choices[0].message.tool_calls else None, @@ -450,7 +450,12 @@ def _batch_complete_text( **kwargs, ) - return [LMOutput(text=res.choices[0].text, finish_reason=res.choices[0].finish_reason) for res in api_responses] + return [ + LMOutput(text="", finish_reason="error") + if res is self.empty_response + else LMOutput(text=res.choices[0].text, finish_reason=res.choices[0].finish_reason) + for res in api_responses + ] def __repr__(self) -> str: return f"{self.__class__.__name__}(model={self.model})" diff --git a/flexeval/core/language_model/openai_batch_api.py b/flexeval/core/language_model/openai_batch_api.py index 0ca208b0..74bbb641 100644 --- a/flexeval/core/language_model/openai_batch_api.py +++ b/flexeval/core/language_model/openai_batch_api.py @@ -257,7 +257,9 @@ def _batch_complete_text( **kwargs, ) return [ - LMOutput(text=res["choices"][0]["message"]["content"], finish_reason=res["choices"][0]["finish_reason"]) + LMOutput(text="", finish_reason="error") + if isinstance(res, str) + else LMOutput(text=res["choices"][0]["message"]["content"], finish_reason=res["choices"][0]["finish_reason"]) for res in api_responses ] @@ -273,7 +275,9 @@ def _batch_generate_chat_response( **kwargs, ) return [ - LMOutput( + LMOutput(text="", finish_reason="error") + if isinstance(res, str) + else LMOutput( text=res["choices"][0]["message"]["content"], finish_reason=res["choices"][0]["finish_reason"], tool_calls=res["choices"][0]["message"].get("tool_calls", None), From 146e93034b22c010bdf09733403f16a17878e5f3 Mon Sep 17 00:00:00 2001 From: Kotaro-Aono Date: Mon, 13 Jul 2026 18:59:33 +0900 Subject: [PATCH 2/4] =?UTF-8?q?=5Fbatch=5Fcompute=5Fchat=5Flog=5Fprobs?= =?UTF-8?q?=E3=82=92=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flexeval/core/language_model/openai_api.py | 16 ++++++++++------ flexeval/core/language_model/openai_batch_api.py | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/flexeval/core/language_model/openai_api.py b/flexeval/core/language_model/openai_api.py index 0a072ff5..f3615c50 100644 --- a/flexeval/core/language_model/openai_api.py +++ b/flexeval/core/language_model/openai_api.py @@ -295,17 +295,21 @@ def _batch_compute_chat_log_probs( ) log_probs = [] - top_logprobs_list = [res.choices[0].logprobs.content[0].top_logprobs for res in api_responses] + top_logprobs_list = [ + None if res is self.empty_response else res.choices[0].logprobs.content[0].top_logprobs + for res in api_responses + ] for index, prompt in enumerate(prompt_list): target_token = response_contents[index] index_in_unique = unique_prompt_list.index(prompt) - log_prob = None # if target token not in top_logprobs, return None for log_prob of the token + log_prob = None # if target token not in top_logprobs, or the request errored, return None top_logprobs = top_logprobs_list[index_in_unique] - for token_logprob in top_logprobs: - if token_logprob.token == target_token: - log_prob = token_logprob.logprob - break + if top_logprobs is not None: + for token_logprob in top_logprobs: + if token_logprob.token == target_token: + log_prob = token_logprob.logprob + break log_probs.append(log_prob) return log_probs diff --git a/flexeval/core/language_model/openai_batch_api.py b/flexeval/core/language_model/openai_batch_api.py index 74bbb641..62bfc942 100644 --- a/flexeval/core/language_model/openai_batch_api.py +++ b/flexeval/core/language_model/openai_batch_api.py @@ -323,17 +323,21 @@ def _batch_compute_chat_log_probs( ) log_probs = [] - top_logprobs_list = [res["choices"][0]["logprobs"]["content"][0]["top_logprobs"] for res in api_responses] + top_logprobs_list = [ + None if isinstance(res, str) else res["choices"][0]["logprobs"]["content"][0]["top_logprobs"] + for res in api_responses + ] for index, prompt in enumerate(prompt_list): target_token = response_contents[index] index_in_unique = unique_prompt_list.index(prompt) - log_prob = None # if target token not in top_logprobs, return None for log_prob of the token + log_prob = None # if target token not in top_logprobs, or the request errored, return None top_logprobs = top_logprobs_list[index_in_unique] - for token_logprob in top_logprobs: - if token_logprob["token"] == target_token: - log_prob = token_logprob["logprob"] - break + if top_logprobs is not None: + for token_logprob in top_logprobs: + if token_logprob["token"] == target_token: + log_prob = token_logprob["logprob"] + break log_probs.append(log_prob) return log_probs From feedb5f22d2883146ba07f83415641352f618ad8 Mon Sep 17 00:00:00 2001 From: Kotaro-Aono Date: Mon, 13 Jul 2026 19:38:08 +0900 Subject: [PATCH 3/4] fix lint-check --- flexeval/core/language_model/openai_batch_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexeval/core/language_model/openai_batch_api.py b/flexeval/core/language_model/openai_batch_api.py index 62bfc942..5c8fcfa7 100644 --- a/flexeval/core/language_model/openai_batch_api.py +++ b/flexeval/core/language_model/openai_batch_api.py @@ -259,7 +259,10 @@ def _batch_complete_text( return [ LMOutput(text="", finish_reason="error") if isinstance(res, str) - else LMOutput(text=res["choices"][0]["message"]["content"], finish_reason=res["choices"][0]["finish_reason"]) + else LMOutput( + text=res["choices"][0]["message"]["content"], + finish_reason=res["choices"][0]["finish_reason"], + ) for res in api_responses ] From 9feddd947012608e2100e108ae79ac0e147df337 Mon Sep 17 00:00:00 2001 From: Kotaro-Aono Date: Tue, 14 Jul 2026 11:57:51 +0900 Subject: [PATCH 4/4] =?UTF-8?q?error=E3=82=92empty=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flexeval/core/language_model/openai_api.py | 6 +++--- flexeval/core/language_model/openai_batch_api.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexeval/core/language_model/openai_api.py b/flexeval/core/language_model/openai_api.py index f3615c50..c6aa2b97 100644 --- a/flexeval/core/language_model/openai_api.py +++ b/flexeval/core/language_model/openai_api.py @@ -226,7 +226,7 @@ def _batch_complete_text( LMOutput( text=res.choices[0].message.content, reasoning_text=get_reasoning_text(res.choices[0].message), - finish_reason="error" if res is self.empty_response else res.choices[0].finish_reason, + finish_reason="empty" if res is self.empty_response else res.choices[0].finish_reason, ) for res in api_responses ] @@ -246,7 +246,7 @@ def _batch_generate_chat_response( LMOutput( text=res.choices[0].message.content, reasoning_text=get_reasoning_text(res.choices[0].message), - finish_reason="error" if res is self.empty_response else res.choices[0].finish_reason, + finish_reason="empty" if res is self.empty_response else res.choices[0].finish_reason, tool_calls=[tool_call.to_dict() for tool_call in res.choices[0].message.tool_calls] if res.choices[0].message.tool_calls else None, @@ -455,7 +455,7 @@ def _batch_complete_text( ) return [ - LMOutput(text="", finish_reason="error") + LMOutput(text="", finish_reason="empty") if res is self.empty_response else LMOutput(text=res.choices[0].text, finish_reason=res.choices[0].finish_reason) for res in api_responses diff --git a/flexeval/core/language_model/openai_batch_api.py b/flexeval/core/language_model/openai_batch_api.py index 5c8fcfa7..6f868edc 100644 --- a/flexeval/core/language_model/openai_batch_api.py +++ b/flexeval/core/language_model/openai_batch_api.py @@ -257,7 +257,7 @@ def _batch_complete_text( **kwargs, ) return [ - LMOutput(text="", finish_reason="error") + LMOutput(text="", finish_reason="empty") if isinstance(res, str) else LMOutput( text=res["choices"][0]["message"]["content"], @@ -278,7 +278,7 @@ def _batch_generate_chat_response( **kwargs, ) return [ - LMOutput(text="", finish_reason="error") + LMOutput(text="", finish_reason="empty") if isinstance(res, str) else LMOutput( text=res["choices"][0]["message"]["content"],