From 7c4a06be05603d980a2100b6698c2adf0891e371 Mon Sep 17 00:00:00 2001 From: Anai-Guo Date: Wed, 2 Sep 2026 21:16:36 -0700 Subject: [PATCH] fix(metrics): batch evaluation still calls the pre-refactor evaluate signatures Both `evaluate_batch` helpers were left behind when their callees were switched to dict inputs, so every batch path raises TypeError. * RatingEval.evaluate_batch calls `self.evaluate(prompt, response, groundtruth)` while `RatingEval.evaluate(self, inputs: dict, predictions: dict)` takes two dicts. SummaryEval.evaluate_batch, right above it in the same file, already calls the two-dict form. * PrometheusEval.evaluate_batch splats entries into three lists and calls `self.evaluate_response(query, result, answer)` while `evaluate_response(self, llm_response: Dict[str, str])` takes one dict. That also silently dropped `response_a`/`response_b`, which `evaluate_response` needs for `relative_grading`. Since the public `PrometheusEval.evaluate()` builds the dicts and delegates to `evaluate_batch`, that entry point is unusable today. Pass the dicts through in both places. `evaluate_response` already reads every key it needs off the entry, so the destructuring is dropped rather than rebuilt. --- langtest/metrics/llm_eval.py | 10 ++++++---- langtest/metrics/prometheus_eval.py | 8 +------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/langtest/metrics/llm_eval.py b/langtest/metrics/llm_eval.py index ab510115b..99fc209ac 100644 --- a/langtest/metrics/llm_eval.py +++ b/langtest/metrics/llm_eval.py @@ -478,11 +478,13 @@ def evaluate_batch(self, examples: List[dict]) -> List[dict]: """ results = [] for example in examples: - prompt = example.get("prompt", "") - response = example.get("response", "") - groundtruth = example.get("groundtruth") if self.include_groundtruth else None + inputs = { + "prompt": example.get("prompt", ""), + "groundtruth": example.get("groundtruth"), + } + predictions = {"response": example.get("response", "")} - result = self.evaluate(prompt, response, groundtruth) + result = self.evaluate(inputs, predictions) results.append(result) return results diff --git a/langtest/metrics/prometheus_eval.py b/langtest/metrics/prometheus_eval.py index cc1b9d63c..2043bfd28 100644 --- a/langtest/metrics/prometheus_eval.py +++ b/langtest/metrics/prometheus_eval.py @@ -163,13 +163,7 @@ def evaluate_batch(self, entries: List[Dict[str, str]]) -> List[Tuple[str, int]] Returns: A list of tuples of feedback and score. """ - queries = [entry.get("query", None) for entry in entries] - results = [entry.get("result", None) for entry in entries] - answers = [entry.get("answer", None) for entry in entries] - return [ - self.evaluate_response(query, result, answer) - for query, result, answer in zip(queries, results, answers) - ] + return [self.evaluate_response(entry) for entry in entries] def evaluate( self,