diff --git a/src/harness/evaluators/__init__.py b/src/harness/evaluators/__init__.py index bb1192a..e6f8732 100644 --- a/src/harness/evaluators/__init__.py +++ b/src/harness/evaluators/__init__.py @@ -4,6 +4,7 @@ from harness.evaluators.contains import ContainsEvaluator from harness.evaluators.exact_match import ExactMatchEvaluator from harness.evaluators.json_match import JSONMatchEvaluator +from harness.evaluators.regex_match import RegexMatchEvaluator from harness.evaluators.tool_usage import ToolUsageEvaluator __all__ = [ @@ -13,4 +14,5 @@ "ContainsEvaluator", "JSONMatchEvaluator", "ToolUsageEvaluator", + "RegexMatchEvaluator", ] diff --git a/src/harness/evaluators/regex_match.py b/src/harness/evaluators/regex_match.py new file mode 100644 index 0000000..56c4426 --- /dev/null +++ b/src/harness/evaluators/regex_match.py @@ -0,0 +1,56 @@ +"""Configurable regular-expression evaluation of agent output.""" + +from __future__ import annotations + +import re + +from harness.core.agent import AgentOutput +from harness.core.result import EvaluationResult +from harness.core.testcase import TestCase +from harness.evaluators.base import EvaluatorRegistry + + +@EvaluatorRegistry.register("regex_match") +class RegexMatchEvaluator: + """Search or fully match output using a caller-supplied regular expression.""" + + name: str = "regex_match" + + def __init__( + self, pattern: str = r".*", case_sensitive: bool = True, full_match: bool = False + ) -> None: + self.pattern = pattern + self.case_sensitive = case_sensitive + self.full_match = full_match + self._flags = 0 if case_sensitive else re.IGNORECASE + + def evaluate(self, test_case: TestCase, output: AgentOutput) -> EvaluationResult: + details = {"pattern": self.pattern, "flags": self._flags, "match": None} + if output.is_error: + return EvaluationResult( + evaluator=self.name, + score=0.0, + passed=False, + explanation=f"Agent produced an error: {output.error}", + details=details, + ) + try: + regex = re.compile(self.pattern, self._flags) + match = regex.fullmatch(output.text) if self.full_match else regex.search(output.text) + except re.error as exc: + return EvaluationResult( + evaluator=self.name, + score=0.0, + passed=False, + explanation=f"Invalid regular expression pattern: {exc}", + details=details, + ) + return EvaluationResult( + evaluator=self.name, + score=1.0 if match is not None else 0.0, + passed=match is not None, + explanation="Output matches the pattern." + if match is not None + else "Output does not match the pattern.", + details={**details, "match": match.group(0) if match is not None else None}, + ) diff --git a/tests/unit/evaluators/test_regex_match.py b/tests/unit/evaluators/test_regex_match.py new file mode 100644 index 0000000..e3cc274 --- /dev/null +++ b/tests/unit/evaluators/test_regex_match.py @@ -0,0 +1,56 @@ +"""Regex evaluator matching, error paths and registry integration.""" + +import re + +import pytest + +from harness.core.agent import AgentOutput +from harness.core.testcase import TestCase +from harness.evaluators import EvaluatorRegistry, RegexMatchEvaluator + + +@pytest.mark.parametrize( + ("pattern", "text", "sensitive", "full", "passed", "match"), + [ + (r"ORD-[0-9]{5}", "Order ORD-12345 confirmed", True, False, True, "ORD-12345"), + (r"ORD-[0-9]{5}", "No order", True, False, False, None), + ("hello", "HELLO", False, False, True, "HELLO"), + ("hello", "HELLO", True, False, False, None), + ("hello", "hello", True, True, True, "hello"), + ("hello", "hello world", True, True, False, None), + ("", "", True, True, True, ""), + ], +) +def test_matches(pattern, text, sensitive, full, passed, match): + evaluator = RegexMatchEvaluator(pattern, case_sensitive=sensitive, full_match=full) + result = evaluator.evaluate( + TestCase(id="regex", name="Regex", input=""), AgentOutput(output=text) + ) + assert result.evaluator == "regex_match" + assert result.passed is passed + assert result.score == float(passed) + assert result.details["pattern"] == pattern + assert result.details["flags"] == (0 if sensitive else re.IGNORECASE) + assert result.details["match"] == match + + +def test_invalid_pattern_returns_failed_result(): + result = RegexMatchEvaluator("[").evaluate( + TestCase(id="regex", name="Regex", input=""), AgentOutput(output="text") + ) + assert result.score == 0.0 + assert not result.passed + assert result.explanation.startswith("Invalid regular expression pattern:") + + +def test_agent_error_takes_precedence_even_over_invalid_pattern(): + result = RegexMatchEvaluator("[").evaluate( + TestCase(id="regex", name="Regex", input=""), AgentOutput(error="test failure") + ) + assert result.score == 0.0 + assert not result.passed + assert result.explanation == "Agent produced an error: test failure" + + +def test_registered_plugin(): + assert isinstance(EvaluatorRegistry.get("regex_match"), RegexMatchEvaluator)