feat: add configurable regex_match evaluator

src/harness/evaluators/__init__.py

@@ -4,6 +4,7 @@
 from harness.evaluators.contains import ContainsEvaluator
 from harness.evaluators.exact_match import ExactMatchEvaluator
 from harness.evaluators.json_match import JSONMatchEvaluator
+from harness.evaluators.regex_match import RegexMatchEvaluator
 from harness.evaluators.tool_usage import ToolUsageEvaluator
 
 __all__ = [
@@ -13,4 +14,5 @@
     "ContainsEvaluator",
     "JSONMatchEvaluator",
     "ToolUsageEvaluator",
+    "RegexMatchEvaluator",
 ]

src/harness/evaluators/regex_match.py

@@ -0,0 +1,56 @@
+"""Configurable regular-expression evaluation of agent output."""
+
+from __future__ import annotations
+
+import re
+
+from harness.core.agent import AgentOutput
+from harness.core.result import EvaluationResult
+from harness.core.testcase import TestCase
+from harness.evaluators.base import EvaluatorRegistry
+
+
+@EvaluatorRegistry.register("regex_match")
+class RegexMatchEvaluator:
+    """Search or fully match output using a caller-supplied regular expression."""
+
+    name: str = "regex_match"
+
+    def __init__(
+        self, pattern: str = r".*", case_sensitive: bool = True, full_match: bool = False
+    ) -> None:
+        self.pattern = pattern
+        self.case_sensitive = case_sensitive
+        self.full_match = full_match
+        self._flags = 0 if case_sensitive else re.IGNORECASE
+
+    def evaluate(self, test_case: TestCase, output: AgentOutput) -> EvaluationResult:
+        details = {"pattern": self.pattern, "flags": self._flags, "match": None}
+        if output.is_error:
+            return EvaluationResult(
+                evaluator=self.name,
+                score=0.0,
+                passed=False,
+                explanation=f"Agent produced an error: {output.error}",
+                details=details,
+            )
+        try:
+            regex = re.compile(self.pattern, self._flags)
+            match = regex.fullmatch(output.text) if self.full_match else regex.search(output.text)
+        except re.error as exc:
+            return EvaluationResult(
+                evaluator=self.name,
+                score=0.0,
+                passed=False,
+                explanation=f"Invalid regular expression pattern: {exc}",
+                details=details,
+            )
+        return EvaluationResult(
+            evaluator=self.name,
+            score=1.0 if match is not None else 0.0,
+            passed=match is not None,
+            explanation="Output matches the pattern."
+            if match is not None
+            else "Output does not match the pattern.",
+            details={**details, "match": match.group(0) if match is not None else None},
+        )

tests/unit/evaluators/test_regex_match.py

@@ -0,0 +1,56 @@
+"""Regex evaluator matching, error paths and registry integration."""
+
+import re
+
+import pytest
+
+from harness.core.agent import AgentOutput
+from harness.core.testcase import TestCase
+from harness.evaluators import EvaluatorRegistry, RegexMatchEvaluator
+
+
+@pytest.mark.parametrize(
+    ("pattern", "text", "sensitive", "full", "passed", "match"),
+    [
+        (r"ORD-[0-9]{5}", "Order ORD-12345 confirmed", True, False, True, "ORD-12345"),
+        (r"ORD-[0-9]{5}", "No order", True, False, False, None),
+        ("hello", "HELLO", False, False, True, "HELLO"),
+        ("hello", "HELLO", True, False, False, None),
+        ("hello", "hello", True, True, True, "hello"),
+        ("hello", "hello world", True, True, False, None),
+        ("", "", True, True, True, ""),
+    ],
+)
+def test_matches(pattern, text, sensitive, full, passed, match):
+    evaluator = RegexMatchEvaluator(pattern, case_sensitive=sensitive, full_match=full)
+    result = evaluator.evaluate(
+        TestCase(id="regex", name="Regex", input=""), AgentOutput(output=text)
+    )
+    assert result.evaluator == "regex_match"
+    assert result.passed is passed
+    assert result.score == float(passed)
+    assert result.details["pattern"] == pattern
+    assert result.details["flags"] == (0 if sensitive else re.IGNORECASE)
+    assert result.details["match"] == match
+
+
+def test_invalid_pattern_returns_failed_result():
+    result = RegexMatchEvaluator("[").evaluate(
+        TestCase(id="regex", name="Regex", input=""), AgentOutput(output="text")
+    )
+    assert result.score == 0.0
+    assert not result.passed
+    assert result.explanation.startswith("Invalid regular expression pattern:")
+
+
+def test_agent_error_takes_precedence_even_over_invalid_pattern():
+    result = RegexMatchEvaluator("[").evaluate(
+        TestCase(id="regex", name="Regex", input=""), AgentOutput(error="test failure")
+    )
+    assert result.score == 0.0
+    assert not result.passed
+    assert result.explanation == "Agent produced an error: test failure"
+
+
+def test_registered_plugin():
+    assert isinstance(EvaluatorRegistry.get("regex_match"), RegexMatchEvaluator)