From 5a34fd657c7d6405ddf230fc5da4b5f82a32722d Mon Sep 17 00:00:00 2001
From: "Sharad." <sharadvc@users.noreply.github.com>
Date: Sat, 12 Sep 2026 14:33:35 +0000
Subject: [PATCH] fix: rubric invalid criterion regex returns fail result

Catch invalid RegExp patterns in rubric criteria instead of throwing,
matching regex and json-schema scorer behavior.

Fixes #33
---
 src/scorers/rubric.ts | 23 ++++++++++++++++++++---
 tests/scorers.test.ts |  9 +++++++++
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/src/scorers/rubric.ts b/src/scorers/rubric.ts
index 75e7337..3d8b01c 100644
--- a/src/scorers/rubric.ts
+++ b/src/scorers/rubric.ts
@@ -17,12 +17,21 @@ export interface RubricCriterion {
   pattern?: string;
 }
 
-function criterionSatisfied(output: string, c: RubricCriterion): boolean {
+function criterionSatisfied(
+  output: string,
+  c: RubricCriterion,
+): boolean | { invalidPattern: string } {
   const hay = normalize(output);
   if (c.allOf && !c.allOf.every((s) => hay.includes(normalize(s)))) return false;
   if (c.anyOf && !c.anyOf.some((s) => hay.includes(normalize(s)))) return false;
   if (c.noneOf && c.noneOf.some((s) => hay.includes(normalize(s)))) return false;
-  if (c.pattern && !new RegExp(c.pattern, "i").test(output)) return false;
+  if (c.pattern) {
+    try {
+      if (!new RegExp(c.pattern, "i").test(output)) return false;
+    } catch (err) {
+      return { invalidPattern: (err as Error).message };
+    }
+  }
   // A criterion with no checks is treated as satisfied (documentation only).
   return true;
 }
@@ -50,7 +59,15 @@ export const rubricScorer: Scorer = {
     for (const c of criteria) {
       const points = c.points ?? 1;
       total += points;
-      if (criterionSatisfied(ctx.output, c)) {
+      const satisfied = criterionSatisfied(ctx.output, c);
+      if (typeof satisfied === "object") {
+        return result(spec, {
+          score: 0,
+          passed: false,
+          reason: `invalid regex in criterion "${c.description}": ${satisfied.invalidPattern}`,
+        });
+      }
+      if (satisfied) {
         earned += points;
       } else {
         failed.push(c.description);
diff --git a/tests/scorers.test.ts b/tests/scorers.test.ts
index 78bf373..1740037 100644
--- a/tests/scorers.test.ts
+++ b/tests/scorers.test.ts
@@ -192,6 +192,15 @@ describe("rubric", () => {
     expect(r.score).toBeCloseTo(0.5);
     expect(r.passed).toBe(true);
   });
+  it("returns a fail result for an invalid criterion pattern regex", async () => {
+    const r = await run(
+      rubricScorer,
+      { type: "rubric", criteria: [{ description: "x", pattern: "([" }] },
+      ctx("anything"),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.reason).toMatch(/invalid regex/i);
+  });
 });
 
 describe("exact-match trim:false preserves internal whitespace", () => {