feat: add word-count scorer

CHANGELOG.md

@@ -8,10 +8,11 @@ All notable changes to this project are documented here. The format is based on
 
 ### Added
 
+- New `word-count` scorer: scores output length against `minWords`/`maxWords`
+  (and/or char) bounds with full credit inside and linear falloff outside.
 - New `refusal` scorer: asserts a reply reads as a refusal using a normalized
   refusal-phrase set, or — inverted with `expectRefuse: false` — that it reads
   as a direct answer.
-
 - New `tool-call` scorer: passes when the output is a JSON tool call whose
   `name` is on the allowlist and whose `arguments` is a plain object, with
   optional per-tool argument schemas (`allowedTools`, `schemas`).

README.md

@@ -144,6 +144,7 @@ score is the weighted mean of the scorer scores.
 | `latency` | call latency within budget | `budgetMs` |
 | `cost` | estimated call cost within budget | `budgetUsd` (provider must set `costPer1kTokens` so `costUsd` is reported) |
 | `rubric` | weighted criteria score >= threshold | `criteria[]`, `threshold` |
+| `word-count` | output length within bounds (linear falloff outside) | `minWords`, `maxWords`, `minChars`, `maxChars` |
 
 Two scorers are pluggable and ship with **deterministic offline fallbacks** so tests and
 the mock provider need no network:

src/scorers/registry.ts

@@ -11,6 +11,7 @@ import { llmJudgeScorer } from "./llm-judge.js";
 import { latencyScorer } from "./latency.js";
 import { costScorer } from "./cost.js";
 import { rubricScorer } from "./rubric.js";
+import { wordCountScorer } from "./word-count.js";
 
 /** A registry mapping scorer ids to their implementations. */
 export class ScorerRegistry {
@@ -58,6 +59,7 @@ export const builtinScorers: Scorer[] = [
   latencyScorer,
   costScorer,
   rubricScorer,
+  wordCountScorer,
 ];
 
 /** Build a registry preloaded with every built-in scorer. */

src/scorers/word-count.ts

@@ -0,0 +1,83 @@
+import type { Scorer, ScoreContext, ScorerSpec } from "../types.js";
+import { result } from "./util.js";
+
+/**
+ * Passes when the output's length is within word (and/or char) bounds.
+ *
+ * The score is full credit inside the bounds and falls off linearly to zero
+ * outside them, mirroring {@link latencyScorer}'s curve: the farther past the
+ * bound, the lower the credit, reaching 0 at `2x` the bound.
+ *
+ * Options:
+ *  - `minWords`: minimum word count (optional)
+ *  - `maxWords`: maximum word count (optional)
+ *  - `minChars`: minimum character count (optional)
+ *  - `maxChars`: maximum character count (optional)
+ */
+export const wordCountScorer: Scorer = {
+  type: "word-count",
+  score(spec: ScorerSpec, ctx: ScoreContext) {
+    const bounds = readBounds(spec);
+    if (!bounds) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: "at least one of minWords/maxWords/minChars/maxChars required",
+      });
+    }
+    const words = ctx.output.trim().split(/\s+/).filter(Boolean).length;
+    const chars = [...ctx.output].length;
+
+    const tests = [
+      { kind: "words", value: words, min: bounds.minWords, max: bounds.maxWords },
+      { kind: "chars", value: chars, min: bounds.minChars, max: bounds.maxChars },
+    ];
+
+    const offending = tests.find((t) => t.min != null || t.max != null)
+      ? tests.filter((t) => t.min != null || t.max != null)
+      : [];
+    for (const t of offending) {
+      if (t.min != null && t.value < t.min) return falloff(spec, t.kind, t.value, `below min ${t.min}`, t.min, false);
+      if (t.max != null && t.value > t.max) return falloff(spec, t.kind, t.value, `above max ${t.max}`, t.max, true);
+    }
+
+    return result(spec, {
+      score: 1,
+      passed: true,
+      reason: `${words} word(s), ${chars} char(s) within bounds`,
+    });
+  },
+};
+
+function falloff(
+  spec: ScorerSpec,
+  kind: string,
+  value: number,
+  detail: string,
+  bound: number,
+  isMax: boolean,
+) {
+  const damage = isMax ? (value - bound) / bound : (bound - value) / bound;
+  const score = Math.max(0, 1 - damage);
+  return result(spec, {
+    score,
+    passed: false,
+    reason: `${kind}: ${value}, ${detail} (credit ${score.toFixed(2)})`,
+  });
+}
+
+function readBounds(spec: ScorerSpec): { minWords?: number; maxWords?: number; minChars?: number; maxChars?: number } | null {
+  const num = (v: unknown): number | undefined =>
+    typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined;
+  const bounds = {
+    minWords: num(spec.minWords),
+    maxWords: num(spec.maxWords),
+    minChars: num(spec.minChars),
+    maxChars: num(spec.maxChars),
+  };
+  const any = bounds.minWords != null || bounds.maxWords != null || bounds.minChars != null || bounds.maxChars != null;
+  if (!any) return null;
+  if (bounds.minWords != null && bounds.maxWords != null && bounds.minWords > bounds.maxWords) return null;
+  if (bounds.minChars != null && bounds.maxChars != null && bounds.minChars > bounds.maxChars) return null;
+  return bounds;
+}

tests/scorers.test.ts

@@ -13,6 +13,7 @@ import { llmJudgeScorer } from "../src/scorers/llm-judge.js";
 import { latencyScorer } from "../src/scorers/latency.js";
 import { costScorer } from "../src/scorers/cost.js";
 import { rubricScorer } from "../src/scorers/rubric.js";
+import { wordCountScorer } from "../src/scorers/word-count.js";
 
 const provider = new MockProvider();
 
@@ -481,3 +482,56 @@ describe("exact-match trim:false preserves internal whitespace", () => {
     expect(r.passed).toBe(false);
   });
 });
+
+describe("word-count", () => {
+  it("fails a terse answer below minWords with partial credit", async () => {
+    const words = "Paris is a lovely city located in the heart of Europe on the river Seine."
+      .split(/\s+/).length;
+    expect(words).toBeGreaterThanOrEqual(5); // 15+ words, ensure the test stays meaningful
+    void words;
+    const r = await run(
+      wordCountScorer,
+      { type: "word-count", minWords: 5 },
+      ctx("Paris"),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.score).toBeLessThan(1);
+    expect(r.score).toBeGreaterThan(0);
+  });
+
+  it("passes with full credit inside the bounds", async () => {
+    const r = await run(
+      wordCountScorer,
+      { type: "word-count", minWords: 5, maxWords: 100 },
+      ctx("the quick brown fox jumps over the lazy dog next week"),
+    );
+    expect(r.passed).toBe(true);
+    expect(r.score).toBe(1);
+  });
+
+  it("fails above maxWords with partial credit", async () => {
+    const long = Array.from({ length: 60 }, (_, i) => `word${i}`).join(" ");
+    const r = await run(
+      wordCountScorer,
+      { type: "word-count", maxWords: 50 },
+      ctx(long),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.score).toBeLessThan(1);
+  });
+
+  it("enforces char bounds", async () => {
+    const r = await run(
+      wordCountScorer,
+      { type: "word-count", maxChars: 20 },
+      ctx("this sentence is far too long for a twenty character budget"),
+    );
+    expect(r.passed).toBe(false);
+  });
+
+  it("fails when no bounds are configured", async () => {
+    const r = await run(wordCountScorer, { type: "word-count" }, ctx("hello world"));
+    expect(r.passed).toBe(false);
+    expect(r.reason).toMatch(/required/);
+  });
+});