diff --git a/CHANGELOG.md b/CHANGELOG.md index 603c76a..6f34311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,11 @@ All notable changes to this project are documented here. The format is based on ### Added +- New `word-count` scorer: scores output length against `minWords`/`maxWords` + (and/or char) bounds with full credit inside and linear falloff outside. - New `refusal` scorer: asserts a reply reads as a refusal using a normalized refusal-phrase set, or — inverted with `expectRefuse: false` — that it reads as a direct answer. - - New `tool-call` scorer: passes when the output is a JSON tool call whose `name` is on the allowlist and whose `arguments` is a plain object, with optional per-tool argument schemas (`allowedTools`, `schemas`). diff --git a/README.md b/README.md index 55015ce..d949d5c 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,7 @@ score is the weighted mean of the scorer scores. | `latency` | call latency within budget | `budgetMs` | | `cost` | estimated call cost within budget | `budgetUsd` (provider must set `costPer1kTokens` so `costUsd` is reported) | | `rubric` | weighted criteria score >= threshold | `criteria[]`, `threshold` | +| `word-count` | output length within bounds (linear falloff outside) | `minWords`, `maxWords`, `minChars`, `maxChars` | Two scorers are pluggable and ship with **deterministic offline fallbacks** so tests and the mock provider need no network: diff --git a/src/scorers/registry.ts b/src/scorers/registry.ts index 52ffe53..f09a132 100644 --- a/src/scorers/registry.ts +++ b/src/scorers/registry.ts @@ -11,6 +11,7 @@ import { llmJudgeScorer } from "./llm-judge.js"; import { latencyScorer } from "./latency.js"; import { costScorer } from "./cost.js"; import { rubricScorer } from "./rubric.js"; +import { wordCountScorer } from "./word-count.js"; /** A registry mapping scorer ids to their implementations. */ export class ScorerRegistry { @@ -58,6 +59,7 @@ export const builtinScorers: Scorer[] = [ latencyScorer, costScorer, rubricScorer, + wordCountScorer, ]; /** Build a registry preloaded with every built-in scorer. */ diff --git a/src/scorers/word-count.ts b/src/scorers/word-count.ts new file mode 100644 index 0000000..53532a0 --- /dev/null +++ b/src/scorers/word-count.ts @@ -0,0 +1,83 @@ +import type { Scorer, ScoreContext, ScorerSpec } from "../types.js"; +import { result } from "./util.js"; + +/** + * Passes when the output's length is within word (and/or char) bounds. + * + * The score is full credit inside the bounds and falls off linearly to zero + * outside them, mirroring {@link latencyScorer}'s curve: the farther past the + * bound, the lower the credit, reaching 0 at `2x` the bound. + * + * Options: + * - `minWords`: minimum word count (optional) + * - `maxWords`: maximum word count (optional) + * - `minChars`: minimum character count (optional) + * - `maxChars`: maximum character count (optional) + */ +export const wordCountScorer: Scorer = { + type: "word-count", + score(spec: ScorerSpec, ctx: ScoreContext) { + const bounds = readBounds(spec); + if (!bounds) { + return result(spec, { + score: 0, + passed: false, + reason: "at least one of minWords/maxWords/minChars/maxChars required", + }); + } + const words = ctx.output.trim().split(/\s+/).filter(Boolean).length; + const chars = [...ctx.output].length; + + const tests = [ + { kind: "words", value: words, min: bounds.minWords, max: bounds.maxWords }, + { kind: "chars", value: chars, min: bounds.minChars, max: bounds.maxChars }, + ]; + + const offending = tests.find((t) => t.min != null || t.max != null) + ? tests.filter((t) => t.min != null || t.max != null) + : []; + for (const t of offending) { + if (t.min != null && t.value < t.min) return falloff(spec, t.kind, t.value, `below min ${t.min}`, t.min, false); + if (t.max != null && t.value > t.max) return falloff(spec, t.kind, t.value, `above max ${t.max}`, t.max, true); + } + + return result(spec, { + score: 1, + passed: true, + reason: `${words} word(s), ${chars} char(s) within bounds`, + }); + }, +}; + +function falloff( + spec: ScorerSpec, + kind: string, + value: number, + detail: string, + bound: number, + isMax: boolean, +) { + const damage = isMax ? (value - bound) / bound : (bound - value) / bound; + const score = Math.max(0, 1 - damage); + return result(spec, { + score, + passed: false, + reason: `${kind}: ${value}, ${detail} (credit ${score.toFixed(2)})`, + }); +} + +function readBounds(spec: ScorerSpec): { minWords?: number; maxWords?: number; minChars?: number; maxChars?: number } | null { + const num = (v: unknown): number | undefined => + typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined; + const bounds = { + minWords: num(spec.minWords), + maxWords: num(spec.maxWords), + minChars: num(spec.minChars), + maxChars: num(spec.maxChars), + }; + const any = bounds.minWords != null || bounds.maxWords != null || bounds.minChars != null || bounds.maxChars != null; + if (!any) return null; + if (bounds.minWords != null && bounds.maxWords != null && bounds.minWords > bounds.maxWords) return null; + if (bounds.minChars != null && bounds.maxChars != null && bounds.minChars > bounds.maxChars) return null; + return bounds; +} \ No newline at end of file diff --git a/tests/scorers.test.ts b/tests/scorers.test.ts index ecf01b5..e12b9bb 100644 --- a/tests/scorers.test.ts +++ b/tests/scorers.test.ts @@ -13,6 +13,7 @@ import { llmJudgeScorer } from "../src/scorers/llm-judge.js"; import { latencyScorer } from "../src/scorers/latency.js"; import { costScorer } from "../src/scorers/cost.js"; import { rubricScorer } from "../src/scorers/rubric.js"; +import { wordCountScorer } from "../src/scorers/word-count.js"; const provider = new MockProvider(); @@ -481,3 +482,56 @@ describe("exact-match trim:false preserves internal whitespace", () => { expect(r.passed).toBe(false); }); }); + +describe("word-count", () => { + it("fails a terse answer below minWords with partial credit", async () => { + const words = "Paris is a lovely city located in the heart of Europe on the river Seine." + .split(/\s+/).length; + expect(words).toBeGreaterThanOrEqual(5); // 15+ words, ensure the test stays meaningful + void words; + const r = await run( + wordCountScorer, + { type: "word-count", minWords: 5 }, + ctx("Paris"), + ); + expect(r.passed).toBe(false); + expect(r.score).toBeLessThan(1); + expect(r.score).toBeGreaterThan(0); + }); + + it("passes with full credit inside the bounds", async () => { + const r = await run( + wordCountScorer, + { type: "word-count", minWords: 5, maxWords: 100 }, + ctx("the quick brown fox jumps over the lazy dog next week"), + ); + expect(r.passed).toBe(true); + expect(r.score).toBe(1); + }); + + it("fails above maxWords with partial credit", async () => { + const long = Array.from({ length: 60 }, (_, i) => `word${i}`).join(" "); + const r = await run( + wordCountScorer, + { type: "word-count", maxWords: 50 }, + ctx(long), + ); + expect(r.passed).toBe(false); + expect(r.score).toBeLessThan(1); + }); + + it("enforces char bounds", async () => { + const r = await run( + wordCountScorer, + { type: "word-count", maxChars: 20 }, + ctx("this sentence is far too long for a twenty character budget"), + ); + expect(r.passed).toBe(false); + }); + + it("fails when no bounds are configured", async () => { + const r = await run(wordCountScorer, { type: "word-count" }, ctx("hello world")); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/required/); + }); +});