diff --git a/README.md b/README.md index 883a5f1..ec3bf15 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ score is the weighted mean of the scorer scores. | `embedding-similarity` | cosine similarity >= threshold | `expected`, `threshold` | | `llm-judge` | a judge model scores >= threshold | `criteria`, `expected`, `threshold`, `model` | | `latency` | call latency within budget | `budgetMs` | -| `cost` | estimated call cost within budget | `budgetUsd` | +| `cost` | estimated call cost within budget | `budgetUsd` (provider must set `costPer1kTokens` so `costUsd` is reported) | | `rubric` | weighted criteria score >= threshold | `criteria[]`, `threshold` | Two scorers are pluggable and ship with **deterministic offline fallbacks** so tests and diff --git a/src/scorers/cost.ts b/src/scorers/cost.ts index 5458d86..5cdd977 100644 --- a/src/scorers/cost.ts +++ b/src/scorers/cost.ts @@ -6,6 +6,9 @@ import { result } from "./util.js"; * * Options: * - `budgetUsd`: the maximum acceptable cost in USD (required) + * + * Requires the provider to report `costUsd` (configure `costPer1kTokens` on OpenAI-compatible + * and Anthropic providers). */ export const costScorer: Scorer = { type: "cost", @@ -14,7 +17,14 @@ export const costScorer: Scorer = { if (!Number.isFinite(budgetUsd) || budgetUsd <= 0) { return result(spec, { score: 0, passed: false, reason: "invalid or missing budgetUsd" }); } - const actual = ctx.response.costUsd ?? 0; + if (ctx.response.costUsd === undefined) { + return result(spec, { + score: 0, + passed: false, + reason: "provider reported no cost (set costPer1kTokens on the provider)", + }); + } + const actual = ctx.response.costUsd; const passed = actual <= budgetUsd; const score = actual <= budgetUsd ? 1 : Math.max(0, 1 - (actual - budgetUsd) / budgetUsd); return result(spec, { diff --git a/tests/scorers.test.ts b/tests/scorers.test.ts index 78bf373..e7f15e5 100644 --- a/tests/scorers.test.ts +++ b/tests/scorers.test.ts @@ -176,6 +176,12 @@ describe("latency and cost budgets", () => { const r = await run(costScorer, { type: "cost", budgetUsd: 0.00001 }, ctx("x", { costUsd: 0.001 })); expect(r.passed).toBe(false); }); + it("cost fails when provider reported no cost", async () => { + const r = await run(costScorer, { type: "cost", budgetUsd: 1 }, ctx("x", { costUsd: undefined })); + expect(r.passed).toBe(false); + expect(r.score).toBe(0); + expect(r.reason).toMatch(/no cost/i); + }); }); describe("rubric", () => {