diff --git a/CHANGELOG.md b/CHANGELOG.md index c1d7369..ff30409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ All notable changes to this project are documented here. The format is based on ### Added +- New `tool-call` scorer: passes when the output is a JSON tool call whose + `name` is on the allowlist and whose `arguments` is a plain object, with + optional per-tool argument schemas (`allowedTools`, `schemas`). - New `json-nonempty` scorer: fails schema-valid-but-empty outputs (`{}`, `{"answer": ""}`, all-null payloads) unless the output has at least one non-empty leaf value (`minKeys`, `rejectBlankStrings`, `rejectNulls`). diff --git a/README.md b/README.md index 33318aa..ee9c850 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ score is the weighted mean of the scorer scores. | `contains` | all substrings present (partial credit) | `value` / `values`, `caseSensitive` | | `not-contains` | no banned substring present | `value` / `values`, `caseSensitive` | | `json-schema` | output is valid JSON matching a schema | `schema` | +| `tool-call` | output is a JSON tool call with a name on the allowlist | `allowedTools`, `schemas` | | `json-nonempty` | output is valid JSON with non-empty leaf values | `schema`, `minKeys`, `rejectBlankStrings`, `rejectNulls` | | `embedding-similarity` | cosine similarity >= threshold | `expected`, `threshold` | | `llm-judge` | a judge model scores >= threshold | `criteria`, `expected`, `threshold`, `model` | diff --git a/package-lock.json b/package-lock.json index 97bafe1..3954930 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "evalgate", - "version": "0.1.0", + "name": "@royalpinto007/evalgate", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "evalgate", - "version": "0.1.0", + "name": "@royalpinto007/evalgate", + "version": "0.1.2", "license": "MIT", "dependencies": { "yaml": "^2.4.5" @@ -23,7 +23,7 @@ "vitest": "^1.6.0" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/@esbuild/aix-ppc64": { diff --git a/src/scorers/registry.ts b/src/scorers/registry.ts index d230dc5..43564b7 100644 --- a/src/scorers/registry.ts +++ b/src/scorers/registry.ts @@ -3,6 +3,7 @@ import { exactMatchScorer } from "./exact-match.js"; import { regexScorer } from "./regex.js"; import { containsScorer, notContainsScorer } from "./contains.js"; import { jsonSchemaScorer } from "./json-schema.js"; +import { toolCallScorer } from "./tool-call.js"; import { jsonNonemptyScorer } from "./json-nonempty.js"; import { embeddingSimilarityScorer } from "./embedding-similarity.js"; import { llmJudgeScorer } from "./llm-judge.js"; @@ -48,6 +49,7 @@ export const builtinScorers: Scorer[] = [ containsScorer, notContainsScorer, jsonSchemaScorer, + toolCallScorer, jsonNonemptyScorer, embeddingSimilarityScorer, llmJudgeScorer, diff --git a/src/scorers/tool-call.ts b/src/scorers/tool-call.ts new file mode 100644 index 0000000..02d0355 --- /dev/null +++ b/src/scorers/tool-call.ts @@ -0,0 +1,89 @@ +import type { Scorer, ScoreContext, ScorerSpec } from "../types.js"; +import { result } from "./util.js"; +import { validate, type JsonSchema } from "./json-schema.js"; + +/** + * Passes when the output parses as JSON describing a callable tool action, + * with a `name` on an allowlist and an `arguments` plain object. + * + * Options: + * - `allowedTools`: list of tool names the model may call (required) + * - `schemas`: per-tool JSON Schema applied to `arguments` (optional) + */ +export const toolCallScorer: Scorer = { + type: "tool-call", + score(spec: ScorerSpec, ctx: ScoreContext) { + const allowedTools = spec.allowedTools as string[] | undefined; + if (!Array.isArray(allowedTools) || allowedTools.length === 0) { + return result(spec, { + score: 0, + passed: false, + reason: "allowedTools must be a non-empty list", + }); + } + + let parsed: unknown; + try { + parsed = JSON.parse(ctx.output); + } catch (err) { + return result(spec, { + score: 0, + passed: false, + reason: `output is not valid JSON: ${(err as Error).message}`, + }); + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return result(spec, { + score: 0, + passed: false, + reason: "output must be a JSON object with name and arguments", + }); + } + + const call = parsed as Record; + const name = call.name; + if (typeof name !== "string") { + return result(spec, { + score: 0, + passed: false, + reason: "output must have a string name", + }); + } + if (!allowedTools.includes(name)) { + return result(spec, { + score: 0, + passed: false, + reason: `tool "${name}" is not in allowedTools`, + }); + } + + const args = call.arguments; + if (typeof args !== "object" || args === null || Array.isArray(args)) { + return result(spec, { + score: 0, + passed: false, + reason: "arguments must be a plain object", + }); + } + + const schemas = (spec.schemas ?? {}) as Record; + const schema = schemas[name]; + if (schema) { + const errors = validate(args, schema); + if (errors.length > 0) { + return result(spec, { + score: 0, + passed: false, + reason: errors.join("; "), + }); + } + } + + return result(spec, { + score: 1, + passed: true, + reason: `calls allowed tool "${name}"`, + }); + }, +}; \ No newline at end of file diff --git a/tests/scorers.test.ts b/tests/scorers.test.ts index a0f1e65..4c73640 100644 --- a/tests/scorers.test.ts +++ b/tests/scorers.test.ts @@ -5,6 +5,7 @@ import { exactMatchScorer } from "../src/scorers/exact-match.js"; import { regexScorer } from "../src/scorers/regex.js"; import { containsScorer, notContainsScorer } from "../src/scorers/contains.js"; import { jsonSchemaScorer, validate } from "../src/scorers/json-schema.js"; +import { toolCallScorer } from "../src/scorers/tool-call.js"; import { jsonNonemptyScorer } from "../src/scorers/json-nonempty.js"; import { embeddingSimilarityScorer, cosineSimilarity } from "../src/scorers/embedding-similarity.js"; import { llmJudgeScorer } from "../src/scorers/llm-judge.js"; @@ -203,6 +204,74 @@ describe("json-nonempty", () => { }); }); +describe("tool-call", () => { + it("passes a tool call on the allowlist", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx('{"name": "get_weather", "arguments": {"city": "London"}}'), + ); + expect(r.passed).toBe(true); + expect(r.score).toBe(1); + }); + + it("fails a tool call not on the allowlist", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx('{"name": "rm_rf", "arguments": {}}'), + ); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/not in allowedTools/); + }); + + it("fails non-JSON output", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx("{not json"), + ); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/not valid JSON/); + }); + + it("fails when arguments is not a plain object", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx('{"name": "get_weather", "arguments": "London"}'), + ); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/plain object/); + }); + + it("fails when allowedTools is missing", async () => { + const r = await run(toolCallScorer, { type: "tool-call" }, ctx('{"name": "get_weather", "arguments": {}}')); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/allowedTools/); + }); + + it("validates arguments against a per-tool schema", async () => { + const r = await run( + toolCallScorer, + { + type: "tool-call", + allowedTools: ["get_weather"], + schemas: { + get_weather: { + type: "object", + required: ["city"], + properties: { city: { type: "string" } }, + }, + }, + }, + ctx('{"name": "get_weather", "arguments": {}}'), + ); + expect(r.passed).toBe(false); + expect(r.reason).toContain("required"); + }); +}); + describe("embedding-similarity", () => { it("scores identical text near 1", async () => { const r = await run(