From 52fdd2aeb7a0baeb4f0e5ef3b3819eec0667b446 Mon Sep 17 00:00:00 2001 From: MapleTheBot Date: Thu, 17 Sep 2026 08:53:02 +0530 Subject: [PATCH] feat: add tool-call scorer Parses output as JSON and requires a tool call: name on the allowedTools allowlist and arguments as a plain object, with optional per-tool argument schemas reusing validate(). Closes #39 --- CHANGELOG.md | 8 ++++ README.md | 1 + src/scorers/registry.ts | 2 + src/scorers/tool-call.ts | 89 ++++++++++++++++++++++++++++++++++++++++ tests/scorers.test.ts | 69 +++++++++++++++++++++++++++++++ 5 files changed, 169 insertions(+) create mode 100644 src/scorers/tool-call.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dcec3a..7528926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Added + +- New `tool-call` scorer: passes when the output is a JSON tool call whose + `name` is on the allowlist and whose `arguments` is a plain object, with + optional per-tool argument schemas (`allowedTools`, `schemas`). + ## [0.1.2] - 2026-09-05 ### Fixed diff --git a/README.md b/README.md index b744fce..3612e17 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ score is the weighted mean of the scorer scores. | `contains` | all substrings present (partial credit) | `value` / `values`, `caseSensitive` | | `not-contains` | no banned substring present | `value` / `values`, `caseSensitive` | | `json-schema` | output is valid JSON matching a schema | `schema` | +| `tool-call` | output is a JSON tool call with a name on the allowlist | `allowedTools`, `schemas` | | `embedding-similarity` | cosine similarity >= threshold | `expected`, `threshold` | | `llm-judge` | a judge model scores >= threshold | `criteria`, `expected`, `threshold`, `model` | | `latency` | call latency within budget | `budgetMs` | diff --git a/src/scorers/registry.ts b/src/scorers/registry.ts index 4f6a4cf..11970cb 100644 --- a/src/scorers/registry.ts +++ b/src/scorers/registry.ts @@ -3,6 +3,7 @@ import { exactMatchScorer } from "./exact-match.js"; import { regexScorer } from "./regex.js"; import { containsScorer, notContainsScorer } from "./contains.js"; import { jsonSchemaScorer } from "./json-schema.js"; +import { toolCallScorer } from "./tool-call.js"; import { embeddingSimilarityScorer } from "./embedding-similarity.js"; import { llmJudgeScorer } from "./llm-judge.js"; import { latencyScorer } from "./latency.js"; @@ -47,6 +48,7 @@ export const builtinScorers: Scorer[] = [ containsScorer, notContainsScorer, jsonSchemaScorer, + toolCallScorer, embeddingSimilarityScorer, llmJudgeScorer, latencyScorer, diff --git a/src/scorers/tool-call.ts b/src/scorers/tool-call.ts new file mode 100644 index 0000000..02d0355 --- /dev/null +++ b/src/scorers/tool-call.ts @@ -0,0 +1,89 @@ +import type { Scorer, ScoreContext, ScorerSpec } from "../types.js"; +import { result } from "./util.js"; +import { validate, type JsonSchema } from "./json-schema.js"; + +/** + * Passes when the output parses as JSON describing a callable tool action, + * with a `name` on an allowlist and an `arguments` plain object. + * + * Options: + * - `allowedTools`: list of tool names the model may call (required) + * - `schemas`: per-tool JSON Schema applied to `arguments` (optional) + */ +export const toolCallScorer: Scorer = { + type: "tool-call", + score(spec: ScorerSpec, ctx: ScoreContext) { + const allowedTools = spec.allowedTools as string[] | undefined; + if (!Array.isArray(allowedTools) || allowedTools.length === 0) { + return result(spec, { + score: 0, + passed: false, + reason: "allowedTools must be a non-empty list", + }); + } + + let parsed: unknown; + try { + parsed = JSON.parse(ctx.output); + } catch (err) { + return result(spec, { + score: 0, + passed: false, + reason: `output is not valid JSON: ${(err as Error).message}`, + }); + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return result(spec, { + score: 0, + passed: false, + reason: "output must be a JSON object with name and arguments", + }); + } + + const call = parsed as Record; + const name = call.name; + if (typeof name !== "string") { + return result(spec, { + score: 0, + passed: false, + reason: "output must have a string name", + }); + } + if (!allowedTools.includes(name)) { + return result(spec, { + score: 0, + passed: false, + reason: `tool "${name}" is not in allowedTools`, + }); + } + + const args = call.arguments; + if (typeof args !== "object" || args === null || Array.isArray(args)) { + return result(spec, { + score: 0, + passed: false, + reason: "arguments must be a plain object", + }); + } + + const schemas = (spec.schemas ?? {}) as Record; + const schema = schemas[name]; + if (schema) { + const errors = validate(args, schema); + if (errors.length > 0) { + return result(spec, { + score: 0, + passed: false, + reason: errors.join("; "), + }); + } + } + + return result(spec, { + score: 1, + passed: true, + reason: `calls allowed tool "${name}"`, + }); + }, +}; \ No newline at end of file diff --git a/tests/scorers.test.ts b/tests/scorers.test.ts index 6f4afe7..a32d447 100644 --- a/tests/scorers.test.ts +++ b/tests/scorers.test.ts @@ -5,6 +5,7 @@ import { exactMatchScorer } from "../src/scorers/exact-match.js"; import { regexScorer } from "../src/scorers/regex.js"; import { containsScorer, notContainsScorer } from "../src/scorers/contains.js"; import { jsonSchemaScorer, validate } from "../src/scorers/json-schema.js"; +import { toolCallScorer } from "../src/scorers/tool-call.js"; import { embeddingSimilarityScorer, cosineSimilarity } from "../src/scorers/embedding-similarity.js"; import { llmJudgeScorer } from "../src/scorers/llm-judge.js"; import { latencyScorer } from "../src/scorers/latency.js"; @@ -136,6 +137,74 @@ describe("json-schema", () => { }); }); +describe("tool-call", () => { + it("passes a tool call on the allowlist", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx('{"name": "get_weather", "arguments": {"city": "London"}}'), + ); + expect(r.passed).toBe(true); + expect(r.score).toBe(1); + }); + + it("fails a tool call not on the allowlist", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx('{"name": "rm_rf", "arguments": {}}'), + ); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/not in allowedTools/); + }); + + it("fails non-JSON output", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx("{not json"), + ); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/not valid JSON/); + }); + + it("fails when arguments is not a plain object", async () => { + const r = await run( + toolCallScorer, + { type: "tool-call", allowedTools: ["get_weather"] }, + ctx('{"name": "get_weather", "arguments": "London"}'), + ); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/plain object/); + }); + + it("fails when allowedTools is missing", async () => { + const r = await run(toolCallScorer, { type: "tool-call" }, ctx('{"name": "get_weather", "arguments": {}}')); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/allowedTools/); + }); + + it("validates arguments against a per-tool schema", async () => { + const r = await run( + toolCallScorer, + { + type: "tool-call", + allowedTools: ["get_weather"], + schemas: { + get_weather: { + type: "object", + required: ["city"], + properties: { city: { type: "string" } }, + }, + }, + }, + ctx('{"name": "get_weather", "arguments": {}}'), + ); + expect(r.passed).toBe(false); + expect(r.reason).toContain("required"); + }); +}); + describe("embedding-similarity", () => { it("scores identical text near 1", async () => { const r = await run(