diff --git a/src/suite.ts b/src/suite.ts index 214c139..f2644bb 100644 --- a/src/suite.ts +++ b/src/suite.ts @@ -1,6 +1,6 @@ import { readFile } from "node:fs/promises"; import { parse as parseYaml } from "yaml"; -import type { EvalCase, EvalSuite, ScorerSpec } from "./types.js"; +import type { EvalCase, EvalSuite, Message, ScorerSpec } from "./types.js"; /** Raised when a suite fails structural validation. */ export class SuiteValidationError extends Error { @@ -32,6 +32,25 @@ function assert(condition: unknown, message: string): asserts condition { if (!condition) throw new SuiteValidationError(message); } +const MESSAGE_ROLES = new Set(["system", "user", "assistant"]); + +function validateMessages( + messages: unknown[], + caseId: string, +): EvalCase["input"]["messages"] { + assert(messages.length > 0, `case "${caseId}" input.messages must not be empty`); + return messages.map((m, i) => { + assert(m && typeof m === "object", `case "${caseId}" messages[${i}] must be an object`); + const msg = m as Record; + assert( + typeof msg.role === "string" && MESSAGE_ROLES.has(msg.role as Message["role"]), + `case "${caseId}" messages[${i}].role must be system, user, or assistant`, + ); + assert(typeof msg.content === "string", `case "${caseId}" messages[${i}].content must be a string`); + return { role: msg.role as Message["role"], content: msg.content }; + }); +} + /** Validate an arbitrary object into a typed {@link EvalSuite}. */ export function validateSuite(data: unknown): EvalSuite { assert(data && typeof data === "object", "suite must be an object"); @@ -104,6 +123,9 @@ function validateCase(data: unknown, index: number, ids: Set): EvalCase const hasPrompt = typeof input.prompt === "string"; const hasMessages = Array.isArray(input.messages); assert(hasPrompt || hasMessages, `case "${c.id}" input needs a prompt or messages`); + if (hasPrompt) { + assert((input.prompt as string).length > 0, `case "${c.id}" input.prompt must not be empty`); + } assert(Array.isArray(c.scorers), `case "${c.id}" requires a scorers array`); assert((c.scorers as unknown[]).length > 0, `case "${c.id}" needs at least one scorer`); @@ -115,7 +137,7 @@ function validateCase(data: unknown, index: number, ids: Set): EvalCase description: typeof c.description === "string" ? c.description : undefined, input: { prompt: hasPrompt ? (input.prompt as string) : undefined, - messages: hasMessages ? (input.messages as EvalCase["input"]["messages"]) : undefined, + messages: hasMessages ? validateMessages(input.messages as unknown[], c.id as string) : undefined, }, model: typeof c.model === "string" ? c.model : undefined, provider: typeof c.provider === "string" ? c.provider : undefined, diff --git a/tests/suite.test.ts b/tests/suite.test.ts index 70dcc91..63bf023 100644 --- a/tests/suite.test.ts +++ b/tests/suite.test.ts @@ -97,6 +97,39 @@ cases: ).toThrow(new RegExp(`suite\\.${field}`)); }); + it("rejects an empty-string prompt", () => { + expect(() => + validateSuite({ + name: "d", + cases: [{ id: "x", input: { prompt: "" }, scorers: [{ type: "regex" }] }], + }), + ).toThrow(/prompt/); + }); + + it("rejects an empty messages array", () => { + expect(() => + validateSuite({ + name: "d", + cases: [{ id: "x", input: { messages: [] }, scorers: [{ type: "regex" }] }], + }), + ).toThrow(/messages/); + }); + + it("rejects unknown message roles", () => { + expect(() => + validateSuite({ + name: "d", + cases: [ + { + id: "x", + input: { messages: [{ role: "bogus", content: "hi" }] }, + scorers: [{ type: "regex" }], + }, + ], + }), + ).toThrow(/role/); + }); + it.each([ ["temperature", Number.NaN], ["maxTokens", -1],