diff --git a/src/cli/index.ts b/src/cli/index.ts index 5e679ac..3293ac4 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -64,6 +64,17 @@ function registryFor(args: ParsedArgs) { return registry; } +/** Shared runSuite options for run, baseline, and compare. */ +export function runOptionsFrom(args: ParsedArgs) { + return { + providers: registryFor(args), + defaultProvider: strFlag(args, "provider"), + defaultModel: strFlag(args, "model"), + concurrency: numFlag(args, "concurrency", 1), + filterTags: strFlag(args, "tags")?.split(",").map((s) => s.trim()).filter(Boolean), + }; +} + async function writeArtifacts(args: ParsedArgs, run: RunResult): Promise { const out = strFlag(args, "out"); if (out) await writeFile(out, JSON.stringify(run, null, 2) + "\n", "utf8"); @@ -77,13 +88,7 @@ async function cmdRun(args: ParsedArgs): Promise { const suitePath = args._[0]; if (!suitePath) throw new Error("run requires a suite path"); const suite = await loadSuite(suitePath); - const run = await runSuite(suite, { - providers: registryFor(args), - defaultProvider: strFlag(args, "provider"), - defaultModel: strFlag(args, "model"), - concurrency: numFlag(args, "concurrency", 1), - filterTags: strFlag(args, "tags")?.split(",").map((s) => s.trim()).filter(Boolean), - }); + const run = await runSuite(suite, runOptionsFrom(args)); if (boolFlag(args, "json")) console.log(JSON.stringify(run, null, 2)); else console.log(renderRunTerminal(run)); @@ -92,16 +97,12 @@ async function cmdRun(args: ParsedArgs): Promise { return run.passed || boolFlag(args, "no-fail") ? 0 : 1; } -async function cmdBaseline(args: ParsedArgs): Promise { +export async function cmdBaseline(args: ParsedArgs): Promise { const suitePath = args._[0]; if (!suitePath) throw new Error("baseline requires a suite path"); const out = strFlag(args, "out") ?? "evalgate.baseline.json"; const suite = await loadSuite(suitePath); - const run = await runSuite(suite, { - providers: registryFor(args), - defaultProvider: strFlag(args, "provider"), - defaultModel: strFlag(args, "model"), - }); + const run = await runSuite(suite, runOptionsFrom(args)); await writeFile(out, JSON.stringify(run, null, 2) + "\n", "utf8"); console.log(`Saved baseline for "${run.suite}" -> ${out} (mean score ${(run.score * 100).toFixed(1)}%)`); return 0; @@ -123,11 +124,7 @@ export async function cmdCompare(args: ParsedArgs): Promise { head = await loadResult(headPath); } else if (suitePath) { const suite = await loadSuite(suitePath); - head = await runSuite(suite, { - providers: registryFor(args), - defaultProvider: strFlag(args, "provider"), - defaultModel: strFlag(args, "model"), - }); + head = await runSuite(suite, runOptionsFrom(args)); const out = strFlag(args, "out"); if (out) await writeFile(out, JSON.stringify(head, null, 2) + "\n", "utf8"); } else { diff --git a/tests/cli-run-options.test.ts b/tests/cli-run-options.test.ts new file mode 100644 index 0000000..7c7da4a --- /dev/null +++ b/tests/cli-run-options.test.ts @@ -0,0 +1,52 @@ +import { describe, it, expect } from "vitest"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseArgs } from "../src/cli/args.js"; +import { runOptionsFrom, cmdBaseline } from "../src/cli/index.js"; + +describe("runOptionsFrom", () => { + it("returns concurrency and tag filter from CLI flags", () => { + const args = parseArgs(["suite.yaml", "--tags", "smoke,fast", "--concurrency", "4", "--degrade"]); + const opts = runOptionsFrom(args); + expect(opts.concurrency).toBe(4); + expect(opts.filterTags).toEqual(["smoke", "fast"]); + }); +}); + +describe("cmdBaseline", () => { + it("honors --tags when building a baseline", async () => { + const dir = await mkdtemp(join(tmpdir(), "evalgate-baseline-")); + const suitePath = join(dir, "suite.eval.yaml"); + const outPath = join(dir, "base.json"); + await writeFile( + suitePath, + `name: tagged +provider: mock +model: mock +cases: + - id: smoke-case + tags: [smoke] + input: { prompt: "exactly: hi" } + expected: "hi" + scorers: [{ type: exact-match }] + - id: other + input: { prompt: "exactly: bye" } + expected: "bye" + scorers: [{ type: exact-match }] +`, + "utf8", + ); + + await cmdBaseline({ + _: [suitePath], + flags: { tags: "smoke", out: outPath }, + }); + + const saved = JSON.parse(await readFile(outPath, "utf8")) as { total: number; cases: { id: string }[] }; + expect(saved.total).toBe(1); + expect(saved.cases[0]!.id).toBe("smoke-case"); + + await rm(dir, { recursive: true, force: true }); + }); +});