From 319e9de958784e979df2b98c0d171b656ca1bb0e Mon Sep 17 00:00:00 2001 From: "Sharad." Date: Sat, 12 Sep 2026 14:39:37 +0000 Subject: [PATCH] docs(cli): document --concurrency, --junit, and --degrade in help Fixes #4 Co-authored-by: Sharad. --- src/cli/help.ts | 32 ++++++++++++++++++++++++++++++++ src/cli/index.ts | 35 +++-------------------------------- tests/cli-help.test.ts | 10 ++++++++++ 3 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 src/cli/help.ts create mode 100644 tests/cli-help.test.ts diff --git a/src/cli/help.ts b/src/cli/help.ts new file mode 100644 index 0000000..d8e6a61 --- /dev/null +++ b/src/cli/help.ts @@ -0,0 +1,32 @@ +export const CLI_HELP = `evalgate - the build fails when your prompt gets dumber. + +Usage: + evalgate run Run a suite and print a report. + evalgate baseline Run a suite and save it as a baseline. + evalgate compare Run a suite and compare it to a baseline. + evalgate compare Compare two existing result files. + evalgate init [file] Write a starter suite you can edit. + evalgate list List the available scorers and providers. + +Common flags: + --provider Override the provider (default: mock). + --model Override the model. + --tags Only run cases with one of these tags. + --concurrency Run cases in parallel (default: 1). + --out Write the JSON result artifact here. + --md Write a Markdown report here. + --junit Write a JUnit XML report here. + --json Print the JSON result to stdout. + --no-fail Do not exit non-zero on failure/regression. + --degrade Use degraded mock outputs (for regression demos). + +compare flags: + --base Baseline result JSON (required for compare). + --head Candidate result JSON (skips running the suite). + --tolerance Allowed score drop before it counts as a regression. + --comment Upsert a PR comment via the GitHub API (needs token). + +Other: + --version, -v Print version. + --help, -h Print this help. +`; diff --git a/src/cli/index.ts b/src/cli/index.ts index 5e679ac..75dc060 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -15,6 +15,7 @@ import { defaultScorerRegistry } from "../scorers/registry.js"; import { MockProvider } from "../providers/mock.js"; import { contextFromEnv, upsertComment } from "../github.js"; import type { RunResult } from "../types.js"; +import { CLI_HELP } from "./help.js"; const require = createRequire(import.meta.url); @@ -26,36 +27,6 @@ function version(): string { } } -const HELP = `evalgate - the build fails when your prompt gets dumber. - -Usage: - evalgate run Run a suite and print a report. - evalgate baseline Run a suite and save it as a baseline. - evalgate compare Run a suite and compare it to a baseline. - evalgate compare Compare two existing result files. - evalgate init [file] Write a starter suite you can edit. - evalgate list List the available scorers and providers. - -Common flags: - --provider Override the provider (default: mock). - --model Override the model. - --tags Only run cases with one of these tags. - --out Write the JSON result artifact here. - --md Write a Markdown report here. - --json Print the JSON result to stdout. - --no-fail Do not exit non-zero on failure/regression. - -compare flags: - --base Baseline result JSON (required for compare). - --head Candidate result JSON (skips running the suite). - --tolerance Allowed score drop before it counts as a regression. - --comment Upsert a PR comment via the GitHub API (needs token). - -Other: - --version, -v Print version. - --help, -h Print this help. -`; - /** Build a provider registry, allowing a --degrade toggle for the mock. */ function registryFor(args: ParsedArgs) { const degrade = boolFlag(args, "degrade"); @@ -215,7 +186,7 @@ async function main(): Promise { } const command = args._.shift(); if (!command || boolFlag(args, ["help", "h"]) || command === "help") { - console.log(HELP); + console.log(CLI_HELP); return 0; } @@ -232,7 +203,7 @@ async function main(): Promise { return cmdInit(args); default: console.error(`[evalgate] unknown command "${command}"\n`); - console.log(HELP); + console.log(CLI_HELP); return 2; } } diff --git a/tests/cli-help.test.ts b/tests/cli-help.test.ts new file mode 100644 index 0000000..40eb074 --- /dev/null +++ b/tests/cli-help.test.ts @@ -0,0 +1,10 @@ +import { describe, it, expect } from "vitest"; +import { CLI_HELP } from "../src/cli/help.js"; + +describe("CLI help text", () => { + it("documents --concurrency, --junit, and --degrade", () => { + expect(CLI_HELP).toContain("--concurrency"); + expect(CLI_HELP).toContain("--junit"); + expect(CLI_HELP).toContain("--degrade"); + }); +});