Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions src/cli/help.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
export const CLI_HELP = `evalgate - the build fails when your prompt gets dumber.

Usage:
evalgate run <suite> Run a suite and print a report.
evalgate baseline <suite> Run a suite and save it as a baseline.
evalgate compare <suite> Run a suite and compare it to a baseline.
evalgate compare Compare two existing result files.
evalgate init [file] Write a starter suite you can edit.
evalgate list List the available scorers and providers.

Common flags:
--provider <name> Override the provider (default: mock).
--model <name> Override the model.
--tags <a,b> Only run cases with one of these tags.
--concurrency <n> Run cases in parallel (default: 1).
--out <file> Write the JSON result artifact here.
--md <file> Write a Markdown report here.
--junit <file> Write a JUnit XML report here.
--json Print the JSON result to stdout.
--no-fail Do not exit non-zero on failure/regression.
--degrade Use degraded mock outputs (for regression demos).

compare flags:
--base <file> Baseline result JSON (required for compare).
--head <file> Candidate result JSON (skips running the suite).
--tolerance <n> Allowed score drop before it counts as a regression.
--comment Upsert a PR comment via the GitHub API (needs token).

Other:
--version, -v Print version.
--help, -h Print this help.
`;
35 changes: 3 additions & 32 deletions src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { defaultScorerRegistry } from "../scorers/registry.js";
import { MockProvider } from "../providers/mock.js";
import { contextFromEnv, upsertComment } from "../github.js";
import type { RunResult } from "../types.js";
import { CLI_HELP } from "./help.js";

const require = createRequire(import.meta.url);

Expand All @@ -26,36 +27,6 @@ function version(): string {
}
}

const HELP = `evalgate - the build fails when your prompt gets dumber.

Usage:
evalgate run <suite> Run a suite and print a report.
evalgate baseline <suite> Run a suite and save it as a baseline.
evalgate compare <suite> Run a suite and compare it to a baseline.
evalgate compare Compare two existing result files.
evalgate init [file] Write a starter suite you can edit.
evalgate list List the available scorers and providers.

Common flags:
--provider <name> Override the provider (default: mock).
--model <name> Override the model.
--tags <a,b> Only run cases with one of these tags.
--out <file> Write the JSON result artifact here.
--md <file> Write a Markdown report here.
--json Print the JSON result to stdout.
--no-fail Do not exit non-zero on failure/regression.

compare flags:
--base <file> Baseline result JSON (required for compare).
--head <file> Candidate result JSON (skips running the suite).
--tolerance <n> Allowed score drop before it counts as a regression.
--comment Upsert a PR comment via the GitHub API (needs token).

Other:
--version, -v Print version.
--help, -h Print this help.
`;

/** Build a provider registry, allowing a --degrade toggle for the mock. */
function registryFor(args: ParsedArgs) {
const degrade = boolFlag(args, "degrade");
Expand Down Expand Up @@ -215,7 +186,7 @@ async function main(): Promise<number> {
}
const command = args._.shift();
if (!command || boolFlag(args, ["help", "h"]) || command === "help") {
console.log(HELP);
console.log(CLI_HELP);
return 0;
}

Expand All @@ -232,7 +203,7 @@ async function main(): Promise<number> {
return cmdInit(args);
default:
console.error(`[evalgate] unknown command "${command}"\n`);
console.log(HELP);
console.log(CLI_HELP);
return 2;
}
}
Expand Down
10 changes: 10 additions & 0 deletions tests/cli-help.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import { describe, it, expect } from "vitest";
import { CLI_HELP } from "../src/cli/help.js";

describe("CLI help text", () => {
it("documents --concurrency, --junit, and --degrade", () => {
expect(CLI_HELP).toContain("--concurrency");
expect(CLI_HELP).toContain("--junit");
expect(CLI_HELP).toContain("--degrade");
});
});
Loading