diff --git a/README.md b/README.md index 68214d5..3e0971f 100644 --- a/README.md +++ b/README.md @@ -355,7 +355,7 @@ scoring live in the template and the instance. | Action | Description | |--------|-------------| -| `init [dir]` | Materialize a fresh instance from the template (no git history, fresh `git init`), register the detected base repo, install runner dependencies | +| `init [dir]` | Materialize a fresh instance from the template (no git history, fresh `git init`), register the detected base repo, install runner dependencies, check the machine (`bench doctor`). Without `dir`, inside a product repo the instance goes to `../-bench` | | `update [dir]` | Upgrade the instance to a newer template: runtime zone replaced wholesale, skills proposed as a reviewable diff, company content untouched | | Flag | Description | @@ -363,30 +363,42 @@ scoring live in the template and the instance. | `--template-version ` | Template tag to install (default: latest) | | `--tool ` | Agent tool for skill placement (`claude-code`, `cursor`, `copilot`, `codex`, `devin-desktop`, `gemini`, `kiro`, `factory`, `generic`) | | `--yes` | Run non-interactively, accepting defaults | -| `--deep` | Clone the detected base repo with full history (default: shallow, HEAD only) | +| `--shallow` | Clone the detected base repo at HEAD only (default: full history, which PR mining reads) | +| `--deep` | Full history — the default since template 0.34.0; kept for older templates | ```bash # Create an instance next to your product repo (run inside it to auto-register) -10x bench-kit init my-benchmark +cd my-product && 10x bench-kit init # → ../my-product-bench + +# Non-interactive (CI, agent shells): pick the tool explicitly +10x bench-kit init --tool claude-code --yes # Pin the template version -10x bench-kit init my-benchmark --template-version v0.8.0 +10x bench-kit init my-benchmark --template-version v0.33.0 -# Keep the full history in the base repo clone under .repos/ -10x bench-kit init my-benchmark --deep +# HEAD-only base repo clone (huge history you do not want to copy) +10x bench-kit init my-benchmark --shallow # Upgrade an existing instance to the latest template 10x bench-kit update ``` A detected base repo is cloned into the instance's `.repos//` (gitignored) -as a working copy for the authoring skills. That clone is **shallow** — HEAD only -— because the skills read the file tree, not the history; `--deep` keeps the full -history, and `git fetch --unshallow` inside the clone adds it later. +as a working copy for the authoring skills. It keeps the **full history**, because +PR mining (the default source of tasks) reads it; cloning from your local working +copy hardlinks the objects, so it takes seconds. `--shallow` keeps HEAD only, and +`git fetch --unshallow` inside the clone adds the history later. + +When init finishes, it prints the exact first prompt to give your agent inside the +instance (and `data.nextStep` in `--json` mode). + +Maintainers can point init/update at an unreleased template with +`BENCH_KIT_TEMPLATE_URL= 10x bench-kit init … --template-version `. Re-running `init` on an existing instance is a **repair**: missing template -files are restored, company content is never touched. After `update`, run the -instance's `bench validate` to confirm the content still matches the new kit. +files are restored, company content is never touched. `update` ends the same +way as init: with the prompt for your agent (`bench-status`, which re-validates +the instance against the new kit) and a link to the installed release's notes. ### Global Flags diff --git a/package.json b/package.json index 72016ea..97e111f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@przeprogramowani/10x-cli", - "version": "1.30.2", + "version": "1.31.0", "description": "Open-source CLI for 10xDevs course content", "repository": { "type": "git", diff --git a/src/commands/bench-kit.ts b/src/commands/bench-kit.ts index 2fd5b63..9c85903 100644 --- a/src/commands/bench-kit.ts +++ b/src/commands/bench-kit.ts @@ -11,7 +11,9 @@ * version, a template that changes its layout ships its own migration. * * Trust boundary: the bootstrap is executed only from a clone of - * TEMPLATE_REPO_URL and only from a ref the user asked for. (npm ci + * TEMPLATE_REPO_URL (or the BENCH_KIT_TEMPLATE_URL the user exported — + * a maintainer hook for testing an unreleased kit branch) and only from a + * ref the user asked for. (npm ci * inside the clone already runs lifecycle scripts, so this executes no * new class of code.) */ @@ -19,7 +21,7 @@ import { spawn } from "node:child_process"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, join, resolve } from "node:path"; +import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; import type { CAC } from "cac"; import { ExitCodes, @@ -40,6 +42,16 @@ import { export const TEMPLATE_REPO_URL = "https://github.com/przeprogramowani/10x-bench-kit"; +/** + * Template source: BENCH_KIT_TEMPLATE_URL when set (any git URL or local + * path — e.g. a kit checkout plus `--template-version `), else the + * official repo. Mirrors API_BASE_URL: an explicit, user-set override. + */ +export function templateRepoUrl(): string { + const override = process.env.BENCH_KIT_TEMPLATE_URL?.trim(); + return override === undefined || override === "" ? TEMPLATE_REPO_URL : override; +} + /** The kit's side of the contract — the only kit-internal path the CLI knows. */ export const BOOTSTRAP_ENTRY = join(".bench-kit", "bootstrap", "index.mjs"); @@ -73,8 +85,10 @@ export interface BootstrapRequest { templateRef: string; templateSource: string; detectedBaseRepo: DetectedBaseRepo | null; - /** True with `--deep`: the base repo clone keeps full history (default: shallow). */ + /** True with `--deep`: full history (the kit's default since 0.34.0; older kits default shallow). */ deepClone?: boolean; + /** True with `--shallow`: clone HEAD only (kits 0.34.0+). */ + shallowClone?: boolean; now: string; } @@ -115,13 +129,67 @@ export interface BootstrapResponse { zones?: { workflows?: SyncCounts; skills?: SyncCounts; shared?: SyncCounts }; warnings?: string[]; nextSteps?: string[]; + /** + * The first thing to ask the agent in the new instance. The kit owns it + * (it knows its skill names); absent in templates older than 0.34.0, + * where the CLI falls back to FALLBACK_FIRST_PROMPT. + */ + nextStep?: { skill: string; prompt: string }; + /** `bench doctor` run by the kit right after init (0.34.0+); null when it could not run. */ + doctor?: DoctorSummary | null; +} + +export interface DoctorSummary { + missing: number; + warns: number; + ok: string[]; + problems: { name: string; status: string; detail: string; fix?: string }[]; +} + +/** + * Environment check rendered BEFORE the next step: an agent session only + * sees the env it was launched with, so a missing key must be fixed here, + * not discovered inside the agent. + */ +function doctorLines(doctor: DoctorSummary | null | undefined): string[] { + if (doctor == null) return []; + const missing = doctor.problems.filter((p) => p.status !== "WARN"); + const lines = [ + "", + missing.length === 0 + ? `Environment ready: ${doctor.ok.length} checks passed${doctor.warns > 0 ? `, ${doctor.warns} optional warning${doctor.warns === 1 ? "" : "s"} (publication)` : ""}.` + : `Environment: ${missing.length} problem${missing.length === 1 ? "" : "s"} to fix BEFORE opening the agent (it inherits this shell's env):`, + ]; + for (const problem of missing) { + lines.push(` ✗ ${problem.name}: ${problem.detail}${problem.fix ? `\n → ${problem.fix}` : ""}`); + } + return lines; +} + +/** First prompt for templates that do not send `nextStep` (0.33.0 and older). */ +/** The agent prompt after an update: bench-status re-checks the instance (validate included). */ +export const UPDATE_PROMPT = "Use bench-status to check this instance after the bench-kit update, then commit the update as bench changes proposes."; + +/** + * Link to the installed release's notes — only for a browsable https source + * (a file:// or ssh override has no web view), pinned to the tag rather than + * a branch name that may not exist. + */ +export function releaseNotesLines(version: string | undefined): string[] { + const base = templateRepoUrl().replace(/\.git$/, ""); + if (!version || !base.startsWith("https://")) return []; + return [`Release notes: ${base}/blob/v${version}/CHANGELOG.md`]; } +export const FALLBACK_FIRST_PROMPT = + "Use bench-measure to set up this instance and run the first smoke measurement."; + interface BenchKitFlags extends GlobalFlags { templateVersion?: string; tool?: string; yes?: boolean; deep?: boolean; + shallow?: boolean; } /** @@ -169,10 +237,12 @@ export function registerBenchKitCommand(cli: CAC): void { .option("--template-version ", "Template tag to install (default: latest)") .option("--tool ", `Agent tool for skill placement (${Object.keys(PROFILES).join(", ")})`) .option("--yes", "Run non-interactively, accepting defaults") - .option("--deep", "Clone the detected base repo with full history (default: shallow, HEAD only)") - .example("10x bench-kit init my-benchmark") - .example("10x bench-kit init my-benchmark --template-version v0.1.0") - .example("10x bench-kit init my-benchmark --deep") + .option("--shallow", "Clone the detected base repo at HEAD only (default: full history, which PR mining reads)") + .option("--deep", "Clone the detected base repo with full history (the default for templates 0.34.0+)") + .example("10x bench-kit init # inside your product repo: creates ../-bench") + .example("10x bench-kit init --tool claude-code --yes") + .example("10x bench-kit init ../my-bench --template-version v0.33.0") + .example("10x bench-kit init --shallow") .example("10x bench-kit update") .action(async (action: string, dir: string | undefined, options: BenchKitFlags) => { const ctx = resolveContext(options); @@ -200,11 +270,22 @@ export async function runBenchKitInit( options: BenchKitFlags, deps: BenchKitDeps = defaultDeps, ): Promise { - const targetDir = resolve(dirArg ?? "."); const requestedRef = normalizeRef(ctx, options.templateVersion); await preflight(ctx, deps); + // Run from inside a product repo with no dir: the instance goes NEXT to + // the repo (`../-bench`). Inside it, it would be an untracked + // nested repo that one `git add -A` commits into the product. + const cwdIsInstance = dirArg === undefined && readInstanceVersion(process.cwd()) !== null; + const detected = cwdIsInstance ? null : await deps.detectBaseRepo(process.cwd()); + const targetDir = + dirArg !== undefined + ? resolve(dirArg) + : detected !== null + ? join(dirname(detected.rootDir), `${detected.name}-bench`) + : resolve("."); + const existingVersion = readInstanceVersion(targetDir); if (existingVersion !== null && requestedRef !== null) { outputError( @@ -227,9 +308,8 @@ export async function runBenchKitInit( // Running init from inside a product repo is the common flow — detect // it here (network side), let the bootstrap decide what to register. - const detected = existingVersion !== null ? null : await deps.detectBaseRepo(process.cwd()); let detectedBaseRepo: DetectedBaseRepo | null = null; - if (detected !== null) { + if (detected !== null && existingVersion === null) { const https = toHttpsUrl(detected.url); const httpsReachable = https !== null && (await deps.remoteReachable(https)); if (httpsReachable) verbose(ctx, `repo answers over https — bootstrap may prefer ${https}`); @@ -249,9 +329,9 @@ export async function runBenchKitInit( toolProfiles: allToolProfiles(), cwd: process.cwd(), templateRef: requestedRef ?? "latest", - templateSource: TEMPLATE_REPO_URL, + templateSource: templateRepoUrl(), detectedBaseRepo, - ...(options.deep === true ? { deepClone: true } : {}), + ...(options.shallow === true ? { shallowClone: true } : { deepClone: true }), now: deps.now().toISOString(), }; verbose(ctx, `running template bootstrap (${BOOTSTRAP_ENTRY})`); @@ -289,6 +369,7 @@ export async function runBenchKitInit( } const repair = res.mode === "repair"; + const firstPrompt = res.nextStep?.prompt ?? FALLBACK_FIRST_PROMPT; const toolId = res.tool ?? toolChoice.id; const toolName = PROFILES[toolId]?.displayName ?? toolId; const skillRoot = res.skillRoot ?? skillRootFor(toolId); @@ -332,7 +413,10 @@ export async function runBenchKitInit( ? "Initialized a fresh git repository with an initial commit." : "Initialized a fresh git repository (initial commit skipped — commit the files yourself).", ...(res.warnings ?? []).map((warning) => `Warning: ${warning}`), - `Next: cd ${targetDir} and run skill *bench-measure* in your agent.`, + ...doctorLines(res.doctor), + "", + `Next: cd ${displayDir(targetDir)}, open it in ${toolName} and ask:`, + ` "${firstPrompt}"`, ]; output(ctx, humanLines.join("\n"), { dir: targetDir, @@ -349,6 +433,8 @@ export async function runBenchKitInit( gitInitialized: res.gitInitialized ?? false, committed: res.committed ?? false, warnings: res.warnings ?? [], + nextStep: repair ? null : { prompt: firstPrompt, dir: targetDir }, + doctor: res.doctor ?? null, }); } finally { rmSync(scratch, { recursive: true, force: true }); @@ -414,7 +500,7 @@ export async function runBenchKitUpdate( toolProfiles: allToolProfiles(), cwd: process.cwd(), templateRef: requestedRef ?? "latest", - templateSource: TEMPLATE_REPO_URL, + templateSource: templateRepoUrl(), detectedBaseRepo: null, now: deps.now().toISOString(), }; @@ -457,7 +543,9 @@ export async function runBenchKitUpdate( ` ${"AGENTS.md".padEnd(23)}${describeSync(zones.shared)} — proposal, review before committing`, " tasks/, evaluation-pool/, bench.config.yaml untouched (company zone)", ...(res.warnings ?? []).map((warning) => `Warning: ${warning}`), - "Next: review 'git diff', run 'bench validate' (it flags any schema changes to fix), then commit via PR.", + ...releaseNotesLines(res.templateVersion), + "Next: review 'git diff' (skills and AGENTS.md are proposals), then in the instance ask your agent:", + ` "${UPDATE_PROMPT}"`, ].join("\n"), { dir: targetDir, @@ -471,6 +559,7 @@ export async function runBenchKitUpdate( runnerDeps: res.runnerDeps, zones: res.zones, warnings: res.warnings ?? [], + nextStep: { prompt: UPDATE_PROMPT, dir: targetDir }, }, ); } finally { @@ -478,6 +567,13 @@ export async function runBenchKitUpdate( } } +/** `targetDir` relative to cwd when that is shorter (what the user types after `cd`). */ +function displayDir(targetDir: string): string { + const rel = relative(process.cwd(), targetDir); + if (rel === "") return "."; + return rel.length < targetDir.length && !isAbsolute(rel) ? rel : targetDir; +} + function describeSync(counts: SyncCounts | undefined): string { if (counts === undefined) return "0 added, 0 updated, 0 unchanged"; return `${counts.added} added, ${counts.updated} updated, ${counts.unchanged} unchanged`; @@ -548,7 +644,7 @@ function normalizeRef(ctx: OutputContext, raw: string | undefined): string | nul "invalid_template_version", `'${raw}' is not a valid template tag.`, ExitCodes.USAGE, - "Pass a tag name, for example '10x bench-kit init --template-version v0.1.0'.", + "Pass a tag name, for example '10x bench-kit init --template-version v0.33.0'.", ); } return ref; @@ -579,13 +675,17 @@ async function cloneTemplateOrDie( scratch: string, action: "init" | "update", ): Promise { - verbose(ctx, `cloning ${TEMPLATE_REPO_URL} (${ref ?? "latest"}) into ${scratch}`); + const source = templateRepoUrl(); + if (source !== TEMPLATE_REPO_URL) { + process.stderr.write(`Using template source from BENCH_KIT_TEMPLATE_URL: ${source}\n`); + } + verbose(ctx, `cloning ${source} (${ref ?? "latest"}) into ${scratch}`); const clone = await deps.cloneTemplate(ref, scratch); if (!clone.ok) { outputError( ctx, "clone_failed", - `Could not download the template from ${TEMPLATE_REPO_URL}.`, + `Could not download the template from ${source}.`, ExitCodes.ERROR, clone.error ? `Git said: ${clone.error.trim()}` @@ -683,7 +783,7 @@ const defaultDeps: BenchKitDeps = { cloneTemplate(ref, destDir) { const args = ["clone", "--depth", "1"]; if (ref !== null) args.push("--branch", ref); - args.push(TEMPLATE_REPO_URL, destDir); + args.push(templateRepoUrl(), destDir); return run("git", args); }, runGit(args, cwd) { diff --git a/tests/bench-kit-command.test.ts b/tests/bench-kit-command.test.ts index 3d5af59..ef551be 100644 --- a/tests/bench-kit-command.test.ts +++ b/tests/bench-kit-command.test.ts @@ -29,7 +29,9 @@ import { type BootstrapRequest, type BootstrapResponse, CONTRACT_VERSION, + FALLBACK_FIRST_PROMPT, registerBenchKitCommand, + releaseNotesLines, runBenchKitInit, runBenchKitUpdate, } from "../src/commands/bench-kit"; @@ -331,32 +333,117 @@ describe("10x bench-kit init", () => { expect(envelope.data.baseRepo.name).toBe("shop-app"); }); - it("asks the bootstrap for a full-history clone only with --deep", async () => { + it("asks for full history by default and HEAD only with --shallow", async () => { const template = buildTemplateFixture(); + const defaultFake = fakeDeps(template, okInitResponse()); + const byDefault = await captureStreams(() => + runBenchKitInit( + JSON_CTX, + join(tempDir("bench-kit-target-"), "instance"), + {}, + defaultFake.deps, + ), + ); + expect(byDefault.exitCode).toBeUndefined(); + // deepClone keeps older kits (0.28–0.33, shallow by default) on full history too. + expect(defaultFake.bootstrapCalls[0]!.request.deepClone).toBe(true); + expect(defaultFake.bootstrapCalls[0]!.request.shallowClone).toBeUndefined(); + const shallowFake = fakeDeps(template, okInitResponse()); const shallow = await captureStreams(() => runBenchKitInit( JSON_CTX, join(tempDir("bench-kit-target-"), "instance"), - {}, + { shallow: true }, shallowFake.deps, ), ); + expect(shallow.exitCode).toBeUndefined(); + expect(shallowFake.bootstrapCalls[0]!.request.shallowClone).toBe(true); expect(shallowFake.bootstrapCalls[0]!.request.deepClone).toBeUndefined(); + }); + + it("prints the kit's first prompt as the next step, in human and JSON form", async () => { + const template = buildTemplateFixture(); + const prompt = "Use bench-measure to set up this instance."; + const response = { ...okInitResponse(), nextStep: { skill: "bench-measure", prompt } }; + const target = join(tempDir("bench-kit-target-"), "instance"); + + const json = await captureStreams(() => + runBenchKitInit(JSON_CTX, target, {}, fakeDeps(template, response).deps), + ); + expect(parseEnvelope(json.stdout).data.nextStep).toEqual({ prompt, dir: target }); - const deepFake = fakeDeps(template, okInitResponse()); - const deep = await captureStreams(() => + const human = await captureStreams(() => runBenchKitInit( - JSON_CTX, + { json: false, verbose: false }, join(tempDir("bench-kit-target-"), "instance"), - { deep: true }, - deepFake.deps, + { tool: "claude-code" }, + fakeDeps(template, response).deps, ), ); + expect(human.stderr).toContain(`open it in Claude Code and ask:\n "${prompt}"`); + expect(human.stderr).not.toContain("bench-wiring"); + }); + + it("defaults the instance to ../-bench when run inside a product repo", async () => { + const template = buildTemplateFixture(); + const parent = tempDir("bench-kit-parent-"); + const { deps, bootstrapCalls } = fakeDeps(template, okInitResponse(), { + detectBaseRepo: () => + Promise.resolve({ + rootDir: join(parent, "shop-app"), + name: "shop-app", + url: "https://github.com/acme/shop-app.git", + headCommit: "a".repeat(40), + }), + }); + + const result = await captureStreams(() => runBenchKitInit(JSON_CTX, undefined, {}, deps)); - expect(deep.exitCode).toBeUndefined(); - expect(deepFake.bootstrapCalls[0]!.request.deepClone).toBe(true); + expect(result.exitCode).toBeUndefined(); + expect(bootstrapCalls[0]!.request.targetDir).toBe(join(parent, "shop-app-bench")); + expect(parseEnvelope(result.stdout).data.dir).toBe(join(parent, "shop-app-bench")); + }); + + it("prints the kit's doctor findings before the next step", async () => { + const template = buildTemplateFixture(); + const doctor = { + missing: 1, + warns: 1, + ok: ["engine", "node"], + problems: [ + { name: "OPENROUTER_API_KEY", status: "BRAK", detail: "not in env", fix: "export OPENROUTER_API_KEY" }, + { name: "instance remote", status: "WARN", detail: "no origin" }, + ], + }; + const result = await captureStreams(() => + runBenchKitInit( + { json: false, verbose: false }, + join(tempDir("bench-kit-target-"), "instance"), + { tool: "claude-code" }, + fakeDeps(template, { ...okInitResponse(), doctor }).deps, + ), + ); + + expect(result.stderr).toContain("1 problem to fix BEFORE opening the agent"); + expect(result.stderr).toContain("✗ OPENROUTER_API_KEY: not in env\n → export OPENROUTER_API_KEY"); + expect(result.stderr).not.toContain("instance remote"); + expect(result.stderr.indexOf("OPENROUTER_API_KEY")).toBeLessThan(result.stderr.indexOf("Next:")); + }); + + it("falls back to the bench-measure prompt for templates without nextStep", async () => { + const template = buildTemplateFixture(); + const result = await captureStreams(() => + runBenchKitInit( + JSON_CTX, + join(tempDir("bench-kit-target-"), "instance"), + {}, + fakeDeps(template, okInitResponse()).deps, + ), + ); + expect(parseEnvelope(result.stdout).data.nextStep.prompt).toBe(FALLBACK_FIRST_PROMPT); }); it("degrades to a hint when the base repo clone fails (init still succeeds)", async () => { @@ -502,6 +589,20 @@ describe("10x bench-kit init", () => { }); describe("10x bench-kit update", () => { + it("links release notes at the installed tag, and only for an https source", () => { + const saved = process.env.BENCH_KIT_TEMPLATE_URL; + try { + delete process.env.BENCH_KIT_TEMPLATE_URL; + const [line] = releaseNotesLines("0.34.0"); + expect(line).toMatch(/^Release notes: https:\/\/.+\/blob\/v0\.34\.0\/CHANGELOG\.md$/); + process.env.BENCH_KIT_TEMPLATE_URL = "file:///tmp/kit"; + expect(releaseNotesLines("0.34.0")).toEqual([]); + } finally { + if (saved === undefined) delete process.env.BENCH_KIT_TEMPLATE_URL; + else process.env.BENCH_KIT_TEMPLATE_URL = saved; + } + }); + function existingInstance(version = "0.9.0"): string { const target = tempDir("bench-kit-target-"); mkdirSync(join(target, ".bench-kit"), { recursive: true }); @@ -545,6 +646,7 @@ describe("10x bench-kit update", () => { expect(envelope.data.fromVersion).toBe("0.9.0"); expect(envelope.data.templateVersion).toBe("0.10.0"); expect(envelope.data.zones.skills.updated).toBe(3); + expect(envelope.data.nextStep.prompt).toContain("bench-status"); }); it("reports up-to-date without a zone report", async () => {