diff --git a/apps/server/src/git/GitManager.ts b/apps/server/src/git/GitManager.ts index d1f48bfed3681..1a98e4ec9fa02 100644 --- a/apps/server/src/git/GitManager.ts +++ b/apps/server/src/git/GitManager.ts @@ -743,9 +743,12 @@ export const make = Effect.gen(function* () { } return result; }); - const readWorktreesDirectory = serverSettingsService.getSettings.pipe( - Effect.map((settings) => settings.worktreesDirectory), - Effect.orElseSucceed(() => ""), + const readWorktreeCreationSettings = serverSettingsService.getSettings.pipe( + Effect.map((settings) => ({ + worktreesDirectory: settings.worktreesDirectory, + worktreeTemplates: settings.enableWorktreeTemplates, + })), + Effect.orElseSucceed(() => ({ worktreesDirectory: "", worktreeTemplates: false })), ); const createWorktree: GitManager["Service"]["createWorktree"] = Effect.fn( "GitManager.createWorktree", @@ -757,9 +760,8 @@ export const make = Effect.gen(function* () { Effect.map(({ settings }) => settings.worktreeSubmodules), Effect.orElseSucceed(() => null), ); - const worktreesDirectory = yield* readWorktreesDirectory; const result = yield* gitCore.createWorktree(input, { - worktreesDirectory, + ...(yield* readWorktreeCreationSettings), ...options, submodules, }); @@ -2980,7 +2982,7 @@ export const make = Effect.gen(function* () { path: null, }, { - worktreesDirectory: yield* readWorktreesDirectory, + ...(yield* readWorktreeCreationSettings), submodules: yield* projectSettingsFor(input).pipe( Effect.map(({ settings }) => settings.worktreeSubmodules), Effect.orElseSucceed(() => null), diff --git a/apps/server/src/git/GitWorkflowService.ts b/apps/server/src/git/GitWorkflowService.ts index bf89c7cc1baaa..731a146c0f1ce 100644 --- a/apps/server/src/git/GitWorkflowService.ts +++ b/apps/server/src/git/GitWorkflowService.ts @@ -113,6 +113,7 @@ export class GitWorkflowService extends Context.Service< readonly removeWorktree: ( input: VcsRemoveWorktreeInput, ) => Effect.Effect; + readonly captureWorktreeTemplate: (worktreePath: string) => Effect.Effect; readonly pruneWorktrees: (input: { readonly cwd: string; }) => Effect.Effect; @@ -461,6 +462,7 @@ export const make = Effect.gen(function* () { Effect.andThen(git.removeWorktree(input)), ); }), + captureWorktreeTemplate: git.captureWorktreeTemplate, pruneWorktrees: Effect.fn("GitWorkflowService.pruneWorktrees")(function (input) { return ensureGitCommand("GitWorkflowService.pruneWorktrees", input.cwd).pipe( Effect.andThen(git.pruneWorktrees(input)), diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 4ab38c4b305e3..a1292c8de4e77 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -230,6 +230,17 @@ const make = Effect.gen(function* () { cause, }); + // A worktree whose setup succeeded seeds later ones with the same lockfiles, off the launch's path. + const captureWorktreeTemplate = (worktreePath: string) => + serverSettings.getSettings.pipe( + Effect.flatMap((settings) => + settings.enableWorktreeTemplates ? git.captureWorktreeTemplate(worktreePath) : Effect.void, + ), + Effect.ignore, + Effect.forkIn(preparationScope), + Effect.asVoid, + ); + const readReceipt = (input: ThreadLaunchInput, commandId: CommandId) => receipts .getByCommandId(commandId) @@ -606,12 +617,15 @@ const make = Effect.gen(function* () { }, })); if (setup.completion) { + const createdHere = createdWorktreePath; const awaitCompletion = Effect.gen(function* () { const completion = yield* setup.completion!; yield* setupTracker.stage(threadId, "setup-script", { status: completion.exitCode === 0 ? "done" : "failed", detail: `exited with ${completion.exitCode ?? "no exit code"}`, }); + if (completion.exitCode === 0 && createdHere !== null) + yield* captureWorktreeTemplate(createdHere); if (completion.exitCode !== 0 && !setup.async) return yield* mapError( input, diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index d46c195155f3c..4674484adb68d 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -144,6 +144,8 @@ export interface CreateWorktreeOptions { /** Null or omitted defers to the checkout's t3.json. */ readonly submodules?: WorktreeSubmodules | null; readonly worktreesDirectory?: string; + /** Clone a captured copy-on-write template when one matches; false also removes the templates. */ + readonly worktreeTemplates?: boolean; } export interface GitCommitProgress { @@ -422,6 +424,8 @@ export class GitVcsDriver extends Context.Service< readonly removeWorktree: ( input: VcsRemoveWorktreeInput, ) => Effect.Effect; + /** Keeps this set-up worktree as the template for its lockfiles; logs and skips on any failure. */ + readonly captureWorktreeTemplate: (worktreePath: string) => Effect.Effect; readonly pruneWorktrees: (input: { readonly cwd: string; }) => Effect.Effect; diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 5013d9194b155..7c8c458c06dfd 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -45,6 +45,7 @@ import * as GitVcsDriver from "./GitVcsDriver.ts"; import { removeEmptyDirectory } from "./removeEmptyDirectory.ts"; import { explicitBareRepositoryEnv, gitConfigEnv } from "./VcsProcess.ts"; import { removeLinkedWorktree } from "./worktreeRemoval.ts"; +import { makeWorktreeTemplates } from "./worktreeTemplates.ts"; import { resolveWorktreesDirectory } from "../worktreesDirectory.ts"; import { parseRemoteNames, @@ -1234,6 +1235,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), ); + const templates = yield* makeWorktreeTemplates((operation, cwd, args, options) => + runGitStdoutWithOptions(operation, cwd, args, options), + ); + const branchExists = (cwd: string, refName: string): Effect.Effect => executeGit( "GitVcsDriver.branchExists", @@ -4157,7 +4162,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* "git worktree add failed", ); } - const args = ["worktree", "add", "--", worktreePath, input.newRefName ?? input.refName]; + if (options?.worktreeTemplates === false) yield* templates.removeAll(worktreePath); + const template = options?.worktreeTemplates + ? yield* templates.find({ cwd: input.cwd, refName: input.refName, worktreePath }) + : null; + const args = [ + "worktree", + "add", + ...(template ? ["--no-checkout"] : []), + "--", + worktreePath, + input.newRefName ?? input.refName, + ]; const progress = options?.progress; const onCheckoutProgress = progress?.onCheckoutProgress; @@ -4200,6 +4216,26 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (progress?.onWorktreeClaimed) { yield* progress.onWorktreeClaimed(worktreePath); } + if (template) { + yield* templates.clone(template, worktreePath, onCheckoutProgress).pipe( + Effect.catch((cause) => + Effect.logWarning("worktree template clone failed; checking out instead", { + worktreePath, + cause, + }).pipe( + Effect.andThen(templates.clear(worktreePath)), + Effect.andThen( + runGit( + "GitVcsDriver.createWorktree.checkout", + worktreePath, + ["-c", `checkout.workers=${checkoutWorkers}`, "reset", "--hard", "-q"], + { timeoutMs: null }, + ), + ), + ), + ), + ); + } // worktree add leaves submodules empty, potentially omitting project skills, tools or source. const hasSubmodules = yield* fileSystem @@ -4986,6 +5022,17 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* Effect.ensuring(invalidateWorktreeCreationCaches(input.cwd)), ); }), + captureWorktreeTemplate: Effect.fn("GitVcsDriver.captureWorktreeTemplate")( + function* (worktreePath) { + yield* templates + .capture(worktreePath) + .pipe( + Effect.catch((cause) => + Effect.logWarning("worktree template capture skipped", { worktreePath, cause }), + ), + ); + }, + ), fetchRefIntoBranch: Effect.fnUntraced(function* (input) { return yield* withListRefsInvalidation(input.cwd, fetchRefIntoBranch(input)); }), diff --git a/apps/server/src/vcs/worktreeTemplates.test.ts b/apps/server/src/vcs/worktreeTemplates.test.ts new file mode 100644 index 0000000000000..f1e90946dfa2d --- /dev/null +++ b/apps/server/src/vcs/worktreeTemplates.test.ts @@ -0,0 +1,243 @@ +// @effect-diagnostics nodeBuiltinImport:off - realpath and readlink check that cloned symlinks stay inside the clone. +import * as NodeFS from "node:fs"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import * as ServerConfig from "../config.ts"; +import { makeGitVcsDriverCore } from "./GitVcsDriverCore.ts"; + +const layerTest = ServerConfig.layerTest(process.cwd(), { + prefix: "t3-worktree-templates-test-", +}).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); + +type CopyMode = "plain" | "unsupported" | "fail-clone"; + +/** + * Records Git arguments and stands in for the filesystem: "plain" drops the clone flags so the tests run on any + * filesystem, "unsupported" fails every clone like ext4 does, "fail-clone" fails only clones out of a template. + */ +const makeDriver = Effect.fnUntraced(function* (mode: { current: CopyMode }) { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const gitArgs: Array> = []; + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "cp") { + if (ChildProcess.isStandardCommand(command)) gitArgs.push(command.args.map(String)); + return delegate.spawn(command); + } + const args = command.args.map(String); + const failing = + mode.current === "unsupported" || + (mode.current === "fail-clone" && args.some((arg) => arg.includes("/.t3-templates/"))); + const copy = failing + ? ChildProcess.make("false", [], command.options) + : ChildProcess.make( + "cp", + args.filter((arg) => arg !== "--reflink=always" && arg !== "-c"), + command.options, + ); + return delegate.spawn(copy); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const git = (cwd: string, args: ReadonlyArray) => + driver + .execute({ operation: "worktreeTemplates.test", cwd, args, timeoutMs: 10_000 }) + .pipe(Effect.map((result) => result.stdout.trim())); + return { driver, git, gitArgs }; +}); + +const setup = Effect.fnUntraced(function* (mode: CopyMode = "plain") { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const copyMode = { current: mode }; + const { driver, git, gitArgs } = yield* makeDriver(copyMode); + const root = NodeFS.realpathSync(yield* fs.makeTempDirectoryScoped({ prefix: "t3-templates-" })); + const repo = path.join(root, "repo"); + yield* fs.makeDirectory(repo); + yield* driver.initRepo({ cwd: repo }); + yield* git(repo, ["config", "user.email", "test@test.com"]); + yield* git(repo, ["config", "user.name", "Test"]); + yield* fs.makeDirectory(path.join(repo, "src")); + yield* fs.writeFileString(path.join(repo, "src/index.ts"), "export const one = 1;\n"); + yield* fs.writeFileString(path.join(repo, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n"); + yield* fs.writeFileString(path.join(repo, ".gitignore"), "node_modules/\n"); + yield* git(repo, ["add", "."]); + yield* git(repo, ["commit", "-m", "initial"]); + const branch = yield* git(repo, ["branch", "--show-current"]); + const worktrees = path.join(root, "worktrees"); + const templates = path.join(worktrees, ".t3-templates"); + + const commit = Effect.fnUntraced(function* (file: string, contents: string) { + yield* fs.writeFileString(path.join(repo, file), contents); + yield* git(repo, ["commit", "-qam", `edit ${file}`]); + }); + const create = (name: string, worktreeTemplates = true) => + driver.createWorktree( + { cwd: repo, refName: branch, newRefName: `t3/${name}`, path: path.join(worktrees, name) }, + { worktreeTemplates }, + ); + /** What a successful `pnpm install` leaves behind, including links and pnpm's absolute state path. */ + const install = Effect.fnUntraced(function* (worktree: string) { + const pkg = path.join(worktree, "node_modules/.pnpm/dep@1.0.0/node_modules/dep"); + yield* fs.makeDirectory(pkg, { recursive: true }); + yield* fs.writeFileString(path.join(pkg, "index.js"), "module.exports = 1;\n"); + NodeFS.symlinkSync(".pnpm/dep@1.0.0/node_modules/dep", path.join(worktree, "node_modules/dep")); + yield* fs.writeFileString( + path.join(worktree, "node_modules/.pnpm-workspace-state-v1.json"), + JSON.stringify({ projects: { [worktree]: {} }, dir: `${worktree}/node_modules` }), + ); + }); + const listTemplates = fs + .readDirectory(templates) + .pipe(Effect.map((names) => names.filter((name) => !name.startsWith(".")))); + + return { + fs, + path, + driver, + git, + gitArgs, + copyMode, + worktrees, + templates, + commit, + create, + install, + listTemplates, + }; +}); + +it.effect("clones a captured template with dependencies and a stat-clean index", () => + Effect.gen(function* () { + const { fs, path, driver, git, gitArgs, create, install, listTemplates } = yield* setup(); + const first = yield* create("first"); + yield* install(first.worktree.path); + yield* driver.captureWorktreeTemplate(first.worktree.path); + assert.equal((yield* listTemplates).length, 1); + + gitArgs.length = 0; + const second = (yield* create("second")).worktree.path; + + assert.isTrue( + gitArgs.some((args) => args.includes("worktree") && args.includes("--no-checkout")), + ); + // diff-files compares stat data only: an entry whose stat differs from the index is listed. + assert.equal(yield* git(second, ["diff-files", "--name-only"]), ""); + assert.equal(yield* git(second, ["status", "--porcelain"]), ""); + assert.equal(yield* git(second, ["branch", "--show-current"]), "t3/second"); + const dep = path.join(second, "node_modules/dep"); + assert.equal(NodeFS.readlinkSync(dep), ".pnpm/dep@1.0.0/node_modules/dep"); + assert.isTrue(NodeFS.realpathSync(dep).startsWith(`${second}/`)); + const state = yield* fs.readFileString( + path.join(second, "node_modules/.pnpm-workspace-state-v1.json"), + ); + assert.deepStrictEqual(JSON.parse(state), { + projects: { [second]: {} }, + dir: `${second}/node_modules`, + }); + assert.isFalse(state.includes(first.worktree.path)); + }).pipe(Effect.scoped, Effect.provide(layerTest)), +); + +it.effect("checks out normally and keeps no templates where the filesystem cannot clone", () => + Effect.gen(function* () { + const { fs, path, driver, git, gitArgs, templates, create, install } = + yield* setup("unsupported"); + const first = yield* create("first"); + yield* install(first.worktree.path); + yield* driver.captureWorktreeTemplate(first.worktree.path); + assert.isFalse(yield* fs.exists(templates)); + + gitArgs.length = 0; + const second = (yield* create("second")).worktree.path; + assert.isFalse(gitArgs.some((args) => args.includes("--no-checkout"))); + assert.equal(yield* git(second, ["status", "--porcelain"]), ""); + assert.isFalse(yield* fs.exists(path.join(second, "node_modules"))); + }).pipe(Effect.scoped, Effect.provide(layerTest)), +); + +it.effect("reuses a template for a newer commit and rebuilds when the lockfile changes", () => + Effect.gen(function* () { + const { fs, path, driver, git, gitArgs, commit, create, install, listTemplates } = + yield* setup(); + const first = yield* create("first"); + yield* install(first.worktree.path); + yield* driver.captureWorktreeTemplate(first.worktree.path); + const [oldest] = yield* listTemplates; + + // Same lockfile, newer source: the template is cloned and Git updates only what changed. + yield* commit("src/index.ts", "export const one = 2;\n"); + gitArgs.length = 0; + const newer = (yield* create("newer")).worktree.path; + assert.isTrue(gitArgs.some((args) => args.includes("--no-checkout"))); + assert.equal( + yield* fs.readFileString(path.join(newer, "src/index.ts")), + "export const one = 2;\n", + ); + assert.equal(yield* git(newer, ["status", "--porcelain"]), ""); + assert.isTrue(yield* fs.exists(path.join(newer, "node_modules/dep"))); + + // A new lockfile has no template: plain checkout, then its own template after setup. + yield* commit("pnpm-lock.yaml", "lockfileVersion: '9.0'\npackages: {}\n"); + gitArgs.length = 0; + const relocked = (yield* create("relocked")).worktree.path; + assert.isFalse(gitArgs.some((args) => args.includes("--no-checkout"))); + assert.isFalse(yield* fs.exists(path.join(relocked, "node_modules"))); + yield* install(relocked); + yield* driver.captureWorktreeTemplate(relocked); + assert.equal((yield* listTemplates).length, 2); + + // A third lockfile evicts the oldest template. + yield* commit("pnpm-lock.yaml", "lockfileVersion: '9.0'\npackages: { a: {} }\n"); + const third = (yield* create("third")).worktree.path; + yield* install(third); + yield* driver.captureWorktreeTemplate(third); + const remaining = yield* listTemplates; + assert.equal(remaining.length, 2); + assert.notInclude(remaining, oldest); + }).pipe(Effect.scoped, Effect.provide(layerTest)), +); + +it.effect("falls back to a checkout when cloning the template fails", () => + Effect.gen(function* () { + const { fs, path, driver, git, copyMode, create, install } = yield* setup(); + const first = yield* create("first"); + yield* install(first.worktree.path); + yield* driver.captureWorktreeTemplate(first.worktree.path); + + copyMode.current = "fail-clone"; + const second = (yield* create("second")).worktree.path; + assert.equal(yield* git(second, ["status", "--porcelain"]), ""); + assert.equal( + yield* fs.readFileString(path.join(second, "src/index.ts")), + "export const one = 1;\n", + ); + assert.isFalse(yield* fs.exists(path.join(second, "node_modules"))); + }).pipe(Effect.scoped, Effect.provide(layerTest)), +); + +it.effect("removes a cloned worktree, and turning templates off removes the templates", () => + Effect.gen(function* () { + const { fs, driver, git, templates, create, install, worktrees } = yield* setup(); + const first = yield* create("first"); + yield* install(first.worktree.path); + yield* driver.captureWorktreeTemplate(first.worktree.path); + const second = (yield* create("second")).worktree.path; + + yield* driver.removeWorktree({ cwd: first.worktree.path, path: second, force: true }); + assert.isFalse(yield* fs.exists(second)); + assert.notInclude(yield* git(first.worktree.path, ["worktree", "list"]), second); + assert.isTrue(yield* fs.exists(templates)); + + yield* create("stock", false); + assert.isFalse(yield* fs.exists(templates)); + assert.isTrue((yield* fs.readDirectory(worktrees)).includes("stock")); + }).pipe(Effect.scoped, Effect.provide(layerTest)), +); diff --git a/apps/server/src/vcs/worktreeTemplates.ts b/apps/server/src/vcs/worktreeTemplates.ts new file mode 100644 index 0000000000000..303f98707ca28 --- /dev/null +++ b/apps/server/src/vcs/worktreeTemplates.ts @@ -0,0 +1,409 @@ +// @effect-diagnostics nodeBuiltinImport:off - Git's index stores nanosecond ctime/mtime, dev, ino, uid and gid; only node:fs lstat reports them all. +import * as NodeCrypto from "node:crypto"; +import * as NodeFSP from "node:fs/promises"; + +import * as Arr from "effect/Array"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import type { GitCommandError } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + +import type { CreateWorktreeProgress } from "./GitVcsDriver.ts"; + +/** Beside the worktrees of one repository, so templates and clones share a filesystem. */ +const TEMPLATES_DIRECTORY = ".t3-templates"; +const MANIFEST = ".t3-template.json"; +const LOCKFILES = ["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lock", "bun.lockb"]; +const KEPT_TEMPLATES = 2; +// One cp is bound by per-file create plus clone; four in parallel took the clone from ~14 s to ~4 s on t3code. +// Shared by every clone of this server, so launching many worktrees at once does not multiply the load. +const CLONE_CONCURRENCY = 4; +const CLONE_BATCH = 64; +const PNPM_WORKSPACE_STATE = ["node_modules", ".pnpm-workspace-state-v1.json"]; + +export interface WorktreeTemplate { + readonly directory: string; + /** The commit the template's tracked files are at. */ + readonly commit: string; + /** The worktree the template's dependencies came from; pnpm's state file names it. */ + readonly source: string; + readonly targetCommit: string; +} + +class WorktreeTemplateError extends Data.TaggedError("WorktreeTemplateError")<{ + readonly detail: string; +}> {} + +type Git = ( + operation: string, + cwd: string, + args: ReadonlyArray, + options?: { readonly env?: NodeJS.ProcessEnv; readonly timeoutMs?: number | null }, +) => Effect.Effect; + +const Manifest = Schema.fromJsonString( + Schema.Struct({ commit: Schema.String, source: Schema.String }), +); +const decodeManifest = Schema.decodeUnknownEffect(Manifest); +const encodeManifest = Schema.encodeEffect(Manifest); +const encodeJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.String)); + +/** + * Copy-on-write templates for new worktrees, keyed by the lockfiles at the start commit. A worktree whose setup + * succeeded is captured once (tracked files at its commit plus its node_modules); later worktrees with the same + * lockfiles clone the template instead of checking out and installing from scratch. Every step fails without + * side effects outside the template directory, so callers fall back to Git's checkout. + */ +export const makeWorktreeTemplates = Effect.fnUntraced(function* (git: Git) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const platform = yield* HostProcess.Platform; + const cloneFlags = platform === "darwin" ? ["-c", "-a"] : ["-a", "--reflink=always"]; + const probes = new Map(); + const clonePermits = yield* Semaphore.make(CLONE_CONCURRENCY); + + const templatesDirectory = (worktreePath: string) => + path.join(path.dirname(worktreePath), TEMPLATES_DIRECTORY); + + // `--reflink=always` and `-c` fail without copying data where the filesystem cannot clone. + const cp = (sources: ReadonlyArray, target: string) => + spawner + .exitCode( + ChildProcess.make("cp", [...cloneFlags, ...sources, target], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }), + ) + .pipe( + Effect.flatMap((code) => + code === 0 + ? Effect.void + : Effect.fail(new WorktreeTemplateError({ detail: `cp exited with ${code}` })), + ), + ); + + /** Clones one file once per directory and server run; false on Windows and on filesystems without clones. */ + const probe = Effect.fnUntraced(function* (directory: string) { + const cached = probes.get(directory); + if (cached !== undefined) return cached; + const supported = + (platform === "linux" || platform === "darwin") && + (yield* Effect.gen(function* () { + yield* fs.makeDirectory(directory, { recursive: true }); + const scratch = yield* fs.makeTempDirectory({ directory, prefix: ".probe-" }); + const source = path.join(scratch, "probe"); + yield* fs.writeFileString(source, "t3"); + return yield* cp([source], `${source}-clone`).pipe( + Effect.as(true), + Effect.ensuring(fs.remove(scratch, { recursive: true }).pipe(Effect.ignore)), + ); + }).pipe(Effect.orElseSucceed(() => false))); + // Leave nothing behind where clones are unsupported. + if ( + !supported && + (yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [""]))).length === 0 + ) + yield* fs.remove(directory, { recursive: true }).pipe(Effect.ignore); + probes.set(directory, supported); + return supported; + }); + + const resolveCommit = (cwd: string, ref: string) => + git("WorktreeTemplates.resolveCommit", cwd, [ + "rev-parse", + "--verify", + "--end-of-options", + `${ref}^{commit}`, + ]).pipe(Effect.map((stdout) => stdout.trim())); + + const templatePath = (cwd: string, commit: string, directory: string) => + git("WorktreeTemplates.lockfiles", cwd, ["ls-tree", commit, "--", ...LOCKFILES]).pipe( + Effect.map((lockfiles) => + path.join( + directory, + NodeCrypto.createHash("sha256").update(lockfiles).digest("hex").slice(0, 16), + ), + ), + ); + + // pnpm's virtual store holds most files; its package directories make even work units for parallel cp. + const expand = Effect.fnUntraced(function* (source: string, targetParent: string) { + const pnpm = path.join(source, ".pnpm"); + if (path.basename(source) !== "node_modules" || !(yield* fs.exists(pnpm))) { + return [[source, targetParent] as const]; + } + const target = path.join(targetParent, "node_modules"); + yield* fs.makeDirectory(path.join(target, ".pnpm"), { recursive: true }); + return [ + ...(yield* fs.readDirectory(source)) + .filter((name) => name !== ".pnpm") + .map((name) => [path.join(source, name), target] as const), + ...(yield* fs.readDirectory(pnpm)).map( + (name) => [path.join(pnpm, name), path.join(target, ".pnpm")] as const, + ), + ]; + }); + + const cloneEntries = Effect.fnUntraced(function* ( + entries: ReadonlyArray, + onProgress?: CreateWorktreeProgress["onCheckoutProgress"], + ) { + const jobs = (yield* Effect.forEach(entries, ([source, target]) => + expand(source, target), + )).flat(); + const batches = [...Map.groupBy(jobs, ([, target]) => target)].flatMap(([target, group]) => + Arr.chunksOf( + group.map(([source]) => source), + CLONE_BATCH, + ).map((sources) => ({ target, sources })), + ); + const completed = yield* Ref.make(0); + yield* Effect.forEach( + batches, + ({ target, sources }) => + clonePermits + .withPermits(1)(cp(sources, target)) + .pipe( + Effect.andThen(Ref.updateAndGet(completed, (count) => count + 1)), + Effect.flatMap((count) => + onProgress + ? onProgress({ + percent: Math.floor((count * 100) / batches.length), + completed: count, + total: batches.length, + }) + : Effect.void, + ), + ), + { concurrency: CLONE_CONCURRENCY, discard: true }, + ); + }); + + const find = Effect.fn("WorktreeTemplates.find")( + function* (input: { + readonly cwd: string; + readonly refName: string; + readonly worktreePath: string; + }) { + const directory = templatesDirectory(input.worktreePath); + if (!(yield* probe(directory))) return null; + const targetCommit = yield* resolveCommit(input.cwd, input.refName); + const template = yield* templatePath(input.cwd, targetCommit, directory); + const manifest = yield* fs + .readFileString(path.join(template, MANIFEST)) + .pipe(Effect.flatMap(decodeManifest)); + return { directory: template, ...manifest, targetCommit } satisfies WorktreeTemplate; + }, + Effect.orElseSucceed(() => null), + ); + + /** Fills a `--no-checkout` worktree from the template and leaves its index stat-clean. */ + const clone = Effect.fn("WorktreeTemplates.clone")(function* ( + template: WorktreeTemplate, + worktreePath: string, + onProgress?: CreateWorktreeProgress["onCheckoutProgress"], + ) { + const names = (yield* fs.readDirectory(template.directory)).filter((name) => name !== MANIFEST); + yield* cloneEntries( + names.map((name) => [path.join(template.directory, name), worktreePath] as const), + onProgress, + ); + const gitDir = (yield* git("WorktreeTemplates.gitDir", worktreePath, [ + "rev-parse", + "--absolute-git-dir", + ])).trim(); + yield* git("WorktreeTemplates.readTree", worktreePath, ["read-tree", template.commit]); + yield* Effect.tryPromise({ + try: () => restatIndex(path.join(gitDir, "index"), worktreePath), + catch: (cause) => new WorktreeTemplateError({ detail: String(cause) }), + }); + if (template.commit !== template.targetCommit) { + // Same lockfiles, newer commit: Git rewrites only the paths that differ. + yield* git( + "WorktreeTemplates.update", + worktreePath, + ["read-tree", "-m", "-u", template.commit, template.targetCommit], + { timeoutMs: null }, + ); + } + const state = path.join(worktreePath, ...PNPM_WORKSPACE_STATE); + if (yield* fs.exists(state)) { + const [from, to] = [template.source, worktreePath].map((root) => + encodeJsonString(root).slice(0, -1), + ); + const contents = yield* fs.readFileString(state); + yield* fs.writeFileString( + state, + contents.replaceAll(`${from}"`, `${to}"`).replaceAll(`${from}/`, `${to}/`), + ); + } + }); + + /** Empties a worktree except its `.git` file, so a failed clone can fall back to a checkout. */ + const clear = (worktreePath: string) => + fs.readDirectory(worktreePath).pipe( + Effect.flatMap((names) => + Effect.forEach( + names.filter((name) => name !== ".git"), + (name) => fs.remove(path.join(worktreePath, name), { recursive: true }), + { discard: true }, + ), + ), + Effect.ignore, + ); + + /** Turning templates off removes them the next time this repository gets a worktree. */ + const removeAll = (worktreePath: string) => + fs.remove(templatesDirectory(worktreePath), { recursive: true }).pipe(Effect.ignore); + + /** + * Captures `worktreePath` as the template for its lockfiles unless one exists. Tracked files come from its + * commit, not its working tree, so an agent's edits never reach the template; dependencies are cloned as + * they are. + */ + const capture = Effect.fn("WorktreeTemplates.capture")(function* (worktreePath: string) { + const directory = templatesDirectory(worktreePath); + if (!(yield* probe(directory))) return; + // Submodule checkouts point into this worktree's own Git directory. + if (yield* fs.exists(path.join(worktreePath, ".gitmodules"))) return; + const commit = yield* resolveCommit(worktreePath, "HEAD"); + const template = yield* templatePath(worktreePath, commit, directory); + if (yield* fs.exists(template)) return; + const dependencies = (yield* git("WorktreeTemplates.ignored", worktreePath, [ + "ls-files", + "-z", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + ])) + .split("\0") + .filter((entry) => entry === "node_modules/" || entry.endsWith("/node_modules/")); + + const staging = yield* fs.makeTempDirectory({ directory, prefix: ".staging-" }); + yield* Effect.gen(function* () { + const env = { GIT_INDEX_FILE: `${staging}.index` }; + yield* git("WorktreeTemplates.stageIndex", worktreePath, ["read-tree", commit], { env }); + yield* git( + "WorktreeTemplates.stageFiles", + worktreePath, + ["--work-tree", staging, "checkout-index", "--all", "--force"], + { env, timeoutMs: null }, + ); + const entries = dependencies.map( + (entry) => + [ + path.join(worktreePath, entry.slice(0, -1)), + path.dirname(path.join(staging, entry.slice(0, -1))), + ] as const, + ); + yield* Effect.forEach( + entries, + ([, parent]) => fs.makeDirectory(parent, { recursive: true }), + { + discard: true, + }, + ); + yield* cloneEntries(entries); + yield* fs.writeFileString( + path.join(staging, MANIFEST), + yield* encodeManifest({ commit, source: worktreePath }), + ); + yield* fs.rename(staging, template); + }).pipe( + Effect.ensuring(fs.remove(`${staging}.index`, { force: true }).pipe(Effect.ignore)), + Effect.onError(() => fs.remove(staging, { recursive: true }).pipe(Effect.ignore)), + ); + + // Keep the newest templates by capture time. Staging leftovers of other captures go too. + const names = yield* fs.readDirectory(directory); + const captured = yield* Effect.forEach( + names.filter((name) => !name.startsWith(".")), + (name) => + fs.stat(path.join(directory, name, MANIFEST)).pipe( + Effect.map((info) => ({ + name, + time: Option.match(info.mtime, { onNone: () => 0, onSome: (date) => date.getTime() }), + })), + Effect.orElseSucceed(() => ({ name, time: 0 })), + ), + ); + const evicted = [ + ...names.filter((name) => name.startsWith(".staging-")), + ...captured + .toSorted((a, b) => b.time - a.time) + .slice(KEPT_TEMPLATES) + .map(({ name }) => name), + ]; + yield* Effect.forEach( + evicted, + (name) => fs.remove(path.join(directory, name), { recursive: true }).pipe(Effect.ignore), + { discard: true }, + ); + }); + + return { find, clone, clear, removeAll, capture }; +}); + +/** Rewrites each entry's stat data in the SHA-1 index v2/v3 at `indexPath` from the files under `root`. */ +const restatIndex = async (indexPath: string, root: string) => { + const index = await NodeFSP.readFile(indexPath); + const body = index.subarray(0, -20); + const version = body.readUInt32BE(4); + const digest = NodeCrypto.createHash("sha1").update(body).digest(); + if (!digest.equals(index.subarray(-20)) || (version !== 2 && version !== 3)) { + throw new Error("unsupported index format"); + } + const parts = [Buffer.from(body.subarray(0, 12))]; + let offset = 12; + for (let count = body.readUInt32BE(8); count > 0; count--) { + const flags = body.readUInt16BE(offset + 60); + const header = 62 + (version === 3 && flags & 0x4000 ? 2 : 0); + const nameEnd = body.indexOf(0, offset + header); + const next = offset + (Math.floor((nameEnd - offset) / 8) + 1) * 8; + const entry = Buffer.from(body.subarray(offset, next)); + // Gitlinks carry no stat data. + if ((entry.readUInt32BE(24) & 0o170000) !== 0o160000) { + const name = body.subarray(offset + header, nameEnd).toString("utf8"); + const stat = await NodeFSP.lstat(`${root}/${name}`, { bigint: true }); + const times = [stat.ctimeNs, stat.mtimeNs].flatMap((ns) => [ + ns / 1_000_000_000n, + ns % 1_000_000_000n, + ]); + const fields: Array = [ + ...times.map((value, at) => [at * 4, value] as const), + [16, stat.dev], + [20, stat.ino], + [28, stat.uid], + [32, stat.gid], + [36, stat.size], + ]; + for (const [at, value] of fields) entry.writeUInt32BE(Number(value & 0xffffffffn), at); + } + parts.push(entry); + offset = next; + } + // Keep the cache tree; other optional extensions hold stat data or offsets that no longer apply. + while (offset < body.length) { + const size = 8 + body.readUInt32BE(offset + 4); + const signature = body.subarray(offset, offset + 4).toString("latin1"); + if (!/^[A-Z]/.test(signature)) throw new Error(`index extension ${signature} is required`); + if (signature === "TREE") parts.push(Buffer.from(body.subarray(offset, offset + size))); + offset += size; + } + const out = Buffer.concat(parts); + await NodeFSP.writeFile( + indexPath, + Buffer.concat([out, NodeCrypto.createHash("sha1").update(out).digest()]), + ); +}; diff --git a/apps/web/src/components/settings/WorktreesSettings.tsx b/apps/web/src/components/settings/WorktreesSettings.tsx index a771e92ceca43..15b166afb024f 100644 --- a/apps/web/src/components/settings/WorktreesSettings.tsx +++ b/apps/web/src/components/settings/WorktreesSettings.tsx @@ -51,6 +51,37 @@ function StartFromRemoteRow() { ); } +function TemplateClonesRow() { + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + return ( + updateSettings({ enableWorktreeTemplates: false })} + /> + ) : null + } + control={ + + updateSettings({ enableWorktreeTemplates: Boolean(checked) }) + } + aria-label="Clone new worktrees from templates" + /> + } + /> + ); +} + export function WorktreesSettingsPanel() { const { scope } = useSettingsScope(); const isProjectScope = scope.kind === "project" || scope.kind === "checkout"; @@ -60,8 +91,14 @@ export function WorktreesSettingsPanel() { {!isProjectScope && } - + + {!isProjectScope && } diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index e61a064431c55..6a1d8b37a815b 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -249,6 +249,15 @@ export const SETTINGS_SEARCH_ITEMS = [ scope: "environment-defaults", searchTerms: ["worktree location folder directory path drive external disk"], }, + { + id: "worktree-templates", + title: "Clone new worktrees from templates", + to: "/settings/worktrees", + scope: "environment-defaults", + searchTerms: [ + "cow reflink clonefile apfs btrfs xfs template node_modules install setup fast worktree", + ], + }, { id: "storage-artifacts", title: "Artifacts and logs", diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index ba481be9f5b00..d664f3a8300f5 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -193,6 +193,18 @@ such as another drive, set **Settings → Worktrees → Worktree location** to a `D:\worktrees` or `~/worktrees`. The setting is per machine. Existing worktrees stay where they are, and cleanup covers both the default folder and the custom one. +## Worktree templates + +**Settings → Worktrees → Advanced → Clone new worktrees from templates** is off by default. When +it is on and the worktree folder's filesystem supports copy-on-write clones (APFS, Btrfs, XFS), +T3 keeps a template of a worktree after its setup script succeeds: the files at its commit plus +its `node_modules`. A new worktree whose lockfiles match a template is cloned from it, and its +setup script then finds the dependencies already installed. Progress shows as the checkout step. +T3 checks the filesystem once per worktree folder and keeps the two most recent templates per +repository in a `.t3-templates` folder beside the worktrees. Anywhere clones are unsupported, or +if a clone fails, worktrees are created as usual. Turning the setting off deletes the templates +the next time that repository gets a worktree. + ## Storage cleanup Open **Settings → Worktrees → Cleanup** to enable automatic cleanup on one machine or all connected diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 4bb3119561b08..d0515b025c282 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1195,6 +1195,8 @@ export const ServerSettings = Schema.Struct({ previousWorktreesDirectories: Schema.Array(TrimmedString).pipe( Schema.withDecodingDefault(Effect.succeed([])), ), + /** Clone new worktrees from copy-on-write templates where the filesystem supports it. */ + enableWorktreeTemplates: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), responseStreamingMode: ResponseStreamingMode.pipe( Schema.withDecodingDefault(Effect.succeed("paragraph" as const)), ), @@ -1604,6 +1606,7 @@ export const ServerSettingsPatch = Schema.Struct({ }), ), worktreesDirectory: Schema.optionalKey(TrimmedString), + enableWorktreeTemplates: Schema.optionalKey(Schema.Boolean), responseStreamingMode: Schema.optionalKey(ResponseStreamingMode), enableProviderUpdateChecks: Schema.optionalKey(Schema.Boolean), continueThreadsAfterServerUpdate: Schema.optionalKey(Schema.Boolean),