diff --git a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx index 710db5df26e63..6ba284dc513db 100644 --- a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx +++ b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx @@ -9,6 +9,7 @@ import { AutoSettleDaysField } from "./components/AutoSettleDaysField"; import { ScreenScrollView as ScrollView } from "../../components/ui/ScreenScrollView"; import { useAtomSet, useAtomValue } from "@effect/atom-react"; import { AsyncResult } from "effect/reactivity"; +import type { ComponentProps } from "react"; import { View } from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; @@ -45,7 +46,20 @@ export function SettingsLabsRouteScreen() { contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }} > - + + @@ -228,48 +242,64 @@ export function NewThreadSettingsSection() { ); } -/** Per machine, since each machine's own setting decides whether it links and answers links. */ -function BetaSettingsSection() { +type MachineBetaSetting = "linkedEnvironmentsEnabled" | "fastWorktreeCheckout"; + +/** One switch per connected machine that offers the beta, since each machine's own setting decides. */ +function MachineBetaSection({ + title, + icon, + capability, + setting, + description, +}: { + readonly title: string; + readonly icon: ComponentProps["icon"]; + readonly capability: "peerLinks" | "fastWorktreeCheckout"; + readonly setting: MachineBetaSetting; + readonly description: string; +}) { const { environments } = useEnvironments(); const supported = environments.filter( (environment) => environment.connection.phase === "connected" && - environment.serverConfig?.environment.capabilities.peerLinks === true, + environment.serverConfig?.environment.capabilities[capability] === true, ); const writable = useEnvironmentsWithScope(supported, AuthSettingsWriteScope); const updateSettings = useAtomCommand(serverEnvironment.updateSettings, { - label: "linked environments beta update", + label: `${setting} beta update`, reportFailure: true, }); if (supported.length === 0) return null; return ( - + {supported.map((environment) => { const policy = environment.serverConfig?.managedPolicy; - const managed = isManagedSettingPath(policy, "linkedEnvironmentsEnabled"); + const managed = isManagedSettingPath(policy, setting); return ( void updateSettings({ environmentId: environment.environmentId, - input: { patch: { linkedEnvironmentsEnabled: value } }, + input: { + patch: + setting === "linkedEnvironmentsEnabled" + ? { linkedEnvironmentsEnabled: value } + : { fastWorktreeCheckout: value }, + }, }) } /> ); })} - - Let agents on a machine work on your other machines, and theirs on it. Turn it on for each - machine. While off, nothing new starts over a link; running tasks finish. - + {description} ); } diff --git a/apps/mobile/src/features/settings/settings-search.ts b/apps/mobile/src/features/settings/settings-search.ts index 6c82f6c317d9d..b3a2e0b68ab97 100644 --- a/apps/mobile/src/features/settings/settings-search.ts +++ b/apps/mobile/src/features/settings/settings-search.ts @@ -104,6 +104,8 @@ const SETTINGS_SEARCH_TERMS = { "beta", "linked environments", "link machine", + "fast worktree checkout", + "clone cache", "tool calls in chat", "hide tool calls", "commands edits searches", diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 468cc89ad01e6..d4883688c08ef 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -42,6 +42,7 @@ export interface ServerDerivedPaths { readonly environmentThemesDir: string; readonly providerStatusCacheDir: string; readonly worktreesDir: string; + readonly worktreeCheckoutCacheDir: string; readonly attachmentsDir: string; readonly browserArtifactsDir: string; readonly logsDir: string; @@ -158,6 +159,7 @@ export const deriveServerPaths = Effect.fn(function* ( environmentThemesDir: join(stateDir, "themes"), providerStatusCacheDir, worktreesDir: join(baseDir, "worktrees"), + worktreeCheckoutCacheDir: join(providerStatusCacheDir, "worktree-checkouts"), attachmentsDir, browserArtifactsDir: join(stateDir, "browser-artifacts"), logsDir, diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index ddce94a44e363..f080cc3588b42 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -239,6 +239,7 @@ export const make = Effect.gen(function* () { projectWorktreeCleanup: true, settledWorktreeCleanup: true, worktreesDirectory: true, + ...(hostPlatform === "darwin" ? { fastWorktreeCheckout: true } : {}), threadAutoSettlementScope: true, threadRestartContinuation: true, projectSettingsOverrides: true, diff --git a/apps/server/src/git/GitManager.ts b/apps/server/src/git/GitManager.ts index a944922d530b5..e6b4bdcfe4230 100644 --- a/apps/server/src/git/GitManager.ts +++ b/apps/server/src/git/GitManager.ts @@ -750,9 +750,20 @@ export const make = Effect.gen(function* () { } return result; }); - const readWorktreesDirectory = serverSettingsService.getSettings.pipe( - Effect.map((settings) => settings.worktreesDirectory), - Effect.orElseSucceed(() => ""), + const readWorktreeCreationSettings = serverSettingsService.getSettings.pipe( + Effect.map((settings) => ({ + worktreesDirectory: settings.worktreesDirectory, + fastCheckout: settings.fastWorktreeCheckout, + })), + Effect.orElseSucceed(() => ({ worktreesDirectory: "" })), + ); + // Turning fast checkout off deletes its cache right away, not at the next worktree. + yield* serverSettingsService.streamChanges.pipe( + Stream.map((settings) => settings.fastWorktreeCheckout), + Stream.changes, + Stream.filter((enabled) => !enabled), + Stream.runForEach(() => gitCore.clearWorktreeCheckoutCache), + Effect.forkScoped, ); const createWorktree: GitManager["Service"]["createWorktree"] = Effect.fn( "GitManager.createWorktree", @@ -764,9 +775,8 @@ export const make = Effect.gen(function* () { Effect.map(({ settings }) => settings.worktreeSubmodules), Effect.orElseSucceed(() => null), ); - const worktreesDirectory = yield* readWorktreesDirectory; const result = yield* gitCore.createWorktree(input, { - worktreesDirectory, + ...(yield* readWorktreeCreationSettings), ...options, submodules, }); @@ -3075,7 +3085,7 @@ export const make = Effect.gen(function* () { path: null, }, { - worktreesDirectory: yield* readWorktreesDirectory, + ...(yield* readWorktreeCreationSettings), submodules: yield* projectSettingsFor(input).pipe( Effect.map(({ settings }) => settings.worktreeSubmodules), Effect.orElseSucceed(() => null), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index ed51ee642418c..ae0d5ef346d34 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -147,6 +147,7 @@ export function makeReplayServerConfig( settingsPath: path.join(stateDir, "settings.json"), providerStatusCacheDir, worktreesDir, + worktreeCheckoutCacheDir: path.join(providerStatusCacheDir, "worktree-checkouts"), attachmentsDir, browserArtifactsDir: path.join(stateDir, "browser-artifacts"), environmentThemesDir, diff --git a/apps/server/src/vcs/CheckoutCloner.ts b/apps/server/src/vcs/CheckoutCloner.ts new file mode 100644 index 0000000000000..9938cc6a169cf --- /dev/null +++ b/apps/server/src/vcs/CheckoutCloner.ts @@ -0,0 +1,81 @@ +import * as NodeModule from "node:module"; + +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +export class CheckoutCloneError extends Schema.TaggedError()( + "CheckoutCloneError", + { + source: Schema.String, + destination: Schema.String, + errno: Schema.optional(Schema.Number), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return `Could not clone ${this.source} to ${this.destination}${this.errno === undefined ? "" : ` (errno ${this.errno})`}.`; + } +} + +/** + * Copies a file or a whole folder copy-on-write in one call, sharing data blocks until either + * copy changes. The destination must not exist. Undefined where the platform has no such call. + */ +export const CheckoutCloner = Context.Reference< + ((source: string, destination: string) => Effect.Effect) | undefined +>("t3/vcs/CheckoutCloner", { defaultValue: () => undefined }); + +// A single-executable server resolves packages on disk only through `require`. +const requireForFfi = NodeModule.createRequire(import.meta.url); + +/** macOS `clonefile(2)`, which clones a folder hierarchy in the kernel; elsewhere nothing. */ +export const layer = Layer.effect( + CheckoutCloner, + Effect.gen(function* () { + if ((yield* HostProcess.Platform) !== "darwin") return undefined; + const loadClonefile = yield* Effect.cached( + Effect.try({ + try: () => { + const { DataType, load, open } = requireForFfi("ffi-rs") as typeof import("ffi-rs"); + open({ library: "t3-libsystem", path: "/usr/lib/libSystem.B.dylib" }); + return (source: string, destination: string) => + load({ + library: "t3-libsystem", + funcName: "clonefile", + retType: DataType.I32, + paramsType: [DataType.String, DataType.String, DataType.I32], + // CLONE_NOFOLLOW: clone a symlink itself rather than its target. + paramsValue: [source, destination, 0x0001], + errno: true, + // A large folder takes hundreds of milliseconds; keep it off the event loop. + runInNewThread: true, + }); + }, + catch: (cause) => + new CheckoutCloneError({ source: "ffi-rs", destination: "clonefile", cause }), + }), + ); + return Effect.fn("CheckoutCloner.clonefile")(function* (source: string, destination: string) { + const clonefile = yield* loadClonefile.pipe( + Effect.mapError( + (error) => new CheckoutCloneError({ source, destination, cause: error.cause }), + ), + ); + const result = yield* Effect.tryPromise({ + try: () => clonefile(source, destination), + catch: (cause) => new CheckoutCloneError({ source, destination, cause }), + }); + // ffi-rs types the errno form as a union with the plain return value. + if (typeof result === "number" || result.value !== 0) { + return yield* new CheckoutCloneError({ + source, + destination, + ...(typeof result === "number" ? {} : { errno: result.errnoCode }), + }); + } + }); + }), +); diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 0d97d43c3e9d6..59db351198182 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -42,6 +42,7 @@ import { PATCH_RENDER_PREFIX_ARGS, splitNullSeparatedGitStdoutPaths, } from "./GitVcsDriverCore.ts"; +import * as CheckoutCloner from "./CheckoutCloner.ts"; import * as VcsDriver from "./VcsDriver.ts"; import * as VcsProcess from "./VcsProcess.ts"; @@ -144,6 +145,8 @@ export interface CreateWorktreeOptions { /** Null or omitted defers to the checkout's t3.json. */ readonly submodules?: WorktreeSubmodules | null; readonly worktreesDirectory?: string; + /** Clone the checkout from the macOS worktree checkout cache. */ + readonly fastCheckout?: boolean; } export interface GitCommitProgress { @@ -373,6 +376,8 @@ export class GitVcsDriver extends Context.Service< input: VcsCreateWorktreeInput, options?: CreateWorktreeOptions, ) => Effect.Effect; + /** Deletes the macOS worktree checkout cache in the background. */ + readonly clearWorktreeCheckoutCache: Effect.Effect; /** Force-updates local `branch` to the primary remote's `ref`. */ readonly fetchRefIntoBranch: ( input: GitFetchRefInput & { readonly branch: string }, @@ -1594,4 +1599,4 @@ export const make = Effect.gen(function* () { }); export const layerVcs = Layer.effect(VcsDriver.VcsDriver, makeVcsDriver); -export const layer = Layer.effect(GitVcsDriver, make); +export const layer = Layer.effect(GitVcsDriver, make).pipe(Layer.provide(CheckoutCloner.layer)); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index e41aada677570..4fcf1b05fe6f5 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -43,6 +43,7 @@ import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observabil import * as GitVcsDriver from "./GitVcsDriver.ts"; import { removeEmptyDirectory, type RemoveEmptyDirectoryError } from "./removeEmptyDirectory.ts"; import { explicitBareRepositoryEnv, gitConfigEnv } from "./VcsProcess.ts"; +import { makeWorktreeCheckoutCache } from "./worktreeCheckoutCache.ts"; import { removeLinkedWorktree } from "./worktreeRemoval.ts"; import { resolveWorktreesDirectory } from "../worktreesDirectory.ts"; import { @@ -945,7 +946,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const { worktreesDir } = yield* ServerConfig.ServerConfig; + const { worktreesDir, worktreeCheckoutCacheDir } = yield* ServerConfig.ServerConfig; const isServerInternalPath = yield* makeIsServerInternalPath; const digest = yield* Digest.Digest; const hostPlatform = yield* HostProcess.Platform; @@ -1143,6 +1144,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* input.writesSharedConfig ? withSharedConfigLock(input.cwd, execution) : execution, ); + const checkoutCache = yield* makeWorktreeCheckoutCache({ + cacheDir: worktreeCheckoutCacheDir, + execute, + }); + const executeGit = ( operation: string, cwd: string, @@ -4259,7 +4265,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* "git worktree add failed", ); } - const args = ["worktree", "add", "--", worktreePath, newRefName ?? input.refName]; + const checkoutRef = newRefName ?? input.refName; const progress = options?.progress; const onCheckoutProgress = progress?.onCheckoutProgress; @@ -4274,10 +4280,20 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { concurrency: "unbounded" }, ); const checkoutWorkers = configuredCheckoutWorkers ?? "0"; - yield* executeGitWithStableDiagnostics( + const fastCheckout = checkoutCache !== undefined && options?.fastCheckout === true; + const cacheGeneration = fastCheckout ? yield* checkoutCache.currentGeneration : 0; + const stockCheckout = executeGitWithStableDiagnostics( "GitVcsDriver.createWorktree", input.cwd, - ["-c", `checkout.workers=${checkoutWorkers}`, ...args], + [ + "-c", + `checkout.workers=${checkoutWorkers}`, + "worktree", + "add", + "--", + worktreePath, + checkoutRef, + ], { fallbackErrorDetail: "git worktree add failed", // Large checkouts and post-checkout hooks can legitimately run for many @@ -4297,6 +4313,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } : {}), }, + ); + yield* ( + fastCheckout + ? checkoutCache + .create({ cwd: input.cwd, worktreePath, refName: checkoutRef, checkoutWorkers }) + .pipe(Effect.flatMap((cloned) => (cloned ? Effect.void : stockCheckout))) + : stockCheckout ).pipe( Effect.onError(() => newRefName @@ -4308,6 +4331,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : Effect.void, ), ); + if (fastCheckout) { + yield* checkoutCache.schedule(worktreePath, cacheGeneration); + } if (progress?.onWorktreeClaimed) { yield* progress.onWorktreeClaimed(worktreePath); @@ -5097,6 +5123,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* Effect.ensuring(invalidateWorktreeCreationCaches(input.cwd)), ); }), + clearWorktreeCheckoutCache: checkoutCache?.clear ?? Effect.void, fetchRefIntoBranch: Effect.fnUntraced(function* (input) { return yield* withListRefsInvalidation(input.cwd, fetchRefIntoBranch(input)); }), diff --git a/apps/server/src/vcs/gitIndexStat.test.ts b/apps/server/src/vcs/gitIndexStat.test.ts new file mode 100644 index 0000000000000..fd8d1c293d87e --- /dev/null +++ b/apps/server/src/vcs/gitIndexStat.test.ts @@ -0,0 +1,130 @@ +// @effect-diagnostics nodeBuiltinImport:off - Fixtures drive git and cp directly and read raw stat data. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { assert, describe, it } from "@effect/vitest"; +import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; +import * as Effect from "effect/Effect"; + +import { indexEntryStat, parseIndex, restatIndex } from "./gitIndexStat.ts"; + +const git = (cwd: string, args: ReadonlyArray, env: NodeJS.ProcessEnv = {}) => + NodeChildProcess.execFileSync("git", args, { + cwd, + env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1", ...env }, + encoding: "utf8", + }); + +// A repository whose checkout in `source` was written by Git with its own index, then copied +// to `copy` the way a clone would: same contents and mtimes, new inodes and ctimes. +const makeCopiedCheckout = Effect.acquireRelease( + Effect.sync(() => { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "git-index-stat-")); + const repo = NodePath.join(root, "repo"); + NodeFS.mkdirSync(repo); + git(repo, ["init", "-q"]); + git(repo, ["config", "filter.probe.clean", "echo read >> ../clean-runs; cat"]); + NodeFS.writeFileSync(NodePath.join(repo, ".gitattributes"), "*.txt filter=probe\n"); + NodeFS.mkdirSync(NodePath.join(repo, "nested")); + NodeFS.writeFileSync(NodePath.join(repo, "a.txt"), "alpha\n"); + NodeFS.writeFileSync(NodePath.join(repo, "nested", "b.txt"), "beta\n"); + NodeFS.writeFileSync(NodePath.join(repo, "run.sh"), "#!/bin/sh\n", { mode: 0o755 }); + NodeFS.symlinkSync("a.txt", NodePath.join(repo, "link")); + git(repo, ["add", "."]); + git(repo, ["-c", "user.name=T", "-c", "user.email=t@t", "commit", "-qm", "init"]); + const tree = git(repo, ["rev-parse", "HEAD^{tree}"]).trim(); + const source = NodePath.join(root, "source"); + const sourceIndex = NodePath.join(root, "source.index"); + NodeFS.mkdirSync(source); + const env = { GIT_INDEX_FILE: sourceIndex }; + git( + repo, + ["-c", "index.version=2", `--work-tree=${source}`, "read-tree", "--reset", "-u", tree], + env, + ); + // Old mtimes keep every entry clear of Git's racy-clean re-read. + for (const name of ["a.txt", "nested/b.txt", "run.sh", ".gitattributes"]) { + NodeFS.utimesSync(NodePath.join(source, name), 1_000_000, 1_000_000); + } + git( + repo, + ["-c", "index.version=2", `--work-tree=${source}`, "update-index", "-q", "--refresh"], + env, + ); + const copy = NodePath.join(root, "copy"); + NodeChildProcess.execFileSync("cp", ["-a", source, copy]); + // Setup ran the filter itself; only reads after this point count. + NodeFS.rmSync(NodePath.join(root, "clean-runs"), { force: true }); + return { root, repo, source, sourceIndex, copy }; + }), + ({ root }) => Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), +); + +const statAll = (root: string, paths: ReadonlyArray) => + paths.map((entryPath) => { + try { + return indexEntryStat(NodeFS.lstatSync(NodePath.join(root, entryPath), { bigint: true })); + } catch { + return undefined; + } + }); + +// The feature is macOS-only; the fixtures need symlinks. +describe.skipIf(!symlinksSupported)("restatIndex", () => { + it.effect("makes Git trust copied files without reading them", () => + Effect.gen(function* () { + const { root, repo, sourceIndex, copy } = yield* makeCopiedCheckout; + const parsed = parseIndex(NodeFS.readFileSync(sourceIndex)); + assert.isNotNull(parsed); + const paths = parsed!.entries.map((entry) => entry.path); + assert.deepStrictEqual(paths, [".gitattributes", "a.txt", "link", "nested/b.txt", "run.sh"]); + + const restated = restatIndex(parsed!, statAll(copy, paths), { verify: true }); + assert.isNotNull(restated); + const copyIndex = NodePath.join(root, "copy.index"); + NodeFS.writeFileSync(copyIndex, restated!); + const status = git( + repo, + [`--work-tree=${copy}`, "status", "--porcelain", "--untracked-files=no"], + { + GIT_INDEX_FILE: copyIndex, + }, + ); + assert.equal(status, ""); + assert.isFalse(NodeFS.existsSync(NodePath.join(root, "clean-runs"))); + }), + ); + + it.effect("rejects a copy whose file no longer matches what Git recorded", () => + Effect.gen(function* () { + const { sourceIndex, copy } = yield* makeCopiedCheckout; + const parsed = parseIndex(NodeFS.readFileSync(sourceIndex))!; + const paths = parsed.entries.map((entry) => entry.path); + NodeFS.writeFileSync(NodePath.join(copy, "a.txt"), "ALPHA\n"); + assert.isNull(restatIndex(parsed, statAll(copy, paths), { verify: true })); + assert.isNotNull(restatIndex(parsed, statAll(copy, paths), { verify: false })); + }), + ); + + it.effect("rejects two tracked paths that are one file, as on a case-insensitive volume", () => + Effect.gen(function* () { + const { sourceIndex, copy } = yield* makeCopiedCheckout; + const parsed = parseIndex(NodeFS.readFileSync(sourceIndex))!; + const paths = parsed.entries.map((entry) => entry.path); + NodeFS.rmSync(NodePath.join(copy, "nested", "b.txt")); + NodeFS.linkSync(NodePath.join(copy, "a.txt"), NodePath.join(copy, "nested", "b.txt")); + assert.isNull(restatIndex(parsed, statAll(copy, paths), { verify: false })); + }), + ); + + it.effect("leaves indexes it cannot rewrite safely to Git", () => + Effect.gen(function* () { + const { repo, source, sourceIndex } = yield* makeCopiedCheckout; + const env = { GIT_INDEX_FILE: sourceIndex }; + git(repo, [`--work-tree=${source}`, "update-index", "--index-version", "4"], env); + assert.isNull(parseIndex(NodeFS.readFileSync(sourceIndex))); + }), + ); +}); diff --git a/apps/server/src/vcs/gitIndexStat.ts b/apps/server/src/vcs/gitIndexStat.ts new file mode 100644 index 0000000000000..c53fcc28879a0 --- /dev/null +++ b/apps/server/src/vcs/gitIndexStat.ts @@ -0,0 +1,190 @@ +// @effect-diagnostics nodeBuiltinImport:off - Pure helpers; the index checksum is SHA-1 over bytes. +import * as NodeCrypto from "node:crypto"; + +/** What Git compares in an index entry's stat data. Numbers are already truncated to 32 bits. */ +export interface IndexEntryStat { + readonly ctimeSeconds: number; + readonly ctimeNanoseconds: number; + readonly mtimeSeconds: number; + readonly mtimeNanoseconds: number; + readonly dev: number; + readonly ino: number; + readonly uid: number; + readonly gid: number; + readonly size: number; + readonly kind: "file" | "executable" | "symlink" | "directory" | "other"; +} + +export interface ParsedIndex { + readonly bytes: Uint8Array; + readonly entries: ReadonlyArray<{ readonly offset: number; readonly path: string }>; +} + +const ENTRY_FIXED_BYTES = 62; +const SHA1_BYTES = 20; +const MODE_TYPE_MASK = 0o170000; +const MODE_REGULAR = 0o100000; +const MODE_SYMLINK = 0o120000; +const MODE_GITLINK = 0o160000; +const FLAG_ASSUME_VALID = 0x8000; +const FLAG_EXTENDED = 0x4000; +const FLAG_STAGE = 0x3000; +const FLAG_NAME_LENGTH = 0x0fff; +const EXTENDED_SKIP_WORKTREE = 0x4000; +// Extensions that let Git trust an entry without checking the file, or that point at another index. +const UNSAFE_EXTENSIONS = new Set(["link", "FSMN", "sdir", "UNTR"]); + +const decoder = new TextDecoder(); + +/** + * Reads a SHA-1 index that T3 wrote itself (versions 2 and 3). Anything else, including + * conflicts, sparse entries, split indexes and fsmonitor data, returns null so callers use Git. + */ +export function parseIndex(bytes: Uint8Array): ParsedIndex | null { + if (bytes.length < 12 + SHA1_BYTES) return null; + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (decoder.decode(bytes.subarray(0, 4)) !== "DIRC") return null; + const version = view.getUint32(4); + if (version !== 2 && version !== 3) return null; + const expected = NodeCrypto.createHash("sha1") + .update(bytes.subarray(0, bytes.length - SHA1_BYTES)) + .digest(); + if (!expected.equals(bytes.subarray(bytes.length - SHA1_BYTES))) return null; + + const count = view.getUint32(8); + const entries: Array<{ offset: number; path: string }> = []; + let offset = 12; + for (let index = 0; index < count; index++) { + if (offset + ENTRY_FIXED_BYTES > bytes.length - SHA1_BYTES) return null; + const flags = view.getUint16(offset + 60); + if (flags & (FLAG_ASSUME_VALID | FLAG_STAGE)) return null; + let nameStart = offset + ENTRY_FIXED_BYTES; + if (flags & FLAG_EXTENDED) { + if (version < 3 || view.getUint16(nameStart) & EXTENDED_SKIP_WORKTREE) return null; + nameStart += 2; + } + const nameLength = flags & FLAG_NAME_LENGTH; + const nameEnd = + nameLength < FLAG_NAME_LENGTH ? nameStart + nameLength : bytes.indexOf(0, nameStart); + if (nameEnd < 0 || bytes[nameEnd] !== 0) return null; + entries.push({ offset, path: decoder.decode(bytes.subarray(nameStart, nameEnd)) }); + // Entries are NUL-padded to a multiple of eight bytes, with at least one NUL. + offset += (nameEnd - offset + 8) & ~7; + } + + // Extensions follow the entries as a 4-byte signature and a 4-byte length. + while (offset < bytes.length - SHA1_BYTES) { + if (offset + 8 > bytes.length - SHA1_BYTES) return null; + if (UNSAFE_EXTENSIONS.has(decoder.decode(bytes.subarray(offset, offset + 4)))) return null; + offset += 8 + view.getUint32(offset + 4); + } + return offset === bytes.length - SHA1_BYTES ? { bytes, entries } : null; +} + +function kindForMode(mode: number): IndexEntryStat["kind"] { + switch (mode & MODE_TYPE_MASK) { + case MODE_REGULAR: + return mode & 0o111 ? "executable" : "file"; + case MODE_SYMLINK: + return "symlink"; + case MODE_GITLINK: + return "directory"; + default: + return "other"; + } +} + +/** + * Returns a copy of the index with each entry's stat data replaced by `stats[i]`, so Git trusts + * files it did not write itself. With `verify`, an entry whose recorded mtime, size or type + * differs from its file makes the result null: the files no longer match what Git checked out. + * A missing stat (an uninitialized submodule folder) keeps the entry's recorded data. + */ +export function restatIndex( + parsed: ParsedIndex, + stats: ReadonlyArray, + options: { readonly verify: boolean }, +): Uint8Array | null { + const bytes = new Uint8Array(parsed.bytes); + const view = new DataView(bytes.buffer); + // Two tracked paths on one file, as on a case-insensitive volume, cannot both match. + const files = new Set(); + for (const [index, entry] of parsed.entries.entries()) { + const stat = stats[index]; + const offset = entry.offset; + const kind = kindForMode(view.getUint32(offset + 24)); + if (stat === undefined) { + if (kind !== "directory") return null; + continue; + } + if (stat.kind !== kind) return null; + if (kind !== "directory") { + const file = `${stat.dev}:${stat.ino}`; + if (files.has(file)) return null; + files.add(file); + } + if ( + options.verify && + kind !== "directory" && + (view.getUint32(offset + 8) !== stat.mtimeSeconds || + view.getUint32(offset + 12) !== stat.mtimeNanoseconds || + view.getUint32(offset + 36) !== stat.size) + ) { + return null; + } + view.setUint32(offset, stat.ctimeSeconds); + view.setUint32(offset + 4, stat.ctimeNanoseconds); + view.setUint32(offset + 8, stat.mtimeSeconds); + view.setUint32(offset + 12, stat.mtimeNanoseconds); + view.setUint32(offset + 16, stat.dev); + view.setUint32(offset + 20, stat.ino); + view.setUint32(offset + 28, stat.uid); + view.setUint32(offset + 32, stat.gid); + view.setUint32(offset + 36, stat.size); + } + bytes.set( + NodeCrypto.createHash("sha1") + .update(bytes.subarray(0, bytes.length - SHA1_BYTES)) + .digest(), + bytes.length - SHA1_BYTES, + ); + return bytes; +} + +const U32 = 0xffff_ffffn; +const BILLION = 1_000_000_000n; + +/** Converts a bigint `lstat` result the way Git does: each field truncated to 32 bits. */ +export function indexEntryStat(stat: { + readonly ctimeNs: bigint; + readonly mtimeNs: bigint; + readonly dev: bigint; + readonly ino: bigint; + readonly uid: bigint; + readonly gid: bigint; + readonly size: bigint; + readonly mode: bigint; +}): IndexEntryStat { + const type = Number(stat.mode) & MODE_TYPE_MASK; + return { + ctimeSeconds: Number((stat.ctimeNs / BILLION) & U32), + ctimeNanoseconds: Number(stat.ctimeNs % BILLION), + mtimeSeconds: Number((stat.mtimeNs / BILLION) & U32), + mtimeNanoseconds: Number(stat.mtimeNs % BILLION), + dev: Number(stat.dev & U32), + ino: Number(stat.ino & U32), + uid: Number(stat.uid & U32), + gid: Number(stat.gid & U32), + size: Number(stat.size & U32), + kind: + type === MODE_REGULAR + ? Number(stat.mode) & 0o111 + ? "executable" + : "file" + : type === MODE_SYMLINK + ? "symlink" + : type === 0o040000 + ? "directory" + : "other", + }; +} diff --git a/apps/server/src/vcs/worktreeCheckoutCache.test.ts b/apps/server/src/vcs/worktreeCheckoutCache.test.ts new file mode 100644 index 0000000000000..0c81504eedb67 --- /dev/null +++ b/apps/server/src/vcs/worktreeCheckoutCache.test.ts @@ -0,0 +1,344 @@ +// @effect-diagnostics nodeBuiltinImport:off - Fixtures drive git and cp directly and read raw stat data. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { assert, describe, it } from "@effect/vitest"; +import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import * as ServerConfig from "../config.ts"; +import * as CheckoutCloner from "./CheckoutCloner.ts"; +import { makeGitVcsDriverCore } from "./GitVcsDriverCore.ts"; +import { makeWorktreeCheckoutCache } from "./worktreeCheckoutCache.ts"; + +// Tests close their own scope first, so background cache builds finish before the temp folder goes. +const layerTest = ServerConfig.layerTest(process.cwd(), { + prefix: "t3-worktree-checkout-cache-test-", +}).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); + +// macOS clones for real; elsewhere `cp -a` stands in, keeping mtimes like a clone does. +const testCloner = Effect.gen(function* () { + const native = yield* CheckoutCloner.CheckoutCloner.pipe(Effect.provide(CheckoutCloner.layer)); + return ( + native ?? + ((source: string, destination: string) => + Effect.try({ + try: () => void NodeChildProcess.execFileSync("cp", ["-a", source, destination]), + catch: (cause) => new CheckoutCloner.CheckoutCloneError({ source, destination, cause }), + })) + ); +}); + +const sh = (cwd: string, args: ReadonlyArray) => + NodeChildProcess.execFileSync("git", args, { + cwd, + encoding: "utf8", + env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }, + }).trim(); + +const setup = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "worktree-checkout-cache-" }); + const repo = NodePath.join(root, "repo"); + NodeFS.mkdirSync(repo); + sh(repo, ["init", "-q", "-b", "main"]); + sh(repo, ["config", "user.name", "Test"]); + sh(repo, ["config", "user.email", "test@test.com"]); + sh(repo, ["config", "filter.probe.smudge", "cat"]); + sh(repo, ["config", "filter.probe.clean", "cat"]); + NodeFS.writeFileSync( + NodePath.join(repo, ".gitattributes"), + "*.bat text eol=crlf\n*.gen filter=probe\n", + ); + NodeFS.writeFileSync(NodePath.join(repo, "README.md"), "# test\n"); + NodeFS.writeFileSync(NodePath.join(repo, "run.bat"), "echo hi\r\n"); + NodeFS.mkdirSync(NodePath.join(repo, "src")); + NodeFS.writeFileSync(NodePath.join(repo, "src", "tool.sh"), "#!/bin/sh\n", { mode: 0o755 }); + NodeFS.symlinkSync("README.md", NodePath.join(repo, "readme-link")); + sh(repo, ["add", "."]); + sh(repo, ["commit", "-qm", "initial"]); + + const spawned: Array> = []; + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command)) spawned.push(command.args.map(String)); + return delegate.spawn(command); + }); + const clone = yield* testCloner; + const provide = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(CheckoutCloner.CheckoutCloner, clone), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HostProcess.Environment, { + ...process.env, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + }), + ); + const driver = yield* provide(makeGitVcsDriverCore()); + const { worktreeCheckoutCacheDir } = yield* ServerConfig.ServerConfig; + const cache = (yield* provide( + makeWorktreeCheckoutCache({ cacheDir: worktreeCheckoutCacheDir, execute: driver.execute }), + ))!; + const create = (name: string, refName = "main", fastCheckout = true) => + provide( + driver.createWorktree( + { cwd: repo, refName, newRefName: name, path: NodePath.join(root, name) }, + { fastCheckout }, + ), + ).pipe(Effect.map(({ worktree }) => worktree.path)); + // Whether the last creation of `name` checked out with Git rather than cloning. + const checkedOutWithGit = (name: string) => + spawned.some( + (args) => + args.includes("add") && + args.includes(NodePath.join(root, name)) && + !args.includes("--no-checkout"), + ); + return { + root, + repo, + create, + checkedOutWithGit, + cacheDir: worktreeCheckoutCacheDir, + build: (worktreePath: string) => provide(cache.build(worktreePath, 0)), + }; +}); + +// Every tracked file and its type, mode and contents, so two checkouts can be compared. +const snapshot = (worktreePath: string) => + sh(worktreePath, ["ls-files", "-z"]) + .split("\0") + .filter(Boolean) + .map((name) => { + const file = NodePath.join(worktreePath, name); + const stat = NodeFS.lstatSync(file); + return [ + name, + stat.mode, + stat.isSymbolicLink() ? NodeFS.readlinkSync(file) : NodeFS.readFileSync(file, "utf8"), + ]; + }); + +// Git's recorded stat data matches each file, so status will not re-read them. +const indexMatchesFiles = (worktreePath: string) => { + const debug = sh(worktreePath, ["ls-files", "--debug"]); + for (const block of debug.split(/\n(?=\S)/)) { + const name = block.split("\n")[0]!; + const ino = Number(block.match(/ino: (\d+)/)![1]); + const size = Number(block.match(/size: (\d+)/)![1]); + const stat = NodeFS.lstatSync(NodePath.join(worktreePath, name), { bigint: true }); + if (ino !== Number(stat.ino & 0xffff_ffffn) || size !== Number(stat.size)) return false; + } + return true; +}; + +// The feature is macOS-only; the fixtures need symlinks. +describe.skipIf(!symlinksSupported)("worktree checkout cache", () => { + it.effect("clones a cached checkout that matches what git worktree add writes", () => + Effect.gen(function* () { + const fixture = yield* setup; + const stock = yield* fixture.create("stock", "main", false); + yield* fixture.build(stock); + + const fast = yield* fixture.create("fast"); + assert.isFalse(fixture.checkedOutWithGit("fast")); + assert.deepStrictEqual(snapshot(fast), snapshot(stock)); + assert.equal(sh(fast, ["status", "--porcelain"]), ""); + assert.equal(sh(fast, ["rev-parse", "--abbrev-ref", "HEAD"]), "fast"); + assert.isTrue(indexMatchesFiles(fast)); + // The clone has its own files: editing one leaves the cache and the other worktree alone. + NodeFS.writeFileSync(NodePath.join(fast, "README.md"), "changed\n"); + assert.equal(sh(stock, ["status", "--porcelain"]), ""); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + it.effect("moves an older cached checkout to the requested commit", () => + Effect.gen(function* () { + const fixture = yield* setup; + yield* fixture.build(yield* fixture.create("base", "main", false)); + NodeFS.writeFileSync(NodePath.join(fixture.repo, "README.md"), "# newer\n"); + NodeFS.writeFileSync(NodePath.join(fixture.repo, "added.txt"), "new file\n"); + NodeFS.rmSync(NodePath.join(fixture.repo, "run.bat")); + sh(fixture.repo, ["add", "-A"]); + sh(fixture.repo, ["commit", "-qm", "newer"]); + + const fast = yield* fixture.create("fast"); + assert.isFalse(fixture.checkedOutWithGit("fast")); + const stock = yield* fixture.create("stock", "main", false); + assert.deepStrictEqual(snapshot(fast), snapshot(stock)); + assert.equal(sh(fast, ["status", "--porcelain"]), ""); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + it.effect("caches a far newer commit by moving the older cached checkout", () => + Effect.gen(function* () { + const fixture = yield* setup; + yield* fixture.build(yield* fixture.create("base", "main", false)); + NodeFS.mkdirSync(NodePath.join(fixture.repo, "many")); + for (let index = 0; index < 600; index++) { + NodeFS.writeFileSync(NodePath.join(fixture.repo, "many", `${index}.txt`), `${index}\n`); + } + NodeFS.writeFileSync(NodePath.join(fixture.repo, "README.md"), "# newer\n"); + sh(fixture.repo, ["add", "-A"]); + sh(fixture.repo, ["commit", "-qm", "many files"]); + const stock = yield* fixture.create("stock", "main", false); + yield* fixture.build(stock); + + const fast = yield* fixture.create("fast"); + assert.isFalse(fixture.checkedOutWithGit("fast")); + assert.deepStrictEqual(snapshot(fast), snapshot(stock)); + assert.equal(sh(fast, ["status", "--porcelain"]), ""); + assert.isTrue(indexMatchesFiles(fast)); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + it.effect("runs the post-checkout hook as git worktree add would", () => + Effect.gen(function* () { + const fixture = yield* setup; + yield* fixture.build(yield* fixture.create("base", "main", false)); + const marker = NodePath.join(fixture.root, "hook-args"); + const hook = NodePath.join(fixture.repo, ".git", "hooks", "post-checkout"); + NodeFS.writeFileSync(hook, `#!/bin/sh\necho "$@" > "${marker}"\n`, { mode: 0o755 }); + + yield* fixture.create("fast"); + assert.isFalse(fixture.checkedOutWithGit("fast")); + const commit = sh(fixture.repo, ["rev-parse", "main"]); + assert.equal(NodeFS.readFileSync(marker, "utf8").trim(), `${"0".repeat(40)} ${commit} 1`); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + it.effect("checks out with Git when the checkout adds a post-index-change hook", () => + Effect.gen(function* () { + const fixture = yield* setup; + const marker = NodePath.join(fixture.root, "index-hook-runs"); + NodeFS.mkdirSync(NodePath.join(fixture.repo, ".hooks")); + const hook = NodePath.join(fixture.repo, ".hooks", "post-index-change"); + NodeFS.writeFileSync(hook, `#!/bin/sh\necho ran >> "${marker}"\n`); + sh(fixture.repo, ["add", "--chmod=+x", hook]); + sh(fixture.repo, ["commit", "-qm", "tracked hooks"]); + yield* fixture.build(yield* fixture.create("base", "main", false)); + sh(fixture.repo, ["config", "core.hooksPath", ".hooks"]); + NodeFS.rmSync(marker, { force: true }); + + yield* fixture.create("fallback"); + assert.isTrue(fixture.checkedOutWithGit("fallback")); + assert.isTrue(NodeFS.existsSync(marker)); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + it.effect("clones the branch Git checks out when a tag has the same name", () => + Effect.gen(function* () { + const fixture = yield* setup; + sh(fixture.repo, ["tag", "fast"]); + NodeFS.writeFileSync(NodePath.join(fixture.repo, "README.md"), "# newer\n"); + sh(fixture.repo, ["commit", "-qam", "newer"]); + const stock = yield* fixture.create("stock", "main", false); + yield* fixture.build(stock); + + const fast = yield* fixture.create("fast"); + assert.isFalse(fixture.checkedOutWithGit("fast")); + assert.deepStrictEqual(snapshot(fast), snapshot(stock)); + assert.equal(sh(fast, ["status", "--porcelain"]), ""); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + it.effect("does not cache checkouts whose attributes name a filter", () => + Effect.gen(function* () { + const fixture = yield* setup; + NodeFS.writeFileSync(NodePath.join(fixture.repo, ".gitattributes"), "*.bin filter=lfs\n"); + NodeFS.writeFileSync(NodePath.join(fixture.repo, "asset.bin"), "pointer\n"); + sh(fixture.repo, ["add", "-A"]); + sh(fixture.repo, ["commit", "-qm", "lfs"]); + yield* fixture.build(yield* fixture.create("stock", "main", false)); + yield* fixture.create("next"); + assert.isTrue(fixture.checkedOutWithGit("next")); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + const cacheEntry = (fixture: { cacheDir: string }) => { + const [bucket] = NodeFS.readdirSync(fixture.cacheDir); + const [entry] = NodeFS.readdirSync(NodePath.join(fixture.cacheDir, bucket!)); + return NodePath.join(fixture.cacheDir, bucket!, entry!, "files"); + }; + + it.effect.each([ + { + name: "a cached file changed", + change: (fixture: { cacheDir: string; repo: string; root: string }) => + NodeFS.writeFileSync(NodePath.join(cacheEntry(fixture), "README.md"), "tampered with\n"), + }, + { + name: "a file was added to the cache", + change: (fixture: { cacheDir: string; repo: string; root: string }) => + NodeFS.writeFileSync(NodePath.join(cacheEntry(fixture), "src", "extra.txt"), "extra\n"), + }, + { + name: "the repository's checkout settings changed", + change: (fixture: { cacheDir: string; repo: string; root: string }) => + sh(fixture.repo, ["config", "core.autocrlf", "true"]), + }, + { + // Moving the cached checkout forward would keep README.md's old line endings. + name: "attributes changed since the cached commit", + change: (fixture: { cacheDir: string; repo: string; root: string }) => { + NodeFS.appendFileSync( + NodePath.join(fixture.repo, ".gitattributes"), + "*.md text eol=crlf\n", + ); + sh(fixture.repo, ["commit", "-qam", "crlf markdown"]); + }, + }, + { + // On a case-insensitive volume the new file would land in the cached `src` folder. + name: "a new folder's name differs only in case", + change: (fixture: { cacheDir: string; repo: string; root: string }) => { + // Plumbing, because a case-insensitive volume cannot hold both folders. + NodeFS.writeFileSync(NodePath.join(fixture.root, "b.txt"), "b\n"); + const blob = sh(fixture.repo, ["hash-object", "-w", NodePath.join(fixture.root, "b.txt")]); + sh(fixture.repo, ["update-index", "--add", "--cacheinfo", `100644,${blob},Src/b.txt`]); + sh(fixture.repo, ["commit", "-qm", "case folder"]); + }, + }, + { + // Git would run this hook during its own checkout, with its own environment. + name: "the move adds a post-index-change hook", + change: (fixture: { cacheDir: string; repo: string; root: string }) => { + const hook = NodePath.join(fixture.repo, ".hooks", "post-index-change"); + NodeFS.mkdirSync(NodePath.dirname(hook)); + NodeFS.writeFileSync(hook, "#!/bin/sh\n"); + sh(fixture.repo, ["add", "--chmod=+x", hook]); + sh(fixture.repo, ["commit", "-qm", "hook"]); + sh(fixture.repo, ["config", "core.hooksPath", ".hooks"]); + }, + }, + { + // The cached tree has the rule but no matching file; the move would run the filter itself. + name: "the move adds a file that uses a filter", + change: (fixture: { cacheDir: string; repo: string; root: string }) => { + NodeFS.writeFileSync(NodePath.join(fixture.repo, "asset.gen"), "generated\n"); + sh(fixture.repo, ["add", "-A"]); + sh(fixture.repo, ["commit", "-qm", "generated asset"]); + }, + }, + ])("checks out with Git when $name", ({ change }) => + Effect.gen(function* () { + const fixture = yield* setup; + yield* fixture.build(yield* fixture.create("base", "main", false)); + change(fixture); + + const created = yield* fixture.create("fallback"); + assert.isTrue(fixture.checkedOutWithGit("fallback")); + const stock = yield* fixture.create("stock", "main", false); + assert.deepStrictEqual(snapshot(created), snapshot(stock)); + assert.equal(sh(created, ["status", "--porcelain"]), ""); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); +}); diff --git a/apps/server/src/vcs/worktreeCheckoutCache.ts b/apps/server/src/vcs/worktreeCheckoutCache.ts new file mode 100644 index 0000000000000..a72d6d859fc54 --- /dev/null +++ b/apps/server/src/vcs/worktreeCheckoutCache.ts @@ -0,0 +1,845 @@ +// @effect-diagnostics nodeBuiltinImport:off - Git index stat data needs bigint lstat, which FileSystem lacks. +import * as NodeCrypto from "node:crypto"; +import * as NodeFSP from "node:fs/promises"; + +import { GitCommandError } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import * as CheckoutCloner from "./CheckoutCloner.ts"; +import type * as GitVcsDriver from "./GitVcsDriver.ts"; +import { indexEntryStat, parseIndex, restatIndex, type IndexEntryStat } from "./gitIndexStat.ts"; + +const KEEP_PER_REPOSITORY = 3; +// Moving a cached checkout costs about 0.1 ms per changed file when a worktree is created, while +// replacing it means writing and later deleting a whole checkout, which slows the next clone. +const REBUILD_AFTER_CHANGED_FILES = 500; +const UNUSED_REPOSITORY_MS = 30 * 24 * 60 * 60 * 1000; +// Older leftovers came from a crash, not from work in progress. A full build of a huge repository +// can take minutes, and its staging folder's own mtime does not change while files land inside. +const ABANDONED_STAGING_MS = 24 * 60 * 60 * 1000; +const ABANDONED_TRASH_MS = 10 * 60 * 1000; +const STAT_BATCH = 512; +const TREE_ID = /^[0-9a-f]{40}$/; +// `-`: checkouts of one tree under different settings never replace each other. +const ENTRY_NAME = /^([0-9a-f]{40})-([0-9a-f]{16})$/; +const GIT_FALSE = new Set(["false", "no", "off", "0"]); +const MAX_LIST_BYTES = 256 * 1024 * 1024; +const NULL_OID = "0".repeat(40); +// 2.42 names its system attributes file, which the fingerprint needs; `git hook run` is from 2.36. +const MIN_GIT = [2, 42] as const; +// Settings that change what a checkout writes. Read in the new worktree, so conditional includes apply. +const CHECKOUT_CONFIG = + "^(core\\.(autocrlf|eol|symlinks|safecrlf|checkroundtripencoding|precomposeunicode|ignorecase|filemode|attributesfile|sparsecheckout)|filter\\.|lfs\\.|extensions\\.)"; +// Keeps cache indexes in the one format the stat rewriter reads. +const CACHE_INDEX_CONFIG = [ + "index.version=2", + "index.skipHash=false", + "core.splitIndex=false", + "core.untrackedCache=false", + "core.fsmonitor=false", + "core.sparseCheckout=false", + // Private builds must not run the repository's hooks. + "core.hooksPath=/dev/null", + "checkout.workers=0", +].flatMap((setting) => ["-c", setting]); + +// How a case- and normalization-insensitive volume such as default APFS compares names. +const foldCase = (file: string) => file.normalize("NFC").toLowerCase(); + +// `check-attr --all -z` output: any `filter` attribute counts, as `unset` can also name a driver. +function namesFilter(output: string): boolean { + const fields = output.split("\0"); + for (let index = 1; index < fields.length; index += 3) { + if (fields[index] === "filter") return true; + } + return false; +} + +/** Every folder that holds one of `paths`, including the root as "". */ +function parentDirectories(paths: Iterable): Set { + const directories = new Set([""]); + for (const entryPath of paths) { + for (let slash = entryPath.lastIndexOf("/"); slash > 0;) { + const directory = entryPath.slice(0, slash); + if (directories.has(directory)) break; + directories.add(directory); + slash = directory.lastIndexOf("/"); + } + } + return directories; +} + +class WorktreeCheckoutCacheError extends Schema.TaggedError()( + "WorktreeCheckoutCacheError", + { detail: Schema.String }, +) { + override get message(): string { + return this.detail; + } +} + +interface CacheEntry { + readonly directory: string; + readonly tree: string; + readonly fingerprint: string; + readonly mtimeMs: number; +} + +/** + * Clones a private, Git-written checkout with APFS copy-on-write, then rewrites the index's stat + * data so Git trusts the clones without re-reading them. An older cached checkout is moved to the + * requested tree with `read-tree -m -u`. Undefined where the platform cannot clone a folder. + */ +export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make")(function* (input: { + readonly cacheDir: string; + readonly execute: GitVcsDriver.GitVcsDriver["Service"]["execute"]; +}) { + const clone = yield* CheckoutCloner.CheckoutCloner; + if (clone === undefined) return undefined; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const scope = yield* Effect.scope; + const homeDirectory = yield* HostProcess.HomeDirectory; + const environment = yield* HostProcess.Environment; + const builds = yield* Semaphore.make(1); + const pendingBuilds = new Set(); + // Bumped by `clear`, so a build that started before it does not publish its result. + let generation = 0; + const { cacheDir, execute } = input; + + const git = ( + operation: string, + cwd: string, + args: ReadonlyArray, + env?: NodeJS.ProcessEnv, + ) => + execute({ + operation: `WorktreeCheckoutCache.git.${operation}`, + cwd, + args, + allowNonZeroExit: true, + timeoutMs: null, + ...(env ? { env } : {}), + }).pipe( + Effect.map((result) => (result.exitCode === 0 ? result.stdout : null)), + Effect.orElseSucceed(() => null), + ); + + const gitSupported = yield* Effect.cached( + git("version", homeDirectory, ["version"]).pipe( + Effect.map((stdout) => { + const match = stdout?.match(/git version (\d+)\.(\d+)/); + if (!match) return false; + const [major, minor] = [Number(match[1]), Number(match[2])]; + return major > MIN_GIT[0] || (major === MIN_GIT[0] && minor >= MIN_GIT[1]); + }), + ), + ); + + const describe = Effect.fn("WorktreeCheckoutCache.describe")(function* ( + cwd: string, + rev: string, + ) { + const stdout = yield* git("describe", cwd, [ + "rev-parse", + "--path-format=absolute", + "--git-common-dir", + "--show-object-format", + "--git-path", + "hooks/post-index-change", + `${rev}^{commit}`, + `${rev}^{tree}`, + ]); + const [commonDir, objectFormat, indexHook, commit, tree] = stdout?.trim().split("\n") ?? []; + if ( + !commonDir || + objectFormat !== "sha1" || + !indexHook || + !commit || + !tree || + !TREE_ID.test(tree) + ) { + return null; + } + const realCommonDir = yield* fileSystem + .realPath(commonDir) + .pipe(Effect.orElseSucceed(() => commonDir)); + const repositoryKey = NodeCrypto.createHash("sha256") + .update(realCommonDir) + .digest("hex") + .slice(0, 16); + return { + commonDir, + commit, + tree, + indexHook: path.resolve(cwd, indexHook), + bucket: path.join(cacheDir, repositoryKey), + }; + }); + + // Empty when `GIT_ATTR_NOSYSTEM` turns the system attributes file off. + const systemAttributePaths = yield* Effect.cached( + git("systemAttributes", homeDirectory, ["var", "GIT_ATTR_SYSTEM"]).pipe( + Effect.map((stdout) => (stdout?.trim() ? [stdout.trim()] : [])), + ), + ); + + // Null for setups the cache cannot reproduce, such as sparse checkouts. + const fingerprint = Effect.fn("WorktreeCheckoutCache.fingerprint")(function* ( + cwd: string, + commonDir: string, + ) { + const config = + (yield* git("fingerprint", cwd, ["config", "--null", "--get-regexp", CHECKOUT_CONFIG])) ?? ""; + const pairs = config + .split("\0") + .filter(Boolean) + .map((pair) => { + const newline = pair.indexOf("\n"); + return newline < 0 + ? ([pair, ""] as const) + : ([pair.slice(0, newline), pair.slice(newline + 1)] as const); + }); + const sparse = pairs.findLast(([key]) => key === "core.sparsecheckout")?.[1]; + if (sparse !== undefined && !GIT_FALSE.has(sparse.toLowerCase())) return null; + const attributesFile = pairs + .findLast(([key]) => key === "core.attributesfile")?.[1] + ?.replace(/^~(?=$|\/)/, homeDirectory); + // Git reads a relative path from each checkout, which differs between the cache and the worktree. + if (attributesFile !== undefined && !path.isAbsolute(attributesFile)) return null; + const attributePaths = [ + ...(yield* systemAttributePaths), + path.join(commonDir, "info", "attributes"), + attributesFile ?? + path.join( + environment.XDG_CONFIG_HOME || path.join(homeDirectory, ".config"), + "git", + "attributes", + ), + ]; + const attributes = yield* Effect.forEach(attributePaths, (file) => + fileSystem.readFileString(file).pipe(Effect.orElseSucceed(() => "")), + ); + const hash = NodeCrypto.createHash("sha256").update(config); + for (const contents of attributes) hash.update("\0").update(contents); + return hash.digest("hex").slice(0, 16); + }); + + const listEntries = Effect.fn("WorktreeCheckoutCache.listEntries")(function* (bucket: string) { + const names = yield* fileSystem.readDirectory(bucket).pipe(Effect.orElseSucceed(() => [])); + const entries = yield* Effect.forEach( + names.flatMap((name) => { + const match = ENTRY_NAME.exec(name); + return match ? [{ name, tree: match[1]!, fingerprint: match[2]! }] : []; + }), + ({ name, tree, fingerprint: entryFingerprint }) => { + const directory = path.join(bucket, name); + return fileSystem.stat(directory).pipe( + Effect.map((info): CacheEntry => ({ + directory, + tree, + fingerprint: entryFingerprint, + mtimeMs: Option.match(info.mtime, { + onNone: () => 0, + onSome: (date) => date.getTime(), + }), + })), + Effect.option, + ); + }, + { concurrency: 8 }, + ); + return entries + .flatMap((entry) => (entry._tag === "Some" ? [entry.value] : [])) + .toSorted((left, right) => right.mtimeMs - left.mtimeMs); + }); + + const touch = (target: string) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + Effect.tryPromise({ + try: () => NodeFSP.utimes(target, now / 1000, now / 1000), + catch: (cause) => + new WorktreeCheckoutCacheError({ detail: `could not touch ${target}: ${cause}` }), + }), + ), + Effect.ignore, + ); + + // Untraced: one stat per tracked file, already covered by the caller's span. + const statEntries = Effect.fnUntraced(function* (root: string, paths: ReadonlyArray) { + return yield* Effect.tryPromise({ + try: async () => { + const stats: Array = []; + for (let start = 0; start < paths.length; start += STAT_BATCH) { + const batch = await Promise.all( + paths.slice(start, start + STAT_BATCH).map((entryPath) => + NodeFSP.lstat(`${root}/${entryPath}`, { bigint: true }).then( + indexEntryStat, + (error) => { + if (error?.code === "ENOENT") return undefined; + throw error; + }, + ), + ), + ); + stats.push(...batch); + } + return stats; + }, + catch: (cause) => + new WorktreeCheckoutCacheError({ detail: `could not stat ${root}: ${cause}` }), + }); + }); + + /** + * Writes `target` from the index at `source` with stat data from the files under `root`, and + * returns what it read. Null when the index or the files cannot be trusted. + */ + const restate = Effect.fn("WorktreeCheckoutCache.restate")(function* (input: { + readonly source: string; + readonly target: string; + readonly root: string; + readonly verify: boolean; + }) { + const parsed = parseIndex(yield* fileSystem.readFile(input.source)); + if (parsed === null) return null; + const paths = parsed.entries.map((entry) => entry.path); + const stats = yield* statEntries(input.root, paths); + const restated = restatIndex(parsed, stats, { verify: input.verify }); + if (restated === null) return null; + const temporary = `${input.target}.t3-${NodeCrypto.randomUUID()}`; + yield* Effect.uninterruptible( + fileSystem + .writeFile(temporary, restated) + .pipe(Effect.andThen(fileSystem.rename(temporary, input.target))), + ); + return { paths, stats }; + }); + + // Untraced: one stat per folder, already covered by the caller's span. + const directoryTimes = Effect.fnUntraced(function* ( + root: string, + directories: ReadonlyArray, + ) { + return yield* Effect.tryPromise({ + try: () => + Promise.all( + directories.map((directory) => + NodeFSP.lstat(directory === "" ? root : `${root}/${directory}`, { bigint: true }).then( + (stat) => String(stat.mtimeNs), + () => "missing", + ), + ), + ), + catch: (cause) => + new WorktreeCheckoutCacheError({ detail: `could not stat folders in ${root}: ${cause}` }), + }); + }); + + // Adding or removing a file changes its folder's mtime, so this catches files the index lacks. + const recordDirectories = Effect.fn("WorktreeCheckoutCache.recordDirectories")(function* ( + entryDirectory: string, + recorded: { + readonly paths: ReadonlyArray; + readonly stats: ReadonlyArray; + }, + ) { + const directories = parentDirectories(recorded.paths); + for (const [index, entryPath] of recorded.paths.entries()) { + if (recorded.stats[index]?.kind === "directory") directories.add(entryPath); + } + // Folders whose names differ only in case share one spelling on disk, which depends on history. + const folded = new Set([...directories].map((directory) => foldCase(directory))); + if (folded.size !== directories.size) { + return yield* new WorktreeCheckoutCacheError({ detail: "folder names differ only in case" }); + } + const list = [...directories]; + // Backdated, so any later change moves the time even on a filesystem with a coarse clock. + const root = path.join(entryDirectory, "files"); + yield* Effect.tryPromise({ + try: () => + Promise.all( + list.map((directory) => + NodeFSP.utimes(directory === "" ? root : `${root}/${directory}`, 1_000_000, 1_000_000), + ), + ), + catch: (cause) => + new WorktreeCheckoutCacheError({ detail: `could not backdate cached folders: ${cause}` }), + }); + const times = yield* directoryTimes(root, list); + yield* fileSystem.writeFileString( + path.join(entryDirectory, "directories"), + list.map((directory, index) => `${times[index]}\0${directory}\0`).join(""), + ); + }); + + const directoriesUnchanged = Effect.fn("WorktreeCheckoutCache.directoriesUnchanged")( + function* (entry: CacheEntry) { + const recorded = (yield* fileSystem.readFileString(path.join(entry.directory, "directories"))) + .split("\0") + .slice(0, -1); + const times = recorded.filter((_, index) => index % 2 === 0); + const directories = recorded.filter((_, index) => index % 2 === 1); + const current = yield* directoryTimes(path.join(entry.directory, "files"), directories); + return directories.length > 0 && current.every((time, index) => time === times[index]); + }, + (effect) => Effect.orElseSucceed(effect, () => false), + ); + + const changedPaths = Effect.fn("WorktreeCheckoutCache.changedPaths")(function* ( + cwd: string, + from: string, + to: string, + ) { + const stdout = yield* git("changes", cwd, [ + "diff-tree", + "-r", + "-z", + "--name-status", + "--no-renames", + from, + to, + ]); + if (stdout === null) return null; + const fields = stdout.split("\0"); + const added: Array = []; + const deleted: Array = []; + // Added or changed: what `read-tree -m -u` writes itself. + const written: Array = []; + let count = 0; + let attributes = false; + for (let index = 0; index + 1 < fields.length; index += 2) { + const [status, file] = [fields[index]!, fields[index + 1]!]; + count++; + // `read-tree -m -u` leaves unchanged files alone, even when their attributes changed. + if (file === ".gitattributes" || file.endsWith("/.gitattributes")) attributes = true; + if (status === "A") added.push(file); + if (status === "D") deleted.push(file); + else written.push(file); + } + return { count, attributes, added, deleted, written }; + }); + + // A smudge filter's output can depend on the environment and the network, as with Git LFS. + const usesFilter = Effect.fn("WorktreeCheckoutCache.usesFilter")(function* ( + cwd: string, + bucket: string, + tree: string, + ) { + // An index of the tree lets `check-attr` read its `.gitattributes` on any Git version. + const index = path.join(bucket, `.attributes-${NodeCrypto.randomUUID()}`); + const env = { GIT_INDEX_FILE: index }; + const run = (args: ReadonlyArray, stdin?: string) => + execute({ + operation: "WorktreeCheckoutCache.git.filters", + cwd, + args, + env, + allowNonZeroExit: true, + maxOutputBytes: MAX_LIST_BYTES, + ...(stdin === undefined ? {} : { stdin }), + }).pipe(Effect.map((result) => (result.exitCode === 0 ? result.stdout : null))); + return yield* Effect.gen(function* () { + if ((yield* run([...CACHE_INDEX_CONFIG, "read-tree", tree])) === null) return true; + const files = yield* run(["ls-files", "-z"]); + if (files === null) return true; + const output = yield* run(["check-attr", "--cached", "--all", "-z", "--stdin"], files); + return output === null || namesFilter(output); + }).pipe( + Effect.orElseSucceed(() => true), + Effect.ensuring(fileSystem.remove(index).pipe(Effect.ignore)), + ); + }); + + // Attributes come from the cloned `.gitattributes` files, which the move leaves unchanged. + const writesWithFilter = Effect.fn("WorktreeCheckoutCache.writesWithFilter")(function* ( + cwd: string, + files: ReadonlyArray, + ) { + if (files.length === 0) return false; + const output = yield* execute({ + operation: "WorktreeCheckoutCache.git.filters", + cwd, + args: ["check-attr", "--all", "-z", "--stdin"], + stdin: files.map((file) => `${file}\0`).join(""), + allowNonZeroExit: true, + maxOutputBytes: MAX_LIST_BYTES, + }).pipe( + Effect.map((result) => (result.exitCode === 0 ? result.stdout : null)), + Effect.orElseSucceed(() => null), + ); + return output === null || namesFilter(output); + }); + + // Renamed first so nothing clones a half-deleted checkout; deleting a large one takes seconds. + const discard = Effect.fn("WorktreeCheckoutCache.discard")( + function* (target: string) { + const trashed = `${target}.trash-${NodeCrypto.randomUUID()}`; + yield* fileSystem.rename(target, trashed); + yield* fileSystem.remove(trashed, { recursive: true }); + }, + // Uninterruptible: deleting is not resumable, and a half-deleted folder must not be left named. + (effect) => Effect.ignore(Effect.uninterruptible(effect)), + ); + + const gitDirOf = Effect.fn("WorktreeCheckoutCache.gitDirOf")(function* (worktreePath: string) { + const contents = yield* fileSystem.readFileString(path.join(worktreePath, ".git")); + const gitDir = contents.match(/^gitdir: (.+)$/m)?.[1]; + if (gitDir === undefined) { + return yield* new WorktreeCheckoutCacheError({ + detail: `${worktreePath} is not a linked worktree`, + }); + } + return path.resolve(worktreePath, gitDir.trim()); + }); + + /** + * Creates the worktree from the cache. False means nothing was created and the caller should + * run `git worktree add`; any partial worktree has been removed again. + */ + const create = Effect.fn("WorktreeCheckoutCache.create")(function* (input: { + readonly cwd: string; + readonly worktreePath: string; + readonly refName: string; + readonly checkoutWorkers: string; + }) { + if (!(yield* gitSupported)) return false; + const worktreePath = path.resolve(input.cwd, input.worktreePath); + const requested = yield* describe(input.cwd, input.refName); + if (requested === null) return false; + const entries = yield* listEntries(requested.bucket); + if (entries.length === 0) return false; + // A clone only works within one volume. + const parent = path.dirname(worktreePath); + yield* fileSystem.makeDirectory(parent, { recursive: true }).pipe(Effect.ignore); + const [bucketInfo, parentInfo] = yield* Effect.all([ + fileSystem.stat(requested.bucket), + fileSystem.stat(parent), + ]).pipe(Effect.orElseSucceed(() => [null, null] as const)); + if (bucketInfo === null || parentInfo === null || bucketInfo.dev !== parentInfo.dev) { + return false; + } + + const populate = Effect.gen(function* () { + // What Git checked out, which can differ from `requested` when a tag shares the branch name. + const target = yield* describe(worktreePath, "HEAD"); + if (target === null || (yield* fileSystem.exists(target.indexHook))) return null; + const worktreeFingerprint = yield* fingerprint(worktreePath, target.commonDir); + if (worktreeFingerprint === null) return null; + const compatible = entries.filter((entry) => entry.fingerprint === worktreeFingerprint); + let base = compatible.find((entry) => entry.tree === target.tree); + let changes: Effect.Success> = null; + for (const entry of base === undefined ? compatible : []) { + changes = yield* changedPaths(worktreePath, entry.tree, target.tree); + if (changes !== null && !changes.attributes) { + base = entry; + break; + } + } + if (base === undefined) return null; + // A cached file changed after Git wrote it; never hand it out again. + const reject = (entry: CacheEntry) => + Effect.logWarning("worktree checkout cache entry changed; discarding it", { + directory: entry.directory, + }).pipe(Effect.andThen(discard(entry.directory)), Effect.as(null)); + if (!(yield* directoriesUnchanged(base))) return yield* reject(base); + yield* Effect.all([touch(base.directory), touch(requested.bucket)], { discard: true }); + + const files = path.join(base.directory, "files"); + // The worktree folder already holds Git's `.git` file, so its entries are cloned one by one. + yield* Effect.forEach( + yield* fileSystem.readDirectory(files), + (name) => clone(path.join(files, name), path.join(worktreePath, name)), + { discard: true }, + ); + // Checked again: a file added during the clone would otherwise come along unnoticed. + if (!(yield* directoriesUnchanged(base))) return yield* reject(base); + // A tracked hooks folder only exists once the files do; Git would have run this hook. + if (yield* fileSystem.exists(target.indexHook)) return null; + const restated = yield* restate({ + source: path.join(base.directory, "index"), + target: path.join(yield* gitDirOf(worktreePath), "index"), + root: worktreePath, + verify: true, + }); + if (restated === null) return yield* reject(base); + if (base.tree === target.tree || changes === null) return target; + // On a case-insensitive volume an added path can land on a file the cached tree kept. + const deleted = new Set(changes.deleted); + const kept = restated.paths.filter((file) => !deleted.has(file)); + const names = new Set(kept.map((file) => foldCase(file))); + // A new folder whose name differs only in case would keep the cached spelling. + const keptFolders = parentDirectories(kept); + for (const [index, file] of restated.paths.entries()) { + if (restated.stats[index]?.kind === "directory" && !deleted.has(file)) + keptFolders.add(file); + } + const spellings = new Map( + [...keptFolders].map((directory) => [foldCase(directory), directory]), + ); + for (const file of changes.added) { + if (names.has(foldCase(file))) return null; + names.add(foldCase(file)); + for (const directory of parentDirectories([file])) { + const existing = spellings.get(foldCase(directory)); + if (existing !== undefined && existing !== directory) return null; + spellings.set(foldCase(directory), directory); + } + } + // A smudge filter run here would see the cached index, not the one Git builds from scratch. + if (yield* writesWithFilter(worktreePath, changes.written)) return null; + // Hooks off: a hook the move adds runs with Git's checkout environment there, so its + // presence afterwards means the caller falls back to that checkout. + const moved = yield* git("update", worktreePath, [ + "-c", + `checkout.workers=${input.checkoutWorkers}`, + "-c", + "core.hooksPath=/dev/null", + "read-tree", + "-m", + "-u", + base.tree, + target.tree, + ]); + if (moved === null || (yield* fileSystem.exists(target.indexHook))) return null; + return target; + }); + const rollback = git("rollback", input.cwd, [ + "worktree", + "remove", + "--force", + "--", + worktreePath, + ]).pipe( + Effect.flatMap((removed) => + removed === null + ? Effect.logWarning("could not remove a partially created worktree", { worktreePath }) + : Effect.void, + ), + ); + // Registration and the rollback that undoes it can't be separated by an interruption. + const target = yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const registered = yield* git("register", input.cwd, [ + "worktree", + "add", + "--no-checkout", + "--", + worktreePath, + input.refName, + ]); + if (registered === null) return null; + return yield* restore(populate).pipe( + Effect.catch((cause) => + Effect.logWarning("worktree checkout cache failed; using git worktree add", { + cause, + }).pipe(Effect.as(null)), + ), + Effect.onExit((exit) => + Exit.isSuccess(exit) && exit.value !== null ? Effect.void : rollback, + ), + ); + }), + ); + if (target === null) return false; + + // `git worktree add --no-checkout` skips this hook; a normal add runs it. + const hook = yield* execute({ + operation: "GitVcsDriver.createWorktree.postCheckout", + cwd: worktreePath, + args: [ + "hook", + "run", + "--ignore-missing", + "post-checkout", + "--", + NULL_OID, + target.commit, + "1", + ], + env: { LANGUAGE: "C" }, + allowNonZeroExit: true, + timeoutMs: null, + }); + if (hook.exitCode !== 0) { + return yield* new GitCommandError({ + operation: "GitVcsDriver.createWorktree", + command: "git worktree add", + cwd: input.cwd, + detail: "git worktree add failed: the post-checkout hook failed", + exitCode: hook.exitCode, + stderrLength: hook.stderr.length, + }); + } + return true; + }); + + const removeIfAbandoned = Effect.fn("WorktreeCheckoutCache.removeIfAbandoned")( + function* (target: string, maxAgeMs: number) { + const info = yield* fileSystem.stat(target); + const modified = Option.getOrUndefined(info.mtime)?.getTime() ?? 0; + if ((yield* Clock.currentTimeMillis) - modified > maxAgeMs) { + yield* fileSystem.remove(target, { recursive: true }); + } + }, + (effect) => Effect.ignore(effect), + ); + + /** Caches the tree checked out in `worktreePath`, unless a cached checkout is already close to it. */ + const build = Effect.fn("WorktreeCheckoutCache.build")(function* ( + worktreePath: string, + // From before the worktree was checked out, so a `clear` during the checkout also counts. + generationAtStart: number, + ) { + if (!(yield* gitSupported)) return; + const target = yield* describe(worktreePath, "HEAD"); + if (target === null) return; + const entryFingerprint = yield* fingerprint(worktreePath, target.commonDir); + if (entryFingerprint === null) return; + const entryName = `${target.tree}-${entryFingerprint}`; + yield* fileSystem.makeDirectory(target.bucket, { recursive: true }); + const entries = yield* listEntries(target.bucket); + const compatible = entries.filter((entry) => entry.fingerprint === entryFingerprint); + if (compatible.some((entry) => entry.tree === target.tree)) return; + // Skip when a cached checkout is close enough to move forward cheaply. + let base: CacheEntry | undefined; + for (const entry of compatible) { + const changes = yield* changedPaths(worktreePath, entry.tree, target.tree); + if (changes === null || changes.attributes) continue; + if (changes.count < REBUILD_AFTER_CHANGED_FILES) return; + base ??= entry; + } + if (yield* usesFilter(worktreePath, target.bucket, target.tree)) return; + + for (const name of yield* fileSystem.readDirectory(target.bucket)) { + if (!ENTRY_NAME.test(name)) { + const leftover = path.join(target.bucket, name); + yield* removeIfAbandoned( + leftover, + name.includes(".trash-") ? ABANDONED_TRASH_MS : ABANDONED_STAGING_MS, + ); + } + } + const staging = path.join(target.bucket, `.staging-${NodeCrypto.randomUUID()}`); + const stagingFiles = path.join(staging, "files"); + const stagingIndex = path.join(staging, "index"); + yield* fileSystem.makeDirectory(staging); + const withIndex = (args: ReadonlyArray) => + git("build", worktreePath, [...CACHE_INDEX_CONFIG, `--work-tree=${stagingFiles}`, ...args], { + GIT_INDEX_FILE: stagingIndex, + }); + + yield* Effect.gen(function* () { + let built = false; + if (base !== undefined && (yield* directoriesUnchanged(base))) { + const from = base; + built = yield* Effect.gen(function* () { + yield* Effect.uninterruptible(clone(path.join(from.directory, "files"), stagingFiles)); + if (!(yield* directoriesUnchanged(from))) return false; + const restated = yield* restate({ + source: path.join(from.directory, "index"), + target: stagingIndex, + root: stagingFiles, + verify: true, + }); + if (restated === null) return false; + return (yield* withIndex(["read-tree", "-m", "-u", from.tree, target.tree])) !== null; + }).pipe(Effect.orElseSucceed(() => false)); + if (!built) { + yield* fileSystem.remove(stagingFiles, { recursive: true }).pipe(Effect.ignore); + yield* fileSystem.remove(stagingIndex).pipe(Effect.ignore); + } + } + if (!built) { + yield* fileSystem.makeDirectory(stagingFiles); + if ((yield* withIndex(["read-tree", "--reset", "-u", target.tree])) === null) { + return yield* new WorktreeCheckoutCacheError({ detail: "git read-tree failed" }); + } + } + // Record the files' own stat data; Git leaves racily written entries for a later re-read. + const recorded = yield* restate({ + source: stagingIndex, + target: stagingIndex, + root: stagingFiles, + verify: false, + }); + if (recorded === null) { + return yield* new WorktreeCheckoutCacheError({ + detail: "the cache index has an unsupported format or colliding paths", + }); + } + yield* recordDirectories(staging, recorded); + if (generation !== generationAtStart) { + return yield* new WorktreeCheckoutCacheError({ detail: "the cache was cleared" }); + } + yield* Effect.uninterruptible( + fileSystem.rename(staging, path.join(target.bucket, entryName)), + ); + }).pipe(Effect.onError(() => discard(staging))); + + yield* touch(target.bucket); + yield* Effect.forEach( + (yield* listEntries(target.bucket)).slice(KEEP_PER_REPOSITORY), + (entry) => discard(entry.directory), + { discard: true }, + ); + const now = yield* Clock.currentTimeMillis; + for (const name of yield* fileSystem.readDirectory(cacheDir)) { + const bucket = path.join(cacheDir, name); + if (bucket === target.bucket) continue; + if (name.includes(".trash-")) { + yield* removeIfAbandoned(bucket, ABANDONED_TRASH_MS); + continue; + } + const info = yield* fileSystem.stat(bucket).pipe(Effect.option); + const used = info.pipe( + Option.flatMap((value) => value.mtime), + Option.map((date) => date.getTime()), + ); + if (Option.isSome(used) && now - used.value > UNUSED_REPOSITORY_MS) { + yield* discard(bucket); + } + } + }); + + /** Caches the worktree's checkout in the background, so the next worktree can clone it. */ + const schedule = (worktreePath: string, generationAtStart: number) => + Effect.suspend(() => { + if (pendingBuilds.has(worktreePath)) return Effect.void; + pendingBuilds.add(worktreePath); + return build(worktreePath, generationAtStart).pipe( + builds.withPermits(1), + Effect.ignoreCause({ log: true }), + Effect.ensuring(Effect.sync(() => pendingBuilds.delete(worktreePath))), + Effect.forkIn(scope), + Effect.asVoid, + ); + }); + + /** Deletes every cached checkout in the background; builds already running are dropped. */ + const clear = Effect.suspend(() => { + generation++; + return fileSystem.exists(cacheDir); + }).pipe( + Effect.flatMap((exists) => + exists + ? discard(cacheDir).pipe(builds.withPermits(1), Effect.forkIn(scope), Effect.asVoid) + : Effect.void, + ), + Effect.ignore, + ); + + /** Pass to `schedule` from before a checkout starts. */ + const currentGeneration = Effect.sync(() => generation); + + return { create, build, schedule, clear, currentGeneration }; +}); diff --git a/apps/web/src/components/settings/LabsSettingsPanel.tsx b/apps/web/src/components/settings/LabsSettingsPanel.tsx index 3ab93229b2c5a..a591d6c8d68fc 100644 --- a/apps/web/src/components/settings/LabsSettingsPanel.tsx +++ b/apps/web/src/components/settings/LabsSettingsPanel.tsx @@ -14,7 +14,7 @@ import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsL import { searchableSetting } from "./settingsSearch"; import { LegacyFeaturesSection } from "./SettingsPanels"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; -import { useOptionalSettingsScope } from "./SettingsScopeContext"; +import { useOptionalSettingsScope, useSettingsScope } from "./SettingsScopeContext"; import { firstManagedDenial, usePrimaryManagedPolicy } from "~/state/managedPolicy"; /** Features still settling in, and legacy behavior kept for people who rely on it. */ @@ -29,6 +29,7 @@ export function LabsSettingsPanel() { {hasCloudPublicConfig() ? : } + @@ -247,3 +248,36 @@ function SimlockHostsRow() { /> ); } + +/** Shows only when every machine in scope is a Mac, since only macOS servers honor it. */ +function FastWorktreeCheckoutRow() { + const { scope, connectedEnvironments } = useSettingsScope(); + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + if ( + scope.kind === "project" || + scope.kind === "checkout" || + connectedEnvironments.length === 0 || + connectedEnvironments.some( + (environment) => + environment.serverConfig?.environment.capabilities.fastWorktreeCheckout !== true, + ) + ) + return null; + return ( + updateSettings({ fastWorktreeCheckout: Boolean(checked) })} + aria-label="Fast worktree checkout" + /> + } + /> + ); +} diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 60c89ffc6b732..c2e099d54c667 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -68,6 +68,7 @@ export interface SettingsSearchItem { readonly wslAvailableOnly?: boolean; readonly managedTunnelOnly?: boolean; readonly peerLinksOnly?: boolean; + readonly fastWorktreeCheckoutOnly?: boolean; readonly secondary?: boolean; readonly requiresThreadAutoSettlement?: boolean; } @@ -82,6 +83,7 @@ export interface SettingsSearchAvailability { readonly hasThreadAutoSettlement: boolean; readonly managedTunnelActive?: boolean; readonly hasPeerLinkHost?: boolean; + readonly hasFastWorktreeCheckoutEnvironment?: boolean; } export const SETTINGS_SECTION_LABELS: Readonly> = { @@ -495,6 +497,14 @@ export const SETTINGS_SEARCH_ITEMS = [ scope: "environment", searchTerms: ["beta alpha simlock callstack simulator emulator ios android pool lease gateway"], }, + { + id: "worktree-fast-checkout", + title: "Fast worktree checkout", + to: "/settings/labs", + scope: "environment", + fastWorktreeCheckoutOnly: true, + searchTerms: ["beta fast worktree checkout clone apfs copy-on-write cache macos speed"], + }, { id: "voice-transcription-destination", title: "Transcribe with", @@ -1444,7 +1454,8 @@ export function filterAvailableSettingsSearchItems( (!item.wslAvailableOnly || availability.isWslSettingsRowVisible) && (!item.requiresThreadAutoSettlement || availability.hasThreadAutoSettlement) && (!item.managedTunnelOnly || availability.managedTunnelActive === true) && - (!item.peerLinksOnly || availability.hasPeerLinkHost === true), + (!item.peerLinksOnly || availability.hasPeerLinkHost === true) && + (!item.fastWorktreeCheckoutOnly || availability.hasFastWorktreeCheckoutEnvironment === true), ); } diff --git a/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts b/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts index 834e7e03a7ad7..77aca0eb0abcf 100644 --- a/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts +++ b/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts @@ -28,6 +28,13 @@ export function useAvailableSettingsSearchItems() { hasServerConfig: environment.serverConfig !== null, }), ) === true; + // Any machine that offers it; the row then shows for a settings scope where all selected ones do. + const hasFastWorktreeCheckoutEnvironment = + environments.some( + (environment) => + environment.connection.phase === "connected" && + environment.serverConfig?.environment.capabilities.fastWorktreeCheckout === true, + ) === true; const hasThreadAutoSettlement = getThreadAutoSettlementSearchAvailability(environments).eligibleEnvironmentIds.length > 0; const primaryEnvironmentId = usePrimaryEnvironmentId(); @@ -57,5 +64,6 @@ export function useAvailableSettingsSearchItems() { hasThreadAutoSettlement, managedTunnelActive, hasPeerLinkHost, + hasFastWorktreeCheckoutEnvironment, }); } diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index ef3b5eac32413..8880dcb3e64ca 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -201,6 +201,19 @@ such as another drive, set **Settings → Worktrees → Worktree location** to a `D:\worktrees` or `~/worktrees`. The setting is per machine. Existing worktrees stay where they are, and cleanup covers both the default folder and the custom one. +## Fast worktree checkout on macOS + +Fast worktree checkout is a beta. Turn it on for each Mac in **Settings → Labs → Fast worktree +checkout**. T3 Code then keeps a few recent checkouts of each repository in the `caches` folder of +the T3 home directory. A new worktree is cloned from one of them with APFS copy-on-write, which is +about twice as fast as writing every file and makes the first `git status` in it fast as well. +Turning it off deletes the cache. + +The first worktree of a repository uses Git's normal checkout and fills the cache. It needs Git 2.42 +or newer. Git's normal checkout is also used for worktrees on a different drive from the T3 home +directory, sparse checkouts, repositories whose attributes use filters such as Git LFS, and +repositories whose checkout settings changed. + ## Storage cleanup Open **Settings → Worktrees → Cleanup** to enable automatic cleanup on one machine or all connected diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index cd5e6833468e8..8e3f2416477e2 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -101,6 +101,8 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ projectWorktreeCleanup: Schema.optionalKey(Schema.Boolean), settledWorktreeCleanup: Schema.optionalKey(Schema.Boolean), worktreesDirectory: Schema.optionalKey(Schema.Boolean), + /** Sent by macOS servers, which honor the fastWorktreeCheckout setting. */ + fastWorktreeCheckout: Schema.optionalKey(Schema.Boolean), threadAutoSettlementScope: Schema.optionalKey(Schema.Boolean), threadRestartContinuation: Schema.optionalKey(Schema.Boolean), projectSettingsOverrides: Schema.optionalKey(Schema.Boolean), diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index f6409a87074fd..b1076a017f989 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1259,6 +1259,8 @@ export const ServerSettings = Schema.Struct({ ), worktreesDirectory: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))), + /** macOS only, Labs beta: clone new worktrees from a cached checkout instead of writing every file. */ + fastWorktreeCheckout: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), /** Server-maintained history keeps old worktrees eligible for cleanup and review diffs. */ previousWorktreesDirectories: Schema.Array(TrimmedString).pipe( Schema.withDecodingDefault(Effect.succeed([])), @@ -1687,6 +1689,7 @@ export const ServerSettingsPatch = Schema.Struct({ }), ), worktreesDirectory: Schema.optionalKey(TrimmedString), + fastWorktreeCheckout: Schema.optionalKey(Schema.Boolean), responseStreamingMode: Schema.optionalKey(ResponseStreamingMode), enableProviderUpdateChecks: Schema.optionalKey(Schema.Boolean), continueThreadsAfterServerUpdate: Schema.optionalKey(Schema.Boolean),