diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 5b84236db14c0..94be10c97fc9e 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -7,6 +7,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { assert, it, describe } from "@effect/vitest"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -3960,6 +3961,98 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect( + "reports a new repository without waiting on a huge untracked file's line count", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* git(cwd, ["init", "-b", "main"]); + yield* writeTextFile(cwd, "small.txt", "line\n"); + yield* writeTextFile(cwd, "large.data", ""); + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + yield* fileSystem.truncate(pathService.join(cwd, "large.data"), 8 * 1024 * 1024 * 1024); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const totalsDiffSettled = yield* Deferred.make(); + const largeFileDiffAbandonedAt = yield* Deferred.make(); + // Stands in for `git diff --numstat` reading gigabytes of untracked file content. + const readsLargeFile = (command: ChildProcess.Command) => { + const indexFile = + ChildProcess.isStandardCommand(command) && + command.args.includes("--numstat") && + command.options.env?.GIT_INDEX_FILE; + return ( + typeof indexFile === "string" && + NodeFS.readFileSync(indexFile).includes(Buffer.from("large.data")) + ); + }; + const spawner = ChildProcessSpawner.make((command) => + readsLargeFile(command) + ? Deferred.succeed(totalsDiffSettled, undefined).pipe( + Effect.as( + ChildProcessSpawner.makeHandle({ + ...makeSuccessfulHandle(""), + isRunning: Effect.succeed(true), + exitCode: Effect.never.pipe( + Effect.onInterrupt(() => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + Deferred.succeed(largeFileDiffAbandonedAt, now), + ), + ), + ), + ), + }), + ), + ) + : delegate.spawn(command), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const startedAt = yield* Clock.currentTimeMillis; + + const status = yield* driver + .statusDetailsLocal(cwd, { includeDivergence: false, includeBranchChanges: true }) + .pipe(Effect.forkChild); + yield* Effect.raceFirst(Deferred.await(totalsDiffSettled), Fiber.await(status)); + // Git's default command timeout, which is what releases the status on main. + yield* TestClock.adjust("30 seconds"); + const reported = yield* Fiber.join(status); + + const statusHeldForMs = (yield* Deferred.isDone(largeFileDiffAbandonedAt)) + ? (yield* Deferred.await(largeFileDiffAbandonedAt)) - startedAt + : 0; + assert.isAtMost(statusHeldForMs, 5_000, "status waited on the untracked line count"); + assert.isTrue(reported.isRepo); + assert.strictEqual(reported.branch, "main"); + }).pipe(Effect.provide(layerServerConfig)), + ); + + it.effect("counts totals when a large unstaged copy of a removed file stays on disk", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + yield* writeTextFile(cwd, "asset.data", "old\n"); + yield* git(cwd, ["add", "asset.data"]); + yield* git(cwd, ["commit", "-m", "add asset"]); + yield* git(cwd, ["rm", "--cached", "asset.data"]); + yield* fileSystem.truncate(pathService.join(cwd, "asset.data"), 257 * 1024 * 1024); + yield* writeTextFile(cwd, "ordinary.txt", "new\n"); + + const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + + assert.deepStrictEqual(status.branchChanges, { + baseRef: null, + insertions: 1, + deletions: 1, + }); + }), + ); + it.effect("reports changes to a file named HEAD", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 12ce999f25e18..9df5cb337242a 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -77,6 +77,8 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; +// git diff --numstat reads every untracked byte (~5 ms/MB), so status totals skip larger untracked input. +const UNTRACKED_TOTALS_MAX_BYTES = 256 * 1024 * 1024; // The listing walks the whole untracked tree before the cap applies, so a folder whose listing // failed (too many files, output limit, timeout) is not listed again on every status poll. const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; @@ -3129,6 +3131,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareReviewIndex = Effect.fn("prepareReviewIndex")(function* ( cwd: string, untrackedPaths: ReadonlyArray, + maxBytes?: number, ) { const [stagedDeletionsStdout, indexValue] = yield* Effect.all( [ @@ -3158,6 +3161,19 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean)); const pathsToAdd = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); if (pathsToAdd.length === 0) return undefined; + if (maxBytes !== undefined) { + const sizes = yield* Effect.forEach( + pathsToAdd, + (relativePath) => + fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( + Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), + Effect.orElseSucceed(() => 0), + ), + { concurrency: 16 }, + ); + // null: over the byte budget, so the caller reports totals as unavailable. + if (sizes.reduce((total, size) => total + size, 0) > maxBytes) return null; + } const indexPath = path.isAbsolute(indexValue.trim()) ? indexValue.trim() @@ -3227,6 +3243,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, + maxBytes?: number, ) { const untracked = yield* executeGit( "GitVcsDriver.review.listUntracked", @@ -3248,7 +3265,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (paths.length === 0) return { env: undefined }; if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return { unavailable: "Too many untracked files to count." }; - const env = yield* prepareReviewIndex(cwd, paths).pipe( + const env = yield* prepareReviewIndex(cwd, paths, maxBytes).pipe( Effect.catchTags({ PlatformError: (cause) => Effect.fail( @@ -3262,6 +3279,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }), ); + if (env === null) return { unavailable: "Untracked files are too large to count." }; return { env }; }); const prepareUntrackedReviewIndex = ( @@ -3361,7 +3379,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs); + const untracked = yield* prepareUntrackedReviewIndexOrReason( + cwd, + pathspecs, + undefined, + UNTRACKED_TOTALS_MAX_BYTES, + ); if ("unavailable" in untracked) { const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; untrackedTotalsFailures.delete(cwd);