From 3123830626e1bdef88f84b894025763f6a4a15e6 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 19:39:29 +0000 Subject: [PATCH] fix(server): review diffs no longer read oversized untracked files Review previews put every untracked file into the temporary review index, so git read all of their bytes; only the status totals had a size budget. The preview now uses the same 256 MiB budget. Files that do not fit stay out of the index, smallest files are kept first, and the source is reported the way an unlistable untracked tree already is: no file list, marked truncated, with the bounded patch of everything that did fit. Staged deletions are dropped before sizes are counted, so their worktree copies cannot spend the budget. Co-authored-by: luis-7654 <103049391+luis-7654@users.noreply.github.com> Co-Authored-By: Claude Fable 5.2 --- apps/server/src/vcs/GitVcsDriverCore.test.ts | 33 +++++++++ apps/server/src/vcs/GitVcsDriverCore.ts | 73 +++++++++++--------- 2 files changed, 72 insertions(+), 34 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index d46666cb468d0..dcddf8147f9dd 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -4323,6 +4323,39 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { insertions: 1, deletions: 1, }); + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + assert.deepStrictEqual( + preview.sources.find((source) => source.kind === "working-tree")?.files, + [ + { path: "asset.data", previousPath: null, additions: 0, deletions: 1 }, + { path: "ordinary.txt", previousPath: null, additions: 1, deletions: 0 }, + ], + ); + }), + ); + + it.effect("keeps an oversized untracked file out of the review diff", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + yield* writeTextFile(cwd, "ordinary.txt", "new\n"); + yield* writeTextFile(cwd, "large.data", ""); + // Sparse: the logical size is over the budget without allocating it. + yield* fileSystem.truncate(pathService.join(cwd, "large.data"), 257 * 1024 * 1024); + + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + + const source = preview.sources.find((source) => source.kind === "working-tree")!; + assert.isUndefined(source.files); + assert.isUndefined(source.patchesDeferred); + assert.isTrue(source.truncated); + assert.include(source.diff, "+# tracked change"); + assert.include(source.diff, "b/ordinary.txt"); + assert.notInclude(source.diff, "large.data"); }), ); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 32042a0ff5e5f..50bc37c6e79ec 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -81,8 +81,8 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; -// git diff --numstat reads every untracked byte (~5 ms/MB), so status totals skip larger untracked input. -const UNTRACKED_TOTALS_MAX_BYTES = 256 * 1024 * 1024; +// git diff reads every untracked byte (~5 ms/MB), so larger untracked input stays out of the review index. +const REVIEW_UNTRACKED_MAX_BYTES = 256 * 1024 * 1024; // The listing walks the whole untracked tree before the cap applies, so a folder whose listing // failed (too many files, output limit, timeout) is not listed again on every status poll. const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; @@ -3242,7 +3242,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareReviewIndex = Effect.fn("prepareReviewIndex")(function* ( cwd: string, untrackedPaths: ReadonlyArray, - maxBytes?: number, + allowPartial: boolean, ) { const [stagedDeletionsStdout, indexValue] = yield* Effect.all( [ @@ -3270,21 +3270,28 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { concurrency: 2 }, ); const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean)); - const pathsToAdd = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); - if (pathsToAdd.length === 0) return undefined; - if (maxBytes !== undefined) { - const sizes = yield* Effect.forEach( - pathsToAdd, - (relativePath) => - fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( - Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), - Effect.orElseSucceed(() => 0), - ), - { concurrency: 16 }, - ); - // null: over the byte budget, so the caller reports totals as unavailable. - if (sizes.reduce((total, size) => total + size, 0) > maxBytes) return null; + // Staged deletions never enter the index, so their worktree copies must not spend the budget. + const candidates = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); + const sizes = yield* Effect.forEach( + candidates, + (relativePath) => + fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( + Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), + Effect.orElseSucceed(() => 0), + Effect.map((size) => ({ relativePath, size })), + ), + { concurrency: 16 }, + ); + // Smallest first, so which files fit does not depend on how their names sort. + const pathsToAdd: string[] = []; + let bytes = 0; + for (const { relativePath, size } of sizes.toSorted((left, right) => left.size - right.size)) { + bytes += size; + if (bytes > REVIEW_UNTRACKED_MAX_BYTES) break; + pathsToAdd.push(relativePath); } + const omitted = pathsToAdd.length < candidates.length; + if (pathsToAdd.length === 0 || (omitted && !allowPartial)) return { env: undefined, omitted }; const indexPath = path.isAbsolute(indexValue.trim()) ? indexValue.trim() @@ -3337,7 +3344,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { env, stdin: `${pathsToAdd.join("\0")}\0` }, ); - return env; + return { env, omitted }; }); const readEmptyTreeHash = Effect.fn("readEmptyTreeHash")(function* (cwd: string) { @@ -3353,8 +3360,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, - onlyPath?: string, - maxBytes?: number, + onlyPath: string | undefined, + allowPartial: boolean, ) { const untracked = yield* executeGit( "GitVcsDriver.review.listUntracked", @@ -3373,10 +3380,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* (candidate) => onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); - if (paths.length === 0) return { env: undefined }; + if (paths.length === 0) return { env: undefined, omitted: false }; if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return { unavailable: "Too many untracked files to count." }; - const env = yield* prepareReviewIndex(cwd, paths, maxBytes).pipe( + const index = yield* prepareReviewIndex(cwd, paths, allowPartial).pipe( Effect.catchTags({ PlatformError: (cause) => Effect.fail( @@ -3390,15 +3397,16 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }), ); - if (env === null) return { unavailable: "Untracked files are too large to count." }; - return { env }; + if (index.omitted && !allowPartial) + return { unavailable: "Untracked files are too large to count." }; + return index; }); const prepareUntrackedReviewIndex = ( cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, ) => - prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe( + prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath, true).pipe( Effect.map((result) => ("unavailable" in result ? null : result)), ); @@ -3490,12 +3498,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndexOrReason( - cwd, - pathspecs, - undefined, - UNTRACKED_TOTALS_MAX_BYTES, - ); + const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs, undefined, false); if ("unavailable" in untracked) { const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; untrackedTotalsFailures.delete(cwd); @@ -3846,9 +3849,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); const [dirtyTrackedResult, baseResult] = yield* Effect.gen(function* () { const untracked = input.trackedOnly - ? { env: undefined } + ? { env: undefined, omitted: false } : yield* prepareUntrackedReviewIndex(cwd, pathArgs, input.file?.path); - const metadataOnly = input.metadataOnly === true && !input.file && untracked !== null; + // Untracked files left out make the file list partial, so the bounded patch is sent instead. + const complete = untracked !== null && !untracked.omitted; + const metadataOnly = input.metadataOnly === true && !input.file && complete; const [dirty, base] = review.mergeBase === dirtyRef ? yield* readTrackedDiff(dirtyRef, untracked?.env, metadataOnly).pipe( @@ -3861,7 +3866,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { concurrency: 2 }, ); - if (untracked !== null) return [dirty, base] as const; + if (complete) return [dirty, base] as const; const incomplete = (ref: string | null, result: typeof dirty) => ref === null ? result : { ...result, files: undefined, stdoutTruncated: true }; return [incomplete(dirtyRef, dirty), incomplete(review.mergeBase, base)] as const;