diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index d46666cb468d0..dcddf8147f9dd 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -4323,6 +4323,39 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { insertions: 1, deletions: 1, }); + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + assert.deepStrictEqual( + preview.sources.find((source) => source.kind === "working-tree")?.files, + [ + { path: "asset.data", previousPath: null, additions: 0, deletions: 1 }, + { path: "ordinary.txt", previousPath: null, additions: 1, deletions: 0 }, + ], + ); + }), + ); + + it.effect("keeps an oversized untracked file out of the review diff", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + yield* writeTextFile(cwd, "ordinary.txt", "new\n"); + yield* writeTextFile(cwd, "large.data", ""); + // Sparse: the logical size is over the budget without allocating it. + yield* fileSystem.truncate(pathService.join(cwd, "large.data"), 257 * 1024 * 1024); + + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + + const source = preview.sources.find((source) => source.kind === "working-tree")!; + assert.isUndefined(source.files); + assert.isUndefined(source.patchesDeferred); + assert.isTrue(source.truncated); + assert.include(source.diff, "+# tracked change"); + assert.include(source.diff, "b/ordinary.txt"); + assert.notInclude(source.diff, "large.data"); }), ); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 32042a0ff5e5f..50bc37c6e79ec 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -81,8 +81,8 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; -// git diff --numstat reads every untracked byte (~5 ms/MB), so status totals skip larger untracked input. -const UNTRACKED_TOTALS_MAX_BYTES = 256 * 1024 * 1024; +// git diff reads every untracked byte (~5 ms/MB), so larger untracked input stays out of the review index. +const REVIEW_UNTRACKED_MAX_BYTES = 256 * 1024 * 1024; // The listing walks the whole untracked tree before the cap applies, so a folder whose listing // failed (too many files, output limit, timeout) is not listed again on every status poll. const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; @@ -3242,7 +3242,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareReviewIndex = Effect.fn("prepareReviewIndex")(function* ( cwd: string, untrackedPaths: ReadonlyArray, - maxBytes?: number, + allowPartial: boolean, ) { const [stagedDeletionsStdout, indexValue] = yield* Effect.all( [ @@ -3270,21 +3270,28 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { concurrency: 2 }, ); const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean)); - const pathsToAdd = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); - if (pathsToAdd.length === 0) return undefined; - if (maxBytes !== undefined) { - const sizes = yield* Effect.forEach( - pathsToAdd, - (relativePath) => - fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( - Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), - Effect.orElseSucceed(() => 0), - ), - { concurrency: 16 }, - ); - // null: over the byte budget, so the caller reports totals as unavailable. - if (sizes.reduce((total, size) => total + size, 0) > maxBytes) return null; + // Staged deletions never enter the index, so their worktree copies must not spend the budget. + const candidates = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); + const sizes = yield* Effect.forEach( + candidates, + (relativePath) => + fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( + Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), + Effect.orElseSucceed(() => 0), + Effect.map((size) => ({ relativePath, size })), + ), + { concurrency: 16 }, + ); + // Smallest first, so which files fit does not depend on how their names sort. + const pathsToAdd: string[] = []; + let bytes = 0; + for (const { relativePath, size } of sizes.toSorted((left, right) => left.size - right.size)) { + bytes += size; + if (bytes > REVIEW_UNTRACKED_MAX_BYTES) break; + pathsToAdd.push(relativePath); } + const omitted = pathsToAdd.length < candidates.length; + if (pathsToAdd.length === 0 || (omitted && !allowPartial)) return { env: undefined, omitted }; const indexPath = path.isAbsolute(indexValue.trim()) ? indexValue.trim() @@ -3337,7 +3344,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { env, stdin: `${pathsToAdd.join("\0")}\0` }, ); - return env; + return { env, omitted }; }); const readEmptyTreeHash = Effect.fn("readEmptyTreeHash")(function* (cwd: string) { @@ -3353,8 +3360,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, - onlyPath?: string, - maxBytes?: number, + onlyPath: string | undefined, + allowPartial: boolean, ) { const untracked = yield* executeGit( "GitVcsDriver.review.listUntracked", @@ -3373,10 +3380,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* (candidate) => onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); - if (paths.length === 0) return { env: undefined }; + if (paths.length === 0) return { env: undefined, omitted: false }; if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return { unavailable: "Too many untracked files to count." }; - const env = yield* prepareReviewIndex(cwd, paths, maxBytes).pipe( + const index = yield* prepareReviewIndex(cwd, paths, allowPartial).pipe( Effect.catchTags({ PlatformError: (cause) => Effect.fail( @@ -3390,15 +3397,16 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }), ); - if (env === null) return { unavailable: "Untracked files are too large to count." }; - return { env }; + if (index.omitted && !allowPartial) + return { unavailable: "Untracked files are too large to count." }; + return index; }); const prepareUntrackedReviewIndex = ( cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, ) => - prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe( + prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath, true).pipe( Effect.map((result) => ("unavailable" in result ? null : result)), ); @@ -3490,12 +3498,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndexOrReason( - cwd, - pathspecs, - undefined, - UNTRACKED_TOTALS_MAX_BYTES, - ); + const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs, undefined, false); if ("unavailable" in untracked) { const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; untrackedTotalsFailures.delete(cwd); @@ -3846,9 +3849,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); const [dirtyTrackedResult, baseResult] = yield* Effect.gen(function* () { const untracked = input.trackedOnly - ? { env: undefined } + ? { env: undefined, omitted: false } : yield* prepareUntrackedReviewIndex(cwd, pathArgs, input.file?.path); - const metadataOnly = input.metadataOnly === true && !input.file && untracked !== null; + // Untracked files left out make the file list partial, so the bounded patch is sent instead. + const complete = untracked !== null && !untracked.omitted; + const metadataOnly = input.metadataOnly === true && !input.file && complete; const [dirty, base] = review.mergeBase === dirtyRef ? yield* readTrackedDiff(dirtyRef, untracked?.env, metadataOnly).pipe( @@ -3861,7 +3866,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { concurrency: 2 }, ); - if (untracked !== null) return [dirty, base] as const; + if (complete) return [dirty, base] as const; const incomplete = (ref: string | null, result: typeof dirty) => ref === null ? result : { ...result, files: undefined, stdoutTruncated: true }; return [incomplete(dirtyRef, dirty), incomplete(review.mergeBase, base)] as const;