diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 9eb932e1d66c..2dddf8fe830a 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -6945,3 +6945,60 @@ describe("Windows long path configuration", () => { } }); }); + +it.layer(layerTest)("branch change totals after a failed untracked listing", (it) => { + it.effect("do not list untracked files again on every status read", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* makeTmpDir("git-untracked-totals-"); + yield* initRepoWithCommit(cwd); + const many = path.join(cwd, "many"); + yield* fs.makeDirectory(many); + yield* Effect.forEach( + Array.from({ length: 5_001 }, (_, index) => index), + (index) => fs.writeFileString(path.join(many, `file-${index}`), ""), + { concurrency: 64, discard: true }, + ); + + let listings = 0; + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawner = ChildProcessSpawner.make((command) => + Effect.suspend(() => { + if ( + ChildProcess.isStandardCommand(command) && + command.args.includes("ls-files") && + command.args.includes("--others") + ) + listings++; + return delegate.spawn(command); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + + assert.strictEqual((yield* read).branchChanges, undefined, "too many files to count"); + assert.strictEqual(listings, 1); + yield* read; + yield* read; + assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff"); + + yield* TestClock.adjust("1 minute"); + yield* read; + assert.strictEqual(listings, 2, "it is retried once the backoff passes"); + yield* TestClock.adjust("1 minute"); + yield* read; + assert.strictEqual(listings, 2, "and the next backoff is longer"); + + // A change at the untracked-roots level scans again at once and counts again. + yield* fs.remove(many, { recursive: true }); + yield* writeTextFile(cwd, "notes.txt", "one\ntwo\n"); + const status = yield* read; + assert.strictEqual(listings, 3); + assert.deepStrictEqual(status.branchChanges, { baseRef: null, insertions: 2, deletions: 0 }); + }), + ); +}); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 9bd289d62738..c6d4bbb1ce63 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -77,6 +77,10 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; +// The listing walks the whole untracked tree before the cap applies, so a folder whose listing +// failed (too many files, output limit, timeout) is not listed again on every status poll. +const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; +const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000; const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024; const REVIEW_BRANCH_COMMIT_LIMIT = 100; const REVIEW_BRANCH_SUBJECT_LIMIT = 300; @@ -1623,6 +1627,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); const statusRemoteRefreshFailures = new Map(); + const untrackedTotalsFailures = new Map< + string, + { + readonly roots: string; + readonly count: number; + readonly retryAt: number; + readonly detail: string; + } + >(); const statusRemoteRefreshFailureKey = (cacheKey: StatusRemoteRefreshCacheKey) => `${cacheKey.gitCommonDir}\0${cacheKey.remoteName}`; const recordStatusRemoteRefreshFailure = Effect.fn( @@ -3149,7 +3162,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return stdout.trim(); }); - const prepareUntrackedReviewIndex = Effect.fn("prepareUntrackedReviewIndex")(function* ( + const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, @@ -3159,15 +3172,21 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd, ["ls-files", "--others", "--exclude-standard", "-z", "--", ...pathArgs], { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ).pipe(Effect.orElseSucceed(() => null)); - if (untracked === null) return null; + ).pipe( + Effect.catchTags({ + GitCommandError: (error) => + Effect.succeed({ unavailable: `Could not list untracked files: ${error.detail}` }), + }), + ); + if ("unavailable" in untracked) return untracked; // ls-files lists a nested repository with a trailing slash; its diff path has none. const paths = splitNullSeparatedGitStdoutPaths(untracked).filter( (candidate) => onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); if (paths.length === 0) return { env: undefined }; - if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return null; + if (paths.length > REVIEW_UNTRACKED_MAX_FILES) + return { unavailable: "Too many untracked files to count." }; const env = yield* prepareReviewIndex(cwd, paths).pipe( Effect.catchTags({ PlatformError: (cause) => @@ -3184,6 +3203,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); return { env }; }); + const prepareUntrackedReviewIndex = ( + cwd: string, + pathArgs: ReadonlyArray, + onlyPath?: string, + ) => + prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe( + Effect.map((result) => ("unavailable" in result ? null : result)), + ); const resolveReviewMergeBase = Effect.fn("resolveReviewMergeBase")(function* ( cwd: string, @@ -3257,16 +3284,46 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* roots.length <= 128 && roots.reduce((length, root) => length + root.length + 16, 0) < 8_000 ? roots.map((root) => `:(top,literal)${root}`) : []; + // Changed untracked roots skip the backoff and scan again. + const rootsKey = roots.join("\0"); + const remembered = untrackedTotalsFailures.get(cwd); + if ( + remembered !== undefined && + remembered.roots === rootsKey && + (yield* Clock.currentTimeMillis) < remembered.retryAt + ) { + return yield* new GitCommandError({ + operation: "GitVcsDriver.readBranchChangeTotals", + command: "git ls-files", + cwd, + detail: remembered.detail, + }); + } // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndex(cwd, pathspecs); - if (untracked === null) { + const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs); + if ("unavailable" in untracked) { + const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; + untrackedTotalsFailures.delete(cwd); + untrackedTotalsFailures.set(cwd, { + roots: rootsKey, + count, + retryAt: + (yield* Clock.currentTimeMillis) + + Math.min(UNTRACKED_TOTALS_RETRY_MIN_MS * 2 ** (count - 1), UNTRACKED_TOTALS_RETRY_MAX_MS), + detail: untracked.unavailable, + }); + if (untrackedTotalsFailures.size > STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY) { + const oldestKey = untrackedTotalsFailures.keys().next().value; + if (oldestKey !== undefined) untrackedTotalsFailures.delete(oldestKey); + } return yield* new GitCommandError({ operation: "GitVcsDriver.readBranchChangeTotals", command: "git ls-files", cwd, - detail: "Too many untracked files to count.", + detail: untracked.unavailable, }); } + untrackedTotalsFailures.delete(cwd); const readNumstat = (ref: string) => executeGit( "GitVcsDriver.readBranchChangeTotals",