From 1601077de104e2405c31f48b228eb3318e3c9772 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Wed, 11 Nov 2026 19:45:08 +0200 Subject: [PATCH 1/2] fix(server): branch totals stop relisting untracked files after a failed listing Co-Authored-By: Claude Opus 5.5 --- apps/server/src/vcs/GitVcsDriverCore.test.ts | 102 +++++++++++++++++++ apps/server/src/vcs/GitVcsDriverCore.ts | 74 ++++++++++++-- 2 files changed, 169 insertions(+), 7 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 9eb932e1d66c..bae6265924ef 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -6945,3 +6945,105 @@ describe("Windows long path configuration", () => { } }); }); + +// The listing walks the whole untracked tree before the 5,000-path cap applies, so a folder +// that failed once costs the same on every status poll. Branch totals remember the failure and +// back off until the untracked roots change. +it.layer(layerTest)("branch change totals after a failed untracked listing", (it) => { + // A listing that fails (here: output past the limit) backs off like one over the cap. + it.effect("do not repeat an untracked listing that failed", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* makeTmpDir("git-untracked-error-"); + yield* initRepoWithCommit(cwd); + yield* fs.writeFileString(path.join(cwd, "untracked.txt"), "new\n"); + let listings = 0; + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + // Stands in for a listing that floods its output past the 16 MiB limit. + const flood = ChildProcess.make(process.execPath, [ + "-e", + "const b = Buffer.alloc(1 << 20, 120); (function w() { process.stdout.write(b, w); })();", + ]); + const spawner = ChildProcessSpawner.make((command) => + Effect.suspend(() => { + if ( + ChildProcess.isStandardCommand(command) && + command.args.includes("ls-files") && + command.args.includes("--others") + ) { + listings++; + return delegate.spawn(flood); + } + return delegate.spawn(command); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + assert.strictEqual((yield* read).branchChanges, undefined); + yield* read; + assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff"); + yield* TestClock.adjust("1 minute"); + yield* read; + assert.strictEqual(listings, 2); + }), + ); + + it.effect("do not list untracked files again on every status read", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* makeTmpDir("git-untracked-totals-"); + yield* initRepoWithCommit(cwd); + const many = path.join(cwd, "many"); + yield* fs.makeDirectory(many); + yield* Effect.forEach( + Array.from({ length: 5_001 }, (_, index) => index), + (index) => fs.writeFileString(path.join(many, `file-${index}`), ""), + { concurrency: 64, discard: true }, + ); + + let listings = 0; + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawner = ChildProcessSpawner.make((command) => + Effect.suspend(() => { + if ( + ChildProcess.isStandardCommand(command) && + command.args.includes("ls-files") && + command.args.includes("--others") + ) + listings++; + return delegate.spawn(command); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + + assert.strictEqual((yield* read).branchChanges, undefined, "too many files to count"); + assert.strictEqual(listings, 1); + yield* read; + yield* read; + assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff"); + + yield* TestClock.adjust("1 minute"); + yield* read; + assert.strictEqual(listings, 2, "it is retried once the backoff passes"); + yield* TestClock.adjust("1 minute"); + yield* read; + assert.strictEqual(listings, 2, "and the next backoff is longer"); + + // A change at the untracked-roots level scans again at once and counts again. + yield* fs.remove(many, { recursive: true }); + yield* writeTextFile(cwd, "notes.txt", "one\ntwo\n"); + const status = yield* read; + assert.strictEqual(listings, 3); + assert.deepStrictEqual(status.branchChanges, { baseRef: null, insertions: 2, deletions: 0 }); + }), + ); +}); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 9bd289d62738..90adaf28b75a 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -77,6 +77,10 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; +// The listing walks the whole untracked tree before the cap applies, so a folder whose listing +// failed (too many files, output limit, timeout) is not listed again on every status poll. +const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; +const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000; const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024; const REVIEW_BRANCH_COMMIT_LIMIT = 100; const REVIEW_BRANCH_SUBJECT_LIMIT = 300; @@ -1623,6 +1627,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); const statusRemoteRefreshFailures = new Map(); + const untrackedTotalsFailures = new Map< + string, + { + readonly roots: string; + readonly count: number; + readonly retryAt: number; + readonly detail: string; + } + >(); const statusRemoteRefreshFailureKey = (cacheKey: StatusRemoteRefreshCacheKey) => `${cacheKey.gitCommonDir}\0${cacheKey.remoteName}`; const recordStatusRemoteRefreshFailure = Effect.fn( @@ -3149,7 +3162,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return stdout.trim(); }); - const prepareUntrackedReviewIndex = Effect.fn("prepareUntrackedReviewIndex")(function* ( + // `unavailable` says why the untracked files could not be indexed. + const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, @@ -3159,15 +3173,21 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd, ["ls-files", "--others", "--exclude-standard", "-z", "--", ...pathArgs], { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ).pipe(Effect.orElseSucceed(() => null)); - if (untracked === null) return null; + ).pipe( + Effect.catchTags({ + GitCommandError: (error) => + Effect.succeed({ unavailable: `Could not list untracked files: ${error.detail}` }), + }), + ); + if ("unavailable" in untracked) return untracked; // ls-files lists a nested repository with a trailing slash; its diff path has none. const paths = splitNullSeparatedGitStdoutPaths(untracked).filter( (candidate) => onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); if (paths.length === 0) return { env: undefined }; - if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return null; + if (paths.length > REVIEW_UNTRACKED_MAX_FILES) + return { unavailable: "Too many untracked files to count." }; const env = yield* prepareReviewIndex(cwd, paths).pipe( Effect.catchTags({ PlatformError: (cause) => @@ -3184,6 +3204,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); return { env }; }); + const prepareUntrackedReviewIndex = ( + cwd: string, + pathArgs: ReadonlyArray, + onlyPath?: string, + ) => + prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe( + Effect.map((result) => ("unavailable" in result ? null : result)), + ); const resolveReviewMergeBase = Effect.fn("resolveReviewMergeBase")(function* ( cwd: string, @@ -3257,16 +3285,48 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* roots.length <= 128 && roots.reduce((length, root) => length + root.length + 16, 0) < 8_000 ? roots.map((root) => `:(top,literal)${root}`) : []; + // Until the retry time, the same untracked roots get the remembered failure without a scan; + // different roots (files added or removed at status level) scan again at once. + const rootsKey = roots.join("\0"); + const remembered = untrackedTotalsFailures.get(cwd); + if ( + remembered !== undefined && + remembered.roots === rootsKey && + (yield* Clock.currentTimeMillis) < remembered.retryAt + ) { + return yield* new GitCommandError({ + operation: "GitVcsDriver.readBranchChangeTotals", + command: "git ls-files", + cwd, + detail: remembered.detail, + }); + } // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndex(cwd, pathspecs); - if (untracked === null) { + const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs); + if ("unavailable" in untracked) { + const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; + // Bounded like the remote refresh failures: the oldest folder is forgotten first. + untrackedTotalsFailures.delete(cwd); + untrackedTotalsFailures.set(cwd, { + roots: rootsKey, + count, + retryAt: + (yield* Clock.currentTimeMillis) + + Math.min(UNTRACKED_TOTALS_RETRY_MIN_MS * 2 ** (count - 1), UNTRACKED_TOTALS_RETRY_MAX_MS), + detail: untracked.unavailable, + }); + if (untrackedTotalsFailures.size > STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY) { + const oldestKey = untrackedTotalsFailures.keys().next().value; + if (oldestKey !== undefined) untrackedTotalsFailures.delete(oldestKey); + } return yield* new GitCommandError({ operation: "GitVcsDriver.readBranchChangeTotals", command: "git ls-files", cwd, - detail: "Too many untracked files to count.", + detail: untracked.unavailable, }); } + untrackedTotalsFailures.delete(cwd); const readNumstat = (ref: string) => executeGit( "GitVcsDriver.readBranchChangeTotals", From 39d90e7e42a93e0fc60b026acd4d8a171e5cfa56 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 19:15:21 +0000 Subject: [PATCH 2/2] test(server): keep one untracked-totals backoff case and trim comments AI-assisted; verified with the tests and tools listed above. --- apps/server/src/vcs/GitVcsDriverCore.test.ts | 45 -------------------- apps/server/src/vcs/GitVcsDriverCore.ts | 5 +-- 2 files changed, 1 insertion(+), 49 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index bae6265924ef..2dddf8fe830a 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -6946,52 +6946,7 @@ describe("Windows long path configuration", () => { }); }); -// The listing walks the whole untracked tree before the 5,000-path cap applies, so a folder -// that failed once costs the same on every status poll. Branch totals remember the failure and -// back off until the untracked roots change. it.layer(layerTest)("branch change totals after a failed untracked listing", (it) => { - // A listing that fails (here: output past the limit) backs off like one over the cap. - it.effect("do not repeat an untracked listing that failed", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const cwd = yield* makeTmpDir("git-untracked-error-"); - yield* initRepoWithCommit(cwd); - yield* fs.writeFileString(path.join(cwd, "untracked.txt"), "new\n"); - let listings = 0; - const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; - // Stands in for a listing that floods its output past the 16 MiB limit. - const flood = ChildProcess.make(process.execPath, [ - "-e", - "const b = Buffer.alloc(1 << 20, 120); (function w() { process.stdout.write(b, w); })();", - ]); - const spawner = ChildProcessSpawner.make((command) => - Effect.suspend(() => { - if ( - ChildProcess.isStandardCommand(command) && - command.args.includes("ls-files") && - command.args.includes("--others") - ) { - listings++; - return delegate.spawn(flood); - } - return delegate.spawn(command); - }), - ); - const driver = yield* makeGitVcsDriverCore().pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.provide(layerServerConfig), - ); - const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); - assert.strictEqual((yield* read).branchChanges, undefined); - yield* read; - assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff"); - yield* TestClock.adjust("1 minute"); - yield* read; - assert.strictEqual(listings, 2); - }), - ); - it.effect("do not list untracked files again on every status read", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 90adaf28b75a..c6d4bbb1ce63 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -3162,7 +3162,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return stdout.trim(); }); - // `unavailable` says why the untracked files could not be indexed. const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, @@ -3285,8 +3284,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* roots.length <= 128 && roots.reduce((length, root) => length + root.length + 16, 0) < 8_000 ? roots.map((root) => `:(top,literal)${root}`) : []; - // Until the retry time, the same untracked roots get the remembered failure without a scan; - // different roots (files added or removed at status level) scan again at once. + // Changed untracked roots skip the backoff and scan again. const rootsKey = roots.join("\0"); const remembered = untrackedTotalsFailures.get(cwd); if ( @@ -3305,7 +3303,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs); if ("unavailable" in untracked) { const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; - // Bounded like the remote refresh failures: the oldest folder is forgotten first. untrackedTotalsFailures.delete(cwd); untrackedTotalsFailures.set(cwd, { roots: rootsKey,