Use git ls-files -s instead of ls-tree for full-tree enumeration

GVFS/GVFS.Common/Git/DiffTreeResult.cs

@@ -195,6 +195,62 @@ public static bool IsLsTreeLineOfType(string line, string typeMarker)
             return line.IndexOf(typeMarker, TypeMarkerStartIndex, typeMarker.Length, StringComparison.OrdinalIgnoreCase) == TypeMarkerStartIndex;
         }
 
+        /// <summary>
+        /// Parse the output of calling git ls-files -s (staging info).
+        /// This reads from the index, which is much faster than ls-tree on large repos.
+        /// ls-files only returns file entries (no tree entries).
+        /// Entries with stage != 0 (unmerged) are skipped to avoid duplicate/conflicting adds.
+        /// </summary>
+        public static DiffTreeResult ParseFromLsFilesStagingLine(string line)
+        {
+            if (string.IsNullOrEmpty(line))
+            {
+                throw new ArgumentException("Line to parse cannot be null or empty", nameof(line));
+            }
+
+            /*
+             * Example output lines from ls-files -s
+             *
+             * 100644 44c5f5cba4b29d31c2ad06eed51ea02af76c27c0 0\tReadme.md
+             * 100755 196142fbb753c0a3c7c6690323db7aa0a11f41ec 0\tScripts/BuildGVFSForMac.sh
+             * ^-mode ^-sha                                    ^stage
+             *                                                   ^-tab
+             *                                                     ^-path
+             *
+             * Format: <mode> <sha> <stage>\t<path>
+             * Mode is 6 chars, space, SHA is 40 chars, space, stage digit(s), tab, path
+             *
+             * During a merge conflict, the same path can appear multiple times with
+             * stage 1 (common ancestor), 2 (ours), and 3 (theirs). We only want
+             * stage 0 (normal) entries. In GVFS-mounted repos merge conflicts should
+             * not occur, but we filter defensively.
+             */
+
+            int tabIndex = line.IndexOf('\t');
+            if (tabIndex < 0 || line.Length < 50)
+            {
+                return null;
+            }
+
+            // Stage is between the SHA and the tab: "<mode> <sha> <stage>\t<path>"
+            // Position 48 = 6 (mode) + 1 (space) + 40 (sha) + 1 (space)
+            int stageStart = 7 + GVFSConstants.ShaStringLength + 1;
+            string stageStr = line.Substring(stageStart, tabIndex - stageStart);
+            if (stageStr != "0")
+            {
+                return null;
+            }
+
+            DiffTreeResult blobAdd = new DiffTreeResult();
+            blobAdd.TargetMode = Convert.ToUInt16(line.Substring(0, 6), 8);
+            blobAdd.TargetIsSymLink = blobAdd.TargetMode == SymLinkFileIndexEntry;
+            blobAdd.TargetSha = line.Substring(7, GVFSConstants.ShaStringLength);
+            blobAdd.TargetPath = ConvertPathToUtf8Path(line.Substring(tabIndex + 1));
+            blobAdd.Operation = Operations.Add;
+
+            return blobAdd;
+        }
+
         private static string AppendPathSeparatorIfNeeded(string path)
         {
             return path.Last() == Path.DirectorySeparatorChar ? path : path + Path.DirectorySeparatorChar;

GVFS/GVFS.Common/Git/GitProcess.cs

@@ -760,6 +760,18 @@ public Result LsTree(string treeish, Action<string> parseStdOutLine, bool recurs
                 parseStdOutLine);
         }
 
+        /// <summary>
+        /// Runs git ls-files -s to list all tracked files with their mode, SHA, and path.
+        /// Reads from the index (fast) rather than walking tree objects (slow).
+        /// </summary>
+        public Result LsFilesStaging(Action<string> parseStdOutLine)
+        {
+            return this.InvokeGitInWorkingDirectoryRoot(
+                "ls-files -s",
+                useReadObjectHook: false,
+                parseStdOutLine: parseStdOutLine);
+        }
+
         public Result LsFiles(Action<string> parseStdOutLine)
         {
             return this.InvokeGitInWorkingDirectoryRoot(

GVFS/GVFS.Common/Prefetch/Git/DiffHelper.cs

@@ -119,21 +119,52 @@ public void PerformDiff(string sourceTreeSha, string targetTreeSha)
                 {
                     this.UpdatedWholeTree = true;
 
-                    // Nothing is checked out (fresh git init), so we must search the entire tree.
-                    GitProcess.Result result = this.git.LsTree(
-                        targetTreeSha,
-                        line => this.EnqueueOperationsFromLsTreeLine(activity, line),
-                        recursive: true,
-                        showAllTrees: true);
-
-                    if (result.ExitCodeIsFailure)
+                    // Prefer ls-files -s over ls-tree -r -t for full-tree enumeration.
+                    // ls-files reads the git index (~6.5s on a 2.5M-file repo) while
+                    // ls-tree walks every tree object (~24s on the same repo).
+                    // ls-files reflects the index (HEAD), so we can only use it when
+                    // targetTreeSha matches HEAD's tree. When they differ (e.g.,
+                    // FastFetch checking out a different commit), fall back to ls-tree.
+                    //
+                    // ls-files only returns file entries (not tree/directory entries).
+                    // This is safe because the ls-files path only fires for gvfs prefetch
+                    // on a GVFS-mounted repo where directories are virtualized by PrjFlt
+                    // and don't need explicit creation. FastFetch force-checkout (which
+                    // needs directory operations) won't match HEAD and falls back to ls-tree.
+                    bool usedLsFiles = false;
+                    if (this.TargetMatchesHeadTree(targetTreeSha))
                     {
-                        this.HasFailures = true;
-                        metadata.Add("Errors", result.Errors);
-                        metadata.Add("Output", result.Output.Length > 1024 ? result.Output.Substring(1024) : result.Output);
+                        GitProcess.Result result = this.git.LsFilesStaging(
+                            line => this.EnqueueOperationsFromLsFilesStagingLine(activity, line));
+
+                        if (result.ExitCodeIsSuccess)
+                        {
+                            usedLsFiles = true;
+                            metadata.Add("Operation", "LsFilesStaging");
+                        }
+                        else
+                        {
+                            this.tracer.RelatedWarning("ls-files -s failed, falling back to ls-tree: " + result.Errors);
+                        }
                     }
 
-                    metadata.Add("Operation", "LsTree");
+                    if (!usedLsFiles)
+                    {
+                        GitProcess.Result result = this.git.LsTree(
+                            targetTreeSha,
+                            line => this.EnqueueOperationsFromLsTreeLine(activity, line),
+                            recursive: true,
+                            showAllTrees: true);
+
+                        if (result.ExitCodeIsFailure)
+                        {
+                            this.HasFailures = true;
+                            metadata.Add("Errors", result.Errors);
+                            metadata.Add("Output", result.Output.Length > 1024 ? result.Output.Substring(1024) : result.Output);
+                        }
+
+                        metadata.Add("Operation", "LsTree");
+                    }
                 }
                 else
                 {
@@ -235,6 +266,47 @@ private void FlushStagedQueues()
             }
         }
 
+        /// <summary>
+        /// Check whether targetTreeSha matches HEAD's tree SHA so we can safely
+        /// use git ls-files -s (which reads the index reflecting HEAD) instead of
+        /// git ls-tree (which walks a specific tree object).
+        ///
+        /// Note: callers may pass either a tree SHA or a commit SHA as targetTreeSha
+        /// (git ls-tree auto-peels commits). We resolve both sides to tree SHAs
+        /// before comparing.
+        /// </summary>
+        private bool TargetMatchesHeadTree(string targetTreeSha)
+        {
+            try
+            {
+                using (LibGit2Repo repo = new LibGit2Repo(this.tracer, this.enlistment.WorkingDirectoryBackingRoot))
+                {
+                    string headTreeSha = repo.GetTreeSha("HEAD");
+
+                    // targetTreeSha may be a commit SHA (callers like BlobPrefetcher
+                    // pass commit IDs). Resolve it to a tree SHA for comparison.
+                    string targetResolvedTreeSha = repo.GetTreeSha(targetTreeSha) ?? targetTreeSha;
+
+                    if (headTreeSha != null && string.Equals(headTreeSha, targetResolvedTreeSha, StringComparison.OrdinalIgnoreCase))
+                    {
+                        return true;
+                    }
+
+                    this.tracer.RelatedInfo(
+                        "TargetMatchesHeadTree: target {0} (tree {1}) != HEAD tree {2}, will use ls-tree",
+                        targetTreeSha,
+                        targetResolvedTreeSha,
+                        headTreeSha ?? "(null)");
+                    return false;
+                }
+            }
+            catch (Exception e)
+            {
+                this.tracer.RelatedWarning("TargetMatchesHeadTree: failed to resolve HEAD tree: " + e.Message);
+                return false;
+            }
+        }
+
         private void EnqueueOperationsFromLsTreeLine(ITracer activity, string line)
         {
             DiffTreeResult result = DiffTreeResult.ParseFromLsTreeLine(line);
@@ -268,6 +340,24 @@ private void EnqueueOperationsFromLsTreeLine(ITracer activity, string line)
             }
         }
 
+        private void EnqueueOperationsFromLsFilesStagingLine(ITracer activity, string line)
+        {
+            DiffTreeResult result = DiffTreeResult.ParseFromLsFilesStagingLine(line);
+            if (result == null)
+            {
+                this.tracer.RelatedError("Unrecognized ls-files -s line: {0}", line);
+                return;
+            }
+
+            if (!this.ShouldIncludeResult(result))
+            {
+                return;
+            }
+
+            // ls-files -s only returns file entries, never trees
+            this.EnqueueFileAddOperation(activity, result);
+        }
+
         private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string line)
         {
             if (!line.StartsWith(":"))

GVFS/GVFS.UnitTests/Prefetch/DiffTreeResultTests.cs

@@ -38,6 +38,15 @@ public class DiffTreeResultTests
         private static readonly string InvalidLineFromLsTree = $"040000 bad {TestSha1}\t{TestTreePath1}";
         private static readonly string SymLinkLineFromLsTree = $"120000 blob {TestSha1}\t{TestTreePath1}";
 
+        // ls-files -s test data
+        private static readonly string BlobLineFromLsFilesStaging = $"100644 {TestSha1} 0\t{TestTreePath1}";
+        private static readonly string ExecutableBlobFromLsFilesStaging = $"100755 {TestSha1} 0\t{TestBlobPath1}";
+        private static readonly string SymLinkFromLsFilesStaging = $"120000 {TestSha1} 0\t{TestTreePath1}";
+        private static readonly string BlobWithSpacesFromLsFilesStaging = $"100644 {TestSha1} 0\t{TestBlobPath1}";
+        private static readonly string UnmergedStage1FromLsFilesStaging = $"100644 {TestSha1} 1\t{TestTreePath1}";
+        private static readonly string UnmergedStage2FromLsFilesStaging = $"100644 {Test2Sha1} 2\t{TestTreePath1}";
+        private static readonly string UnmergedStage3FromLsFilesStaging = $"100644 {TestSha1} 3\t{TestTreePath1}";
+
         [TestCase]
         [Category(CategoryConstants.ExceptionExpected)]
         public void ParseFromDiffTreeLine_NullLine()
@@ -341,6 +350,90 @@ public void ParseFromDiffTreeLine_BlobLineWithTreePath()
             this.ValidateDiffTreeResult(expected, result);
         }
 
+        [TestCase]
+        [Category(CategoryConstants.ExceptionExpected)]
+        public void ParseFromLsFilesStagingLine_NullLine()
+        {
+            Assert.Throws<ArgumentException>(() => DiffTreeResult.ParseFromLsFilesStagingLine(null));
+        }
+
+        [TestCase]
+        [Category(CategoryConstants.ExceptionExpected)]
+        public void ParseFromLsFilesStagingLine_EmptyLine()
+        {
+            Assert.Throws<ArgumentException>(() => DiffTreeResult.ParseFromLsFilesStagingLine(string.Empty));
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_InvalidLine()
+        {
+            DiffTreeResult.ParseFromLsFilesStagingLine("short").ShouldBeNull();
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_BlobLine()
+        {
+            DiffTreeResult expected = new DiffTreeResult()
+            {
+                Operation = DiffTreeResult.Operations.Add,
+                SourceIsDirectory = false,
+                TargetIsDirectory = false,
+                TargetPath = TestTreePath1.Replace('/', Path.DirectorySeparatorChar),
+                SourceSha = null,
+                TargetSha = TestSha1
+            };
+
+            DiffTreeResult result = DiffTreeResult.ParseFromLsFilesStagingLine(BlobLineFromLsFilesStaging);
+            this.ValidateDiffTreeResult(expected, result);
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_ExecutableBlob()
+        {
+            DiffTreeResult result = DiffTreeResult.ParseFromLsFilesStagingLine(ExecutableBlobFromLsFilesStaging);
+            result.ShouldNotBeNull();
+            result.Operation.ShouldEqual(DiffTreeResult.Operations.Add);
+            result.TargetMode.ShouldEqual(Convert.ToUInt16("100755", 8));
+            result.TargetSha.ShouldEqual(TestSha1);
+            result.TargetPath.ShouldEqual(TestBlobPath1.Replace('/', Path.DirectorySeparatorChar));
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_SymLink()
+        {
+            DiffTreeResult result = DiffTreeResult.ParseFromLsFilesStagingLine(SymLinkFromLsFilesStaging);
+            result.ShouldNotBeNull();
+            result.TargetIsSymLink.ShouldBeTrue();
+            result.TargetSha.ShouldEqual(TestSha1);
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_PathWithSpaces()
+        {
+            DiffTreeResult result = DiffTreeResult.ParseFromLsFilesStagingLine(BlobWithSpacesFromLsFilesStaging);
+            result.ShouldNotBeNull();
+            result.TargetPath.ShouldEqual(TestBlobPath1.Replace('/', Path.DirectorySeparatorChar));
+            result.TargetSha.ShouldEqual(TestSha1);
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_UnmergedStage1_ReturnsNull()
+        {
+            DiffTreeResult.ParseFromLsFilesStagingLine(UnmergedStage1FromLsFilesStaging).ShouldBeNull();
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_UnmergedStage2_ReturnsNull()
+        {
+            DiffTreeResult.ParseFromLsFilesStagingLine(UnmergedStage2FromLsFilesStaging).ShouldBeNull();
+        }
+
+        [TestCase]
+        public void ParseFromLsFilesStagingLine_UnmergedStage3_ReturnsNull()
+        {
+            DiffTreeResult.ParseFromLsFilesStagingLine(UnmergedStage3FromLsFilesStaging).ShouldBeNull();
+        }
+
         [TestCase("040000 tree 73b881d52b607b0f3e9e620d36f556d3d233a11d\tGVFS", DiffTreeResult.TreeMarker, true)]
         [TestCase("040000 tree 73b881d52b607b0f3e9e620d36f556d3d233a11d\tGVFS", DiffTreeResult.BlobMarker, false)]
         [TestCase("100644 blob 44c5f5cba4b29d31c2ad06eed51ea02af76c27c0\tReadme.md", DiffTreeResult.BlobMarker, true)]