fix(shallow): honour the shallow boundary across clone, gc, fsck and rev-list

Taskfile.pkl

@@ -332,8 +332,12 @@ local e2e: Task = new {
 
 local testSubdir: Task = new {
   name = "test-subdir"
-  description = "subdir-clone/push integration tests (t/ directory)"
-  cmd = "bash t/run-tests.sh t900"
+  description = "subdir-clone/push and transport integration tests (t/ directory)"
+  // The filter is a substring match, so `t900` alone silently skips t902x.
+  cmd = #"""
+    bash t/run-tests.sh t900
+    bash t/run-tests.sh t902
+    """#
 }
 
 local testDistributed: Task = new {

modules/bit/cmd/bit/fsck.mbt

@@ -84,6 +84,7 @@ async fn handle_fsck(args : Array[String]) -> Unit raise Error {
     fs,
     ref_tips,
     collect_extra=need_extra,
+    shallow=@repo.read_shallow_boundaries(fs, git_dir),
   )
   errors += result.errors
   // Report missing objects

modules/bit/cmd/bit/rev_list.mbt

@@ -600,6 +600,13 @@ async fn handle_rev_list(args : Array[String]) -> Unit raise Error {
   } else {
     Map([])
   }
+  // Commits on the shallow boundary are grafted: their parents were never
+  // fetched. Following those links emits an id whose object is not in the
+  // store, so the walk must stop here exactly as `log` does.
+  let shallow_boundary : Map[String, Bool] = Map([])
+  for id in @repo.read_shallow_boundaries(fs, git_dir) {
+    shallow_boundary[id.to_hex()] = true
+  }
   let visited : Map[String, Bool] = Map([])
   let result : Array[@bitcore.ObjectId] = []
   // BFS/DFS traversal — collect ALL reachable commits, then sort/slice
@@ -618,7 +625,9 @@ async fn handle_rev_list(args : Array[String]) -> Unit raise Error {
     // Get commit and add parents
     match rev_list_load_commit_info(db, fs, id) {
       Some(info) =>
-        if first_parent && info.parents.length() > 0 {
+        if shallow_boundary.contains(hex) {
+          ()
+        } else if first_parent && info.parents.length() > 0 {
           if !visited.contains(info.parents[0].to_hex()) {
             queue.push(info.parents[0])
           }

modules/bit_io_native/src/upload_pack_process.mbt

@@ -1261,7 +1261,10 @@ pub async fn clone_process_to_fs(
   if wants.length() == 0 {
     return refs
   }
-  let pack = fetch_pack_process(remote, wants, prefer_v2, depth~, filter~)
+  let fetched = fetch_pack_process_result(
+    remote, wants, prefer_v2, depth~, filter~,
+  )
+  let pack = fetched.pack
   let objects = @pack.parse_packfile(pack)
   let git_dir = @bit.join_path(root, ".git")
   let pack_id = read_pack_trailer_id(pack)
@@ -1290,24 +1293,13 @@ pub async fn clone_process_to_fs(
         )
       }
   }
-  if depth > 0 {
-    let shallow_ids : Array[String] = []
-    let seen : Map[String, Bool] = Map([])
-    for id in wants {
-      let hex = id.to_hex()
-      if seen.contains(hex) {
-        continue
-      }
-      seen[hex] = true
-      shallow_ids.push(hex)
-    }
-    if shallow_ids.length() > 0 {
-      fs.write_string(
-        @bit.join_path(git_dir, "shallow"),
-        shallow_ids.join("\n") + "\n",
-      )
-    }
-  }
+  // The boundary is whatever the server reported as shallow, not the refs we
+  // wanted: those coincide only at depth 1. Recording the tips instead makes
+  // every deeper clone look parentless at its tip, which then truncates the
+  // history a repack keeps.
+  @repo.apply_shallow_updates(
+    fs, rfs, git_dir, fetched.shallow, fetched.unshallow,
+  )
   refs
 }
 

modules/bit_lib/src/fsck.mbt

@@ -18,7 +18,15 @@ pub fn fsck_connectivity_check(
   fs : &@bit.RepoFileSystem,
   tips : Array[@bit.ObjectId],
   collect_extra? : Bool = false,
+  shallow? : Array[@bit.ObjectId] = [],
 ) -> FsckResult {
+  // Commits on the shallow boundary are grafted: their parents were never
+  // fetched, so following those links would report a well-formed shallow
+  // clone as corrupt.
+  let boundary : Map[String, Bool] = Map([])
+  for id in shallow {
+    boundary[id.to_hex()] = true
+  }
   let reachable : Map[String, Bool] = Map([])
   let missing : Map[String, Bool] = Map([])
   let root_commits : Array[String] = []
@@ -68,10 +76,12 @@ pub fn fsck_connectivity_check(
             if !reachable.contains(tree_hex) {
               queue.push(info.tree)
             }
-            for parent in info.parents {
-              let phex = parent.to_hex()
-              if !reachable.contains(phex) {
-                queue.push(parent)
+            if !boundary.contains(hex) {
+              for parent in info.parents {
+                let phex = parent.to_hex()
+                if !reachable.contains(phex) {
+                  queue.push(parent)
+                }
               }
             }
           }

modules/bit_lib/src/fsck_test.mbt

@@ -127,3 +127,48 @@ test "fsck_verify_loose_hash valid" {
     ),
   )
 }
+
+///|
+test "fsck_connectivity_check treats a shallow boundary as grafted" {
+  let fs = @bit.TestFs::new()
+  let git_dir = "/repo/.git"
+  fs.mkdir_p(git_dir)
+  fs.mkdir_p(git_dir + "/objects")
+  fs.mkdir_p(git_dir + "/refs")
+  let (tree_id, tree_bytes) = @bit.create_tree([])
+  @bit_lib.write_object_bytes(fs, git_dir, tree_id, tree_bytes)
+  // A boundary commit whose parent was never fetched, exactly what a
+  // `clone --depth` leaves behind.
+  let absent_parent_hex = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+  let absent_parent = @bit.ObjectId::from_hex(absent_parent_hex)
+  let commit = @bit.Commit::new(
+    tree_id,
+    [absent_parent],
+    "A <a@b.c>",
+    1000000000L,
+    "+0000",
+    "A <a@b.c>",
+    1000000000L,
+    "+0000",
+    "boundary\n",
+  )
+  let (commit_id, commit_bytes) = @bit.create_commit(commit)
+  @bit_lib.write_object_bytes(fs, git_dir, commit_id, commit_bytes)
+  let db = @bit_lib.ObjectDb::load(fs, git_dir)
+
+  // Without the boundary the absent parent is reported as corruption.
+  let plain = @bit_lib.fsck_connectivity_check(db, fs, [commit_id])
+  assert_true(plain.errors > 0)
+  assert_true(plain.missing.contains(absent_parent_hex))
+
+  // Declaring the boundary makes the same repository read as clean.
+  let grafted = @bit_lib.fsck_connectivity_check(
+    db,
+    fs,
+    [commit_id],
+    shallow=[commit_id],
+  )
+  assert_eq(grafted.errors, 0)
+  assert_true(!grafted.missing.contains(absent_parent_hex))
+  assert_true(grafted.reachable.contains(tree_id.to_hex()))
+}

modules/bit_lib/src/gc.mbt

@@ -83,7 +83,10 @@ fn repack_git_dir(
   if roots.length() == 0 {
     return None
   }
-  let objects = collect_reachable_objects_from_commits(db, rfs, roots)
+  // A shallow clone's boundary commits have no parents on disk. Walking past
+  // them either aborts the run or, worse, truncates what a repack keeps.
+  let shallow = @repo.read_shallow_boundaries(rfs, git_dir)
+  let objects = collect_reachable_objects_from_commits(db, rfs, roots, shallow~)
   collect_reachable_tag_objects(db, rfs, ref_ids, objects)
   // Reflog-referenced objects are roots too (git keeps them); repacking without
   // them would drop reflog-only objects from the new pack.
@@ -101,10 +104,11 @@ fn gc_git_dir(
   let db = ObjectDb::load(rfs, git_dir)
   let roots = resolve_commit_roots(db, rfs, ref_ids)
   let prune_enabled = roots.length() > 0
+  let shallow = @repo.read_shallow_boundaries(rfs, git_dir)
   let reachable = if roots.length() == 0 {
     []
   } else {
-    collect_reachable_objects_from_commits(db, rfs, roots)
+    collect_reachable_objects_from_commits(db, rfs, roots, shallow~)
   }
   // Also include tag objects that refs point to directly (annotated tags).
   collect_reachable_tag_objects(db, rfs, ref_ids, reachable)
@@ -154,7 +158,8 @@ fn prune_git_dir(
   if roots.length() == 0 {
     return { pruned: [] }
   }
-  let reachable = collect_reachable_objects_from_commits(db, rfs, roots)
+  let shallow = @repo.read_shallow_boundaries(rfs, git_dir)
+  let reachable = collect_reachable_objects_from_commits(db, rfs, roots, shallow~)
   collect_reachable_tag_objects(db, rfs, ref_ids, reachable)
   // Match gc_git_dir: reflog-referenced objects are protected roots, so a bare
   // `bit gc --prune` must not delete reflog-only loose objects.
@@ -374,6 +379,7 @@ fn collect_reflog_referenced_objects(
   git_dir : String,
   objects : Array[@bit.PackObject],
 ) -> Unit raise @bit.GitError {
+  let shallow = @repo.read_shallow_boundaries(fs, git_dir)
   let seen : Map[String, Bool] = Map([])
   for obj in objects {
     seen[@bit.hash_object_content(obj.obj_type, obj.data).to_hex()] = true
@@ -391,7 +397,9 @@ fn collect_reflog_referenced_objects(
         objects.push(obj)
         // If it's a commit, also collect its tree and blobs
         if obj.obj_type == @bit.ObjectType::Commit {
-          let sub = collect_reachable_objects_from_commits(db, fs, [id])
+          let sub = collect_reachable_objects_from_commits(
+            db, fs, [id], shallow~,
+          )
           for s in sub {
             let sh = @bit.hash_object_content(s.obj_type, s.data).to_hex()
             if !seen.contains(sh) {

modules/bit_protocol/src/upload_pack_http_common.mbt

@@ -253,6 +253,8 @@ priv struct PreparedClone {
   pack : Bytes
   pack_id : @bit.ObjectId
   objects : Array[@bit.PackObject]
+  shallow : Array[@bit.ObjectId]
+  unshallow : Array[@bit.ObjectId]
 }
 
 ///|
@@ -276,16 +278,22 @@ async fn prepare_clone_with_http(
   if default_ref is None || wants.length() == 0 {
     return (refs, None)
   }
-  let pack = fetch_pack_with_http(
+  // Keep the shallow/unshallow lines: without them a --depth clone leaves no
+  // boundary on disk, so the repository reports itself complete and a later
+  // `fetch --unshallow` has nothing to deepen from.
+  let result = fetch_pack_with_http_result(
     remote, wants, prefer_v2, depth, filter, http_get, http_post,
   )
+  let pack = result.pack
   (
     refs,
     Some({
       default_ref: default_ref.unwrap(),
       pack,
       pack_id: read_pack_trailer_id_http(pack),
       objects: @pack.parse_packfile(pack),
+      shallow: result.shallow,
+      unshallow: result.unshallow,
     }),
   )
 }
@@ -316,6 +324,9 @@ pub async fn clone_to_fs_with_http(
   let git_dir = @bit.join_path(root, ".git")
   fs.mkdir_p(@bit.join_path(git_dir, "objects/pack"))
   @pack.write_packfile_with_index(fs, git_dir, prepared.pack, prepared.objects)
+  @repo.apply_shallow_updates(
+    fs, rfs, git_dir, prepared.shallow, prepared.unshallow,
+  )
   if filter.is_partial() {
     write_promisor_file(fs, git_dir, remote)
     write_pack_promisor_marker_http(
@@ -384,6 +395,7 @@ pub async fn[
     prepared.pack,
     prepared.objects,
   )
+  @repo.write_shallow_boundaries_async(fs, git_dir, prepared.shallow)
   if filter.is_partial() {
     write_promisor_file_async(fs, git_dir, remote)
     write_pack_promisor_marker_http_async(

modules/bit_repo/src/shallow.mbt

@@ -53,3 +53,35 @@ pub fn apply_shallow_updates(
   let lines = [ for _, id in boundaries => id.to_hex() ]
   fs.write_string(path, lines.join("\n") + "\n")
 }
+
+///|
+/// Record the shallow boundary of a freshly cloned repository.
+///
+/// A clone starts with no `shallow` file, so there is nothing to merge and no
+/// prior boundary an `unshallow` line could lift: writing the reported set is
+/// the whole job. `apply_shallow_updates` covers the incremental fetch case,
+/// which needs to read what is already there.
+pub async fn[FS : @types.AsyncFileSystem] write_shallow_boundaries_async(
+  fs : FS,
+  git_dir : String,
+  shallow : Array[@object.ObjectId],
+) -> Unit raise @object.GitError {
+  if shallow.length() == 0 {
+    return
+  }
+  let seen : Map[String, Bool] = Map([])
+  let lines : Array[String] = []
+  for id in shallow {
+    let hex = id.to_hex()
+    if seen.contains(hex) {
+      continue
+    }
+    seen[hex] = true
+    lines.push(hex)
+  }
+  @types.AsyncFileSystem::write_string(
+    fs,
+    join_path(git_dir, "shallow"),
+    lines.join("\n") + "\n",
+  )
+}