Reverted changes back to prior commit version for _get_all_datafiles

Author: ForeverAngry

pyiceberg/table/maintenance.py

@@ -294,9 +294,27 @@ def _get_protected_snapshot_ids(self, table_metadata: TableMetadata) -> Set[int]
 
     def _get_all_datafiles(self) -> List[DataFile]:
         """Collect all DataFiles in the current snapshot only."""
-        data_file_structs = self.tbl.inspect.data_files()
-        data_files = [DataFile(df) for df in data_file_structs]
-        return data_files
+        datafiles: List[DataFile] = []
+
+        current_snapshot = self.tbl.current_snapshot()
+        if not current_snapshot:
+            return datafiles
+
+        def process_manifest(manifest: ManifestFile) -> list[DataFile]:
+            found: list[DataFile] = []
+            for entry in manifest.fetch_manifest_entry(io=self.tbl.io, discard_deleted=True):
+                if hasattr(entry, "data_file"):
+                    found.append(entry.data_file)
+            return found
+
+        # Scan only the current snapshot's manifests
+        manifests = current_snapshot.manifests(io=self.tbl.io)
+        with ThreadPoolExecutor() as executor:
+            results = executor.map(process_manifest, manifests)
+            for res in results:
+                datafiles.extend(res)
+
+        return datafiles
 
     def deduplicate_data_files(self) -> List[DataFile]:
         """

tests/table/test_dedup_data_file_filepaths.py

@@ -124,9 +124,6 @@ def test_deduplicate_data_files_removes_duplicates_in_current_snapshot(
 ) -> None:
     mt = MaintenanceTable(tbl=prepopulated_table)
 
-    print("=== Before deduplication ===")
-    check_data_files(prepopulated_table)
-
     all_datafiles: List[DataFile] = mt._get_all_datafiles()
     file_names: List[str] = [os.path.basename(df.file_path) for df in all_datafiles]
     # There should be more than one reference before deduplication
@@ -135,9 +132,6 @@ def test_deduplicate_data_files_removes_duplicates_in_current_snapshot(
     )
     removed: List[DataFile] = mt.deduplicate_data_files()
 
-    print("=== After deduplication ===")
-    check_data_files(prepopulated_table)
-
     all_datafiles_after: List[DataFile] = mt._get_all_datafiles()
     file_names_after: List[str] = [os.path.basename(df.file_path) for df in all_datafiles_after]
     # Only one reference should remain after deduplication