Improve comments in to_record_batches

Author: koenvo

pyiceberg/io/pyarrow.py

@@ -1611,23 +1611,27 @@ def to_record_batches(self, tasks: Iterable[FileScanTask]) -> Iterator[pa.Record
         total_row_count = 0
         executor = ExecutorFactory.get_or_create()
 
+        def batches_for_task(task: FileScanTask) -> List[pa.RecordBatch]:
+            # Materialize the iterator here to ensure execution happens within the executor.
+            # Otherwise, the iterator would be lazily consumed later (in the main thread),
+            # defeating the purpose of using executor.map.
+            return list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
+
         limit_reached = False
-        for batches in executor.map(
-            lambda task: list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file)), tasks
-        ):
+        for batches in executor.map(batches_for_task, tasks):
             for batch in batches:
                 current_batch_size = len(batch)
                 if self._limit is not None and total_row_count + current_batch_size >= self._limit:
                     yield batch.slice(0, self._limit - total_row_count)
 
-                    # This break will also cancel all running tasks
                     limit_reached = True
                     break
 
                 yield batch
                 total_row_count += current_batch_size
 
             if limit_reached:
+                # This break will also cancel all running tasks in the executor
                 break
 
     def _record_batches_from_scan_tasks_and_deletes(