Merge branch 'main' into feature/write-to-branch

Author: vinjai

.gitignore

@@ -35,6 +35,7 @@ coverage.xml
 .project
 .settings
 bin/
+.vscode/
 
 # Hive/metastore files
 metastore_db/

dev/docker-compose-integration.yml

@@ -81,7 +81,7 @@ services:
       - AWS_REGION=us-east-1
     entrypoint: >
       /bin/sh -c "
-      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
+      until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
       /usr/bin/mc mb minio/warehouse;
       /usr/bin/mc policy set public minio/warehouse;
       tail -f /dev/null

dev/docker-compose.yml

@@ -38,7 +38,7 @@ services:
       - AWS_REGION=us-east-1
     entrypoint: >
       /bin/sh -c "
-      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
+      until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
       /usr/bin/mc rm -r --force minio/warehouse;
       /usr/bin/mc mb minio/warehouse;
       /usr/bin/mc policy set public minio/warehouse;

poetry.lock

@@ -27,7 +27,7 @@
 VALIDATE_DATA_FILES_EXIST_OPERATIONS = {Operation.OVERWRITE, Operation.REPLACE, Operation.DELETE}
 
 
-def validation_history(
+def _validation_history(
     table: Table,
     from_snapshot: Snapshot,
     to_snapshot: Snapshot,
@@ -100,7 +100,7 @@ def _deleted_data_files(
     if parent_snapshot is None:
         return
 
-    manifests, snapshot_ids = validation_history(
+    manifests, snapshot_ids = _validation_history(
         table,
         parent_snapshot,
         starting_snapshot,

pyiceberg/table/update/validate.py

@@ -80,7 +80,7 @@ huggingface-hub = { version = ">=0.24.0", optional = true }
 psycopg2-binary = { version = ">=2.9.6", optional = true }
 sqlalchemy = { version = "^2.0.18", optional = true }
 getdaft = { version = ">=0.2.12", optional = true }
-cachetools = "^5.5.0"
+cachetools = ">=5.5,<7.0"
 pyiceberg-core = { version = "^0.4.0", optional = true }
 polars = { version = "^1.21.0", optional = true }
 thrift-sasl = { version = ">=0.4.3", optional = true }
@@ -96,11 +96,11 @@ coverage = { version = "^7.4.2", extras = ["toml"] }
 requests-mock = "1.12.1"
 moto = { version = "^5.0.2", extras = ["server"] }
 typing-extensions = "4.13.2"
-pytest-mock = "3.14.0"
-pyspark = "3.5.5"
+pytest-mock = "3.14.1"
+pyspark = "3.5.6"
 cython = "3.1.1"
 deptry = ">=0.14,<0.24"
-datafusion = ">=44,<47"
+datafusion = ">=44,<48"
 docutils = "!=0.21.post1"   # https://github.com/python-poetry/poetry/issues/9248#issuecomment-2026240520
 
 [tool.poetry.group.docs.dependencies]
@@ -109,9 +109,9 @@ mkdocs = "1.6.1"
 griffe = "1.7.3"
 jinja2 = "3.1.6"
 mkdocstrings = "0.29.1"
-mkdocstrings-python = "1.16.10"
+mkdocstrings-python = "1.16.11"
 mkdocs-literate-nav = "0.6.2"
-mkdocs-autorefs = "1.4.1"
+mkdocs-autorefs = "1.4.2"
 mkdocs-gen-files = "0.5.0"
 mkdocs-material = "9.6.14"
 mkdocs-material-extensions = "1.3.1"

pyproject.toml

@@ -25,7 +25,7 @@
 from pyiceberg.manifest import ManifestContent, ManifestEntry, ManifestEntryStatus, ManifestFile
 from pyiceberg.table import Table
 from pyiceberg.table.snapshots import Operation, Snapshot, Summary
-from pyiceberg.table.update.validate import _deleted_data_files, _validate_deleted_data_files, validation_history
+from pyiceberg.table.update.validate import _deleted_data_files, _validate_deleted_data_files, _validation_history
 
 
 @pytest.fixture
@@ -69,7 +69,7 @@ def mock_read_manifest_side_effect(self: Snapshot, io: FileIO) -> list[ManifestF
         return []
 
     with patch("pyiceberg.table.snapshots.Snapshot.manifests", new=mock_read_manifest_side_effect):
-        manifests, snapshots = validation_history(
+        manifests, snapshots = _validation_history(
             table,
             oldest_snapshot,
             newest_snapshot,
@@ -99,7 +99,7 @@ def test_validation_history_fails_on_snapshot_with_no_summary(
     )
     with patch("pyiceberg.table.update.validate.ancestors_between", return_value=[snapshot_with_no_summary]):
         with pytest.raises(ValidationException):
-            validation_history(
+            _validation_history(
                 table,
                 oldest_snapshot,
                 newest_snapshot,
@@ -129,7 +129,7 @@ def mock_read_manifest_side_effect(self: Snapshot, io: FileIO) -> list[ManifestF
     with patch("pyiceberg.table.snapshots.Snapshot.manifests", new=mock_read_manifest_side_effect):
         with patch("pyiceberg.table.update.validate.ancestors_between", return_value=missing_oldest_snapshot):
             with pytest.raises(ValidationException):
-                validation_history(
+                _validation_history(
                     table,
                     oldest_snapshot,
                     newest_snapshot,