Merge branch 'main' into bugfix/upsert-complex-type

Author: koenvo

dev/Dockerfile

@@ -39,20 +39,20 @@ WORKDIR ${SPARK_HOME}
 # Remember to also update `tests/conftest`'s spark setting
 ENV SPARK_VERSION=3.5.4
 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
-ENV ICEBERG_VERSION=1.8.0
+ENV ICEBERG_VERSION=1.9.0-SNAPSHOT
 ENV PYICEBERG_VERSION=0.9.0
 
 RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
  && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
  && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
 
 # Download iceberg spark runtime
-RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
+RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.0-SNAPSHOT/iceberg-spark-runtime-3.5_2.12-1.9.0-20250408.001846-43.jar \
       -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar
 
 
 # Download AWS bundle
-RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
+RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.9.0-SNAPSHOT/iceberg-aws-bundle-1.9.0-20250408.002722-86.jar \
       -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
 
 COPY spark-defaults.conf /opt/spark/conf

poetry.lock

@@ -22,8 +22,10 @@
     DelimitedList,
     Group,
     MatchFirst,
+    ParseException,
     ParserElement,
     ParseResults,
+    QuotedString,
     Suppress,
     Word,
     alphanums,
@@ -79,7 +81,16 @@
 LIKE = CaselessKeyword("like")
 
 unquoted_identifier = Word(alphas + "_", alphanums + "_$")
-quoted_identifier = Suppress('"') + unquoted_identifier + Suppress('"')
+quoted_identifier = QuotedString('"', escChar="\\", unquoteResults=True)
+
+
+@quoted_identifier.set_parse_action
+def validate_quoted_identifier(result: ParseResults) -> str:
+    if "." in result[0]:
+        raise ParseException("Expected '\"', found '.'")
+    return result[0]
+
+
 identifier = MatchFirst([unquoted_identifier, quoted_identifier]).set_results_name("identifier")
 column = DelimitedList(identifier, delim=".", combine=False).set_results_name("column")
 

pyiceberg/expressions/parser.py

@@ -1441,11 +1441,15 @@ def _task_to_record_batches(
 
             # Apply the user filter
             if pyarrow_filter is not None:
-                current_batch = current_batch.filter(pyarrow_filter)
+                # Temporary fix until PyArrow 21 is released ( https://github.com/apache/arrow/pull/46057 )
+                table = pa.Table.from_batches([current_batch])
+                table = table.filter(pyarrow_filter)
                 # skip empty batches
-                if current_batch.num_rows == 0:
+                if table.num_rows == 0:
                     continue
 
+                current_batch = table.combine_chunks().to_batches()[0]
+
             result_batch = _to_requested_schema(
                 projected_schema,
                 file_project_schema,

pyiceberg/io/pyarrow.py

@@ -438,6 +438,15 @@ def update_snapshot(self, snapshot_properties: Dict[str, str] = EMPTY_DICT) -> U
         """
         return UpdateSnapshot(self, io=self._table.io, snapshot_properties=snapshot_properties)
 
+    def update_statistics(self) -> UpdateStatistics:
+        """
+        Create a new UpdateStatistics to update the statistics of the table.
+
+        Returns:
+            A new UpdateStatistics
+        """
+        return UpdateStatistics(transaction=self)
+
     def append(self, df: pa.Table, snapshot_properties: Dict[str, str] = EMPTY_DICT) -> None:
         """
         Shorthand API for appending a PyArrow table to a table transaction.

pyiceberg/table/__init__.py

@@ -234,8 +234,8 @@ class BucketTransform(Transform[S, int]):
     _num_buckets: PositiveInt = PrivateAttr()
 
     def __init__(self, num_buckets: int, **data: Any) -> None:
-        self._num_buckets = num_buckets
         super().__init__(f"bucket[{num_buckets}]", **data)
+        self._num_buckets = num_buckets
 
     @property
     def num_buckets(self) -> int:

pyiceberg/transforms.py

@@ -93,7 +93,7 @@ fastavro = "1.10.0"
 coverage = { version = "^7.4.2", extras = ["toml"] }
 requests-mock = "1.12.1"
 moto = { version = "^5.0.2", extras = ["server"] }
-typing-extensions = "4.12.2"
+typing-extensions = "4.13.1"
 pytest-mock = "3.14.0"
 pyspark = "3.5.5"
 cython = "3.0.12"
@@ -104,16 +104,16 @@ docutils = "!=0.21.post1"   # https://github.com/python-poetry/poetry/issues/924
 [tool.poetry.group.docs.dependencies]
 # for mkdocs
 mkdocs = "1.6.1"
-griffe = "1.7.1"
+griffe = "1.7.2"
 jinja2 = "3.1.6"
 mkdocstrings = "0.29.1"
-mkdocstrings-python = "1.16.8"
+mkdocstrings-python = "1.16.10"
 mkdocs-literate-nav = "0.6.2"
 mkdocs-autorefs = "1.4.1"
 mkdocs-gen-files = "0.5.0"
-mkdocs-material = "9.6.10"
+mkdocs-material = "9.6.11"
 mkdocs-material-extensions = "1.3.1"
-mkdocs-section-index = "0.3.9"
+mkdocs-section-index = "0.3.10"
 
 [[tool.mypy.overrides]]
 module = "pytest_mock.*"

pyproject.toml

@@ -230,9 +230,11 @@ def test_quoted_column_with_dots() -> None:
     with pytest.raises(ParseException) as exc_info:
         parser.parse("\"foo.bar\".baz = 'data'")
 
-    assert "Expected '\"', found '.'" in str(exc_info.value)
-
     with pytest.raises(ParseException) as exc_info:
         parser.parse("'foo.bar'.baz = 'data'")
 
     assert "Expected <= | <> | < | >= | > | == | = | !=, found '.'" in str(exc_info.value)
+
+
+def test_quoted_column_with_spaces() -> None:
+    assert EqualTo("Foo Bar", "data") == parser.parse("\"Foo Bar\" = 'data'")

tests/expressions/test_parser.py

@@ -82,3 +82,10 @@ def create_statistics_file(snapshot_id: int, type_name: str) -> StatisticsFile:
         update.remove_statistics(add_snapshot_id_1)
 
     assert len(tbl.metadata.statistics) == 1
+
+    with tbl.transaction() as txn:
+        with txn.update_statistics() as update:
+            update.set_statistics(statistics_file_snap_1)
+            update.set_statistics(statistics_file_snap_2)
+
+    assert len(tbl.metadata.statistics) == 2