Merge branch 'main' into fix/nested-struct-field-filtering

Author: yiftizur

.github/workflows/nightly-pypi-build.yml

@@ -71,7 +71,7 @@ jobs:
 
     steps:
     - name: Download all the artifacts
-      uses: actions/download-artifact@v5
+      uses: actions/download-artifact@v6
       with:
         merge-multiple: true
         path: dist/

.github/workflows/pypi-build-artifacts.yml

@@ -42,7 +42,6 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: |
-            3.9
             3.10
             3.11
             3.12
@@ -69,12 +68,14 @@ jobs:
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it
           # and would require a local rust build chain
           CIBW_TEST_SKIP: "pp*"
+          # Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support
+          CIBW_SKIP: "cp3*t-*"
 
       - name: Add source distribution
         if: startsWith(matrix.os, 'ubuntu')

.github/workflows/python-ci.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.9', '3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v5
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.9', '3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v5

.github/workflows/svn-build-artifacts.yml

@@ -42,7 +42,6 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: |
-            3.9
             3.10
             3.11
             3.12
@@ -64,12 +63,14 @@ jobs:
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it
           # and would require a local rust build chain
           CIBW_TEST_SKIP: "pp*"
+          # Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support
+          CIBW_SKIP: "cp3*t-*"
 
       - name: Add source distribution
         if: startsWith(matrix.os, 'ubuntu')

.pre-commit-config.yaml

@@ -27,13 +27,13 @@ repos:
       - id: check-yaml
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.9
+    rev: v0.14.3
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.17.1
+    rev: v1.18.2
     hooks:
       - id: mypy
         args:

dev/hive/Dockerfile

@@ -13,21 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM openjdk:8-jre-slim AS build
-
-RUN apt-get update -qq && apt-get -qq -y install curl
+FROM apache/hive:4.0.0
 
 ENV HADOOP_VERSION=3.3.6
 ENV AWS_SDK_BUNDLE=1.12.753
 
-RUN curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -Lo /tmp/hadoop-aws-${HADOOP_VERSION}.jar
-RUN curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar -Lo /tmp/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar
+USER root
 
-FROM apache/hive:4.0.0
+# Install curl, download JARs, and cleanup in a single layer
+RUN apt-get update -qq && apt-get -qq -y install curl && \
+    curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -Lo /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar && \
+    curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar -Lo /opt/hive/lib/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
-ENV HADOOP_VERSION=3.3.6
-ENV AWS_SDK_BUNDLE=1.12.753
-
-COPY --from=build /tmp/hadoop-aws-${HADOOP_VERSION}.jar /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar
-COPY --from=build /tmp/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar /opt/hive/lib/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar
 COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml
+
+USER hive

mkdocs/docs/contributing.md

@@ -234,8 +234,6 @@ Deprecated in 0.1.0, will be removed in 0.2.0. The old_property is deprecated. P
 
 For the type annotation the types from the `Typing` package are used.
 
-PyIceberg offers support from Python 3.9 onwards, we can't use the [type hints from the standard collections](https://peps.python.org/pep-0585/).
-
 ### Third party libraries
 
 PyIceberg naturally integrates into the rich Python ecosystem, however it is important to be hesitant adding third party packages. Adding a lot of packages makes the library heavyweight, and causes incompatibilities with other projects if they use a different version of the library. Also, big libraries such as `s3fs`, `adlfs`, `pyarrow`, `thrift` should be optional to avoid downloading everything, while not being sure if is actually being used.

mkdocs/docs/how-to-release.md

@@ -30,6 +30,7 @@ This guide outlines the process for releasing PyIceberg in accordance with the [
 ## Requirements
 
 * A GPG key must be registered and published in the [Apache Iceberg KEYS file](https://downloads.apache.org/iceberg/KEYS). Follow [the instructions for setting up a GPG key and uploading it to the KEYS file](#set-up-gpg-key-and-upload-to-apache-iceberg-keys-file).
+    * Permission to update the `KEYS` artifact in the [Apache release distribution](https://dist.apache.org/repos/dist/release/iceberg/) (requires Iceberg PMC privileges).
 * SVN Access
     * Permission to upload artifacts to the [Apache development distribution](https://dist.apache.org/repos/dist/dev/iceberg/) (requires Apache Committer access).
     * Permission to upload artifacts to the [Apache release distribution](https://dist.apache.org/repos/dist/release/iceberg/) (requires Apache PMC access).
@@ -405,5 +406,12 @@ cd icebergsvn
 echo "" >> KEYS # append a newline
 gpg --list-sigs <YOUR KEY ID HERE> >> KEYS # append signatures
 gpg --armor --export <YOUR KEY ID HERE> >> KEYS # append public key block
-svn commit -m "add key for <YOUR NAME HERE>"
+svn commit -m "add key for <YOUR NAME HERE>" # this requires Iceberg PMC privileges
 ```
+
+<!-- prettier-ignore-start -->
+
+!!! note
+    Updating the `KEYS` artifact in the `release/` distribution requires Iceberg PMC privileges. Please work with a PMC member to update the file.
+
+<!-- prettier-ignore-end -->

mkdocs/gen_doc_stubs.py

@@ -17,8 +17,8 @@
 
 from pathlib import Path
 
-import griffe  # type: ignore
-import mkdocs_gen_files  # type: ignore
+import griffe
+import mkdocs_gen_files
 
 nav = mkdocs_gen_files.Nav()