Merge branch 'apache:main' into feature/add-files-branch

Author: vinjai

.github/workflows/license_check.yml

@@ -22,7 +22,7 @@ on: pull_request
 
 jobs:
   rat:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v5
     - run: dev/check-license

.github/workflows/pypi-build-artifacts.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
 
     steps:
       - uses: actions/checkout@v5
@@ -62,7 +62,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.1.4
+        uses: pypa/cibuildwheel@v3.2.0
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

.github/workflows/python-ci-docs.yml

@@ -32,15 +32,15 @@ concurrency:
 
 jobs:
   docs:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v5
-      - name: Install poetry
-        run: make install-poetry
       - uses: actions/setup-python@v6
         with:
           python-version: 3.12
+      - name: Install poetry
+        run: make install-poetry
       - name: Install
         run: make docs-install
       - name: Build docs

.github/workflows/python-ci.yml

@@ -44,18 +44,18 @@ concurrency:
 
 jobs:
   lint-and-test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         python: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v5
-    - name: Install poetry
-      run: make install-poetry
     - uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python }}
+    - name: Install poetry
+      run: make install-poetry
     - name: Install system dependencies
       run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
     - name: Install
@@ -68,7 +68,7 @@ jobs:
       run: COVERAGE_FAIL_UNDER=85 make coverage-report
 
   integration-test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         python: ['3.9', '3.10', '3.11', '3.12']

.github/workflows/python-release-docs.yml

@@ -27,15 +27,15 @@ concurrency:
 
 jobs:
   docs:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v5
-      - name: Install poetry
-        run: make install-poetry
       - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python }}
+      - name: Install poetry
+        run: make install-poetry
       - name: Install docs
         run: make docs-install
       - name: Build docs

.github/workflows/stale.yml

@@ -29,7 +29,7 @@ permissions:
 jobs:
   stale:
     if: github.repository_owner == 'apache'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/stale@v10.0.0
         with:

.github/workflows/svn-build-artifacts.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
 
     steps:
       - uses: actions/checkout@v5
@@ -57,7 +57,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.1.4
+        uses: pypa/cibuildwheel@v3.2.0
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

.gitignore

@@ -37,13 +37,6 @@ coverage.xml
 bin/
 .vscode/
 
-# Hive/metastore files
-metastore_db/
-
-# Spark/metastore files
-spark-warehouse/
-derby.log
-
 # Python stuff
 .mypy_cache/
 htmlcov

Makefile

@@ -18,17 +18,27 @@
 # Configuration Variables
 # ========================
 
-PYTEST_ARGS ?= -v  # Override with e.g. PYTEST_ARGS="-vv --tb=short"
+PYTEST_ARGS ?= -v -x  # Override with e.g. PYTEST_ARGS="-vv --tb=short"
 COVERAGE ?= 0      # Set COVERAGE=1 to enable coverage: make test COVERAGE=1
 COVERAGE_FAIL_UNDER ?= 85  # Minimum coverage % to pass: make coverage-report COVERAGE_FAIL_UNDER=70
+KEEP_COMPOSE ?= 0  # Set KEEP_COMPOSE=1 to keep containers after integration tests
+
+PIP = python -m pip
+
+POETRY_VERSION = 2.1.4
+POETRY = python -m poetry
 
 ifeq ($(COVERAGE),1)
-  TEST_RUNNER = poetry run coverage run --parallel-mode --source=pyiceberg -m
+  TEST_RUNNER = $(POETRY) run coverage run --parallel-mode --source=pyiceberg -m
 else
-  TEST_RUNNER = poetry run
+  TEST_RUNNER = $(POETRY) run
 endif
 
-POETRY_VERSION = 2.1.4
+ifeq ($(KEEP_COMPOSE),1)
+  CLEANUP_COMMAND = echo "Keeping containers running for debugging (KEEP_COMPOSE=1)"
+else
+  CLEANUP_COMMAND = docker compose -f dev/docker-compose-integration.yml down -v --remove-orphans --timeout 0 2>/dev/null || true
+endif
 
 # ============
 # Help Section
@@ -46,21 +56,21 @@ help: ## Display this help message
 ##@ Setup
 
 install-poetry: ## Ensure Poetry is installed at the specified version
-	@if ! command -v poetry &> /dev/null; then \
+	@if ! command -v ${POETRY} &> /dev/null; then \
 		echo "Poetry not found. Installing..."; \
-		pip install --user poetry==$(POETRY_VERSION); \
+		${PIP} install poetry==$(POETRY_VERSION); \
 	else \
-		INSTALLED_VERSION=$$(pip show poetry | grep Version | awk '{print $$2}'); \
+		INSTALLED_VERSION=$$(${PIP} show poetry | grep Version | awk '{print $$2}'); \
 		if [ "$$INSTALLED_VERSION" != "$(POETRY_VERSION)" ]; then \
 			echo "Updating Poetry to version $(POETRY_VERSION)..."; \
-			pip install --user --upgrade poetry==$(POETRY_VERSION); \
+			${PIP} install --upgrade poetry==$(POETRY_VERSION); \
 		else \
 			echo "Poetry version $(POETRY_VERSION) already installed."; \
 		fi; \
 	fi
 
 install-dependencies: ## Install all dependencies including extras
-	poetry install --all-extras
+	$(POETRY) install --all-extras
 
 install: install-poetry install-dependencies ## Install Poetry and dependencies
 
@@ -74,7 +84,7 @@ check-license: ## Check license headers
 	./dev/check-license
 
 lint: ## Run code linters via pre-commit
-	poetry run pre-commit run --all-files
+	$(POETRY) run pre-commit run --all-files
 
 # ===============
 # Testing Section
@@ -85,7 +95,7 @@ lint: ## Run code linters via pre-commit
 test: ## Run all unit tests (excluding integration)
 	$(TEST_RUNNER) pytest tests/ -m "(unmarked or parametrize) and not integration" $(PYTEST_ARGS)
 
-test-integration: test-integration-setup test-integration-exec ## Run integration tests
+test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests
 
 test-integration-setup: ## Start Docker services for integration tests
 	docker compose -f dev/docker-compose-integration.yml kill
@@ -98,6 +108,12 @@ test-integration-setup: ## Start Docker services for integration tests
 test-integration-exec: ## Run integration tests (excluding provision)
 	$(TEST_RUNNER) pytest tests/ -m integration $(PYTEST_ARGS)
 
+test-integration-cleanup: ## Clean up integration test environment
+	@if [ "${KEEP_COMPOSE}" != "1" ]; then \
+		echo "Cleaning up Docker containers..."; \
+	fi
+	$(CLEANUP_COMMAND)
+
 test-integration-rebuild: ## Rebuild integration Docker services from scratch
 	docker compose -f dev/docker-compose-integration.yml kill
 	docker compose -f dev/docker-compose-integration.yml rm -f
@@ -119,10 +135,10 @@ test-coverage: COVERAGE=1
 test-coverage: test test-integration test-s3 test-adls test-gcs coverage-report ## Run all tests with coverage and report
 
 coverage-report: ## Combine and report coverage
-	poetry run coverage combine
-	poetry run coverage report -m --fail-under=$(COVERAGE_FAIL_UNDER)
-	poetry run coverage html
-	poetry run coverage xml
+	${POETRY} run coverage combine
+	${POETRY} run coverage report -m --fail-under=$(COVERAGE_FAIL_UNDER)
+	${POETRY} run coverage html
+	${POETRY} run coverage xml
 
 # ================
 # Documentation
@@ -131,13 +147,13 @@ coverage-report: ## Combine and report coverage
 ##@ Documentation
 
 docs-install: ## Install docs dependencies
-	poetry install --with docs
+	${POETRY} install --with docs
 
 docs-serve: ## Serve local docs preview (hot reload)
-	poetry run mkdocs serve -f mkdocs/mkdocs.yml
+	${POETRY} run mkdocs serve -f mkdocs/mkdocs.yml
 
 docs-build: ## Build the static documentation site
-	poetry run mkdocs build -f mkdocs/mkdocs.yml --strict
+	${POETRY} run mkdocs build -f mkdocs/mkdocs.yml --strict
 
 # ===================
 # Project Maintenance

dev/Dockerfile

@@ -36,25 +36,51 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
 RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
 WORKDIR ${SPARK_HOME}
 
-# Remember to also update `tests/conftest`'s spark setting
 ENV SPARK_VERSION=3.5.6
-ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
-ENV ICEBERG_VERSION=1.9.1
+ENV SCALA_VERSION=2.12
+ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_${SCALA_VERSION}
+ENV ICEBERG_VERSION=1.10.0
 ENV PYICEBERG_VERSION=0.10.0
+ENV HADOOP_VERSION=3.3.4
+ENV AWS_SDK_VERSION=1.12.753
 
-RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
- && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
- && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
+# Try the primary Apache mirror (downloads.apache.org) first, then fall back to the archive
+RUN set -eux; \
+  FILE=spark-${SPARK_VERSION}-bin-hadoop3.tgz; \
+  URLS="https://downloads.apache.org/spark/spark-${SPARK_VERSION}/${FILE} https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${FILE}"; \
+  for url in $URLS; do \
+    echo "Attempting download: $url"; \
+    if curl --retry 3 --retry-delay 5 -f -s -C - "$url" -o "$FILE"; then \
+      echo "Downloaded from: $url"; \
+      break; \
+    else \
+      echo "Failed to download from: $url"; \
+    fi; \
+  done; \
+  if [ ! -f "$FILE" ]; then echo "Failed to download Spark from all mirrors" >&2; exit 1; fi; \
+  tar xzf "$FILE" --directory /opt/spark --strip-components 1; \
+  rm -rf "$FILE"
+
+# Download Spark Connect server JAR
+RUN curl --retry 5 -s -L https://repo1.maven.org/maven2/org/apache/spark/spark-connect_${SCALA_VERSION}/${SPARK_VERSION}/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar \
+      -Lo /opt/spark/jars/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar
 
 # Download iceberg spark runtime
 RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
       -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar
 
-
 # Download AWS bundle
 RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
       -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
 
+# Download hadoop-aws (required for S3 support)
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
+      -Lo /opt/spark/jars/hadoop-aws-${HADOOP_VERSION}.jar
+
+# Download AWS SDK bundle
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar \
+      -Lo /opt/spark/jars/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar
+
 COPY spark-defaults.conf /opt/spark/conf
 ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"