Merge branch 'main' into test-consolidation

Author: nssalian

.github/workflows/pypi-build-artifacts.yml

@@ -45,6 +45,7 @@ jobs:
             3.10
             3.11
             3.12
+            3.13
 
       - name: Install UV
         uses: astral-sh/setup-uv@v7
@@ -61,14 +62,14 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.3.0
+        uses: pypa/cibuildwheel@v3.3.1
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it

.github/workflows/python-ci.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v6
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/svn-build-artifacts.yml

@@ -45,6 +45,7 @@ jobs:
             3.10
             3.11
             3.12
+            3.13
 
       - name: Install UV
         uses: astral-sh/setup-uv@v7
@@ -56,14 +57,14 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.3.0
+        uses: pypa/cibuildwheel@v3.3.1
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it

Makefile

@@ -70,7 +70,7 @@ setup-venv: ## Create virtual environment
 	uv venv $(PYTHON_ARG)
 
 install-dependencies: setup-venv ## Install all dependencies including extras
-	uv sync $(PYTHON_ARG) --all-extras
+	uv sync $(PYTHON_ARG) --all-extras --reinstall
 
 install: install-uv install-dependencies ## Install uv and dependencies
 
@@ -100,7 +100,7 @@ test-integration: test-integration-setup test-integration-exec test-integration-
 test-integration-setup: ## Start Docker services for integration tests
 	docker compose -f dev/docker-compose-integration.yml kill
 	docker compose -f dev/docker-compose-integration.yml rm -f
-	docker compose -f dev/docker-compose-integration.yml up -d --wait
+	docker compose -f dev/docker-compose-integration.yml up -d --build --wait
 	uv run $(PYTHON_ARG) python dev/provision.py
 
 test-integration-exec: ## Run integration tests (excluding provision)
@@ -148,7 +148,7 @@ docs-install: ## Install docs dependencies (included in default groups)
 	uv sync $(PYTHON_ARG) --group docs
 
 docs-serve: ## Serve local docs preview (hot reload)
-	uv run $(PYTHON_ARG) mkdocs serve -f mkdocs/mkdocs.yml
+	uv run $(PYTHON_ARG) mkdocs serve -f mkdocs/mkdocs.yml --livereload
 
 docs-build: ## Build the static documentation site
 	uv run $(PYTHON_ARG) mkdocs build -f mkdocs/mkdocs.yml --strict

dev/spark/Dockerfile

@@ -18,9 +18,8 @@ ARG BASE_IMAGE_SPARK_VERSION=4.0.1
 FROM apache/spark:${BASE_IMAGE_SPARK_VERSION}
 
 # Dependency versions - keep these compatible
-ARG ICEBERG_VERSION=1.10.0
+ARG ICEBERG_VERSION=1.10.1
 ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13
-ARG SPARK_VERSION=4.0.1
 ARG HADOOP_VERSION=3.4.1
 ARG SCALA_VERSION=2.13
 ARG AWS_SDK_VERSION=2.24.6
@@ -43,7 +42,6 @@ RUN mkdir -p /home/iceberg/spark-events && \
 
 # Required JAR dependencies
 ENV JARS_TO_DOWNLOAD="\
-    org/apache/spark/spark-connect_${SCALA_VERSION}/${SPARK_VERSION}/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar \
     org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
     org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
     org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \

mkdocs/docs/contributing.md

@@ -118,7 +118,7 @@ make lint
 
 In addition to manually running `make lint`, you can install the pre-commit hooks in your local repo with `prek install`. By doing this, linting is run automatically every time you make a commit.
 
-You can bump the integrations to the latest version using `prek auto-update`. This will check if there is a newer version of `{black,mypy,isort,...}` and update the yaml.
+You can bump the integrations to the latest version using `prek auto-update`. This will check if there is a newer version of `{ruff,mypy,...}` and update the yaml.
 
 ## Cleaning
 
@@ -258,6 +258,27 @@ Which will warn:
 Deprecated in 0.1.0, will be removed in 0.2.0. The old_property is deprecated. Please use the something_else property instead.
 ```
 
+### Logging
+
+PyIceberg uses Python's standard logging module. You can control the logging level using either:
+
+**CLI option:**
+
+```bash
+pyiceberg --log-level DEBUG describe my_table
+```
+
+**Environment variable:**
+
+```bash
+export PYICEBERG_LOG_LEVEL=DEBUG
+pyiceberg describe my_table
+```
+
+Valid log levels are: `DEBUG`, `INFO`, `WARNING` (default), `ERROR`, `CRITICAL`.
+
+Debug logging is particularly useful for troubleshooting issues with FileIO implementations, catalog connections, and other integration points.
+
 ### Type annotations
 
 For the type annotation the types from the `Typing` package are used.

pyiceberg/catalog/__init__.py

@@ -285,8 +285,8 @@ def delete_files(io: FileIO, files_to_delete: set[str], file_type: str) -> None:
     for file in files_to_delete:
         try:
             io.delete(file)
-        except OSError as exc:
-            logger.warning(msg=f"Failed to delete {file_type} file {file}", exc_info=exc)
+        except OSError:
+            logger.warning(f"Failed to delete {file_type} file {file}", exc_info=logger.isEnabledFor(logging.DEBUG))
 
 
 def delete_data_files(io: FileIO, manifests_to_delete: list[ManifestFile]) -> None:
@@ -305,8 +305,8 @@ def delete_data_files(io: FileIO, manifests_to_delete: list[ManifestFile]) -> No
             if not deleted_files.get(path, False):
                 try:
                     io.delete(path)
-                except OSError as exc:
-                    logger.warning(msg=f"Failed to delete data file {path}", exc_info=exc)
+                except OSError:
+                    logger.warning(f"Failed to delete data file {path}", exc_info=logger.isEnabledFor(logging.DEBUG))
                 deleted_files[path] = True
 
 
@@ -319,8 +319,8 @@ def _import_catalog(name: str, catalog_impl: str, properties: Properties) -> Cat
         module = importlib.import_module(module_name)
         class_ = getattr(module, class_name)
         return class_(name, **properties)
-    except ModuleNotFoundError as exc:
-        logger.warning(f"Could not initialize Catalog: {catalog_impl}", exc_info=exc)
+    except ModuleNotFoundError:
+        logger.warning(f"Could not initialize Catalog: {catalog_impl}", exc_info=logger.isEnabledFor(logging.DEBUG))
         return None