infra: Add Python 3.13 support (apache#2863)

<!--
Thanks for opening a pull request!
-->

<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->

# Rationale for this change
Closes apache#1372, apache#2530

Based on apache#1377, apache#2536, apache#2658

Adds Python 3.13 throughout the repo
Upgrade `ray` to a version built with python 3.13
Add `filterwarnings` for `ResourceWarnings` coming from `sqlite` and
`ray` for now, will follow up with a proper fix

## Are these changes tested?
Yes
```
PYTHON=3.13 make install
PYTHON=3.13 make test
PYTHON=3.13 make test-integration
```

## Are there any user-facing changes?

<!-- In the case of user-facing changes, please add the changelog label.
-->

---------

Co-authored-by: Saurabh Kumar <github@sa1.me>
Co-authored-by: Saurabh Kumar <557509+sa1@users.noreply.github.com>

Author: kevinjqliu

.github/workflows/pypi-build-artifacts.yml

@@ -45,6 +45,7 @@ jobs:
             3.10
             3.11
             3.12
+            3.13
 
       - name: Install UV
         uses: astral-sh/setup-uv@v7
@@ -68,7 +69,7 @@ jobs:
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it

.github/workflows/python-ci.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v6
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/svn-build-artifacts.yml

@@ -45,6 +45,7 @@ jobs:
             3.10
             3.11
             3.12
+            3.13
 
       - name: Install UV
         uses: astral-sh/setup-uv@v7
@@ -63,7 +64,7 @@ jobs:
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it

pyproject.toml

@@ -28,6 +28,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
     "mmh3>=4.0.0,<6.0.0",
@@ -66,7 +67,7 @@ duckdb = [
     "pyarrow>=17.0.0",
 ]
 ray = [
-    "ray>=2.10.0,<=2.44.0",
+    "ray>=2.10.0,<3.0.0",
     "pyarrow>=17.0.0",
     "pandas>=1.0.0,<3.0.0",
 ]
@@ -160,6 +161,11 @@ filterwarnings = [
   "error",
   # Ignore Python version deprecation warning from google.api_core while we still support 3.10
   "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core",
+  # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections
+  "ignore:unclosed database in <sqlite3.Connection object*:ResourceWarning",
+  # Ignore Ray subprocess cleanup warnings
+  "ignore:unclosed file:ResourceWarning",
+  "ignore:subprocess.*is still running:ResourceWarning",
 ]
 
 [tool.black]

tests/catalog/test_base.py

@@ -17,6 +17,7 @@
 # pylint:disable=redefined-outer-name
 
 
+from collections.abc import Generator
 from pathlib import PosixPath
 
 import pyarrow as pa
@@ -51,8 +52,10 @@
 
 
 @pytest.fixture
-def catalog(tmp_path: PosixPath) -> InMemoryCatalog:
-    return InMemoryCatalog("test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"})
+def catalog(tmp_path: PosixPath) -> Generator[Catalog, None, None]:
+    catalog = InMemoryCatalog("test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"})
+    yield catalog
+    catalog.close()
 
 
 TEST_TABLE_IDENTIFIER = ("com", "organization", "department", "my_table")

tests/conftest.py

@@ -2947,3 +2947,16 @@ def pyarrow_table_with_promoted_types(pyarrow_schema_with_promoted_types: "pa.Sc
         },
         schema=pyarrow_schema_with_promoted_types,
     )
+
+
+@pytest.fixture(scope="session")
+def ray_session() -> Generator[Any, None, None]:
+    """Fixture to manage Ray initialization and shutdown for tests."""
+    import ray
+
+    ray.init(
+        ignore_reinit_error=True,
+        runtime_env={"working_dir": None},  # Prevent Ray from serializing the working directory to workers
+    )
+    yield ray
+    ray.shutdown()

tests/integration/test_reads.py

@@ -21,6 +21,7 @@
 import uuid
 from datetime import datetime, timedelta
 from pathlib import PosixPath
+from typing import Any
 from urllib.parse import urlparse
 
 import pyarrow as pa
@@ -357,16 +358,17 @@ def test_bodo_nan(catalog: Catalog, monkeypatch: pytest.MonkeyPatch) -> None:
 @pytest.mark.integration
 @pytest.mark.filterwarnings("ignore")
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
-def test_ray_nan(catalog: Catalog) -> None:
+def test_ray_nan(catalog: Catalog, ray_session: Any) -> None:
     table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten")
     ray_dataset = table_test_null_nan_rewritten.scan().to_ray()
     assert ray_dataset.count() == 3
     assert math.isnan(ray_dataset.take()[0]["col_numeric"])
 
 
 @pytest.mark.integration
+@pytest.mark.filterwarnings("ignore")
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
-def test_ray_nan_rewritten(catalog: Catalog) -> None:
+def test_ray_nan_rewritten(catalog: Catalog, ray_session: Any) -> None:
     table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten")
     ray_dataset = table_test_null_nan_rewritten.scan(
         row_filter=IsNaN("col_numeric"), selected_fields=("idx", "col_numeric")
@@ -377,17 +379,19 @@ def test_ray_nan_rewritten(catalog: Catalog) -> None:
 
 
 @pytest.mark.integration
+@pytest.mark.filterwarnings("ignore")
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
 @pytest.mark.skip(reason="Fixing issues with NaN's: https://github.com/apache/arrow/issues/34162")
-def test_ray_not_nan_count(catalog: Catalog) -> None:
+def test_ray_not_nan_count(catalog: Catalog, ray_session: Any) -> None:
     table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten")
     ray_dataset = table_test_null_nan_rewritten.scan(row_filter=NotNaN("col_numeric"), selected_fields=("idx",)).to_ray()
     assert ray_dataset.count() == 2
 
 
 @pytest.mark.integration
+@pytest.mark.filterwarnings("ignore")
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
-def test_ray_all_types(catalog: Catalog) -> None:
+def test_ray_all_types(catalog: Catalog, ray_session: Any) -> None:
     table_test_all_types = catalog.load_table("default.test_all_types")
     ray_dataset = table_test_all_types.scan().to_ray()
     pandas_dataframe = table_test_all_types.scan().to_pandas()