Skip to content

Commit d5cad63

Browse files
coca-alexcoca-alex
authored
Merge branch 'apache:main' into fix-storage-credential
2 parents b16b2af + 7d4a8ef commit d5cad63

175 files changed

Lines changed: 19026 additions & 13705 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.asf.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ github:
4242
required_approving_review_count: 1
4343

4444
required_linear_history: true
45-
del_branch_on_merge: true
45+
pull_requests:
46+
# auto-delete head branches after being merged
47+
del_branch_on_merge: true
4648
features:
4749
wiki: true
4850
issues: true
@@ -57,4 +59,5 @@ notifications:
5759
commits: commits@iceberg.apache.org
5860
issues: issues@iceberg.apache.org
5961
pullrequests: issues@iceberg.apache.org
62+
jobs: ci-jobs@iceberg.apache.org
6063
jira_options: link label link label

‎.github/dependabot.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
version: 2
2121
updates:
22-
- package-ecosystem: "pip"
22+
- package-ecosystem: "uv"
2323
directory: "/"
2424
schedule:
2525
interval: "weekly"

‎.github/workflows/check-md-link.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,11 @@ on:
3030
paths:
3131
- '.github/workflows/check-md-link.yml'
3232
- 'mkdocs/**'
33+
workflow_dispatch:
3334

3435
jobs:
3536
markdown-link-check:
3637
runs-on: ubuntu-latest
3738
steps:
3839
- uses: actions/checkout@master
39-
- uses: tcort/github-action-markdown-link-check@v1
40+
- uses: tcort/github-action-markdown-link-check@e7c7a18363c842693fadde5d41a3bd3573a7a225

‎.github/workflows/license_check.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,5 @@ jobs:
2424
rat:
2525
runs-on: ubuntu-latest
2626
steps:
27-
- uses: actions/checkout@v5
27+
- uses: actions/checkout@v6
2828
- run: dev/check-license

‎.github/workflows/nightly-pypi-build.yml‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,21 +31,21 @@ jobs:
3131
outputs:
3232
VERSION: ${{ steps.set-version.outputs.VERSION }}
3333
steps:
34-
- uses: actions/checkout@v5
34+
- uses: actions/checkout@v6
3535
with:
3636
fetch-depth: 1
3737

3838
- uses: actions/setup-python@v6
3939
with:
4040
python-version: 3.12
4141

42-
- name: Install Poetry
43-
run: make install-poetry
42+
- name: Install UV
43+
uses: astral-sh/setup-uv@v7
4444

4545
- name: Set version
4646
id: set-version
4747
run: |
48-
CURRENT_VERSION=$(poetry version --short)
48+
CURRENT_VERSION=$(uv version --short)
4949
TIMESTAMP=$(date +%Y%m%d%H%M%S)
5050
echo "VERSION=${CURRENT_VERSION}.dev${TIMESTAMP}" >> "$GITHUB_OUTPUT"
5151
@@ -71,15 +71,42 @@ jobs:
7171

7272
steps:
7373
- name: Download all the artifacts
74-
uses: actions/download-artifact@v5
74+
uses: actions/download-artifact@v7
7575
with:
7676
merge-multiple: true
7777
path: dist/
7878
- name: List downloaded artifacts
7979
run: ls -R dist/
8080
- name: Publish to TestPyPI
81+
id: publish-testpypi
82+
continue-on-error: true
8183
uses: pypa/gh-action-pypi-publish@release/v1
8284
with:
8385
repository-url: https://test.pypi.org/legacy/
8486
skip-existing: true
8587
verbose: true
88+
- name: Display error message on publish failure
89+
if: steps.publish-testpypi.outcome == 'failure'
90+
run: |
91+
echo "::error::Failed to publish to TestPyPI"
92+
echo ""
93+
echo "⚠️ TestPyPI Publish Failed"
94+
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
95+
echo ""
96+
echo "This may be due to TestPyPI storage limits."
97+
echo "See: https://docs.pypi.org/project-management/storage-limits"
98+
echo ""
99+
echo "To resolve this issue, use the pypi-cleanup utility to clean up old TestPyPI artifacts:"
100+
echo "https://pypi.org/project/pypi-cleanup/"
101+
echo ""
102+
echo " uvx pypi-cleanup --package pyiceberg --host https://test.pypi.org/ \\"
103+
echo " --verbose -d 10 --do-it --username <username>"
104+
echo ""
105+
echo "Requirements:"
106+
echo " • Must be a maintainer for pyiceberg on TestPyPI"
107+
echo " (https://test.pypi.org/project/pyiceberg)"
108+
echo " • Requires TestPyPI password and 2FA"
109+
echo " • ⚠️ ONLY do this for TestPyPI, NOT for production PyPI!"
110+
echo ""
111+
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
112+
exit 1

‎.github/workflows/pypi-build-artifacts.yml‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,52 +32,52 @@ jobs:
3232
runs-on: ${{ matrix.os }}
3333
strategy:
3434
matrix:
35-
os: [ ubuntu-latest, windows-latest, macos-latest ]
35+
os: [ ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest ]
3636

3737
steps:
38-
- uses: actions/checkout@v5
38+
- uses: actions/checkout@v6
3939
with:
4040
fetch-depth: 1
4141

4242
- uses: actions/setup-python@v6
4343
with:
4444
python-version: |
45-
3.9
4645
3.10
4746
3.11
4847
3.12
48+
3.13
4949
50-
- name: Install poetry
51-
run: make install-poetry
50+
- name: Install UV
51+
uses: astral-sh/setup-uv@v7
5252

5353
- name: Set version with RC
5454
env:
5555
VERSION: ${{ inputs.VERSION }}
56-
run: python -m poetry version "${{ env.VERSION }}"
56+
run: uv version "${{ env.VERSION }}"
5757

5858
# Publish the source distribution with the version that's in
5959
# the repository, otherwise the tests will fail
6060
- name: Compile source distribution
61-
run: python3 -m poetry build --format=sdist
62-
if: startsWith(matrix.os, 'ubuntu')
61+
run: uv build --sdist
62+
if: matrix.os == 'ubuntu-latest'
6363

6464
- name: Build wheels
65-
uses: pypa/cibuildwheel@v3.2.0
65+
uses: pypa/cibuildwheel@v3.3.1
6666
with:
6767
output-dir: wheelhouse
6868
config-file: "pyproject.toml"
6969
env:
7070
# Ignore 32 bit architectures
7171
CIBW_ARCHS: "auto64"
72-
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
73-
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
74-
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
75-
# Ignore tests for pypy since not all dependencies are compiled for it
76-
# and would require a local rust build chain
77-
CIBW_TEST_SKIP: "pp*"
72+
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
73+
CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project"
74+
CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py"
75+
# Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support
76+
CIBW_SKIP: "cp3*t-*"
77+
7878

7979
- name: Add source distribution
80-
if: startsWith(matrix.os, 'ubuntu')
80+
if: matrix.os == 'ubuntu-latest'
8181
run: ls -lah dist/* && cp dist/* wheelhouse/
8282

8383
- uses: actions/upload-artifact@v4

‎.github/workflows/python-ci-docs.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,15 @@ jobs:
3535
runs-on: ubuntu-latest
3636

3737
steps:
38-
- uses: actions/checkout@v5
38+
- uses: actions/checkout@v6
3939
- uses: actions/setup-python@v6
4040
with:
4141
python-version: 3.12
42-
- name: Install poetry
43-
run: make install-poetry
42+
- name: Install UV
43+
uses: astral-sh/setup-uv@v7
4444
- name: Install
4545
run: make docs-install
4646
- name: Build docs
4747
run: make docs-build
48+
- name: Run linters
49+
run: make lint

‎.github/workflows/python-ci.yml‎

Lines changed: 104 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,23 +43,26 @@ concurrency:
4343
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
4444

4545
jobs:
46-
lint-and-test:
46+
lint-and-unit-test:
4747
runs-on: ubuntu-latest
4848
strategy:
49+
fail-fast: true
4950
matrix:
50-
python: ['3.9', '3.10', '3.11', '3.12']
51+
python: ['3.10', '3.11', '3.12', '3.13']
5152

5253
steps:
53-
- uses: actions/checkout@v5
54+
- uses: actions/checkout@v6
5455
- uses: actions/setup-python@v6
5556
with:
5657
python-version: ${{ matrix.python }}
57-
- name: Install poetry
58-
run: make install-poetry
58+
- name: Install UV
59+
uses: astral-sh/setup-uv@v7
60+
with:
61+
enable-cache: true
5962
- name: Install system dependencies
6063
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
6164
- name: Install
62-
run: make install-dependencies
65+
run: make install
6366
- name: Run linters
6467
run: make lint
6568
- name: Run unit tests with coverage
@@ -69,43 +72,130 @@ jobs:
6972

7073
integration-test:
7174
runs-on: ubuntu-latest
72-
strategy:
73-
matrix:
74-
python: ['3.9', '3.10', '3.11', '3.12']
75-
7675
steps:
77-
- uses: actions/checkout@v5
76+
- uses: actions/checkout@v6
7877
- uses: actions/setup-python@v6
7978
with:
80-
python-version: ${{ matrix.python }}
79+
python-version: '3.12'
80+
- name: Install UV
81+
uses: astral-sh/setup-uv@v7
82+
with:
83+
enable-cache: true
8184
- name: Install system dependencies
8285
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
8386
- name: Install
8487
run: make install
85-
8688
- name: Run integration tests with coverage
8789
run: COVERAGE=1 make test-integration
8890
- name: Show debug logs
8991
if: ${{ failure() }}
90-
run: docker compose -f dev/docker-compose.yml logs
92+
run: docker compose -f dev/docker-compose-integration.yml logs
93+
- name: Upload coverage data
94+
uses: actions/upload-artifact@v4
95+
with:
96+
name: coverage-integration
97+
path: .coverage*
98+
include-hidden-files: true
9199

100+
integration-test-s3:
101+
runs-on: ubuntu-latest
102+
steps:
103+
- uses: actions/checkout@v6
104+
- uses: actions/setup-python@v6
105+
with:
106+
python-version: '3.12'
107+
- name: Install UV
108+
uses: astral-sh/setup-uv@v7
109+
with:
110+
enable-cache: true
111+
- name: Install system dependencies
112+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
113+
- name: Install
114+
run: make install
92115
- name: Run s3 integration tests with coverage
93116
run: COVERAGE=1 make test-s3
94117
- name: Show debug logs
95118
if: ${{ failure() }}
96119
run: docker compose -f dev/docker-compose.yml logs
120+
- name: Upload coverage data
121+
uses: actions/upload-artifact@v4
122+
with:
123+
name: coverage-s3
124+
path: .coverage*
125+
include-hidden-files: true
97126

127+
integration-test-adls:
128+
runs-on: ubuntu-latest
129+
steps:
130+
- uses: actions/checkout@v6
131+
- uses: actions/setup-python@v6
132+
with:
133+
python-version: '3.12'
134+
- name: Install UV
135+
uses: astral-sh/setup-uv@v7
136+
with:
137+
enable-cache: true
138+
- name: Install system dependencies
139+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
140+
- name: Install
141+
run: make install
98142
- name: Run adls integration tests with coverage
99143
run: COVERAGE=1 make test-adls
100144
- name: Show debug logs
101145
if: ${{ failure() }}
102146
run: docker compose -f dev/docker-compose-azurite.yml logs
147+
- name: Upload coverage data
148+
uses: actions/upload-artifact@v4
149+
with:
150+
name: coverage-adls
151+
path: .coverage*
152+
include-hidden-files: true
103153

154+
integration-test-gcs:
155+
runs-on: ubuntu-latest
156+
steps:
157+
- uses: actions/checkout@v6
158+
- uses: actions/setup-python@v6
159+
with:
160+
python-version: '3.12'
161+
- name: Install UV
162+
uses: astral-sh/setup-uv@v7
163+
with:
164+
enable-cache: true
165+
- name: Install system dependencies
166+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
167+
- name: Install
168+
run: make install
104169
- name: Run gcs integration tests with coverage
105170
run: COVERAGE=1 make test-gcs
106171
- name: Show debug logs
107172
if: ${{ failure() }}
108173
run: docker compose -f dev/docker-compose-gcs-server.yml logs
174+
- name: Upload coverage data
175+
uses: actions/upload-artifact@v4
176+
with:
177+
name: coverage-gcs
178+
path: .coverage*
179+
include-hidden-files: true
109180

181+
integration-coverage-report:
182+
runs-on: ubuntu-latest
183+
needs: [integration-test, integration-test-s3, integration-test-adls, integration-test-gcs]
184+
steps:
185+
- uses: actions/checkout@v6
186+
- uses: actions/setup-python@v6
187+
with:
188+
python-version: '3.12'
189+
- name: Install UV
190+
uses: astral-sh/setup-uv@v7
191+
with:
192+
enable-cache: true
193+
- name: Install dependencies
194+
run: uv sync --group dev
195+
- name: Download all coverage artifacts
196+
uses: actions/download-artifact@v7
197+
with:
198+
pattern: coverage-*
199+
merge-multiple: true
110200
- name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it!
111201
run: COVERAGE_FAIL_UNDER=75 make coverage-report

‎.github/workflows/python-release-docs.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@ jobs:
3030
runs-on: ubuntu-latest
3131

3232
steps:
33-
- uses: actions/checkout@v5
33+
- uses: actions/checkout@v6
3434
- uses: actions/setup-python@v6
3535
with:
3636
python-version: ${{ matrix.python }}
37-
- name: Install poetry
38-
run: make install-poetry
37+
- name: Install UV
38+
uses: astral-sh/setup-uv@v7
3939
- name: Install docs
4040
run: make docs-install
4141
- name: Build docs

0 commit comments

Comments
 (0)