From 8e1fd4dd36f689b835ce113c116f7cabd20ef296 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Wed, 28 Jan 2026 15:56:01 +0800 Subject: [PATCH 1/3] Expand wheel matrix for Python 3.13/3.14 Add macOS/Windows entries for free-threaded variants. Include distinct artifact suffixes and use explicit maturin interpreter settings. Add manylinux entries for the latest Python versions with similar configurations. --- .github/workflows/build.yml | 91 +++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2dc8b96fe..60e6adae1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,8 +74,49 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] - os: [macos-latest, windows-latest] + include: + - os: macos-latest + python-version: "3.10" + artifact-suffix: "" + - os: macos-latest + python-version: "3.11" + artifact-suffix: "" + - os: macos-latest + python-version: "3.12" + artifact-suffix: "" + - os: macos-latest + python-version: "3.13" + artifact-suffix: "" + - os: macos-latest + python-version: "3.14" + artifact-suffix: "" + - os: macos-latest + python-version: "3.13t" + artifact-suffix: "-freethreaded" + - os: macos-latest + python-version: "3.14t" + artifact-suffix: "-freethreaded" + - os: windows-latest + python-version: "3.10" + artifact-suffix: "" + - os: windows-latest + python-version: "3.11" + artifact-suffix: "" + - os: windows-latest + python-version: "3.12" + artifact-suffix: "" + - os: windows-latest + python-version: "3.13" + artifact-suffix: "" + - os: windows-latest + python-version: "3.14" + artifact-suffix: "" + - os: windows-latest + python-version: "3.13t" + artifact-suffix: "-freethreaded" + - os: windows-latest + python-version: "3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 @@ -105,7 +146,7 @@ jobs: - name: Build Python package run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait + uv run --no-project maturin build --release --strip --features substrait --interpreter python - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -121,7 +162,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-${{ matrix.os }} + name: dist-${{ matrix.os }}${{ matrix.artifact-suffix }} path: target/wheels/* build-macos-x86_64: @@ -131,7 +172,21 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + include: + - python-version: "3.10" + artifact-suffix: "" + - python-version: "3.11" + artifact-suffix: "" + - python-version: "3.12" + artifact-suffix: "" + - python-version: "3.13" + artifact-suffix: "" + - python-version: "3.14" + artifact-suffix: "" + - python-version: "3.13t" + artifact-suffix: "-freethreaded" + - python-version: "3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 @@ -161,7 +216,7 @@ jobs: - name: Build Python package run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait + uv run --no-project maturin build --release --strip --features substrait --interpreter python - name: List Mac wheels run: find target/wheels/ @@ -169,13 +224,21 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-macos-aarch64 + name: dist-macos-aarch64${{ matrix.artifact-suffix }} path: target/wheels/* build-manylinux-x86_64: needs: [generate-license] name: Manylinux x86_64 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - interpreter: "3.10 3.11 3.12 3.13 3.14" + artifact-suffix: "" + - interpreter: "3.13t 3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -194,17 +257,26 @@ jobs: target: x86_64 manylinux: auto rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 + interpreter: ${{ matrix.interpreter }} args: --release --manylinux 2014 --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-x86_64 + name: dist-manylinux-x86_64${{ matrix.artifact-suffix }} path: target/wheels/* build-manylinux-aarch64: needs: [generate-license] name: Manylinux arm64 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - interpreter: "3.10 3.11 3.12 3.13 3.14" + artifact-suffix: "" + - interpreter: "3.13t 3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -224,11 +296,12 @@ jobs: # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail manylinux: 2_28 rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 + interpreter: ${{ matrix.interpreter }} args: --release --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-aarch64 + name: dist-manylinux-aarch64${{ matrix.artifact-suffix }} path: target/wheels/* build-sdist: From 6efe8789c92d642bbff2e8b786ffd256a397f437 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Wed, 28 Jan 2026 15:56:35 +0800 Subject: [PATCH 2/3] Update PyO3 dependency and document wheel rationale Rely on explicit abi3-py310 flag for ABI3 compatibility in the PyO3 dependency feature set while retaining extension-module enabled. Document the rationale for free-threaded 3.13/3.14 wheels alongside ABI3 wheel guidance in the release notes. --- Cargo.toml | 1 - dev/release/README.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 364713964..7fe9ee617 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,6 @@ tokio = { version = "1.47", features = [ ] } pyo3 = { version = "0.26", features = [ "extension-module", - "abi3", "abi3-py310", ] } pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] } diff --git a/dev/release/README.md b/dev/release/README.md index 5d2fae5a7..5a76da810 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -130,6 +130,10 @@ datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl datafusion-22.0.0-cp37-abi3-win_amd64.whl ``` +Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and ignore the `abi3` setting. We keep +the `abi3-py310` feature enabled to preserve the stable-ABI wheels for GIL-enabled Python, and we additionally ship +version-specific `cp313t`/`cp314t` wheels for the free-threaded builds. + Upload the wheels to testpypi. ```bash From c03f56d0a1e78426a1513700969cfef3f306920e Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Wed, 4 Feb 2026 14:01:33 +0800 Subject: [PATCH 3/3] Update build.yml to use python-tag for artifact naming and enhance README with artifact naming conventions --- .github/workflows/build.yml | 60 ++++++++++++++++++------------------- dev/release/README.md | 6 ++++ 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60e6adae1..bbaf81183 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,46 +77,46 @@ jobs: include: - os: macos-latest python-version: "3.10" - artifact-suffix: "" + python-tag: cp310 - os: macos-latest python-version: "3.11" - artifact-suffix: "" + python-tag: cp311 - os: macos-latest python-version: "3.12" - artifact-suffix: "" + python-tag: cp312 - os: macos-latest python-version: "3.13" - artifact-suffix: "" + python-tag: cp313 - os: macos-latest python-version: "3.14" - artifact-suffix: "" + python-tag: cp314 - os: macos-latest python-version: "3.13t" - artifact-suffix: "-freethreaded" + python-tag: cp313t - os: macos-latest python-version: "3.14t" - artifact-suffix: "-freethreaded" + python-tag: cp314t - os: windows-latest python-version: "3.10" - artifact-suffix: "" + python-tag: cp310 - os: windows-latest python-version: "3.11" - artifact-suffix: "" + python-tag: cp311 - os: windows-latest python-version: "3.12" - artifact-suffix: "" + python-tag: cp312 - os: windows-latest python-version: "3.13" - artifact-suffix: "" + python-tag: cp313 - os: windows-latest python-version: "3.14" - artifact-suffix: "" + python-tag: cp314 - os: windows-latest python-version: "3.13t" - artifact-suffix: "-freethreaded" + python-tag: cp313t - os: windows-latest python-version: "3.14t" - artifact-suffix: "-freethreaded" + python-tag: cp314t steps: - uses: actions/checkout@v5 @@ -162,7 +162,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-${{ matrix.os }}${{ matrix.artifact-suffix }} + name: dist-${{ matrix.os }}-${{ matrix.python-tag }} path: target/wheels/* build-macos-x86_64: @@ -174,19 +174,19 @@ jobs: matrix: include: - python-version: "3.10" - artifact-suffix: "" + python-tag: cp310 - python-version: "3.11" - artifact-suffix: "" + python-tag: cp311 - python-version: "3.12" - artifact-suffix: "" + python-tag: cp312 - python-version: "3.13" - artifact-suffix: "" + python-tag: cp313 - python-version: "3.14" - artifact-suffix: "" + python-tag: cp314 - python-version: "3.13t" - artifact-suffix: "-freethreaded" + python-tag: cp313t - python-version: "3.14t" - artifact-suffix: "-freethreaded" + python-tag: cp314t steps: - uses: actions/checkout@v5 @@ -224,7 +224,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-macos-aarch64${{ matrix.artifact-suffix }} + name: dist-macos-aarch64-${{ matrix.python-tag }} path: target/wheels/* build-manylinux-x86_64: @@ -236,9 +236,9 @@ jobs: matrix: include: - interpreter: "3.10 3.11 3.12 3.13 3.14" - artifact-suffix: "" + artifact-label: cp310-314 - interpreter: "3.13t 3.14t" - artifact-suffix: "-freethreaded" + artifact-label: cp313t-314t steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -262,7 +262,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-x86_64${{ matrix.artifact-suffix }} + name: dist-manylinux-x86_64-${{ matrix.artifact-label }} path: target/wheels/* build-manylinux-aarch64: @@ -274,9 +274,9 @@ jobs: matrix: include: - interpreter: "3.10 3.11 3.12 3.13 3.14" - artifact-suffix: "" + artifact-label: cp310-314 - interpreter: "3.13t 3.14t" - artifact-suffix: "-freethreaded" + artifact-label: cp313t-314t steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -301,7 +301,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-aarch64${{ matrix.artifact-suffix }} + name: dist-manylinux-aarch64-${{ matrix.artifact-label }} path: target/wheels/* build-sdist: @@ -396,7 +396,7 @@ jobs: - name: Download pre-built Linux wheel uses: actions/download-artifact@v5 with: - name: dist-manylinux-x86_64 + name: dist-manylinux-x86_64-cp310-314 path: wheels/ # Install from the pre-built wheel diff --git a/dev/release/README.md b/dev/release/README.md index 5a76da810..2568caa06 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -222,6 +222,12 @@ cargo publish ### Publishing Python Artifacts to PyPi +GitHub Actions groups wheel artifacts by platform and interpreter tag using the pattern `dist--`. +For example, standard manylinux wheels live under `dist-manylinux-x86_64-cp310-314` while the free-threaded builds +use `dist-manylinux-x86_64-cp313t-314t`. macOS and Windows jobs publish one artifact per CPython version as well +(`dist-macos-latest-cp311`, `dist-windows-latest-cp313t`, etc.). Download the exact tags you intend to push to PyPI, +and remember that the docs workflow currently installs from the `cp310-314` manylinux artifact. + Go to the Test PyPI page of Datafusion, and download [all published artifacts](https://test.pypi.org/project/datafusion/#files) under `dist-release/` directory. Then proceed uploading them using `twine`: