diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2dc8b96fe..bbaf81183 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,8 +74,49 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] - os: [macos-latest, windows-latest] + include: + - os: macos-latest + python-version: "3.10" + python-tag: cp310 + - os: macos-latest + python-version: "3.11" + python-tag: cp311 + - os: macos-latest + python-version: "3.12" + python-tag: cp312 + - os: macos-latest + python-version: "3.13" + python-tag: cp313 + - os: macos-latest + python-version: "3.14" + python-tag: cp314 + - os: macos-latest + python-version: "3.13t" + python-tag: cp313t + - os: macos-latest + python-version: "3.14t" + python-tag: cp314t + - os: windows-latest + python-version: "3.10" + python-tag: cp310 + - os: windows-latest + python-version: "3.11" + python-tag: cp311 + - os: windows-latest + python-version: "3.12" + python-tag: cp312 + - os: windows-latest + python-version: "3.13" + python-tag: cp313 + - os: windows-latest + python-version: "3.14" + python-tag: cp314 + - os: windows-latest + python-version: "3.13t" + python-tag: cp313t + - os: windows-latest + python-version: "3.14t" + python-tag: cp314t steps: - uses: actions/checkout@v5 @@ -105,7 +146,7 @@ jobs: - name: Build Python package run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait + uv run --no-project maturin build --release --strip --features substrait --interpreter python - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -121,7 +162,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-${{ matrix.os }} + name: dist-${{ matrix.os }}-${{ matrix.python-tag }} path: target/wheels/* build-macos-x86_64: @@ -131,7 +172,21 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + include: + - python-version: "3.10" + python-tag: cp310 + - python-version: "3.11" + python-tag: cp311 + - python-version: "3.12" + python-tag: cp312 + - python-version: "3.13" + python-tag: cp313 + - python-version: "3.14" + python-tag: cp314 + - python-version: "3.13t" + python-tag: cp313t + - python-version: "3.14t" + python-tag: cp314t steps: - uses: actions/checkout@v5 @@ -161,7 +216,7 @@ jobs: - name: Build Python package run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait + uv run --no-project maturin build --release --strip --features substrait --interpreter python - name: List Mac wheels run: find target/wheels/ @@ -169,13 +224,21 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-macos-aarch64 + name: dist-macos-aarch64-${{ matrix.python-tag }} path: target/wheels/* build-manylinux-x86_64: needs: [generate-license] name: Manylinux x86_64 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - interpreter: "3.10 3.11 3.12 3.13 3.14" + artifact-label: cp310-314 + - interpreter: "3.13t 3.14t" + artifact-label: cp313t-314t steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -194,17 +257,26 @@ jobs: target: x86_64 manylinux: auto rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 + interpreter: ${{ matrix.interpreter }} args: --release --manylinux 2014 --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-x86_64 + name: dist-manylinux-x86_64-${{ matrix.artifact-label }} path: target/wheels/* build-manylinux-aarch64: needs: [generate-license] name: Manylinux arm64 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - interpreter: "3.10 3.11 3.12 3.13 3.14" + artifact-label: cp310-314 + - interpreter: "3.13t 3.14t" + artifact-label: cp313t-314t steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -224,11 +296,12 @@ jobs: # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail manylinux: 2_28 rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 + interpreter: ${{ matrix.interpreter }} args: --release --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-aarch64 + name: dist-manylinux-aarch64-${{ matrix.artifact-label }} path: target/wheels/* build-sdist: @@ -323,7 +396,7 @@ jobs: - name: Download pre-built Linux wheel uses: actions/download-artifact@v5 with: - name: dist-manylinux-x86_64 + name: dist-manylinux-x86_64-cp310-314 path: wheels/ # Install from the pre-built wheel diff --git a/Cargo.toml b/Cargo.toml index 364713964..7fe9ee617 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,6 @@ tokio = { version = "1.47", features = [ ] } pyo3 = { version = "0.26", features = [ "extension-module", - "abi3", "abi3-py310", ] } pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] } diff --git a/dev/release/README.md b/dev/release/README.md index 5d2fae5a7..2568caa06 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -130,6 +130,10 @@ datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl datafusion-22.0.0-cp37-abi3-win_amd64.whl ``` +Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and ignore the `abi3` setting. We keep +the `abi3-py310` feature enabled to preserve the stable-ABI wheels for GIL-enabled Python, and we additionally ship +version-specific `cp313t`/`cp314t` wheels for the free-threaded builds. + Upload the wheels to testpypi. ```bash @@ -218,6 +222,12 @@ cargo publish ### Publishing Python Artifacts to PyPi +GitHub Actions groups wheel artifacts by platform and interpreter tag using the pattern `dist--`. +For example, standard manylinux wheels live under `dist-manylinux-x86_64-cp310-314` while the free-threaded builds +use `dist-manylinux-x86_64-cp313t-314t`. macOS and Windows jobs publish one artifact per CPython version as well +(`dist-macos-latest-cp311`, `dist-windows-latest-cp313t`, etc.). Download the exact tags you intend to push to PyPI, +and remember that the docs workflow currently installs from the `cp310-314` manylinux artifact. + Go to the Test PyPI page of Datafusion, and download [all published artifacts](https://test.pypi.org/project/datafusion/#files) under `dist-release/` directory. Then proceed uploading them using `twine`: