From 4c1a515ab1f404d0f1a19ad1c7336a1136e4b9cf Mon Sep 17 00:00:00 2001 From: grufoony Date: Tue, 24 Mar 2026 12:57:17 +0100 Subject: [PATCH 1/4] Add HPC compatible build --- .github/workflows/pypi.yml | 52 ++++++++++++++++++++++++++---- CMakeLists.txt | 20 +++++++++--- README.md | 66 ++++++++++++++++++++++++++++++++++++++ setup.py | 6 ++++ 4 files changed, 134 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 09988b52..accccf6e 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -128,13 +128,14 @@ jobs: python -c "import dsf; print(dsf.__version__)" build-wheels-linux: - name: Build wheels on Linux (Python ${{ matrix.python-version }}) + name: Build wheels on Linux (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) needs: [check-version, uv-validate] runs-on: ubuntu-latest if: needs.check-version.outputs.should_build == 'true' strategy: matrix: python-version: ['3.10', '3.12'] + build_variant: ['standard', 'hpc'] steps: - name: Checkout repository @@ -159,6 +160,7 @@ jobs: env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} + DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} run: | rm -rf dist wheelhouse python -m build --wheel @@ -168,6 +170,17 @@ jobs: mkdir -p wheelhouse auditwheel repair dist/*.whl -w wheelhouse + - name: Rename HPC wheels with _hpc suffix + if: matrix.build_variant == 'hpc' + run: | + cd wheelhouse + for whl in *.whl; do + # Insert _hpc before .whl suffix + new_name="${whl%.whl}_hpc.whl" + mv "$whl" "$new_name" + done + ls -la + - name: Test wheel installation run: | python -m pip install wheelhouse/*.whl @@ -176,17 +189,18 @@ jobs: - name: Upload wheels as artifacts uses: actions/upload-artifact@v6 with: - name: wheel-linux-${{ matrix.python-version }} + name: wheel-linux-${{ matrix.python-version }}-${{ matrix.build_variant }} path: wheelhouse/*.whl build-wheels-macos: - name: Build wheels on macOS (Python ${{ matrix.python-version }}) + name: Build wheels on macOS (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) needs: [check-version, uv-validate] runs-on: macos-latest if: needs.check-version.outputs.should_build == 'true' strategy: matrix: python-version: ['3.10', '3.12'] + build_variant: ['standard', 'hpc'] steps: - name: Checkout repository @@ -215,6 +229,7 @@ jobs: env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} + DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} run: python -m build --wheel - name: Repair wheel (bundle libraries) @@ -222,20 +237,32 @@ jobs: mkdir -p wheelhouse delocate-wheel -w wheelhouse -v --require-archs $(uname -m) dist/*.whl + - name: Rename HPC wheels with _hpc suffix + if: matrix.build_variant == 'hpc' + run: | + cd wheelhouse + for whl in *.whl; do + # Insert _hpc before .whl suffix + new_name="${whl%.whl}_hpc.whl" + mv "$whl" "$new_name" + done + ls -la + - name: Upload wheels as artifacts uses: actions/upload-artifact@v6 with: - name: wheel-macos-${{ matrix.python-version }} + name: wheel-macos-${{ matrix.python-version }}-${{ matrix.build_variant }} path: wheelhouse/*.whl build-wheels-windows: - name: Build wheels on Windows (Python ${{ matrix.python-version }}) + name: Build wheels on Windows (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) needs: [check-version, uv-validate] runs-on: windows-latest if: needs.check-version.outputs.should_build == 'true' strategy: matrix: python-version: ['3.10', '3.12'] + build_variant: ['standard', 'hpc'] steps: - name: Checkout repository @@ -266,12 +293,25 @@ jobs: env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} + DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} run: python -m build --wheel + - name: Rename HPC wheels with _hpc suffix + if: matrix.build_variant == 'hpc' + shell: bash + run: | + cd dist + for whl in *.whl; do + # Insert _hpc before .whl suffix + new_name="${whl%.whl}_hpc.whl" + mv "$whl" "$new_name" + done + ls -la + - name: Upload wheels as artifacts uses: actions/upload-artifact@v6 with: - name: wheel-windows-${{ matrix.python-version }} + name: wheel-windows-${{ matrix.python-version }}-${{ matrix.build_variant }} path: dist/*.whl build-sdist: diff --git a/CMakeLists.txt b/CMakeLists.txt index 8127c7c1..50628144 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ option(DSF_BENCHMARKS "Build DSF benchmarks" OFF) option(DSF_BUILD_PIC "Build DSF with position-independent code" OFF) option(BUILD_PYTHON_BINDINGS "Build Python bindings" OFF) option(DSF_OPTIMIZE_ARCH "Optimize for native architecture" ON) +option(DSF_HPC_BUILD "Build with conservative -O3 for HPC cluster portability" OFF) # If CMAKE_BUILD_TYPE not set, default to Debug if(NOT CMAKE_BUILD_TYPE) @@ -50,12 +51,23 @@ set(CMAKE_CXX_EXTENSIONS OFF) # Ensure optimization flags are applied only in Release mode if(CMAKE_BUILD_TYPE MATCHES "Release") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Ofast -flto=auto") - if(DSF_OPTIMIZE_ARCH) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + if(DSF_HPC_BUILD) + # HPC mode: conservative -O3 for maximum portability on HPC clusters + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") + message(STATUS "HPC Build Mode enabled: using -O3 for portability") + else() + # Standard mode: aggressive optimization + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Ofast -flto=auto") + if(DSF_OPTIMIZE_ARCH) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + endif() endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") + if(DSF_HPC_BUILD) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") + endif() endif() elseif(CMAKE_BUILD_TYPE MATCHES "Profile") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") diff --git a/README.md b/README.md index 55e15b53..a989db63 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This rework consists of a full code rewriting, in order to implement more featur ## Table of Contents - [Installation](#installation) - [Installation (from source)](#installation-from-source) +- [Installation (Python - HPC Variant)](#installation-python---hpc-variant) - [Testing](#testing) - [Benchmarking](#benchmarking) - [Citing](#citing) @@ -92,6 +93,71 @@ print(dsf.__version__) If you encounter issues, ensure that the installation path is in your `PYTHONPATH` environment variable. +## Installation (Python - HPC Variant) + +For high-performance computing (HPC) clusters and environments where binary portability is critical, an HPC-optimized wheel variant is available. This variant uses conservative `-O3` optimization instead of architecture-specific tuning (`-Ofast`, `-flto=auto`, `-march=native`), ensuring compatibility across diverse HPC hardware architectures. + +### When to Use HPC Variant +- Deploying on HPC clusters with heterogeneous node architectures +- Avoiding runtime errors due to unsupported CPU instructions +- Maximizing portability across different compute nodes + +### Installation on HPC Systems + +The HPC variant wheels are distributed alongside standard wheels on PyPI with an `_hpc` suffix. You can install the HPC variant manually by downloading directly from PyPI: + +```shell +# Visit https://pypi.org/project/dsf-mobility/ and download the wheel for your Python version and platform +# For example, for Python 3.12 on Linux x86_64: +pip install dsf_mobility-X.Y.Z-cp312-cp312-linux_x86_64_hpc.whl +``` + +Alternatively, you can use `pip download` to select the correct variant: + +```shell +# Download HPC variants only +pip download --only-binary :all: dsf-mobility --python-version 312 --python-tag cp312 --platform linux_x86_64 + +# Then install from the downloaded wheel +pip install dsf_mobility-X.Y.Z-cp312-cp312-linux_x86_64_hpc.whl +``` + +Or if using `uv` package manager: + +```shell +# Create a virtual environment +uv venv + +# Download and install the HPC variant +uv pip install --only-binary :all: dsf-mobility +# Then manually select the _hpc wheel, or use a direct URL +``` + +### Building HPC Variant Locally + +To build the HPC variant locally for development or testing: + +```shell +DSF_HPC_BUILD=1 pip install . +``` + +This uses conservative `-O3` optimization for maximum portability: + +```shell +cmake -B build -DCMAKE_BUILD_TYPE=Release -DDSF_HPC_BUILD=ON +cmake --build build -j$(nproc) +``` + +### Standard vs. HPC Variants + +| Aspect | Standard | HPC | +|--------|----------|-----| +| **Optimization** | `-Ofast -flto=auto` + optional `-march=native` | `-O3` only | +| **Use Case** | Single-system deployments, development | HPC clusters, portable deployments | +| **Performance** | Highest on optimized hardware | Portable across architectures | +| **Portability** | Variable (CPU-specific) | Maximum (all x86_64 CPUs) | +| **Wheel Suffix** | None (`*-linux_x86_64.whl`) | `_hpc` (`*-linux_x86_64_hpc.whl`) | + ## Testing This project uses [Doctest](https://github.com/doctest/doctest) for testing. diff --git a/setup.py b/setup.py index 3ba5eaa3..87ed8c90 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,12 @@ def build_extension(self, ext: CMakeExtension): "-DBUILD_PYTHON_BINDINGS=ON", ] + # Pass DSF_HPC_BUILD environment variable to CMake for HPC-compatible builds + hpc_build = os.environ.get("DSF_HPC_BUILD", "0") + if hpc_build in ("1", "true", "TRUE", "on", "ON"): + cmake_args.append("-DDSF_HPC_BUILD=ON") + print("HPC Build Mode enabled: using conservative -O3 optimization") + if platform.system() == "Windows": cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] if "CMAKE_TOOLCHAIN_FILE" in os.environ: From 378f45bf06c51e0b21109565fa35ebef637d4504 Mon Sep 17 00:00:00 2001 From: grufoony Date: Tue, 24 Mar 2026 13:02:03 +0100 Subject: [PATCH 2/4] Fix workflow --- .github/workflows/pypi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index accccf6e..224d0939 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -160,7 +160,7 @@ jobs: env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} - DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} + DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} run: | rm -rf dist wheelhouse python -m build --wheel @@ -229,7 +229,7 @@ jobs: env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} - DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} + DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} run: python -m build --wheel - name: Repair wheel (bundle libraries) @@ -293,7 +293,7 @@ jobs: env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} - DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' ? '1' : '0' }} + DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} run: python -m build --wheel - name: Rename HPC wheels with _hpc suffix From 33fcb68e92867218413438e05b0b9807971f07e3 Mon Sep 17 00:00:00 2001 From: grufoony Date: Tue, 24 Mar 2026 13:59:24 +0100 Subject: [PATCH 3/4] Copilot fixes --- .github/workflows/pypi.yml | 117 ++++++++++++++++++------------------- CMakeLists.txt | 15 +++-- README.md | 37 ++++++------ setup.py | 21 +++++-- 4 files changed, 103 insertions(+), 87 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 224d0939..dcd36bf1 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -61,23 +61,27 @@ jobs: exit 0 fi - PYPI_URL="https://pypi.org/pypi/dsf-mobility/${VERSION}/json" PYPI_NAME="PyPI" - - echo "Checking if dsf-mobility version ${VERSION} exists on ${PYPI_NAME}..." - - # Check PyPI/TestPyPI API for the specific version - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "${PYPI_URL}") - - if [ "$HTTP_STATUS" = "200" ]; then - echo "Version ${VERSION} already exists on ${PYPI_NAME}" - echo "Version already published; skipping build and publish jobs." - echo "should_build=false" >> "$GITHUB_OUTPUT" - exit 0 - else - echo "Version ${VERSION} does not exist on ${PYPI_NAME} (HTTP ${HTTP_STATUS})" - echo "should_build=true" >> "$GITHUB_OUTPUT" - fi + PACKAGES=("dsf-mobility" "dsf-mobility-hpc") + + for PACKAGE in "${PACKAGES[@]}"; do + PYPI_URL="https://pypi.org/pypi/${PACKAGE}/${VERSION}/json" + echo "Checking if ${PACKAGE} version ${VERSION} exists on ${PYPI_NAME}..." + + # Check PyPI API for the specific version + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "${PYPI_URL}") + + if [ "$HTTP_STATUS" = "200" ]; then + echo "Version ${VERSION} already exists for ${PACKAGE} on ${PYPI_NAME}." + echo "Version already published for at least one package; skipping build and publish jobs." + echo "should_build=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Version ${VERSION} does not exist for ${PACKAGE} on ${PYPI_NAME} (HTTP ${HTTP_STATUS})" + done + + echo "should_build=true" >> "$GITHUB_OUTPUT" uv-validate: name: Validate uv workflows @@ -156,13 +160,33 @@ jobs: python -m pip install --upgrade pip python -m pip install build wheel setuptools pybind11-stubgen auditwheel + - name: Switch package name for HPC distribution + if: matrix.build_variant == 'hpc' + run: | + python - <<'PY' + from pathlib import Path + + pyproject_path = Path("pyproject.toml") + content = pyproject_path.read_text(encoding="utf-8") + old = 'name = "dsf-mobility"' + new = 'name = "dsf-mobility-hpc"' + if old not in content: + raise RuntimeError("Unable to locate project name in pyproject.toml") + pyproject_path.write_text(content.replace(old, new, 1), encoding="utf-8") + print("Switched project name to dsf-mobility-hpc for HPC build") + PY + + - name: Clean previous build artifacts + run: | + rm -rf dist wheelhouse + rm -rf build/temp.* build/lib.* + - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} run: | - rm -rf dist wheelhouse python -m build --wheel - name: Repair wheel (auditwheel) @@ -170,17 +194,6 @@ jobs: mkdir -p wheelhouse auditwheel repair dist/*.whl -w wheelhouse - - name: Rename HPC wheels with _hpc suffix - if: matrix.build_variant == 'hpc' - run: | - cd wheelhouse - for whl in *.whl; do - # Insert _hpc before .whl suffix - new_name="${whl%.whl}_hpc.whl" - mv "$whl" "$new_name" - done - ls -la - - name: Test wheel installation run: | python -m pip install wheelhouse/*.whl @@ -193,14 +206,13 @@ jobs: path: wheelhouse/*.whl build-wheels-macos: - name: Build wheels on macOS (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) + name: Build wheels on macOS (Python ${{ matrix.python-version }}) needs: [check-version, uv-validate] runs-on: macos-latest if: needs.check-version.outputs.should_build == 'true' strategy: matrix: python-version: ['3.10', '3.12'] - build_variant: ['standard', 'hpc'] steps: - name: Checkout repository @@ -225,11 +237,16 @@ jobs: echo "ARCHFLAGS=-arch $(uname -m)" >> $GITHUB_ENV echo "_PYTHON_HOST_PLATFORM=macosx-$(sw_vers -productVersion | cut -d. -f1)-$(uname -m)" >> $GITHUB_ENV + - name: Clean previous build artifacts + run: | + rm -rf dist wheelhouse + rm -rf build/temp.* build/lib.* + - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} - DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} + DSF_HPC_BUILD: "0" run: python -m build --wheel - name: Repair wheel (bundle libraries) @@ -237,32 +254,20 @@ jobs: mkdir -p wheelhouse delocate-wheel -w wheelhouse -v --require-archs $(uname -m) dist/*.whl - - name: Rename HPC wheels with _hpc suffix - if: matrix.build_variant == 'hpc' - run: | - cd wheelhouse - for whl in *.whl; do - # Insert _hpc before .whl suffix - new_name="${whl%.whl}_hpc.whl" - mv "$whl" "$new_name" - done - ls -la - - name: Upload wheels as artifacts uses: actions/upload-artifact@v6 with: - name: wheel-macos-${{ matrix.python-version }}-${{ matrix.build_variant }} + name: wheel-macos-${{ matrix.python-version }} path: wheelhouse/*.whl build-wheels-windows: - name: Build wheels on Windows (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) + name: Build wheels on Windows (Python ${{ matrix.python-version }}) needs: [check-version, uv-validate] runs-on: windows-latest if: needs.check-version.outputs.should_build == 'true' strategy: matrix: python-version: ['3.10', '3.12'] - build_variant: ['standard', 'hpc'] steps: - name: Checkout repository @@ -289,29 +294,23 @@ jobs: python -m pip install --upgrade pip python -m pip install build wheel setuptools pybind11-stubgen + - name: Clean previous build artifacts + shell: bash + run: | + rm -rf dist wheelhouse + rm -rf build/temp.* build/lib.* + - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} - DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} + DSF_HPC_BUILD: "0" run: python -m build --wheel - - name: Rename HPC wheels with _hpc suffix - if: matrix.build_variant == 'hpc' - shell: bash - run: | - cd dist - for whl in *.whl; do - # Insert _hpc before .whl suffix - new_name="${whl%.whl}_hpc.whl" - mv "$whl" "$new_name" - done - ls -la - - name: Upload wheels as artifacts uses: actions/upload-artifact@v6 with: - name: wheel-windows-${{ matrix.python-version }}-${{ matrix.build_variant }} + name: wheel-windows-${{ matrix.python-version }} path: dist/*.whl build-sdist: diff --git a/CMakeLists.txt b/CMakeLists.txt index 50628144..7739f0c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,8 @@ if(NOT CMAKE_BUILD_TYPE) endif() endif() if(BUILD_PYTHON_BINDINGS) + # Build all targets as PIC when producing Python extension modules. + # This prevents non-PIC static dependencies from failing at shared-module link time. set(DSF_BUILD_PIC ON) endif() @@ -63,11 +65,7 @@ if(CMAKE_BUILD_TYPE MATCHES "Release") endif() endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - if(DSF_HPC_BUILD) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") - endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") endif() elseif(CMAKE_BUILD_TYPE MATCHES "Profile") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") @@ -152,6 +150,9 @@ FetchContent_Declare( FetchContent_GetProperties(simdjson) if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) + if(DSF_BUILD_PIC) + set_target_properties(simdjson PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() endif() # Check if the user has TBB installed find_package(TBB REQUIRED CONFIG) @@ -169,8 +170,12 @@ FetchContent_Declare( FetchContent_GetProperties(SQLiteCpp) if(NOT SQLiteCpp_POPULATED) FetchContent_MakeAvailable(SQLiteCpp) + if(DSF_BUILD_PIC) + set_target_properties(SQLiteCpp PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() endif() + add_library(dsf STATIC ${SOURCES}) target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT) if(DSF_BUILD_PIC) diff --git a/README.md b/README.md index a989db63..716a66a3 100644 --- a/README.md +++ b/README.md @@ -104,33 +104,36 @@ For high-performance computing (HPC) clusters and environments where binary port ### Installation on HPC Systems -The HPC variant wheels are distributed alongside standard wheels on PyPI with an `_hpc` suffix. You can install the HPC variant manually by downloading directly from PyPI: +The HPC build is published as a separate PyPI distribution named `dsf-mobility-hpc` (PEP-compliant), with Linux wheels intended for cluster portability. Install it directly with: ```shell -# Visit https://pypi.org/project/dsf-mobility/ and download the wheel for your Python version and platform -# For example, for Python 3.12 on Linux x86_64: -pip install dsf_mobility-X.Y.Z-cp312-cp312-linux_x86_64_hpc.whl +pip install dsf-mobility-hpc ``` -Alternatively, you can use `pip download` to select the correct variant: +Or with `uv`: ```shell -# Download HPC variants only -pip download --only-binary :all: dsf-mobility --python-version 312 --python-tag cp312 --platform linux_x86_64 - -# Then install from the downloaded wheel -pip install dsf_mobility-X.Y.Z-cp312-cp312-linux_x86_64_hpc.whl +uv pip install dsf-mobility-hpc ``` -Or if using `uv` package manager: +If you need to download a wheel explicitly, use: ```shell -# Create a virtual environment -uv venv +pip download --only-binary :all: dsf-mobility-hpc +``` + +### Wheel Filename Pattern on PyPI + +HPC wheel filenames are standard and parseable by pip, for example: + +```text +dsf_mobility_hpc--cp-cp-.whl +``` + +Typical Linux example: -# Download and install the HPC variant -uv pip install --only-binary :all: dsf-mobility -# Then manually select the _hpc wheel, or use a direct URL +```text +dsf_mobility_hpc-5.3.1-cp312-cp312-manylinux_2_17_x86_64.whl ``` ### Building HPC Variant Locally @@ -156,7 +159,7 @@ cmake --build build -j$(nproc) | **Use Case** | Single-system deployments, development | HPC clusters, portable deployments | | **Performance** | Highest on optimized hardware | Portable across architectures | | **Portability** | Variable (CPU-specific) | Maximum (all x86_64 CPUs) | -| **Wheel Suffix** | None (`*-linux_x86_64.whl`) | `_hpc` (`*-linux_x86_64_hpc.whl`) | +| **PyPI Package** | `dsf-mobility` | `dsf-mobility-hpc` | ## Testing This project uses [Doctest](https://github.com/doctest/doctest) for testing. diff --git a/setup.py b/setup.py index 87ed8c90..e0fec8d9 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ """ import os +from distutils import log from pathlib import Path import platform import re @@ -96,10 +97,13 @@ def build_extension(self, ext: CMakeExtension): ] # Pass DSF_HPC_BUILD environment variable to CMake for HPC-compatible builds - hpc_build = os.environ.get("DSF_HPC_BUILD", "0") - if hpc_build in ("1", "true", "TRUE", "on", "ON"): + hpc_build = os.environ.get("DSF_HPC_BUILD", "0").strip().lower() + if hpc_build in {"1", "true", "on", "yes"}: cmake_args.append("-DDSF_HPC_BUILD=ON") - print("HPC Build Mode enabled: using conservative -O3 optimization") + self.announce( + "HPC Build Mode enabled: using conservative -O3 optimization", + level=log.INFO, + ) if platform.system() == "Windows": cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] @@ -124,11 +128,16 @@ def build_extension(self, ext: CMakeExtension): cmake_prefix_path = f"{fmt_prefix};{spdlog_prefix}" cmake_args.append(f"-DCMAKE_PREFIX_PATH={cmake_prefix_path}") - print(f"Added macOS Homebrew prefix paths: {cmake_prefix_path}") + self.announce( + f"Added macOS Homebrew prefix paths: {cmake_prefix_path}", + level=log.INFO, + ) except (subprocess.CalledProcessError, FileNotFoundError): - print( - "Warning: Could not determine Homebrew prefix paths. Make sure Homebrew is installed and dependencies are available." + self.announce( + "Warning: Could not determine Homebrew prefix paths. " + "Make sure Homebrew is installed and dependencies are available.", + level=log.WARN, ) # Fallback to common Homebrew paths cmake_args.append("-DCMAKE_PREFIX_PATH=/opt/homebrew;/usr/local") From 92fd66dd2f43c1869e34f008569e088bff32446d Mon Sep 17 00:00:00 2001 From: grufoony Date: Tue, 24 Mar 2026 14:55:18 +0100 Subject: [PATCH 4/4] Increment development version --- .github/workflows/pypi.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index dcd36bf1..24ef7749 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -39,9 +39,12 @@ jobs: BASE_VERSION="${{ steps.extract.outputs.version }}" if [ "${{ github.event_name }}" = "pull_request" ]; then - # TestPyPI only: use a numeric dev suffix so the version is PEP 440 compliant. - # Concatenating run id and attempt keeps it unique across PRs and re-runs. - PUBLISH_VERSION="${BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}" + # TestPyPI only: bump patch by one, then append a numeric dev suffix + # so the version is PEP 440 compliant and unique across PRs/re-runs. + IFS='.' read -r VERSION_MAJOR VERSION_MINOR VERSION_PATCH <<< "${BASE_VERSION}" + VERSION_PATCH=$((VERSION_PATCH + 1)) + PR_BASE_VERSION="${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}" + PUBLISH_VERSION="${PR_BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}" else # PyPI: publish the exact release version with no run-id suffix. PUBLISH_VERSION="${BASE_VERSION}"