diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 09988b52..24ef7749 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -39,9 +39,12 @@ jobs: BASE_VERSION="${{ steps.extract.outputs.version }}" if [ "${{ github.event_name }}" = "pull_request" ]; then - # TestPyPI only: use a numeric dev suffix so the version is PEP 440 compliant. - # Concatenating run id and attempt keeps it unique across PRs and re-runs. - PUBLISH_VERSION="${BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}" + # TestPyPI only: bump patch by one, then append a numeric dev suffix + # so the version is PEP 440 compliant and unique across PRs/re-runs. + IFS='.' read -r VERSION_MAJOR VERSION_MINOR VERSION_PATCH <<< "${BASE_VERSION}" + VERSION_PATCH=$((VERSION_PATCH + 1)) + PR_BASE_VERSION="${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}" + PUBLISH_VERSION="${PR_BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}" else # PyPI: publish the exact release version with no run-id suffix. PUBLISH_VERSION="${BASE_VERSION}" @@ -61,23 +64,27 @@ jobs: exit 0 fi - PYPI_URL="https://pypi.org/pypi/dsf-mobility/${VERSION}/json" PYPI_NAME="PyPI" - - echo "Checking if dsf-mobility version ${VERSION} exists on ${PYPI_NAME}..." - - # Check PyPI/TestPyPI API for the specific version - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "${PYPI_URL}") - - if [ "$HTTP_STATUS" = "200" ]; then - echo "Version ${VERSION} already exists on ${PYPI_NAME}" - echo "Version already published; skipping build and publish jobs." - echo "should_build=false" >> "$GITHUB_OUTPUT" - exit 0 - else - echo "Version ${VERSION} does not exist on ${PYPI_NAME} (HTTP ${HTTP_STATUS})" - echo "should_build=true" >> "$GITHUB_OUTPUT" - fi + PACKAGES=("dsf-mobility" "dsf-mobility-hpc") + + for PACKAGE in "${PACKAGES[@]}"; do + PYPI_URL="https://pypi.org/pypi/${PACKAGE}/${VERSION}/json" + echo "Checking if ${PACKAGE} version ${VERSION} exists on ${PYPI_NAME}..." + + # Check PyPI API for the specific version + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "${PYPI_URL}") + + if [ "$HTTP_STATUS" = "200" ]; then + echo "Version ${VERSION} already exists for ${PACKAGE} on ${PYPI_NAME}." + echo "Version already published for at least one package; skipping build and publish jobs." + echo "should_build=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Version ${VERSION} does not exist for ${PACKAGE} on ${PYPI_NAME} (HTTP ${HTTP_STATUS})" + done + + echo "should_build=true" >> "$GITHUB_OUTPUT" uv-validate: name: Validate uv workflows @@ -128,13 +135,14 @@ jobs: python -c "import dsf; print(dsf.__version__)" build-wheels-linux: - name: Build wheels on Linux (Python ${{ matrix.python-version }}) + name: Build wheels on Linux (Python ${{ matrix.python-version }} - ${{ matrix.build_variant }}) needs: [check-version, uv-validate] runs-on: ubuntu-latest if: needs.check-version.outputs.should_build == 'true' strategy: matrix: python-version: ['3.10', '3.12'] + build_variant: ['standard', 'hpc'] steps: - name: Checkout repository @@ -155,12 +163,33 @@ jobs: python -m pip install --upgrade pip python -m pip install build wheel setuptools pybind11-stubgen auditwheel + - name: Switch package name for HPC distribution + if: matrix.build_variant == 'hpc' + run: | + python - <<'PY' + from pathlib import Path + + pyproject_path = Path("pyproject.toml") + content = pyproject_path.read_text(encoding="utf-8") + old = 'name = "dsf-mobility"' + new = 'name = "dsf-mobility-hpc"' + if old not in content: + raise RuntimeError("Unable to locate project name in pyproject.toml") + pyproject_path.write_text(content.replace(old, new, 1), encoding="utf-8") + print("Switched project name to dsf-mobility-hpc for HPC build") + PY + + - name: Clean previous build artifacts + run: | + rm -rf dist wheelhouse + rm -rf build/temp.* build/lib.* + - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} + DSF_HPC_BUILD: ${{ matrix.build_variant == 'hpc' && '1' || '0' }} run: | - rm -rf dist wheelhouse python -m build --wheel - name: Repair wheel (auditwheel) @@ -176,7 +205,7 @@ jobs: - name: Upload wheels as artifacts uses: actions/upload-artifact@v6 with: - name: wheel-linux-${{ matrix.python-version }} + name: wheel-linux-${{ matrix.python-version }}-${{ matrix.build_variant }} path: wheelhouse/*.whl build-wheels-macos: @@ -211,10 +240,16 @@ jobs: echo "ARCHFLAGS=-arch $(uname -m)" >> $GITHUB_ENV echo "_PYTHON_HOST_PLATFORM=macosx-$(sw_vers -productVersion | cut -d. -f1)-$(uname -m)" >> $GITHUB_ENV + - name: Clean previous build artifacts + run: | + rm -rf dist wheelhouse + rm -rf build/temp.* build/lib.* + - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} + DSF_HPC_BUILD: "0" run: python -m build --wheel - name: Repair wheel (bundle libraries) @@ -262,10 +297,17 @@ jobs: python -m pip install --upgrade pip python -m pip install build wheel setuptools pybind11-stubgen + - name: Clean previous build artifacts + shell: bash + run: | + rm -rf dist wheelhouse + rm -rf build/temp.* build/lib.* + - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} + DSF_HPC_BUILD: "0" run: python -m build --wheel - name: Upload wheels as artifacts diff --git a/CMakeLists.txt b/CMakeLists.txt index 8127c7c1..7739f0c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ option(DSF_BENCHMARKS "Build DSF benchmarks" OFF) option(DSF_BUILD_PIC "Build DSF with position-independent code" OFF) option(BUILD_PYTHON_BINDINGS "Build Python bindings" OFF) option(DSF_OPTIMIZE_ARCH "Optimize for native architecture" ON) +option(DSF_HPC_BUILD "Build with conservative -O3 for HPC cluster portability" OFF) # If CMAKE_BUILD_TYPE not set, default to Debug if(NOT CMAKE_BUILD_TYPE) @@ -39,6 +40,8 @@ if(NOT CMAKE_BUILD_TYPE) endif() endif() if(BUILD_PYTHON_BINDINGS) + # Build all targets as PIC when producing Python extension modules. + # This prevents non-PIC static dependencies from failing at shared-module link time. set(DSF_BUILD_PIC ON) endif() @@ -50,9 +53,16 @@ set(CMAKE_CXX_EXTENSIONS OFF) # Ensure optimization flags are applied only in Release mode if(CMAKE_BUILD_TYPE MATCHES "Release") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Ofast -flto=auto") - if(DSF_OPTIMIZE_ARCH) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + if(DSF_HPC_BUILD) + # HPC mode: conservative -O3 for maximum portability on HPC clusters + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") + message(STATUS "HPC Build Mode enabled: using -O3 for portability") + else() + # Standard mode: aggressive optimization + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Ofast -flto=auto") + if(DSF_OPTIMIZE_ARCH) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + endif() endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") @@ -140,6 +150,9 @@ FetchContent_Declare( FetchContent_GetProperties(simdjson) if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) + if(DSF_BUILD_PIC) + set_target_properties(simdjson PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() endif() # Check if the user has TBB installed find_package(TBB REQUIRED CONFIG) @@ -157,8 +170,12 @@ FetchContent_Declare( FetchContent_GetProperties(SQLiteCpp) if(NOT SQLiteCpp_POPULATED) FetchContent_MakeAvailable(SQLiteCpp) + if(DSF_BUILD_PIC) + set_target_properties(SQLiteCpp PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() endif() + add_library(dsf STATIC ${SOURCES}) target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT) if(DSF_BUILD_PIC) diff --git a/README.md b/README.md index 55e15b53..716a66a3 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This rework consists of a full code rewriting, in order to implement more featur ## Table of Contents - [Installation](#installation) - [Installation (from source)](#installation-from-source) +- [Installation (Python - HPC Variant)](#installation-python---hpc-variant) - [Testing](#testing) - [Benchmarking](#benchmarking) - [Citing](#citing) @@ -92,6 +93,74 @@ print(dsf.__version__) If you encounter issues, ensure that the installation path is in your `PYTHONPATH` environment variable. +## Installation (Python - HPC Variant) + +For high-performance computing (HPC) clusters and environments where binary portability is critical, an HPC-optimized wheel variant is available. This variant uses conservative `-O3` optimization instead of architecture-specific tuning (`-Ofast`, `-flto=auto`, `-march=native`), ensuring compatibility across diverse HPC hardware architectures. + +### When to Use HPC Variant +- Deploying on HPC clusters with heterogeneous node architectures +- Avoiding runtime errors due to unsupported CPU instructions +- Maximizing portability across different compute nodes + +### Installation on HPC Systems + +The HPC build is published as a separate PyPI distribution named `dsf-mobility-hpc` (PEP-compliant), with Linux wheels intended for cluster portability. Install it directly with: + +```shell +pip install dsf-mobility-hpc +``` + +Or with `uv`: + +```shell +uv pip install dsf-mobility-hpc +``` + +If you need to download a wheel explicitly, use: + +```shell +pip download --only-binary :all: dsf-mobility-hpc +``` + +### Wheel Filename Pattern on PyPI + +HPC wheel filenames are standard and parseable by pip, for example: + +```text +dsf_mobility_hpc--cp-cp-.whl +``` + +Typical Linux example: + +```text +dsf_mobility_hpc-5.3.1-cp312-cp312-manylinux_2_17_x86_64.whl +``` + +### Building HPC Variant Locally + +To build the HPC variant locally for development or testing: + +```shell +DSF_HPC_BUILD=1 pip install . +``` + +This uses conservative `-O3` optimization for maximum portability: + +```shell +cmake -B build -DCMAKE_BUILD_TYPE=Release -DDSF_HPC_BUILD=ON +cmake --build build -j$(nproc) +``` + +### Standard vs. HPC Variants + +| Aspect | Standard | HPC | +|--------|----------|-----| +| **Optimization** | `-Ofast -flto=auto` + optional `-march=native` | `-O3` only | +| **Use Case** | Single-system deployments, development | HPC clusters, portable deployments | +| **Performance** | Highest on optimized hardware | Portable across architectures | +| **Portability** | Variable (CPU-specific) | Maximum (all x86_64 CPUs) | +| **PyPI Package** | `dsf-mobility` | `dsf-mobility-hpc` | + ## Testing This project uses [Doctest](https://github.com/doctest/doctest) for testing. diff --git a/setup.py b/setup.py index 3ba5eaa3..e0fec8d9 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ """ import os +from distutils import log from pathlib import Path import platform import re @@ -95,6 +96,15 @@ def build_extension(self, ext: CMakeExtension): "-DBUILD_PYTHON_BINDINGS=ON", ] + # Pass DSF_HPC_BUILD environment variable to CMake for HPC-compatible builds + hpc_build = os.environ.get("DSF_HPC_BUILD", "0").strip().lower() + if hpc_build in {"1", "true", "on", "yes"}: + cmake_args.append("-DDSF_HPC_BUILD=ON") + self.announce( + "HPC Build Mode enabled: using conservative -O3 optimization", + level=log.INFO, + ) + if platform.system() == "Windows": cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] if "CMAKE_TOOLCHAIN_FILE" in os.environ: @@ -118,11 +128,16 @@ def build_extension(self, ext: CMakeExtension): cmake_prefix_path = f"{fmt_prefix};{spdlog_prefix}" cmake_args.append(f"-DCMAKE_PREFIX_PATH={cmake_prefix_path}") - print(f"Added macOS Homebrew prefix paths: {cmake_prefix_path}") + self.announce( + f"Added macOS Homebrew prefix paths: {cmake_prefix_path}", + level=log.INFO, + ) except (subprocess.CalledProcessError, FileNotFoundError): - print( - "Warning: Could not determine Homebrew prefix paths. Make sure Homebrew is installed and dependencies are available." + self.announce( + "Warning: Could not determine Homebrew prefix paths. " + "Make sure Homebrew is installed and dependencies are available.", + level=log.WARN, ) # Fallback to common Homebrew paths cmake_args.append("-DCMAKE_PREFIX_PATH=/opt/homebrew;/usr/local")