diff --git a/.github/workflows/ci-cuda.yml b/.github/workflows/ci-cuda.yml new file mode 100644 index 00000000000..19254bbbe38 --- /dev/null +++ b/.github/workflows/ci-cuda.yml @@ -0,0 +1,154 @@ +name: CI / CUDA + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + actions: read + contents: read + +env: + CARGO_TERM_COLOR: auto + RUST_BACKTRACE: 1 + +jobs: + cuda-build-lint: + if: github.repository == 'vortex-data/vortex' + name: "CUDA build & lint" + timeout-minutes: 40 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/check-rebuild + with: + command: >- + cargo build --locked --all-features --all-targets + -p vortex-cuda -p vortex-cub -p vortex-nvcomp + -p gpu-scan-cli -p vortex-test-e2e-cuda + - name: Clippy CUDA crates + run: | + cargo clippy --locked --all-features --all-targets \ + -p vortex-cuda \ + -p vortex-cub \ + -p vortex-nvcomp \ + -p gpu-scan-cli \ + -p vortex-test-e2e-cuda \ + -- -D warnings + + cuda-test: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install nextest + uses: taiki-e/install-action@v2 + with: + tool: nextest + - name: Rust Tests + env: + FLAT_LAYOUT_INLINE_ARRAY_NODE: true + run: | + cargo nextest run \ + --locked \ + -p vortex-file \ + -p vortex-cuda \ + -p vortex-cub \ + -p vortex-nvcomp \ + -p vortex-test-e2e-cuda \ + --all-features \ + --no-fail-fast \ + --target x86_64-unknown-linux-gnu \ + --verbose + + cuda-test-sanitizer: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests (${{ matrix.sanitizer }})" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer + strategy: + fail-fast: false + matrix: + include: + - sanitizer: memcheck + runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" + # TODO(joe): try to re-enable racecheck, it is hanging in CI. + # - sanitizer: racecheck + # runner_flags: "--tool racecheck --error-exitcode 1" + - sanitizer: synccheck + runner_flags: "--tool synccheck --error-exitcode 1" + - sanitizer: initcheck + runner_flags: "--tool initcheck --error-exitcode 1" + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build tests + run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run + - name: "CUDA - ${{ matrix.sanitizer }}" + env: + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" + run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu + + cuda-test-cudf: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests (cudf)" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build cudf test library + run: cargo build --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu + - name: Download and run cudf-test-harness + run: | + curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz + cd cudf-test-harness-x86_64 + compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/debug/libvortex_test_e2e_cuda.so diff --git a/.github/workflows/ci-java.yml b/.github/workflows/ci-java.yml new file mode 100644 index 00000000000..c55ac630c61 --- /dev/null +++ b/.github/workflows/ci-java.yml @@ -0,0 +1,67 @@ +name: CI / Java + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + actions: read + contents: read + +env: + CARGO_TERM_COLOR: auto + RUST_BACKTRACE: 1 + +jobs: + build-java: + name: "Java" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=java', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 40 + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + - run: ./gradlew test --parallel + working-directory: ./java + + check-java-publish-build: + runs-on: ${{ matrix.target.runs-on }} + container: + image: "ubuntu:20.04" + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + target: + - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } + - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: ./.github/actions/prepare-java-linux + - uses: actions/setup-java@v5 + with: + distribution: "corretto" + java-version: "17" + - uses: ./.github/actions/setup-rust + with: + targets: ${{ matrix.target.target }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "false" + - run: cargo build --package vortex-jni diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml new file mode 100644 index 00000000000..406f61c2e50 --- /dev/null +++ b/.github/workflows/ci-python.yml @@ -0,0 +1,130 @@ +name: CI / Python + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + actions: read + contents: read + +env: + CARGO_TERM_COLOR: auto + RUST_BACKTRACE: 1 + +jobs: + python-lint: + name: "Python (lint)" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 40 + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) + - name: Python Lint - Format + run: uvx ruff format --check . + - name: Python Lint - Ruff + run: uvx ruff check . + # PyRight needs the project for type information, so use uv run + - name: Python Lint - PyRight + env: + MATURIN_PEP517_ARGS: "--profile dev" + run: uv run basedpyright vortex-python + + python-test: + name: "Python (test)" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 40 + env: + RUST_LOG: "info,uv=debug" + MATURIN_PEP517_ARGS: "--profile dev" + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + + - name: Pytest - Vortex + run: | + uv run --all-packages pytest --benchmark-disable -n auto test/ + working-directory: vortex-python/ + + - name: Setup benchmark environment + run: sudo bash scripts/setup-benchmark.sh + + - name: Pytest Benchmarks - Vortex + run: | + bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ + working-directory: vortex-python/ + + - name: Doctest - PyVortex + run: | + uv run --all-packages make doctest + working-directory: docs/ + + - name: Ensure docs build - PyVortex + run: | + uv run --all-packages make html + working-directory: docs/ + + python-wheel-build: + name: "Python (wheel build)" + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref_name == 'develop' }} + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "false" + - uses: mlugg/setup-zig@v2.2.1 + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + with: + sync: false + prune-cache: false + - name: Ensure wheel and sdist can be built on Linux - PyVortex + shell: bash + run: | + echo "Clearing wheel target directory" + rm -rf ../target/wheels/ + + uv venv + uv tool run maturin@1.10 build --interpreter python3.11 --zig + uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist + + file_count=$(ls -1 ../target/wheels/ | wc -l) + + if [[ $file_count -ne 2 ]]; then + echo "Unexpected number of files detected ${file_count}:" + ls ../target/wheels/ + exit 1 + else + echo "Generated two files" + fi + working-directory: vortex-python/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3da39fae989..ac8ab22598c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,113 +46,6 @@ jobs: -c .yamllint.yaml \ .github/ - python-lint: - name: "Python (lint)" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 40 - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) - - name: Python Lint - Format - run: uvx ruff format --check . - - name: Python Lint - Ruff - run: uvx ruff check . - # PyRight needs the project for type information, so use uv run - - name: Python Lint - PyRight - env: - MATURIN_PEP517_ARGS: "--profile dev" - run: uv run basedpyright vortex-python - - python-test: - name: "Python (test)" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 40 - env: - RUST_LOG: "info,uv=debug" - MATURIN_PEP517_ARGS: "--profile dev" - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - - name: Pytest - Vortex - run: | - uv run --all-packages pytest --benchmark-disable -n auto test/ - working-directory: vortex-python/ - - - name: Setup benchmark environment - run: sudo bash scripts/setup-benchmark.sh - - - name: Pytest Benchmarks - Vortex - run: | - bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ - working-directory: vortex-python/ - - - name: Doctest - PyVortex - run: | - uv run --all-packages make doctest - working-directory: docs/ - - - name: Ensure docs build - PyVortex - run: | - uv run --all-packages make html - working-directory: docs/ - - python-wheel-build: - name: "Python (wheel build)" - runs-on: ubuntu-latest - timeout-minutes: 40 - steps: - - uses: actions/checkout@v6 - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.ref_name == 'develop' }} - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - uses: mlugg/setup-zig@v2.2.1 - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 - with: - sync: false - prune-cache: false - - name: Ensure wheel and sdist can be built on Linux - PyVortex - shell: bash - run: | - echo "Clearing wheel target directory" - rm -rf ../target/wheels/ - - uv venv - uv tool run maturin@1.10 build --interpreter python3.11 --zig - uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist - - file_count=$(ls -1 ../target/wheels/ | wc -l) - - if [[ $file_count -ne 2 ]]; then - echo "Unexpected number of files detected ${file_count}:" - ls ../target/wheels/ - exit 1 - else - echo "Generated two files" - fi - working-directory: vortex-python/ - rust-docs: name: "Rust (docs)" timeout-minutes: 40 @@ -425,135 +318,6 @@ jobs: --target x86_64-unknown-linux-gnu \ -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array - cuda-build-lint: - if: github.repository == 'vortex-data/vortex' - name: "CUDA build & lint" - timeout-minutes: 40 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/check-rebuild - with: - command: >- - cargo build --locked --all-features --all-targets - -p vortex-cuda -p vortex-cub -p vortex-nvcomp - -p gpu-scan-cli -p vortex-test-e2e-cuda - - name: Clippy CUDA crates - run: | - cargo clippy --locked --all-features --all-targets \ - -p vortex-cuda \ - -p vortex-cub \ - -p vortex-nvcomp \ - -p gpu-scan-cli \ - -p vortex-test-e2e-cuda \ - -- -D warnings - - cuda-test: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install nextest - uses: taiki-e/install-action@v2 - with: - tool: nextest - - name: Rust Tests - env: - FLAT_LAYOUT_INLINE_ARRAY_NODE: true - run: | - cargo nextest run \ - --locked \ - -p vortex-file \ - -p vortex-cuda \ - -p vortex-cub \ - -p vortex-nvcomp \ - -p vortex-test-e2e-cuda \ - --all-features \ - --no-fail-fast \ - --target x86_64-unknown-linux-gnu \ - --verbose - - cuda-test-sanitizer: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests (sanitizer)" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: CUDA - memcheck - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool memcheck --leak-check=full --error-exitcode 1 - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu - # TODO(joe): try to re-enable, This is hanging in CI. - # - name: CUDA - racecheck - # env: - # CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool racecheck --error-exitcode 1 - # run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu - - name: CUDA - synccheck - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool synccheck --error-exitcode 1 - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu - - name: CUDA - initcheck - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool initcheck --error-exitcode 1 - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu - - cuda-test-cudf: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests (cudf)" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build cudf test library - run: cargo build --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu - - name: Download and run cudf-test-harness - run: | - curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz - cd cudf-test-harness-x86_64 - compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/debug/libvortex_test_e2e_cuda.so - rust-test-other: name: "Rust tests (${{ matrix.os }})" timeout-minutes: 40 @@ -610,23 +374,6 @@ jobs: alert-title: "Rust tests (${{ matrix.os }}) failed on develop" deduplication-key: ci-rust-test-${{ matrix.os }}-failure - build-java: - name: "Java" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=java', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 40 - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - run: ./gradlew test --parallel - working-directory: ./java - bench-codspeed: strategy: matrix: @@ -819,33 +566,6 @@ jobs: run: | find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{} - check-java-publish-build: - runs-on: ${{ matrix.target.runs-on }} - container: - image: "ubuntu:20.04" - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - target: - - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } - - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - uses: ./.github/actions/prepare-java-linux - - uses: actions/setup-java@v5 - with: - distribution: "corretto" - java-version: "17" - - uses: ./.github/actions/setup-rust - with: - targets: ${{ matrix.target.target }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - run: cargo build --package vortex-jni - compat-check: name: "Compat check" uses: ./.github/workflows/compat-validation.yml