diff --git a/.github/workflows/_build_linux.yml b/.github/workflows/_build_linux.yml index 172f07cfd73..d67cfae8ea7 100644 --- a/.github/workflows/_build_linux.yml +++ b/.github/workflows/_build_linux.yml @@ -8,7 +8,7 @@ on: description: "Build Images" required: true type: string - default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310" + default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux" FASTDEPLOY_ARCHIVE_URL: description: "URL of the compressed FastDeploy code archive." required: true @@ -53,6 +53,11 @@ on: required: false type: string default: "" + PYTHON_VERSION: + description: "Python version to use (3.10, 3.11, 3.12)" + required: false + type: string + default: "3.10" outputs: wheel_path: description: "Output path of the generated wheel" @@ -137,6 +142,7 @@ jobs: PADDLE_WHL_URL: ${{ inputs.PADDLE_WHL_URL }} WITH_NIGHTLY_BUILD: ${{ inputs.WITH_NIGHTLY_BUILD }} FD_UNIFY_BUILD: ${{ inputs.FD_UNIFY_BUILD }} + PYTHON_VERSION: ${{ inputs.PYTHON_VERSION }} run: | set -x runner_name="${{ runner.name }}" @@ -171,7 +177,33 @@ jobs: -e "PADDLE_WHL_URL=${PADDLE_WHL_URL}" \ -e "BRANCH_REF=${BRANCH_REF}" \ -e "CCACHE_MAXSIZE=50G" \ + -e "PYTHON_VERSION=${PYTHON_VERSION}" \ --gpus "\"device=${gpu_id}\"" ${docker_image} /bin/bash -c ' + # Switch Python environment based on PYTHON_VERSION + if [[ "${PYTHON_VERSION}" == "3.11" ]]; then + export LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} + export PATH=/opt/_internal/cpython-3.11.0/bin/:${PATH} + elif [[ "${PYTHON_VERSION}" == "3.12" ]]; then + export LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} + export PATH=/opt/_internal/cpython-3.12.0/bin/:${PATH} + else + # Default to Python 3.10 + export LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} + export PATH=/opt/_internal/cpython-3.10.0/bin/:${PATH} + fi + python --version + + # Isolate CCache by Python version to avoid cache conflicts + export CCACHE_DIR=/root/.ccache/py${PYTHON_VERSION} + export CCACHE_BASEDIR=/workspace/FastDeploy + mkdir -p ${CCACHE_DIR} + # Show CCache statistics (cache size, hit rate, etc.) + ccache -s + + # Isolate pip cache by Python version + export PIP_CACHE_DIR=/root/.cache/pip/py${PYTHON_VERSION} + mkdir -p ${PIP_CACHE_DIR} + if [[ -n "${FD_VERSION}" ]]; then export FASTDEPLOY_VERSION=${FD_VERSION} echo "Custom FastDeploy version: ${FASTDEPLOY_VERSION}" @@ -199,8 +231,15 @@ jobs: pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple python -m pip install --upgrade pip - python -m pip install -r requirements.txt - python -m pip install wheel + # Install jinja2 for custom ops code generation (required for 3.11/3.12) + python -m pip install jinja2 wheel + # Skip cp310-specific wheels for non-3.10 Python versions + if [[ "${PYTHON_VERSION}" != "3.10" ]]; then + grep -v "cp310-cp310" requirements.txt > requirements_filtered.txt + python -m pip install -r requirements_filtered.txt || true + else + python -m pip install -r requirements.txt || true + fi # 编译RDMA export FD_ENABLE_RDMA_COMPILE=1 export FD_UNIFY_BUILD="${FD_UNIFY_BUILD}" diff --git a/.github/workflows/pr_build_and_test.yml b/.github/workflows/pr_build_and_test.yml index 9ffcd75ee5c..6c360d8e545 100644 --- a/.github/workflows/pr_build_and_test.yml +++ b/.github/workflows/pr_build_and_test.yml @@ -16,8 +16,22 @@ jobs: secrets: github-token: ${{ secrets.GITHUB_TOKEN }} - build: - name: FD-Build-Linux +# build_310: +# name: FD-Build-Linux-Py310 +# needs: clone +# uses: ./.github/workflows/_build_linux.yml +# with: +# DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux +# FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} +# COMPILE_ARCH: "90" +# WITH_NIGHTLY_BUILD: "OFF" +# FD_VERSION: "0.0.0" +# PYTHON_VERSION: "3.10" +# secrets: +# github-token: ${{ secrets.GITHUB_TOKEN }} + + build_311: + name: FD-Build-Linux-Py311 needs: clone uses: ./.github/workflows/_build_linux.yml with: @@ -26,86 +40,102 @@ jobs: COMPILE_ARCH: "90" WITH_NIGHTLY_BUILD: "OFF" FD_VERSION: "0.0.0" + PYTHON_VERSION: "3.11" secrets: github-token: ${{ secrets.GITHUB_TOKEN }} - resultshow: - name: Use Build Output - needs: build - runs-on: ubuntu-latest - steps: - - name: Print wheel path - run: | - echo "The built wheel is located at: ${{ needs.build.outputs.wheel_path }}" - - unittest_coverage: - name: Run FastDeploy Unit Tests and Coverage - needs: [clone,build] - uses: ./.github/workflows/_unit_test_coverage.yml - with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev - FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} - FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} - MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" - secrets: - github-token: ${{ secrets.GITHUB_TOKEN }} - - four_cards_test: - name: Run Four Cards Tests - needs: [clone,build] - uses: ./.github/workflows/_gpu_4cards_case_test.yml - with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev - FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} - FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} - MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" - secrets: - github-token: ${{ secrets.GITHUB_TOKEN }} - - logprob_test: - name: Run FastDeploy LogProb Tests - needs: [build] - uses: ./.github/workflows/_logprob_test_linux.yml - with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev - PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz" - FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} - MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" - secrets: - github-token: ${{ secrets.GITHUB_TOKEN }} - - pre_ce_test: - name: Extracted partial CE model tasks to run in CI. - needs: [clone,build] - uses: ./.github/workflows/_pre_ce_test.yml + build_312: + name: FD-Build-Linux-Py312 + needs: clone + uses: ./.github/workflows/_build_linux.yml with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} - FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} - MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + COMPILE_ARCH: "90" + WITH_NIGHTLY_BUILD: "OFF" + FD_VERSION: "0.0.0" + PYTHON_VERSION: "3.12" secrets: github-token: ${{ secrets.GITHUB_TOKEN }} - base_test: - name: Run Base Tests - needs: [clone,build] - uses: ./.github/workflows/_base_test.yml - with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev - FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} - FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} - MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" - secrets: - github-token: ${{ secrets.GITHUB_TOKEN }} + resultshow: + name: Use Build Output + needs: [build_311, build_312] + runs-on: ubuntu-latest + steps: + - name: Print wheel paths + run: | + echo "Py311 wheel: ${{ needs.build_311.outputs.wheel_path }}" + echo "Py312 wheel: ${{ needs.build_312.outputs.wheel_path }}" - stable_test: - name: Run Stable Tests - needs: [clone,build] - uses: ./.github/workflows/_stable_test.yml - with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate - FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} - FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} - MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" - secrets: - github-token: ${{ secrets.GITHUB_TOKEN }} +# unittest_coverage: +# name: Run FastDeploy Unit Tests and Coverage +# needs: [clone,build_310] +# uses: ./.github/workflows/_unit_test_coverage.yml +# with: +# DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev +# FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} +# FASTDEPLOY_WHEEL_URL: ${{ needs.build_310.outputs.wheel_path }} +# MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" +# secrets: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# +# four_cards_test: +# name: Run Four Cards Tests +# needs: [clone,build_310] +# uses: ./.github/workflows/_gpu_4cards_case_test.yml +# with: +# DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev +# FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} +# FASTDEPLOY_WHEEL_URL: ${{ needs.build_310.outputs.wheel_path }} +# MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" +# secrets: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# +# logprob_test: +# name: Run FastDeploy LogProb Tests +# needs: [build_310] +# uses: ./.github/workflows/_logprob_test_linux.yml +# with: +# DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev +# PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz" +# FASTDEPLOY_WHEEL_URL: ${{ needs.build_310.outputs.wheel_path }} +# MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" +# secrets: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# +# pre_ce_test: +# name: Extracted partial CE model tasks to run in CI. +# needs: [clone,build_310] +# uses: ./.github/workflows/_pre_ce_test.yml +# with: +# DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate +# FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} +# FASTDEPLOY_WHEEL_URL: ${{ needs.build_310.outputs.wheel_path }} +# MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" +# secrets: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# +# base_test: +# name: Run Base Tests +# needs: [clone,build_310] +# uses: ./.github/workflows/_base_test.yml +# with: +# DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-paddle-dev +# FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} +# FASTDEPLOY_WHEEL_URL: ${{ needs.build_310.outputs.wheel_path }} +# MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" +# secrets: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# +# stable_test: +# name: Run Stable Tests +# needs: [clone,build_310] +# uses: ./.github/workflows/_stable_test.yml +# with: +# DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate +# FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} +# FASTDEPLOY_WHEEL_URL: ${{ needs.build_310.outputs.wheel_path }} +# MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" +# secrets: +# github-token: ${{ secrets.GITHUB_TOKEN }}