ci.yaml

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: CI

permissions:
  contents: read
  pull-requests: write

on:
  pull_request:
    branches:
    - development
    - '[0-9]+.[0-9]+.x'
    - 'feature/**'
  workflow_dispatch:

  # Run CI also on push to backport release branches - we sometimes push code there by cherry-picking, meaning it
  # doesn't go through CI (no PR)
  # For master all code is coming from development where it's coming from PRs (meaning it's passing CI) but there's the
  # rare scenario where the combination of merged PRs won't pass CI
  push:
    branches:
    - '[0-9]+.[0-9]+.x'

concurrency:
  group:
    ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  lint:
    name: Lint code (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11"]
    steps:
    - uses: actions/checkout@v6
    - name: Set up python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip~=25.0
        pip install -r dev-requirements.txt
    - name: Lint
      run: make lint

    # perform copyright check here, to avoid spinning up another runner needlessly
    - uses: denoland/setup-deno@e95548e56dfa95d4e1a28d6f422fafe75c4c26fb # 2.0.3
      with:
        deno-version: v1.x
    - name: Check
      run: |
        deno run --allow-read https://deno.land/x/license_checker@v3.2.3/main.ts
    - name: Instructions For Adding Copyright
      if: ${{ failure() }}
      run: |
        echo "
        Please add the following lines to the top of your code:

        # Copyright 2024 Iguazio
        #
        # Licensed under the Apache License, Version 2.0 (the "License");
        # you may not use this file except in compliance with the License.
        # You may obtain a copy of the License at
        #
        #   http://www.apache.org/licenses/LICENSE-2.0
        #
        # Unless required by applicable law or agreed to in writing, software
        # distributed under the License is distributed on an "AS IS" BASIS,
        # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
        # See the License for the specific language governing permissions and
        # limitations under the License.
        "

  tests:
    name: Run Dockerized Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11"]
        test-suite: ["server", "client"]
    steps:
    - uses: actions/checkout@v6
    - name: Set up python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip

    - name: Extract git branch
      id: git_info
      run: |
        echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT

    - name: Resolve docker cache tag
      id: docker_cache
      run: |
        export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
        export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
        echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT

    - name: Install automation scripts dependencies
      run: make install-automation-requirements
    - name: Set test suite
      run: |
          if [[ "${{ matrix.test-suite }}" == "server" ]]; then
            echo "UNIT_TESTS_PATH=\"server/\"" >> $GITHUB_ENV
          else
            echo "UNIT_TESTS_IGNORE_PATH=\"server/\"" >> $GITHUB_ENV
          fi
    - name: Set RUN_COVERAGE environment variable
      run: echo "RUN_COVERAGE=true" >> $GITHUB_ENV
    - name: Run Dockerized tests
      run: |
        MLRUN_DOCKER_REGISTRY=ghcr.io/ \
        MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} \
        MLRUN_PYTHON_VERSION=${{ matrix.python-version }} \
        UNIT_TESTS_PATH=$UNIT_TESTS_PATH \
        UNIT_TESTS_IGNORE_PATH=$UNIT_TESTS_IGNORE_PATH \
        RUN_COVERAGE=$RUN_COVERAGE \
        COVERAGE_DIR_SUFFIX="_${{ matrix.test-suite }}" \
        COVERAGE_FILE="tests/coverage_reports/unit_tests_${{ matrix.test-suite }}.coverage" \
        make test-dockerized
    - name: Upload coverage report
      if: env.RUN_COVERAGE == 'true'
      uses: actions/upload-artifact@v7
      with:
        name: coverage_unit_tests_report_${{ matrix.python-version }}_${{ github.run_id }}_${{ matrix.test-suite }}.coverage
        path: /tmp/coverage_reports/unit_tests_${{ matrix.test-suite }}/unit_tests_${{ matrix.test-suite }}.coverage
        retention-days: 30


  integration-tests:
    name: Run Dockerized Integration Tests
    runs-on: ubuntu-latest

    strategy:
      matrix:
        include:
          - python-version: "3.11"
            test-db: mysql
          - python-version: "3.11"
            test-db: postgres

    steps:
    - uses: actions/checkout@v6

    # since github-actions gives us 14G only, and fills it up with some garbage
    - name: Freeing up disk space
      run: |
        "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip

    # TODO: re-use the next 2 actions instead of duplicating
    - name: Extract git branch
      id: git_info
      run: |
        echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT

    - name: Resolve docker cache tag
      id: docker_cache
      run: |
        export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
        export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
        echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT

    - name: Install automation scripts dependencies
      run: make install-automation-requirements

    - name: Set RUN_COVERAGE environment variable
      run: echo "RUN_COVERAGE=true" >> $GITHUB_ENV

    - name: Run Dockerized tests
      run: |
          MLRUN_VERSION=${{ github.sha }} \
          MLRUN_DOCKER_REGISTRY=ghcr.io/ \
          MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} \
          MLRUN_PYTHON_VERSION=${{ matrix.python-version }} \
          MLRUN_TEST_DB=${{ matrix.test-db }} \
          RUN_COVERAGE=$RUN_COVERAGE \
          make test-integration-dockerized
      timeout-minutes: 90

    - name: Upload coverage report
      uses: actions/upload-artifact@v7
      if: env.RUN_COVERAGE == 'true'
      with:
         name: coverage_integration_tests_report_${{ matrix.python-version }}_${{ matrix.test-db }}_${{ github.run_id }}.coverage
         path: /tmp/coverage_reports/integration_tests/integration_tests.coverage
         retention-days: 30

  migrations-tests:
    name: Run Dockerized Migrations Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      # TODO: re-use the next 2 actions instead of duplicating
      - name: Extract git branch
        id: git_info
        run: |
          echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
      - name: Resolve docker cache tag
        id: docker_cache
        run: |
          export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
          export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
          echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
      - name: Run Dockerized DB Migration tests
        run: |
          MLRUN_VERSION=${{ github.sha }} \
          MLRUN_DOCKER_REGISTRY=ghcr.io/ \
          MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} \
          make test-migrations-dockerized RUN_COVERAGE=true
      - name: Upload coverage report
        uses: actions/upload-artifact@v7
        with:
          name: coverage_migration_tests_report-${{ github.run_id }}.coverage
          path: /tmp/coverage_reports/migration_tests/migration_tests.coverage
          retention-days: 30

  package-tests:
    name: Run package tests (Python ${{ matrix.python-version }}, Pipelines ${{ matrix.pipeline-adapter }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - python-version: "3.11"
            pipeline-adapter: "kfp-v1-8"
    env:
      MLRUN_PYTHON_PACKAGE_INSTALLER: uv
    steps:
    - uses: actions/checkout@v6

    - name: Set up python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

    - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # 7.3.1
      with:
          python-version: ${{ matrix.python-version }}
          enable-cache: true

    - name: Install automation scripts dependencies and add mlrun to dev packages
      run: |
        uv venv venv --seed
        source venv/bin/activate
        make install-automation-requirements
        uv pip install -e .

    - name: Test package
      run: |
        source venv/bin/activate
        make test-package
      env:
        MLRUN_PYTHON_VERSION: ${{ matrix.python-version }}

  docs:
    name: Build Project Documentation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      # TODO: re-use the next 2 actions instead of duplicating
      - name: Extract git branch
        id: git_info
        run: |
          echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
      - name: Resolve docker cache tag
        id: docker_cache
        run: |
          export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
          export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
          echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
      - name: Generate HTML docs
        run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make html-docs-dockerized
      - name: Upload generated docs
        uses: actions/upload-artifact@v7
        with:
          name: docs
          path: docs/_build/html

  backward-compatibility-tests:
    name: Run Backward Compatibility Tests
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    env:
      MLRUN_BC_TESTS_HOME_PATH: /home/runner/work/mlrun/mlrun
      MLRUN_BC_TESTS_BASE_CODE_PATH: /home/runner/work/mlrun/mlrun/base/mlrun
      MLRUN_BC_TESTS_HEAD_CODE_PATH: /home/runner/work/mlrun/mlrun/head/mlrun
      MLRUN_DOCKER_REGISTRY: ghcr.io/
    steps:
      - id: resolve_base_branch
        run: |
            echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
      - name: Checkout PR Base (target) Branch
        uses: actions/checkout@v6
        with:
          ref: ${{ steps.resolve_base_branch.outputs.branch }}
          path: base/mlrun
      - name: Checkout Merge Commit (requested branch merged with the target branch)
        uses: actions/checkout@v6
        with:
          path: head/mlrun
      - name: Resolve docker cache tag
        id: docker_cache
        run: |
          export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
          export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
          echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
      - name: Run Backward Compatibility Tests
        run: |
          cd head/mlrun
          MLRUN_VERSION=${{ github.sha }} \
          MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} \
          make test-backward-compatibility-dockerized

  verify-compatible-requirements-one-after-one:
    name: Verify compatible requirements (one after one)
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11"]
    steps:
    - uses: actions/checkout@v6
    - name: Set up python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install requirements ${{ matrix.python-version }}
      run: MLRUN_PYTHON_VERSION=${{ matrix.python-version }} make install-requirements
    - name: Install Complete Requirements ${{ matrix.python-version }}
      run: MLRUN_PYTHON_VERSION=${{ matrix.python-version }} make install-complete-requirements
    - name: Install All Requirements ${{ matrix.python-version }}
      run: MLRUN_PYTHON_VERSION=${{ matrix.python-version }} make install-all-requirements

  coverage:
    name: Full coverage
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ "3.11"]
    needs: [tests, integration-tests, migrations-tests]
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: Install dependencies
        # To save time, install only pytest and coverage-related packages, without mlrun and its dependencies.
        run: make install-dev-requirements
      - name: Download server unit tests coverage report
        uses: actions/download-artifact@v8
        with:
          name: coverage_unit_tests_report_${{ matrix.python-version }}_${{ github.run_id }}_server.coverage
          path: tests/coverage_reports/
      - name: Download client unit tests coverage report
        uses: actions/download-artifact@v8
        with:
          name: coverage_unit_tests_report_${{ matrix.python-version }}_${{ github.run_id }}_client.coverage
          path: tests/coverage_reports/
      - name: Download integration coverage report
        uses: actions/download-artifact@v8
        with:
          path: tests/coverage_reports/

      - name: Download migration coverage report
        uses: actions/download-artifact@v8
        with:
          name: coverage_migration_tests_report-${{ github.run_id }}.coverage
          path: tests/coverage_reports/
      - name: Combine coverage
        run: make coverage-combine UNIT_TEST_COVERAGE_PATHS="tests/coverage_reports/unit_tests_client.coverage tests/coverage_reports/unit_tests_server.coverage"
        # Save it as an artifact:
      - name: Upload coverage combined report
        uses: actions/upload-artifact@v7
        with:
          name: coverage_combined_report-${{ github.run_id }}.coverage
          path: tests/coverage_reports/combined.coverage
          retention-days: 30
      - name: Create diff coverage report
        run: |
          diff-cover tests/coverage_reports/combined.xml \
          --compare-branch origin/development \
          --show-uncovered \
          --html-report tests/coverage_reports/combined_diff.html \
          --markdown-report tests/coverage_reports/diff_coverage.md

      - name: Add coverage to job summary
        if: github.event_name == 'pull_request'
        run: |
          {
            echo "## 📊 Diff Coverage Report"
            echo ""
            echo "<details>"
            echo "<summary>📂 Click to view full coverage details</summary>"
            echo ""
            cat tests/coverage_reports/diff_coverage.md
            echo ""
            echo "</details>"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload markdown coverage report for PR comment
        if: github.event_name == 'pull_request'
        uses: actions/upload-artifact@v7
        with:
          name: coverage-markdown-report
          path: tests/coverage_reports/diff_coverage.md
          retention-days: 30

      - name: Upload coverage html diff report
        uses: actions/upload-artifact@v7
        with:
          name: coverage_combined_diff_report_${{ matrix.python-version }}_${{ github.run_id }}.html
          path: tests/coverage_reports/combined_diff.html
          retention-days: 30

  label-oss-system-test-unknown:
    name: Label PR with Smoke Unknown
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Set Smoke tests Unknown label
        run: |
          PR="${{ github.event.pull_request.number }}"
          API_URL="https://api.github.com/repos/${{ github.repository }}/issues/${PR}/labels"

          REPO_LABELS_URL="https://api.github.com/repos/${{ github.repository }}/labels"

          # Ensure the Unknown label exists with gray color
          curl -sS -X PATCH \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            -d '{"color":"c5def5"}' \
            "${REPO_LABELS_URL}/Smoke tests%3A%20Unknown" || \
          curl -sS -X POST \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            -d '{"name":"Smoke tests: Unknown","color":"c5def5"}' \
            "${REPO_LABELS_URL}"

          # Remove Pass and Fail labels (ignore errors if not present)
          curl -sS -X DELETE \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            "${API_URL}/Smoke tests%3A%20Pass" || true
          curl -sS -X DELETE \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            "${API_URL}/Smoke tests%3A%20Fail" || true

          # Add Unknown label
          curl -sS -X POST \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            -d '{"labels":["Smoke tests: Unknown"]}' \
            "${API_URL}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}