From 26c3c4e43320d4f2bec23572433bcfdbb712890d Mon Sep 17 00:00:00 2001 From: Kai Graeper Date: Wed, 18 Mar 2026 23:05:30 +0100 Subject: [PATCH 1/4] add unified workflow --- .github/workflows/known_good_correct.yml | 2 +- ...yml => reusable_build_and_test_autosd.yml} | 29 ++- ...ml => reusable_build_and_test_ebclfsa.yml} | 29 ++- ....yml => reusable_build_and_test_linux.yml} | 28 ++- ...nx.yml => reusable_build_and_test_qnx.yml} | 29 ++- ...=> reusable_codeql-multiple-repo-scan.yml} | 19 +- .../workflows/reusable_integration-build.yml | 4 +- ...e-test.yml => reusable_smoke-test-bak.yml} | 4 +- .github/workflows/reusable_sw-check.yml | 209 ++++++++++++++++++ ...nd_docs.yml => reusable_test_and_docs.yml} | 43 ++-- .../sw-auto-update-deps-workflow.yml | 56 +++++ .github/workflows/sw-pr-workflow.yml | 29 +++ .github/workflows/test_integration.yml | 56 +++-- scripts/integration_test.py | 16 +- .../known_good_to_workspace_metadata.py | 7 +- .../known_good/override_known_good_repo.py | 8 +- .../update_module_from_known_good.py | 9 +- scripts/known_good/update_module_latest.py | 68 +++--- 18 files changed, 523 insertions(+), 122 deletions(-) rename .github/workflows/{build_and_test_autosd.yml => reusable_build_and_test_autosd.yml} (83%) rename .github/workflows/{build_and_test_ebclfsa.yml => reusable_build_and_test_ebclfsa.yml} (69%) rename .github/workflows/{build_and_test_linux.yml => reusable_build_and_test_linux.yml} (69%) rename .github/workflows/{build_and_test_qnx.yml => reusable_build_and_test_qnx.yml} (79%) rename .github/workflows/{codeql-multiple-repo-scan.yml => reusable_codeql-multiple-repo-scan.yml} (92%) rename .github/workflows/{reusable_smoke-test.yml => reusable_smoke-test-bak.yml} (98%) create mode 100644 .github/workflows/reusable_sw-check.yml rename .github/workflows/{test_and_docs.yml => reusable_test_and_docs.yml} (88%) create mode 100644 .github/workflows/sw-auto-update-deps-workflow.yml create mode 100644 .github/workflows/sw-pr-workflow.yml diff --git a/.github/workflows/known_good_correct.yml b/.github/workflows/known_good_correct.yml index 094db421b30..47516ff44b1 100644 --- a/.github/workflows/known_good_correct.yml +++ b/.github/workflows/known_good_correct.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v6.0.2 - name: Check run: | scripts/known_good/update_module_from_known_good.py --known known_good.json --output-dir-modules bazel_common diff --git a/.github/workflows/build_and_test_autosd.yml b/.github/workflows/reusable_build_and_test_autosd.yml similarity index 83% rename from .github/workflows/build_and_test_autosd.yml rename to .github/workflows/reusable_build_and_test_autosd.yml index 293b8eb21e4..4afeb31add8 100644 --- a/.github/workflows/build_and_test_autosd.yml +++ b/.github/workflows/reusable_build_and_test_autosd.yml @@ -12,13 +12,14 @@ # ******************************************************************************* name: AutoSD - Build & Integration Test on: - pull_request: - types: [opened, reopened, synchronize] - merge_group: - types: [checks_requested] - push: - branches: - - main + workflow_call: + # pull_request: + # types: [opened, reopened, synchronize] + # merge_group: + # types: [checks_requested] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -34,7 +35,19 @@ jobs: - name: Clean disk space uses: eclipse-score/more-disk-space@v1 - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 + - name: download known_good.updated.json artifact + if: github.event_name == 'workflow_call' + uses: actions/download-artifact@v4 + with: + name: known_good.updated.json + - name: update MODULE.bazel files + if: github.event_name == 'workflow_call' + run: | + echo "::group::update bazel_common/score_modules_*.MODULE.bazel" + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" - name: Setup Bazel uses: bazel-contrib/setup-bazel@0.18.0 with: diff --git a/.github/workflows/build_and_test_ebclfsa.yml b/.github/workflows/reusable_build_and_test_ebclfsa.yml similarity index 69% rename from .github/workflows/build_and_test_ebclfsa.yml rename to .github/workflows/reusable_build_and_test_ebclfsa.yml index c0390351546..4d4da3ddfd0 100644 --- a/.github/workflows/build_and_test_ebclfsa.yml +++ b/.github/workflows/reusable_build_and_test_ebclfsa.yml @@ -12,13 +12,14 @@ # ******************************************************************************* name: EB corbos Linux for Safety Applications - Build & Integration Test on: - pull_request: - types: [opened, reopened, synchronize] - merge_group: - types: [checks_requested] - push: - branches: - - main + workflow_call: + # pull_request: + # types: [opened, reopened, synchronize] + # merge_group: + # types: [checks_requested] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -33,7 +34,19 @@ jobs: - name: Clean disk space uses: eclipse-score/more-disk-space@v1 - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 + - name: download known_good.updated.json artifact + if: github.event_name == 'workflow_call' + uses: actions/download-artifact@v4 + with: + name: known_good.updated.json + - name: update MODULE.bazel files + if: github.event_name == 'workflow_call' + run: | + echo "::group::update bazel_common/score_modules_*.MODULE.bazel" + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" - name: Setup Bazel uses: bazel-contrib/setup-bazel@0.18.0 with: diff --git a/.github/workflows/build_and_test_linux.yml b/.github/workflows/reusable_build_and_test_linux.yml similarity index 69% rename from .github/workflows/build_and_test_linux.yml rename to .github/workflows/reusable_build_and_test_linux.yml index 39f9fa2527f..e13907e1b3d 100644 --- a/.github/workflows/build_and_test_linux.yml +++ b/.github/workflows/reusable_build_and_test_linux.yml @@ -12,13 +12,13 @@ # ******************************************************************************* name: Linux - Build & Integration Test on: - workflow_dispatch: - pull_request: - release: - types: [created] - push: - branches: - - main + workflow_call: + # pull_request: + # release: + # types: [created] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -32,7 +32,19 @@ jobs: - name: Clean disk space uses: eclipse-score/more-disk-space@v1 - name: Checkout repository - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v6.0.2 + - name: download known_good.updated.json artifact + if: github.event_name == 'workflow_call' + uses: actions/download-artifact@v4 + with: + name: known_good.updated.json + - name: update MODULE.bazel files + if: github.event_name == 'workflow_call' + run: | + echo "::group::update bazel_common/score_modules_*.MODULE.bazel" + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.8.0 - name: Setup Bazel diff --git a/.github/workflows/build_and_test_qnx.yml b/.github/workflows/reusable_build_and_test_qnx.yml similarity index 79% rename from .github/workflows/build_and_test_qnx.yml rename to .github/workflows/reusable_build_and_test_qnx.yml index b45b09e1420..472c7d7a07a 100644 --- a/.github/workflows/build_and_test_qnx.yml +++ b/.github/workflows/reusable_build_and_test_qnx.yml @@ -12,13 +12,14 @@ # ******************************************************************************* name: QNX8 - Build & Integration Test on: - pull_request_target: - types: [opened, reopened, synchronize] - merge_group: - types: [checks_requested] - push: - branches: - - main + workflow_call: + # pull_request_target: + # types: [opened, reopened, synchronize] + # merge_group: + # types: [checks_requested] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -35,10 +36,22 @@ jobs: - name: Clean disk space uses: eclipse-score/more-disk-space@v1 - name: Checkout repository (Handle all events) - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v6.0.2 with: ref: ${{ github.head_ref || github.event.pull_request.head.ref || github.ref }} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + - name: download known_good.updated.json artifact + if: github.event_name == 'workflow_call' + uses: actions/download-artifact@v4 + with: + name: known_good.updated.json + - name: update MODULE.bazel files + if: github.event_name == 'workflow_call' + run: | + echo "::group::update bazel_common/score_modules_*.MODULE.bazel" + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" - name: Setup Bazel with shared caching uses: bazel-contrib/setup-bazel@0.18.0 with: diff --git a/.github/workflows/codeql-multiple-repo-scan.yml b/.github/workflows/reusable_codeql-multiple-repo-scan.yml similarity index 92% rename from .github/workflows/codeql-multiple-repo-scan.yml rename to .github/workflows/reusable_codeql-multiple-repo-scan.yml index a22531153b2..4be41a79396 100644 --- a/.github/workflows/codeql-multiple-repo-scan.yml +++ b/.github/workflows/reusable_codeql-multiple-repo-scan.yml @@ -12,13 +12,14 @@ # ******************************************************************************* name: "CodeQL - Multi-Repo Source Scan" on: - pull_request: - types: [opened, reopened, synchronize] - merge_group: - types: [checks_requested] - push: - branches: - - main + # pull_request: + # types: [opened, reopened, synchronize] + # merge_group: + # types: [checks_requested] + # push: + # branches: + # - main + workflow_call: release: types: [created] workflow_dispatch: @@ -39,9 +40,9 @@ jobs: contents: read steps: - name: Checkout central repository - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 - name: Checkout CodeQL Coding Standards scripts - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 with: repository: github/codeql-coding-standards path: codeql-coding-standards-repo # Klonen in diesen Ordner diff --git a/.github/workflows/reusable_integration-build.yml b/.github/workflows/reusable_integration-build.yml index c3811fb34d1..c52236b4ada 100644 --- a/.github/workflows/reusable_integration-build.yml +++ b/.github/workflows/reusable_integration-build.yml @@ -96,7 +96,7 @@ jobs: - name: Clean disk space uses: eclipse-score/more-disk-space@v1 - name: Checkout repository - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v6.0.2 with: repository: ${{ env.REFERENCE_INTEGRATION_REPO }} ref: ${{ inputs.target_branch || 'main' }} @@ -121,7 +121,7 @@ jobs: echo "::endgroup::" echo "::group::update score_modules.MODULE.bazel" - python3 scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir bazel_common/ + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common cat bazel_common/score_modules.MODULE.bazel echo "::endgroup::" env: diff --git a/.github/workflows/reusable_smoke-test.yml b/.github/workflows/reusable_smoke-test-bak.yml similarity index 98% rename from .github/workflows/reusable_smoke-test.yml rename to .github/workflows/reusable_smoke-test-bak.yml index afc095df06d..d42e109592d 100644 --- a/.github/workflows/reusable_smoke-test.yml +++ b/.github/workflows/reusable_smoke-test-bak.yml @@ -93,7 +93,7 @@ jobs: known_good_updated: ${{ steps.set_known_good.outputs.known_good_updated }} steps: - name: Checkout repository - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v6.0.2 with: repository: ${{ env.REFERENCE_INTEGRATION_REPO }} ref: ${{ inputs.target_branch }} @@ -163,7 +163,7 @@ jobs: if: always() steps: - name: Checkout repository - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v6.0.2 with: repository: ${{ env.REFERENCE_INTEGRATION_REPO }} ref: ${{ inputs.target_branch }} diff --git a/.github/workflows/reusable_sw-check.yml b/.github/workflows/reusable_sw-check.yml new file mode 100644 index 00000000000..e82bc543f33 --- /dev/null +++ b/.github/workflows/reusable_sw-check.yml @@ -0,0 +1,209 @@ +# Module Smoke Test Workflow +# +# Summary: +# Orchestrates validation of modules by producing a known_good.updated.json +# and then invoking the integration build workflow to compile and test. +# +# Modes of operation (controlled by inputs): +# 1. External module PR (github.repository != reference_integration_repo): +# Overrides `module_name` to the PR commit SHA. +# 2. Nightly / latest (update_to_latest=true): +# Updates all modules to latest HEAD on their branch +# (via update_module_latest.py). +# 3. PR / pinned (update_to_latest=false, default): +# Uses the repository's known_good.json as-is. +# +# What it does: +# - Checks out the reference integration repository +# - Generates known_good.updated.json (see modes above) +# - Uploads known_good.updated.json as an artifact +# - Calls the Module Integration Build workflow with a matrix of configs +# - Publishes a summary to the GitHub Step Summary +# +# Inputs: +# - repo_runner_labels (string, required, default: ubuntu-latest): Runner label. +# - module_name (string, optional): Module to override for external PRs. +# - update_to_latest (boolean, optional, default: false): When true, updates +# all modules to the latest commit on their branch. Used for nightly builds. +# - target_branch (string, required, default: main): +# The ref to checkout via actions/checkout — can be a branch name, tag, or +# commit SHA. This ensures the workflow uses the exact version of the +# integration files you intend. +# +# Repository Variables: +# - reference_integration_repo (optional): Repository providing integration +# workflows and tools (format: owner/repo). Supports private forks. +# Default: eclipse-score/reference_integration +# +# Secrets: +# - REPO_READ_TOKEN (optional): Token for reading private repos; falls back to +# github.token when not provided. +# +# Usage: +# This workflow is reusable and triggered via workflow_call from other workflows. +# +# External module PR: +# jobs: +# smoke: +# uses: eclipse-score/reference_integration/.github/workflows/reusable_sw-check.yml@main +# with: +# module_name: score_baselibs +# target_branch: main +# secrets: +# REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} +# +# Nightly (latest HEAD): +# jobs: +# nightly: +# uses: ./.github/workflows/reusable_sw-check.yml +# with: +# update_to_latest: true +# target_branch: main +# +# PR (pinned known_good.json): +# jobs: +# pr: +# uses: ./.github/workflows/reusable_sw-check.yml +# with: +# target_branch: ${{ github.ref }} +# +# Note: Set the 'reference_integration_repo' repository variable to use a +# private fork (e.g., my-org/reference_integration). +# +# Notes: +# - Extend the matrix in `integration-test` to cover additional configs. +name: Module Smoke Test +on: + workflow_call: + inputs: + repo_runner_labels: + description: 'The runner tag to use for the job' + required: true + type: string + default: 'ubuntu-latest' + module_name: + description: 'Name of the module to override (e.g., score_baselibs). Required for external module PRs.' + required: false + type: string + default: '' + update_to_latest: + description: 'When true, update all modules to latest HEAD on their branch (nightly mode).' + required: false + type: boolean + default: false + target_branch: + description: 'Ref to checkout (branch, tag, or commit SHA).' + required: true + type: string + default: 'main' + outputs: + known_good_updated_file: + description: "Updated known_good.json content" + value: "known_good.updated.json" + secrets: + REPO_READ_TOKEN: + description: 'Token for reading repositories' + required: false +permissions: + contents: write + pull-requests: write + pages: write + id-token: write +env: + REFERENCE_INTEGRATION_REPO: ${{ vars.reference_integration_repo != '' && vars.reference_integration_repo || 'eclipse-score/reference_integration' }} +jobs: + preparation: + name: Preparation + runs-on: ubuntu-latest + outputs: + known_good_updated: ${{ steps.set_known_good.outputs.known_good_updated }} + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.2 + with: + repository: ${{ env.REFERENCE_INTEGRATION_REPO }} + ref: ${{ inputs.target_branch }} + token: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Create updated known_good.json + id: set_known_good + run: | + if [ "${{ github.repository }}" != "${{ env.REFERENCE_INTEGRATION_REPO }}" ]; then + # External module PR: override the specific module to the PR commit + echo "Overriding ${{ inputs.module_name }} with current PR" + python3 scripts/known_good/override_known_good_repo.py \ + --known known_good.json \ + --output known_good.updated.json \ + --module-override ${{ inputs.module_name }}@${{ github.event.repository.clone_url }}@${{ github.sha }} + elif [ "${{ inputs.update_to_latest }}" = "true" ]; then + # Nightly mode: update all modules to latest HEAD on their branch + echo "Updating all modules to latest commits" + echo "::group::get latest commits from module branches" + python3 scripts/known_good/update_module_latest.py --output known_good.updated.json + cat known_good.updated.json + echo "::endgroup::" + else + # PR mode: use the pinned known_good.json from the repository + echo "Using pinned known_good.json from repository" + cp known_good.json known_good.updated.json + fi + env: + GITHUB_TOKEN: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Show updated known_good.json and MODULE.bazel files + run: | + echo "::group::Updated known_good.updated.json" + cat known_good.updated.json + echo "::endgroup::" + + echo "::group::Updated bazel_common/score_modules_*.MODULE.bazel" + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" + - name: Upload updated known_good.json artifact + uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: known_good.updated.json + path: known_good.updated.json + utest: + name: Unit Tests + needs: preparation + # uses: ./.github/workflows/reusable_test_and_docs.yml + # secrets: inherit + runs-on: ubuntu-latest + steps: + - name: not implemented + run: echo "separate doc workflow" + docs: + name: Generate Documentation + needs: utest + steps: + - name: not implemented + run: echo "separate doc workflow" + build_and_test_autosd: + name: build_and_test_autosd + needs: docs + uses: ./.github/workflows/reusable_build_and_test_autosd.yml + secrets: inherit + build_and_test_ebclfsa: + name: build_and_test_ebclfsa + needs: docs + uses: ./.github/workflows/reusable_build_and_test_ebclfsa.yml + secrets: inherit + build_and_test_linux: + name: build_and_test_linux + needs: docs + uses: ./.github/workflows/reusable_build_and_test_linux.yml + secrets: inherit + build_and_test_qnx: + name: build_and_test_qnx + needs: docs + uses: ./.github/workflows/reusable_build_and_test_qnx.yml + secrets: inherit + summary: + name: Publish Summary + runs-on: ubuntu-latest + needs: [build_and_test_autosd, build_and_test_ebclfsa, build_and_test_linux, build_and_test_qnx] + if: always() + steps: + - name: summary + run: | + echo "run publish_integration_summary.py" diff --git a/.github/workflows/test_and_docs.yml b/.github/workflows/reusable_test_and_docs.yml similarity index 88% rename from .github/workflows/test_and_docs.yml rename to .github/workflows/reusable_test_and_docs.yml index 6017213b8d7..3b9345d7ef1 100644 --- a/.github/workflows/test_and_docs.yml +++ b/.github/workflows/reusable_test_and_docs.yml @@ -11,19 +11,20 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* name: Code Quality & Documentation -permissions: - contents: write - pages: write - pull-requests: write - id-token: write +# permissions: +# contents: write +# pages: write +# pull-requests: write +# id-token: write on: - pull_request_target: # Allows forks to trigger the docs build - types: [opened, reopened, synchronize] - push: - branches: - - main - merge_group: - types: [checks_requested] + workflow_call: + # pull_request_target: # Allows forks to trigger the docs build + # types: [opened, reopened, synchronize] + # push: + # branches: + # - main + # merge_group: + # types: [checks_requested] release: types: [created] # Do not flood CI with unneeded previous runs in PR @@ -58,13 +59,25 @@ jobs: sudo apt-get install -y lcov - name: Checkout repository (pull_request_target via workflow_call) if: ${{ github.event_name == 'pull_request_target' }} - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 with: ref: ${{ github.head_ref || github.event.pull_request.head.ref || github.ref }} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - name: Checkout repository if: ${{ github.event_name != 'pull_request_target' }} - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 + - name: download known_good.updated.json artifact + if: github.event_name == 'workflow_call' + uses: actions/download-artifact@v4 + with: + name: known_good.updated.json + - name: update MODULE.bazel files + if: github.event_name == 'workflow_call' + run: | + echo "::group::update bazel_common/score_modules_*.MODULE.bazel" + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" - name: Execute Unit Tests with Coverage Analysis run: | python ./scripts/quality_runners.py @@ -189,7 +202,7 @@ jobs: echo "gh-pages branch exists. Skipping creation." fi - name: Checkout repository - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v6.0.2 - name: Download documentation artifact uses: actions/download-artifact@v4.1.8 with: diff --git a/.github/workflows/sw-auto-update-deps-workflow.yml b/.github/workflows/sw-auto-update-deps-workflow.yml new file mode 100644 index 00000000000..0c285a7d800 --- /dev/null +++ b/.github/workflows/sw-auto-update-deps-workflow.yml @@ -0,0 +1,56 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# Nightly / manual integration tests: updates all modules to latest HEAD on their branch. +name: Integration Tests (Nightly) +on: + workflow_dispatch: + schedule: + - cron: '30 2 * * *' # Every night at 02:30 UTC on main-etas branch +jobs: + sw-check: + uses: ./.github/workflows/reusable_sw-check.yml + secrets: inherit + with: + update_to_latest: true + repo_runner_labels: 'ubuntu-latest' + target_branch: '${{ github.ref }}' + update_known_good: + name: update_known_good + runs-on: ubuntu-latest + needs: sw-check + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + - name: download known_good.updated.json artifact + if: github.event_name == 'workflow_call' + uses: actions/download-artifact@v4 + with: + name: known_good.updated.json + - name: update MODULE.bazel files + if: github.event_name == 'workflow_call' + run: | + echo "::group::update bazel_common/score_modules_*.MODULE.bazel" + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" + # - name: push changes if there are any + # run: | + # git config user.name "github-actions[bot]" + # git config user.email "github-actions[bot]@users.noreply.github.com" + # if [[ -n $(git status --porcelain) ]]; then + # git add . + # git commit -m "Update known_good.json and MODULE.bazel files to latest HEAD" + # git push + # else + # echo "No changes to push" + # fi diff --git a/.github/workflows/sw-pr-workflow.yml b/.github/workflows/sw-pr-workflow.yml new file mode 100644 index 00000000000..7f56cdb5ab4 --- /dev/null +++ b/.github/workflows/sw-pr-workflow.yml @@ -0,0 +1,29 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# Workflow configuration for S-CORE CI - Bazel Build & Test baselibs +# This workflow runs Bazel build and test when triggered by specific pull request events. +name: Integration Tests (PR) +on: + workflow_dispatch: + pull_request_target: + types: [opened, reopened, synchronize] +jobs: + codeql-scan: + uses: ./.github/workflows/reusable_codeql-multiple-repo-scan.yml + secrets: inherit + integration_test: + uses: ./.github/workflows/reusable_sw-check.yml + secrets: inherit + with: + repo_runner_labels: 'ubuntu-latest' + target_branch: '${{ github.ref }}' diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index 34a961e529f..e4d551679e5 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -1,30 +1,44 @@ +# to be deleted after the first successful run, as this workflow is now reusable and can be called from other workflows # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -# Workflow configuration for S-CORE CI - Bazel Build & Test baselibs -# This workflow runs Bazel build and test when triggered by specific pull request events. -name: build latest mains +name: Integration Tests (Nightly) on: workflow_dispatch: pull_request: + types: [opened, reopened, synchronize] push: - # schedule: - # - cron: '30 2 * * *' # Every night at 02:30 UTC on main branch +permissions: + contents: write + pull-requests: write + pages: write + id-token: write + actions: read + packages: read + security-events: write jobs: - integration_test: - if: false # Temporarily disabled until we get things sorted out - uses: ./.github/workflows/reusable_smoke-test.yml + sw-check: + uses: ./.github/workflows/reusable_sw-check.yml secrets: inherit with: + update_to_latest: true repo_runner_labels: 'ubuntu-latest' - module_name: 'reference_integration' - target_branch: ${{ github.ref }} + target_branch: '${{ github.ref }}' + codeql-scan: + uses: ./.github/workflows/reusable_codeql-multiple-repo-scan.yml + secrets: inherit + update_known_good: + name: update_known_good + runs-on: ubuntu-latest + needs: sw-check + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + - name: download known_good.updated.json artifact + uses: actions/download-artifact@v4 + with: + name: known_good.updated.json + - name: update MODULE.bazel files + run: |- + echo "::group::update bazel_common/score_modules_*.MODULE.bazel" + scripts/known_good/update_module_from_known_good.py --known known_good.updated.json --output-dir-modules bazel_common + cat bazel_common/score_modules_*.MODULE.bazel + echo "::endgroup::" diff --git a/scripts/integration_test.py b/scripts/integration_test.py index 219905689c7..b03f5daf4a4 100755 --- a/scripts/integration_test.py +++ b/scripts/integration_test.py @@ -300,14 +300,24 @@ def main(): log_dir.mkdir(parents=True, exist_ok=True) summary_file.parent.mkdir(parents=True, exist_ok=True) - # Load modules from known_good files + # Load modules from known_good files (flatten grouped structure to flat module dict) + def _flatten_modules(grouped: Dict[str, Dict[str, Module]]) -> Dict[str, Module]: + flat: Dict[str, Module] = {} + for group_modules in grouped.values(): + flat.update(group_modules) + return flat + try: - old_modules = load_known_good(Path("known_good.json")).modules if Path("known_good.json").exists() else {} + old_modules = ( + _flatten_modules(load_known_good(Path("known_good.json")).modules) + if Path("known_good.json").exists() + else {} + ) except FileNotFoundError: old_modules = {} try: - new_modules = load_known_good(known_good_file).modules if known_good_file else {} + new_modules = _flatten_modules(load_known_good(known_good_file).modules) if known_good_file else {} except FileNotFoundError as e: raise SystemExit(f"ERROR: {e}") diff --git a/scripts/known_good/known_good_to_workspace_metadata.py b/scripts/known_good/known_good_to_workspace_metadata.py index bb933c8dcca..97f74916c4f 100644 --- a/scripts/known_good/known_good_to_workspace_metadata.py +++ b/scripts/known_good/known_good_to_workspace_metadata.py @@ -42,10 +42,13 @@ def main(): except ValueError as e: raise SystemExit(f"ERROR: {e}") - modules = list(known_good.modules.values()) + # Flatten modules from all groups + all_modules = [] + for group_modules in known_good.modules.values(): + all_modules.extend(group_modules.values()) gita_metadata = [] - for module in modules: + for module in all_modules: if not module.repo: raise RuntimeError(f"Module {module.name}: repo must not be empty") diff --git a/scripts/known_good/override_known_good_repo.py b/scripts/known_good/override_known_good_repo.py index 218d360d0a3..1b678ef42d2 100755 --- a/scripts/known_good/override_known_good_repo.py +++ b/scripts/known_good/override_known_good_repo.py @@ -142,8 +142,14 @@ def apply_overrides(known_good: KnownGood, repo_overrides: List[str]) -> KnownGo Returns: Updated KnownGood instance """ + # Flatten grouped modules into flat dict for override lookup + # (Module instances are shared by reference, so changes propagate back to known_good) + flat_modules: Dict[str, Module] = {} + for group_modules in known_good.modules.values(): + flat_modules.update(group_modules) + # Parse and apply overrides - overrides_applied = parse_and_apply_overrides(known_good.modules, repo_overrides) + overrides_applied = parse_and_apply_overrides(flat_modules, repo_overrides) if overrides_applied == 0: logging.warning("No overrides were applied to any modules") diff --git a/scripts/known_good/update_module_from_known_good.py b/scripts/known_good/update_module_from_known_good.py index e6c82fe3cb3..7e86db7ce5e 100755 --- a/scripts/known_good/update_module_from_known_good.py +++ b/scripts/known_good/update_module_from_known_good.py @@ -56,11 +56,10 @@ def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[s # Generate patches lines if bazel_patches exist patches_lines = "" if module.bazel_patches: - patches_lines = " patches = [\n" + patches_lines = " patch_strip = 1,\n patches = [\n" for patch in module.bazel_patches: patches_lines += f' "{patch}",\n' patches_lines += " ],\n" - patch_strip_line = " patch_strip = 1,\n" if patches_lines else "" if module.version: # If version is provided, use bazel_dep with single_version_override @@ -68,9 +67,8 @@ def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[s f'bazel_dep(name = "{module.name}")\n' "single_version_override(\n" f' module_name = "{module.name}",\n' - f"{patch_strip_line}" - f"{patches_lines}" f' version = "{module.version}",\n' + f"{patches_lines}" ")\n" ) else: @@ -93,17 +91,16 @@ def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[s continue # If no version, use bazel_dep with git_override - # Only include patch_strip if there are patches to apply block = ( f'bazel_dep(name = "{module.name}")\n' "git_override(\n" f' module_name = "{module.name}",\n' f' commit = "{commit}",\n' - f"{patch_strip_line}" f"{patches_lines}" f' remote = "{module.repo}",\n' ")\n" ) + blocks.append(block) return blocks diff --git a/scripts/known_good/update_module_latest.py b/scripts/known_good/update_module_latest.py index 33e238c4f17..d9cc48d1620 100755 --- a/scripts/known_good/update_module_latest.py +++ b/scripts/known_good/update_module_latest.py @@ -22,8 +22,8 @@ Usage: python tools/update_module_latest.py \ - --known-good score_reference_integration/known_good.json \ - [--branch main] [--output updated_known_good.json] + --known-good score_reference_integration/known_good.json \ + [--branch main] [--output updated_known_good.json] Environment: Optionally set GITHUB_TOKEN to increase rate limits / access private repos. @@ -150,33 +150,45 @@ def main(argv: list[str]) -> int: if args.no_gh and shutil.which("gh") is not None: print("INFO: --no-gh specified; ignoring installed 'gh' CLI", file=sys.stderr) - for mod in known_good.modules.values(): - if mod.pin_version: - print(f"{mod.name}: pinned, skipping") - continue - - try: - branch = mod.branch if mod.branch else args.branch - if use_gh: - latest = fetch_latest_commit_gh(mod.owner_repo, branch) - else: - latest = fetch_latest_commit(mod.owner_repo, branch, token) - - old_hash = mod.hash - if latest != old_hash: - mod.hash = latest - mod.version = None # Clear version when hash changes - if mod.version: - print(f"{mod.name}: {mod.version} -> {latest[:8]} (branch {branch})") + for group_name, group_modules in known_good.modules.items(): + for mod in group_modules.values(): + if mod.pin_version: + print(f"{mod.name}: pinned, skipping") + continue + + # Skip version-only modules (single_version_override, no git hash) + if mod.version and not mod.hash: + print(f"{mod.name}: version override ({mod.version}), skipping") + continue + + # Skip modules without a repo URL + if not mod.repo: + print(f"{mod.name}: no repo URL, skipping") + continue + + try: + branch = mod.branch if mod.branch else args.branch + if use_gh: + latest = fetch_latest_commit_gh(mod.owner_repo, branch) + else: + latest = fetch_latest_commit(mod.owner_repo, branch, token) + + old_hash = mod.hash + if latest != old_hash: + old_version = mod.version + mod.hash = latest + mod.version = None # Clear version when hash changes + if old_version: + print(f"{mod.name}: {old_version} -> {latest[:8]} (branch {branch})") + else: + print(f"{mod.name}: {old_hash[:8]} -> {latest[:8]} (branch {branch})") else: - print(f"{mod.name}: {old_hash[:8]} -> {latest[:8]} (branch {branch})") - else: - print(f"{mod.name}: {old_hash[:8]} (no update)") - except Exception as e: # noqa: BLE001 - failures += 1 - print(f"ERROR {mod.name}: {e}", file=sys.stderr) - if args.fail_fast: - break + print(f"{mod.name}: {old_hash[:8]} (no update)") + except Exception as e: # noqa: BLE001 + failures += 1 + print(f"ERROR {mod.name}: {e}", file=sys.stderr) + if args.fail_fast: + break if args.output: try: From 7e897a148cf79fbf7ad0cad9618f0df5a4a7fe7d Mon Sep 17 00:00:00 2001 From: Kai Graeper Date: Wed, 18 Mar 2026 23:09:04 +0100 Subject: [PATCH 2/4] fix --- .github/workflows/reusable_sw-check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable_sw-check.yml b/.github/workflows/reusable_sw-check.yml index e82bc543f33..d07e2c51dd8 100644 --- a/.github/workflows/reusable_sw-check.yml +++ b/.github/workflows/reusable_sw-check.yml @@ -174,6 +174,7 @@ jobs: run: echo "separate doc workflow" docs: name: Generate Documentation + runs-on: ubuntu-latest needs: utest steps: - name: not implemented From 9cd2dbfdff1afda04a2634a1bf8f91f290542d76 Mon Sep 17 00:00:00 2001 From: Kai Graeper Date: Thu, 19 Mar 2026 08:25:29 +0100 Subject: [PATCH 3/4] start run --- .github/workflows/test_integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index e4d551679e5..d30754a5bc9 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -22,6 +22,7 @@ jobs: update_to_latest: true repo_runner_labels: 'ubuntu-latest' target_branch: '${{ github.ref }}' + codeql-scan: uses: ./.github/workflows/reusable_codeql-multiple-repo-scan.yml secrets: inherit From 98e48d211ac476c0f8729541eb721065d5fe1625 Mon Sep 17 00:00:00 2001 From: Kai Graeper Date: Thu, 19 Mar 2026 13:53:33 +0100 Subject: [PATCH 4/4] add concurenncy --- .github/workflows/test_integration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index d30754a5bc9..85aca24c788 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -6,6 +6,9 @@ on: pull_request: types: [opened, reopened, synchronize] push: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true permissions: contents: write pull-requests: write