From f533584d1589c1871054f181bbcd4b5427750421 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Thu, 12 Mar 2026 16:53:56 +0300 Subject: [PATCH 01/12] refactor: replace GH_TOKEN with GitHub App token in workflow files --- .../build-citus-community-nightlies.yml | 22 +++++++++++++-- .../citus-package-all-platforms-test.yml | 18 ++++++++++-- .github/workflows/package-tests.yml | 22 +++++++++++++++ .github/workflows/packaging-methods-tests.yml | 19 +++++++++++-- .../workflows/publish-docker-image-tests.yml | 15 ++++++++++ .github/workflows/statistic-schedule.yml | 15 ++++++++-- .github/workflows/statistic-tests.yml | 23 ++++++++++++++- .github/workflows/tool-tests.yml | 28 ++++++++++++++++++- 8 files changed, 151 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index e4773508..2bc9e35a 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -4,7 +4,6 @@ env: MAIN_BRANCH: "all-citus" PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} DOCKERHUB_USER_NAME: ${{ secrets.DOCKERHUB_USER_NAME }} DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} on: @@ -32,9 +31,22 @@ jobs: - ubuntu/jammy steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 1 path: tools @@ -46,7 +58,10 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Clone build branch - run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} + run: git clone -b "${MAIN_BRANCH}" --depth=1 https://gh-token:${GH_TOKEN}@github.com/citusdata/packaging.git packaging - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -55,6 +70,9 @@ jobs: run: python -m pip install -r tools/packaging_automation/requirements.txt - name: Build packages + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | python -m tools.packaging_automation.citus_package \ --gh_token "${GH_TOKEN}" \ diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index a7692477..d74779f6 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -1,8 +1,6 @@ name: Citus package all platforms tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay @@ -36,8 +34,21 @@ jobs: PLATFORM: ${{ matrix.platform }} steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -49,4 +60,7 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 0d401807..cfee9759 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -18,9 +18,18 @@ jobs: pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} citus_version: ${{ steps.get-citus-version.outputs.citus_version }} steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata - name: Checkout uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 2 - name: Package version id: get-citus-version @@ -60,8 +69,18 @@ jobs: PLATFORM: ${{ matrix.platform }} steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.GITHUB_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -70,6 +89,9 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | export PROJECT_VERSION="${{ github.event.inputs.project_version }}" echo "Citus Version: ${PROJECT_VERSION} " diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 7d53056f..c34623f7 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -1,8 +1,5 @@ name: Packaging helper methods tests -env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - on: push: branches: @@ -15,8 +12,21 @@ jobs: runs-on: ubuntu-latest steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -25,4 +35,7 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package_utils.py diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c14bc3d7..c20c1e62 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -12,8 +12,20 @@ jobs: runs-on: ubuntu-latest steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{steps.app.outputs.token}} + - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -22,4 +34,7 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Build and publish docker images tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_publish_docker.py diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 3dcb4b22..11d3b7f8 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} on: schedule: - cron: "0 16 * * *" @@ -25,8 +24,18 @@ jobs: job_name: [docker_pull_citus, github_clone_citus, homebrew_citus] steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -35,6 +44,8 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Execute 'Fetch Daily Statistics' - run: packaging_automation/bash/daily-statistics-job.sh env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} JOB_NAME: "${{ matrix.JOB_NAME }}" + run: packaging_automation/bash/daily-statistics-job.sh diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 5b9eb9ea..76a201c8 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} PACKAGE_CLOUD_ADMIN_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_ADMIN_API_TOKEN }} on: @@ -21,8 +20,18 @@ jobs: runs-on: ubuntu-latest steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -31,13 +40,25 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Unit tests for "Docker statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_docker_statistics_collector.py - name: Unit tests for "Github clone statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_github_statistics_collector.py - name: Unit tests for "Packagecloud download statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_package_cloud_statistics_collector.py - name: Unit tests for "Homebrew download statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_homebrew_statistics_collector.py diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 8759a7d5..a3d66756 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -1,7 +1,6 @@ name: Tool Tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay MAIN_BRANCH: all-citus @@ -27,9 +26,21 @@ jobs: runs-on: ubuntu-latest steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 0 - name: Set up Python 3.10 @@ -53,9 +64,15 @@ jobs: run: black . --check - name: Unit tests for "Common tools" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py - name: Unit tests for "Update Package Properties" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py # no longer viable, outdated test, skipping to not block the pipeline @@ -63,10 +80,19 @@ jobs: # run: python -m pytest -q packaging_automation/tests/test_prepare_release.py - name: Unit tests for "Update Docker" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_docker.py - name: Unit tests for "Update Pgxn" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py - name: Packaging Warning Handler + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py From 7891e7cbb535531a81e558273072a5793c95eb8d Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Thu, 12 Mar 2026 19:39:42 +0300 Subject: [PATCH 02/12] fix: update GitHub App token secret reference in workflow files --- .github/workflows/citus-package-all-platforms-test.yml | 2 +- .github/workflows/package-tests.yml | 2 +- .github/workflows/packaging-methods-tests.yml | 2 +- .github/workflows/publish-docker-image-tests.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index d74779f6..9e2cd6e7 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -39,7 +39,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata repositories: | tools diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index cfee9759..90eaab1c 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -24,7 +24,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index c34623f7..a82f1e3c 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -17,7 +17,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata repositories: | tools diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c20c1e62..41f34047 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -18,7 +18,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - name: Checkout repository From f17e33f558cc40ded9652caef2edf8fd99557ec4 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Sat, 14 Mar 2026 18:53:44 +0300 Subject: [PATCH 03/12] feat: update workflows to use GitHub App token and improve permissions --- .../workflows/build-citus-community-nightlies.yml | 12 ++++++------ .../workflows/citus-package-all-platforms-test.yml | 9 ++++----- .github/workflows/package-tests.yml | 7 ++++++- .github/workflows/packaging-methods-tests.yml | 10 +++++----- .github/workflows/publish-docker-image-tests.yml | 7 +++++-- .github/workflows/pypi-statistics-schedule.yml | 2 +- .github/workflows/statistic-schedule.yml | 6 +++++- .github/workflows/statistic-tests.yml | 10 +++++----- .github/workflows/tool-tests.yml | 14 +++++--------- packaging_automation/common_tool_methods.py | 3 ++- packaging_automation/tests/test_citus_package.py | 6 ++++-- .../tests/test_citus_package_utils.py | 5 ++++- packaging_automation/tests/test_prepare_release.py | 3 ++- packaging_automation/tests/test_publish_docker.py | 3 ++- packaging_automation/tests/test_update_docker.py | 5 ++++- packaging_automation/tests/test_update_pgxn.py | 5 ++++- 16 files changed, 64 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 2bc9e35a..b2baa007 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -18,6 +18,10 @@ jobs: build_package: name: Build package runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read strategy: fail-fast: false matrix: @@ -39,9 +43,6 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 @@ -49,6 +50,7 @@ jobs: token: ${{ steps.app.outputs.token }} fetch-depth: 1 path: tools + submodules: true # This step is to fetch the images unanonymously to have higher bandwidth - name: Login to Docker Hub @@ -60,8 +62,7 @@ jobs: - name: Clone build branch env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} - run: git clone -b "${MAIN_BRANCH}" --depth=1 https://gh-token:${GH_TOKEN}@github.com/citusdata/packaging.git packaging + run: git clone -b "${MAIN_BRANCH}" --depth=1 https://${GH_TOKEN}@github.com/citusdata/packaging.git packaging - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -72,7 +73,6 @@ jobs: - name: Build packages env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | python -m tools.packaging_automation.citus_package \ --gh_token "${GH_TOKEN}" \ diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 9e2cd6e7..54cfd327 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -17,6 +17,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read strategy: fail-fast: false matrix: @@ -40,10 +44,6 @@ jobs: with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 @@ -62,5 +62,4 @@ jobs: - name: Citus package tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 90eaab1c..a647e268 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -14,6 +14,10 @@ on: jobs: metadata: runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read outputs: pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} citus_version: ${{ steps.get-citus-version.outputs.citus_version }} @@ -31,6 +35,8 @@ jobs: with: token: ${{ steps.app.outputs.token }} fetch-depth: 2 + submodules: true + - name: Package version id: get-citus-version run: | @@ -91,7 +97,6 @@ jobs: - name: Citus package tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | export PROJECT_VERSION="${{ github.event.inputs.project_version }}" echo "Citus Version: ${PROJECT_VERSION} " diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index a82f1e3c..7066b40b 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -10,7 +10,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token id: app @@ -19,14 +22,12 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} + submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -37,5 +38,4 @@ jobs: - name: Citus package tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package_utils.py diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index 41f34047..c94abd96 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -10,7 +10,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token @@ -25,6 +28,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{steps.app.outputs.token}} + submodules: true - name: Install package dependencies @@ -36,5 +40,4 @@ jobs: - name: Build and publish docker images tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_publish_docker.py diff --git a/.github/workflows/pypi-statistics-schedule.yml b/.github/workflows/pypi-statistics-schedule.yml index a8562ddc..d54548d0 100644 --- a/.github/workflows/pypi-statistics-schedule.yml +++ b/.github/workflows/pypi-statistics-schedule.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - + - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 11d3b7f8..2cc15d9e 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -18,6 +18,10 @@ on: jobs: execute_job: runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read strategy: fail-fast: false matrix: @@ -36,6 +40,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} + submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -46,6 +51,5 @@ jobs: - name: Execute 'Fetch Daily Statistics' env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} JOB_NAME: "${{ matrix.JOB_NAME }}" run: packaging_automation/bash/daily-statistics-job.sh diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 76a201c8..89360767 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -18,7 +18,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token id: app @@ -32,6 +35,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} + submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -42,23 +46,19 @@ jobs: - name: Unit tests for "Docker statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_docker_statistics_collector.py - name: Unit tests for "Github clone statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_github_statistics_collector.py - name: Unit tests for "Packagecloud download statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_package_cloud_statistics_collector.py - name: Unit tests for "Homebrew download statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_homebrew_statistics_collector.py diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index a3d66756..3938eacb 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -24,7 +24,10 @@ jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token id: app @@ -33,15 +36,13 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 0 + submodules: true - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -66,13 +67,11 @@ jobs: - name: Unit tests for "Common tools" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py - name: Unit tests for "Update Package Properties" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py # no longer viable, outdated test, skipping to not block the pipeline @@ -82,17 +81,14 @@ jobs: - name: Unit tests for "Update Docker" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_docker.py - name: Unit tests for "Update Pgxn" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py - name: Packaging Warning Handler env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b4b89d64..b7e30068 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -680,7 +680,8 @@ def remove_suffix(initial_str: str, suffix: str) -> str: def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): - run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") + GH_TOKEN = os.environ.get("GH_TOKEN", "") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr( diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 7e62b7ce..0df48b62 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -86,9 +86,11 @@ def setup_module(): "pgxn-citus" if PLATFORM == "pgxn" else PACKAGING_BRANCH_NAME ) if not os.path.exists(PACKAGING_EXEC_FOLDER): + GH_TOKEN = os.environ.get("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git" - f" {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" ) diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 064bfcca..5b187281 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -45,8 +45,11 @@ def setup_module(): if not os.path.exists("packaging_test"): + GH_TOKEN = os.environ.get("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" + f"git clone --branch all-citus-unit-tests https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" ) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 19a637ba..e75743a9 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,8 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") + GH_TOKEN = os.getenv("GH_TOKEN", "") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 740ff446..2d71475e 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -29,7 +29,8 @@ def initialize_env(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + GH_TOKEN = os.getenv("GH_TOKEN", "") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 06069dc6..1d8c510d 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -34,7 +34,10 @@ def setup_module(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + GH_TOKEN = os.getenv("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 572fc5c4..3fb1d62b 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -18,8 +18,11 @@ def setup_module(): if not os.path.exists("packaging_test"): + GH_TOKEN = os.environ.get("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") run( - "git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" + f"git clone --branch pgxn-citus https://{GH_TOKEN}@github.com/citusdata/packaging.git packaging_test" ) From df17e17cc29309b4732d6bc16dcacd3ca13cf22d Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Sat, 14 Mar 2026 19:18:46 +0300 Subject: [PATCH 04/12] feat: update workflows to use GH_TOKEN references --- .github/workflows/build-citus-community-nightlies.yml | 10 +++++++--- .github/workflows/citus-package-all-platforms-test.yml | 6 ++++++ .github/workflows/package-tests.yml | 9 ++++++++- .github/workflows/packaging-methods-tests.yml | 6 ++++++ .github/workflows/publish-docker-image-tests.yml | 6 ++++++ .github/workflows/statistic-schedule.yml | 6 ++++++ .github/workflows/statistic-tests.yml | 6 ++++++ .github/workflows/tool-tests.yml | 6 ++++++ packaging_automation/common_tool_methods.py | 2 +- packaging_automation/tests/test_citus_package.py | 5 +---- packaging_automation/tests/test_citus_package_utils.py | 5 +---- packaging_automation/tests/test_prepare_release.py | 2 +- packaging_automation/tests/test_publish_docker.py | 2 +- packaging_automation/tests/test_update_docker.py | 5 +---- packaging_automation/tests/test_update_pgxn.py | 5 +---- 15 files changed, 58 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index b2baa007..ef76878a 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -44,6 +44,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: @@ -60,9 +66,7 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Clone build branch - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: git clone -b "${MAIN_BRANCH}" --depth=1 https://${GH_TOKEN}@github.com/citusdata/packaging.git packaging + run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 54cfd327..5a7f8474 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -45,6 +45,12 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index a647e268..969787a8 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -30,13 +30,20 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 2 submodules: true - + - name: Package version id: get-citus-version run: | diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 7066b40b..523656e6 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -23,6 +23,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c94abd96..09a07cee 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -24,6 +24,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 2cc15d9e..5cc55c69 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -36,6 +36,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 89360767..39ae0c04 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -31,6 +31,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 3938eacb..75964152 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -37,6 +37,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b7e30068..b15a83c8 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -681,7 +681,7 @@ def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): GH_TOKEN = os.environ.get("GH_TOKEN", "") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/{project_name}.git {checkout_dir}") + run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr( diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 0df48b62..d08f3398 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -86,11 +86,8 @@ def setup_module(): "pgxn-citus" if PLATFORM == "pgxn" else PACKAGING_BRANCH_NAME ) if not os.path.exists(PACKAGING_EXEC_FOLDER): - GH_TOKEN = os.environ.get("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch {packaging_branch_name} https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" ) diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 5b187281..064bfcca 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -45,11 +45,8 @@ def setup_module(): if not os.path.exists("packaging_test"): - GH_TOKEN = os.environ.get("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch all-citus-unit-tests https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" + f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" ) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index e75743a9..15af96ea 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -41,7 +41,7 @@ def initialize_env() -> str: remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): GH_TOKEN = os.getenv("GH_TOKEN", "") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/citus.git {test_base_path_major}") + run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 2d71475e..81631179 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -30,7 +30,7 @@ def initialize_env(): if not os.path.exists("docker"): GH_TOKEN = os.getenv("GH_TOKEN", "") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") + run(f"git clone https://github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 1d8c510d..de286598 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -34,10 +34,7 @@ def setup_module(): if not os.path.exists("docker"): - GH_TOKEN = os.getenv("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") + run(f"git clone https://github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 3fb1d62b..674edaed 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -18,11 +18,8 @@ def setup_module(): if not os.path.exists("packaging_test"): - GH_TOKEN = os.environ.get("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch pgxn-citus https://{GH_TOKEN}@github.com/citusdata/packaging.git packaging_test" + f"git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" ) From 9fd829617c4372e6e3f6caa1b40df35aa2973543 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 15:35:26 +0300 Subject: [PATCH 05/12] refactor: update workflows to use GitHub App --- .../build-citus-community-nightlies.yml | 19 ++++------- .../citus-package-all-platforms-test.yml | 11 ++----- .../delete-packagecloud-packages.yml | 2 +- .github/workflows/package-tests.yml | 33 +++++++++---------- .github/workflows/packaging-methods-tests.yml | 21 ++++-------- .../workflows/publish-docker-image-tests.yml | 22 ++++--------- .github/workflows/statistic-schedule.yml | 22 +++++-------- .github/workflows/statistic-tests.yml | 28 +++++----------- .github/workflows/tool-tests.yml | 31 +++++------------ .../tests/test_citus_package.py | 2 +- 10 files changed, 66 insertions(+), 125 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index ef76878a..745abc19 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -18,10 +18,6 @@ jobs: build_package: name: Build package runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read strategy: fail-fast: false matrix: @@ -38,17 +34,17 @@ jobs: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 @@ -56,7 +52,6 @@ jobs: token: ${{ steps.app.outputs.token }} fetch-depth: 1 path: tools - submodules: true # This step is to fetch the images unanonymously to have higher bandwidth - name: Login to Docker Hub @@ -75,8 +70,6 @@ jobs: run: python -m pip install -r tools/packaging_automation/requirements.txt - name: Build packages - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: | python -m tools.packaging_automation.citus_package \ --gh_token "${GH_TOKEN}" \ diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 5a7f8474..88226ad4 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -17,10 +17,6 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read strategy: fail-fast: false matrix: @@ -40,16 +36,17 @@ jobs: steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} - name: Setup git authentication for GitHub App env: GH_TOKEN: ${{ steps.app.outputs.token }} run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 @@ -66,6 +63,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/.github/workflows/delete-packagecloud-packages.yml b/.github/workflows/delete-packagecloud-packages.yml index c9d55c53..de8a8900 100644 --- a/.github/workflows/delete-packagecloud-packages.yml +++ b/.github/workflows/delete-packagecloud-packages.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 969787a8..3e65e82a 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -14,10 +14,6 @@ on: jobs: metadata: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read outputs: pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} citus_version: ${{ steps.get-citus-version.outputs.citus_version }} @@ -29,20 +25,19 @@ jobs: with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 2 - submodules: true - name: Package version id: get-citus-version @@ -86,11 +81,17 @@ jobs: id: app uses: actions/create-github-app-token@v2 with: - app-id: ${{ secrets.GITHUB_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} - owner: citusdata + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} - - name: Checkout repository + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + + - name: Checkout uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} @@ -102,8 +103,6 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: | export PROJECT_VERSION="${{ github.event.inputs.project_version }}" echo "Citus Version: ${PROJECT_VERSION} " diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 523656e6..f7030f8c 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -10,30 +10,25 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} - submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -42,6 +37,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package_utils.py diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index 09a07cee..22d51c70 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -10,32 +10,26 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{steps.app.outputs.token}} - submodules: true - - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -44,6 +38,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Build and publish docker images tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_publish_docker.py diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 5cc55c69..5aad636a 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -18,10 +18,6 @@ on: jobs: execute_job: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read strategy: fail-fast: false matrix: @@ -30,23 +26,22 @@ jobs: steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} - submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -55,7 +50,6 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Execute 'Fetch Daily Statistics' + run: packaging_automation/bash/daily-statistics-job.sh env: - GH_TOKEN: ${{ steps.app.outputs.token }} JOB_NAME: "${{ matrix.JOB_NAME }}" - run: packaging_automation/bash/daily-statistics-job.sh diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 39ae0c04..702b1766 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -18,30 +18,26 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read + steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} - submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -50,21 +46,13 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Unit tests for "Docker statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_docker_statistics_collector.py - name: Unit tests for "Github clone statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_github_statistics_collector.py - name: Unit tests for "Packagecloud download statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_package_cloud_statistics_collector.py - name: Unit tests for "Homebrew download statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_homebrew_statistics_collector.py diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 75964152..89029dda 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -19,36 +19,33 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 + - name: Install all scripts run: make && sudo make install unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read steps: + - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 0 - submodules: true - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -71,13 +68,9 @@ jobs: run: black . --check - name: Unit tests for "Common tools" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py - name: Unit tests for "Update Package Properties" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py # no longer viable, outdated test, skipping to not block the pipeline @@ -85,16 +78,10 @@ jobs: # run: python -m pytest -q packaging_automation/tests/test_prepare_release.py - name: Unit tests for "Update Docker" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_docker.py - name: Unit tests for "Update Pgxn" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py - name: Packaging Warning Handler - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index d08f3398..4f1c0d35 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -87,7 +87,7 @@ def setup_module(): ) if not os.path.exists(PACKAGING_EXEC_FOLDER): run( - f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" ) From 1fa5d293a5ee722d17eedf15afa3986dfa748734 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 16:21:29 +0300 Subject: [PATCH 06/12] refactor: replace GITHUB_TOKEN with GH_TOKEN in workflow and test files also should refactor links if clone fails --- packaging_automation/citus_package.py | 4 ++-- packaging_automation/common_tool_methods.py | 1 - packaging_automation/tests/test_common_tool_methods.py | 6 +++--- packaging_automation/tests/test_prepare_release.py | 1 - packaging_automation/tests/test_publish_docker.py | 3 +-- packaging_automation/tests/test_update_docker.py | 2 +- .../tests/test_update_package_properties.py | 2 +- packaging_automation/tests/test_update_pgxn.py | 2 +- 8 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 06a81dc7..d6ff3955 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -341,7 +341,7 @@ def build_package( ): docker_image_name = "packaging" if not is_test else "packaging-test" postgres_extension = "all" if postgres_version == "all" else f"pg{postgres_version}" - os.environ["GITHUB_TOKEN"] = github_token + os.environ["GH_TOKEN"] = github_token os.environ["CONTAINER_BUILD_RUN_ENABLED"] = "true" if not os.path.exists(input_output_parameters.output_dir): os.makedirs(input_output_parameters.output_dir) @@ -349,7 +349,7 @@ def build_package( docker_command = ( f"docker run --rm -v {input_output_parameters.output_dir}:/packages -v " f"{input_output_parameters.input_files_dir}:/buildfiles:ro " - f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " + f"-e GH_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b15a83c8..b4b89d64 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -680,7 +680,6 @@ def remove_suffix(initial_str: str, suffix: str) -> str: def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): - GH_TOKEN = os.environ.get("GH_TOKEN", "") run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 2cf7b98f..5f5cece5 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -46,7 +46,7 @@ str_array_to_str, ) -GITHUB_TOKEN = os.getenv("GH_TOKEN") +GH_TOKEN = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[1] TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() TEST_GPG_KEY_NAME = "Citus Data " @@ -264,7 +264,7 @@ def test_prepend_line_in_file(): def test_getprs(): # created at is not seen on Github. Should be checked on API result - g = Github(GITHUB_TOKEN) + g = Github(GH_TOKEN) repository = g.get_repo("citusdata/citus") prs = get_prs_for_patch_release( repository, @@ -277,7 +277,7 @@ def test_getprs(): def test_getprs_with_backlog_label(): - g = Github(GITHUB_TOKEN) + g = Github(GH_TOKEN) repository = g.get_repo("citusdata/citus") prs = get_prs_for_patch_release( repository, diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 15af96ea..19a637ba 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,6 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - GH_TOKEN = os.getenv("GH_TOKEN", "") run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 81631179..740ff446 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -29,8 +29,7 @@ def initialize_env(): if not os.path.exists("docker"): - GH_TOKEN = os.getenv("GH_TOKEN", "") - run(f"git clone https://github.com/citusdata/docker.git") + run("git clone https://github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index de286598..06069dc6 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -34,7 +34,7 @@ def setup_module(): if not os.path.exists("docker"): - run(f"git clone https://github.com/citusdata/docker.git") + run("git clone https://github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_package_properties.py b/packaging_automation/tests/test_update_package_properties.py index c1bcebed..804ee541 100644 --- a/packaging_automation/tests/test_update_package_properties.py +++ b/packaging_automation/tests/test_update_package_properties.py @@ -25,7 +25,7 @@ TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[1]) -GITHUB_TOKEN = os.getenv("GH_TOKEN") +GH_TOKEN = os.getenv("GH_TOKEN") PROJECT_VERSION = os.getenv("PROJECT_VERSION", default="10.2.4") TAG_NAME = os.getenv("TAG_NAME", default="v10.2.4") PROJECT_NAME = os.getenv("PROJECT_NAME", default="citus") diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 674edaed..572fc5c4 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -19,7 +19,7 @@ def setup_module(): if not os.path.exists("packaging_test"): run( - f"git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" + "git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" ) From 361638f29a3a276e6bceb4ac381d3c8bed3e40ab Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 16:53:57 +0300 Subject: [PATCH 07/12] update build_packages to double check token validility --- packaging_automation/citus_package.py | 1 + packaging_automation/tests/test_citus_package_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index d6ff3955..36a9d636 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -401,6 +401,7 @@ def build_packages( input_output_parameters: InputOutputParameters, is_test: bool = False, ) -> None: + os.environ["GH_TOKEN"] = github_token os_name, os_version = decode_os_and_release(platform) release_versions, nightly_versions = get_postgres_versions( platform, input_output_parameters.input_files_dir diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 064bfcca..44c8d0e4 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -46,7 +46,7 @@ def setup_module(): if not os.path.exists("packaging_test"): run( - f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" + f"git clone --branch all-citus-unit-tests https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" ) From 413ed4e3a9f4e9bbd1fe872d17928cc8527a8891 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 17:03:04 +0300 Subject: [PATCH 08/12] changing the way token gets passed --- .github/workflows/build-citus-community-nightlies.yml | 2 +- packaging_automation/tests/test_citus_package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 745abc19..6e1dce4b 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -72,7 +72,7 @@ jobs: - name: Build packages run: | python -m tools.packaging_automation.citus_package \ - --gh_token "${GH_TOKEN}" \ + --gh_token "${{ steps.app.outputs.token }}" \ --platform "${{ matrix.platform }}" \ --build_type "nightly" \ --secret_key "${PACKAGING_SECRET_KEY}" \ diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 4f1c0d35..6059a542 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -60,7 +60,7 @@ TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE") -GH_TOKEN = os.getenv("GH_TOKEN") +GH_TOKEN = os.environ("GH_TOKEN") PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN") REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET") PLATFORM = get_build_platform( From 8ad28a3bce8029c1841d1510b164fe05a84c0d11 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 17:06:49 +0300 Subject: [PATCH 09/12] refactor: update GH_TOKEN usage in build process and tests --- .github/workflows/build-citus-community-nightlies.yml | 2 +- packaging_automation/citus_package.py | 1 - packaging_automation/tests/test_citus_package.py | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 6e1dce4b..745abc19 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -72,7 +72,7 @@ jobs: - name: Build packages run: | python -m tools.packaging_automation.citus_package \ - --gh_token "${{ steps.app.outputs.token }}" \ + --gh_token "${GH_TOKEN}" \ --platform "${{ matrix.platform }}" \ --build_type "nightly" \ --secret_key "${PACKAGING_SECRET_KEY}" \ diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 36a9d636..d6ff3955 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -401,7 +401,6 @@ def build_packages( input_output_parameters: InputOutputParameters, is_test: bool = False, ) -> None: - os.environ["GH_TOKEN"] = github_token os_name, os_version = decode_os_and_release(platform) release_versions, nightly_versions = get_postgres_versions( platform, input_output_parameters.input_files_dir diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 6059a542..02fb6a20 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -60,7 +60,9 @@ TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE") -GH_TOKEN = os.environ("GH_TOKEN") +GH_TOKEN = os.getenv("GH_TOKEN") +if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is required for tests to run") PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN") REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET") PLATFORM = get_build_platform( From adc0d6af5e4a27fff2c1cf64380f55df2364ffa0 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 17:09:42 +0300 Subject: [PATCH 10/12] refactor: add GH_TOKEN environment variable to Citus package tests step --- .github/workflows/citus-package-all-platforms-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 88226ad4..01860d72 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -63,4 +63,6 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s From c3e370db129e5c05d52f6484a1ab732ecacfa142 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 18:29:06 +0300 Subject: [PATCH 11/12] testing sh scripts --- .github/workflows/citus-package-all-platforms-test.yml | 8 ++++---- packaging_automation/citus_package.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 01860d72..7667dbf3 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -42,9 +42,11 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} + - name: Set GH_TOKEN for all steps + run: | + echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: | git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" @@ -63,6 +65,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index d6ff3955..06a81dc7 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -341,7 +341,7 @@ def build_package( ): docker_image_name = "packaging" if not is_test else "packaging-test" postgres_extension = "all" if postgres_version == "all" else f"pg{postgres_version}" - os.environ["GH_TOKEN"] = github_token + os.environ["GITHUB_TOKEN"] = github_token os.environ["CONTAINER_BUILD_RUN_ENABLED"] = "true" if not os.path.exists(input_output_parameters.output_dir): os.makedirs(input_output_parameters.output_dir) @@ -349,7 +349,7 @@ def build_package( docker_command = ( f"docker run --rm -v {input_output_parameters.output_dir}:/packages -v " f"{input_output_parameters.input_files_dir}:/buildfiles:ro " - f"-e GH_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " + f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) From 131e05955f82052c90c41d8d38487e9f620c448d Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 18:40:44 +0300 Subject: [PATCH 12/12] refactor: update git clone commands inside docker --- packaging_automation/common_tool_methods.py | 5 ++++- packaging_automation/tests/test_prepare_release.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b4b89d64..1b50a9c2 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -679,8 +679,11 @@ def remove_suffix(initial_str: str, suffix: str) -> str: def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + raise ValueError("GH_TOKEN environment variable is required") if not os.path.exists(checkout_dir): - run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") + run(f"git clone https://x-access-token:{gh_token}@github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr( diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 19a637ba..0cde6ee7 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,7 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") + run(f"git clone https://x-access-token:{GH_TOKEN}@github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major