From 383f35298517b98dac94f08e90b97f6fe0d6ce48 Mon Sep 17 00:00:00 2001 From: naorpeled Date: Mon, 23 Mar 2026 00:29:01 +0200 Subject: [PATCH 1/7] feat: add ARM64 (linux/arm64) support for multi-arch Docker builds Add QEMU emulation and multi-platform support to both CI workflows: - test.yml: add platform matrix dimension (amd64 + arm64) for full cross-arch testing - publish.yml: build and push multi-arch manifests (linux/amd64,linux/arm64) --- .github/workflows/publish.yml | 4 ++++ .github/workflows/test.yml | 26 ++++++++++++-------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4c54c6d..a51ae8a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -46,6 +46,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -64,6 +67,7 @@ jobs: with: context: . push: true + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99fb817..18836b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,23 +12,19 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - include: - - pg_version: "17.9" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" - - pg_version: "16.13" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" - - pg_version: "15.17" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" - - pg_version: "14.22" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" + platform: + - linux/amd64 + - linux/arm64 + pg_version: ["17.9", "16.13", "15.17", "14.22"] + postgis_version: ["3.6.2"] + pgvector_version: ["0.8.2"] steps: - uses: actions/checkout@v6 + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -37,6 +33,7 @@ jobs: with: context: . load: true + platforms: ${{ matrix.platform }} tags: postgres-test:pg${{ matrix.pg_version }} build-args: | PG_VERSION=${{ matrix.pg_version }} @@ -46,9 +43,10 @@ jobs: - name: Start PostgreSQL container and test extensions run: | IMAGE_TAG="postgres-test:pg${{ matrix.pg_version }}" - echo "Testing image: $IMAGE_TAG" + echo "Testing image: $IMAGE_TAG (platform: ${{ matrix.platform }})" docker run -d --name test-db \ + --platform ${{ matrix.platform }} \ -e POSTGRES_PASSWORD=test \ -e POSTGRES_USER=test \ -e POSTGRES_DB=test \ From 8d814d5823affc8805c50ae1674d3cac9765c553 Mon Sep 17 00:00:00 2001 From: naorpeled Date: Mon, 23 Mar 2026 00:40:54 +0200 Subject: [PATCH 2/7] perf: use native ARM runners instead of QEMU emulation for tests Replace QEMU-based ARM testing with GitHub's native ubuntu-24.04-arm runners for significantly faster builds. Each runner builds natively for its own architecture, eliminating emulation overhead. --- .github/workflows/test.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 18836b1..b2b03d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,12 +9,10 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} strategy: matrix: - platform: - - linux/amd64 - - linux/arm64 + runner: [ubuntu-latest, ubuntu-24.04-arm] pg_version: ["17.9", "16.13", "15.17", "14.22"] postgis_version: ["3.6.2"] pgvector_version: ["0.8.2"] @@ -22,9 +20,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -33,7 +28,6 @@ jobs: with: context: . load: true - platforms: ${{ matrix.platform }} tags: postgres-test:pg${{ matrix.pg_version }} build-args: | PG_VERSION=${{ matrix.pg_version }} @@ -43,10 +37,9 @@ jobs: - name: Start PostgreSQL container and test extensions run: | IMAGE_TAG="postgres-test:pg${{ matrix.pg_version }}" - echo "Testing image: $IMAGE_TAG (platform: ${{ matrix.platform }})" + echo "Testing image: $IMAGE_TAG (runner: ${{ matrix.runner }})" docker run -d --name test-db \ - --platform ${{ matrix.platform }} \ -e POSTGRES_PASSWORD=test \ -e POSTGRES_USER=test \ -e POSTGRES_DB=test \ From b3d35b8a941a1660bf6bdc758859e392e11019c0 Mon Sep 17 00:00:00 2001 From: naorpeled Date: Mon, 23 Mar 2026 00:42:46 +0200 Subject: [PATCH 3/7] fix: scope git ref tag to latest matrix entry to prevent race condition The type=ref,event=tag metadata generates the same tag (e.g. v1.0.0) for all matrix entries. Restrict it to the latest entry only, matching how the 'latest' tag is already handled. --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a51ae8a..8b690c0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -60,7 +60,7 @@ jobs: tags: | type=raw,value=postgres-${{ matrix.pg_version }}-postgis-${{ matrix.postgis_version }}-pgvector-${{ matrix.pgvector_version }} type=raw,value=latest,enable=${{ matrix.latest }} - type=ref,event=tag + type=ref,event=tag,enable=${{ matrix.latest }} - name: Build and push Docker image uses: docker/build-push-action@v7 From ca612b3aba4897ca82cb514d9091ed63f47fb590 Mon Sep 17 00:00:00 2001 From: naorpeled Date: Mon, 23 Mar 2026 00:44:30 +0200 Subject: [PATCH 4/7] perf: use native ARM runners for publish instead of QEMU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split publish into build + merge jobs: - build: matrix of PG version × runner (ubuntu-latest, ubuntu-24.04-arm), each pushes a single-platform image by digest - merge: downloads digests per PG version and creates multi-arch manifests Eliminates QEMU emulation overhead for ARM builds. --- .github/workflows/publish.yml | 102 ++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 24 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8b690c0..f404385 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,10 +7,74 @@ on: env: REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/${{ github.event.repository.name }} jobs: - build-and-push: + build: + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + + strategy: + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + pg_version: ["17.9", "16.13", "15.17", "14.22"] + postgis_version: ["3.6.2"] + pgvector_version: ["0.8.2"] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v7 + with: + context: . + push: true + labels: ${{ steps.meta.outputs.labels }} + build-args: | + PG_VERSION=${{ matrix.pg_version }} + POSTGIS_VERSION=${{ matrix.postgis_version }} + PGVECTOR_VERSION=${{ matrix.pgvector_version }} + cache-from: type=gha,scope=${{ matrix.pg_version }}-${{ matrix.runner }} + cache-to: type=gha,mode=max,scope=${{ matrix.pg_version }}-${{ matrix.runner }} + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digest-${{ matrix.pg_version }}-${{ matrix.runner }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: runs-on: ubuntu-latest + needs: build permissions: contents: read packages: write @@ -36,8 +100,12 @@ jobs: latest: false steps: - - name: Checkout repository - uses: actions/checkout@v6 + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digest-${{ matrix.pg_version }}-* + merge-multiple: true + path: /tmp/digests - name: Log in to GitHub Container Registry uses: docker/login-action@v4 @@ -46,33 +114,19 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v6 with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=raw,value=postgres-${{ matrix.pg_version }}-postgis-${{ matrix.postgis_version }}-pgvector-${{ matrix.pgvector_version }} type=raw,value=latest,enable=${{ matrix.latest }} type=ref,event=tag,enable=${{ matrix.latest }} - - name: Build and push Docker image - uses: docker/build-push-action@v7 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - PG_VERSION=${{ matrix.pg_version }} - POSTGIS_VERSION=${{ matrix.postgis_version }} - PGVECTOR_VERSION=${{ matrix.pgvector_version }} - cache-from: type=gha - cache-to: type=gha,mode=max + - name: Create multi-arch manifest and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) From 4b75469bfb4c916a4e9d19f34422e49ab884e1b8 Mon Sep 17 00:00:00 2001 From: naorpeled Date: Mon, 23 Mar 2026 00:51:41 +0200 Subject: [PATCH 5/7] refactor: address review feedback across all files Dockerfile: - Remove redundant apt-get install of already-installed build tools - Add --depth 1 to pgvector git clone - Purge lsb-release and gnupg in cleanup step Workflows (shared): - Add versions.json as single source of truth for version matrix - Both workflows read from versions.json via load-versions job - Add fail-fast: false to all matrix strategies - Add timeout-minutes to all jobs - Add GHA build cache with per-version per-runner scoping test.yml: - Replace sleep 15 with pg_isready health check polling - Add PostGIS functional test (ST_Point) - Add version assertions (verify installed matches requested) - Simplify container cleanup to docker rm -f publish.yml: - Add explicit platform field mapped from runner - Add name= to push-by-digest output for unambiguous registry reference - Add imagetools inspect verification after manifest creation --- .github/workflows/publish.yml | 59 ++++++++++++++++---------- .github/workflows/test.yml | 79 +++++++++++++++++++++++++++++------ Dockerfile | 7 ++-- versions.json | 26 ++++++++++++ 4 files changed, 132 insertions(+), 39 deletions(-) create mode 100644 versions.json diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f404385..6d05494 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,18 +10,42 @@ env: IMAGE_NAME: ${{ github.repository_owner }}/${{ github.event.repository.name }} jobs: + load-versions: + runs-on: ubuntu-latest + outputs: + pg_versions: ${{ steps.set-matrix.outputs.pg_versions }} + postgis_versions: ${{ steps.set-matrix.outputs.postgis_versions }} + pgvector_versions: ${{ steps.set-matrix.outputs.pgvector_versions }} + versions: ${{ steps.set-matrix.outputs.versions }} + steps: + - uses: actions/checkout@v6 + - id: set-matrix + run: | + echo "pg_versions=$(jq -c '[.[].pg_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" + echo "postgis_versions=$(jq -c '[.[].postgis_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" + echo "pgvector_versions=$(jq -c '[.[].pgvector_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" + echo "versions=$(jq -c '.' versions.json)" >> "$GITHUB_OUTPUT" + build: + needs: load-versions runs-on: ${{ matrix.runner }} + timeout-minutes: 30 permissions: contents: read packages: write strategy: + fail-fast: false matrix: runner: [ubuntu-latest, ubuntu-24.04-arm] - pg_version: ["17.9", "16.13", "15.17", "14.22"] - postgis_version: ["3.6.2"] - pgvector_version: ["0.8.2"] + pg_version: ${{ fromJSON(needs.load-versions.outputs.pg_versions) }} + postgis_version: ${{ fromJSON(needs.load-versions.outputs.postgis_versions) }} + pgvector_version: ${{ fromJSON(needs.load-versions.outputs.pgvector_versions) }} + include: + - runner: ubuntu-latest + platform: linux/amd64 + - runner: ubuntu-24.04-arm + platform: linux/arm64 steps: - name: Checkout repository @@ -49,6 +73,7 @@ jobs: with: context: . push: true + platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} build-args: | PG_VERSION=${{ matrix.pg_version }} @@ -56,7 +81,7 @@ jobs: PGVECTOR_VERSION=${{ matrix.pgvector_version }} cache-from: type=gha,scope=${{ matrix.pg_version }}-${{ matrix.runner }} cache-to: type=gha,mode=max,scope=${{ matrix.pg_version }}-${{ matrix.runner }} - outputs: type=image,push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | @@ -74,30 +99,16 @@ jobs: merge: runs-on: ubuntu-latest - needs: build + timeout-minutes: 10 + needs: [load-versions, build] permissions: contents: read packages: write strategy: + fail-fast: false matrix: - include: - - pg_version: "17.9" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" - latest: true - - pg_version: "16.13" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" - latest: false - - pg_version: "15.17" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" - latest: false - - pg_version: "14.22" - postgis_version: "3.6.2" - pgvector_version: "0.8.2" - latest: false + include: ${{ fromJSON(needs.load-versions.outputs.versions) }} steps: - name: Download digests @@ -130,3 +141,7 @@ jobs: docker buildx imagetools create \ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) + + - name: Verify multi-arch manifest + run: | + docker buildx imagetools inspect $(jq -cr '.tags[0]' <<< "$DOCKER_METADATA_OUTPUT_JSON") diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2b03d5..9523341 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,14 +8,31 @@ on: workflow_dispatch: jobs: + load-versions: + runs-on: ubuntu-latest + outputs: + pg_versions: ${{ steps.set-matrix.outputs.pg_versions }} + postgis_versions: ${{ steps.set-matrix.outputs.postgis_versions }} + pgvector_versions: ${{ steps.set-matrix.outputs.pgvector_versions }} + steps: + - uses: actions/checkout@v6 + - id: set-matrix + run: | + echo "pg_versions=$(jq -c '[.[].pg_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" + echo "postgis_versions=$(jq -c '[.[].postgis_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" + echo "pgvector_versions=$(jq -c '[.[].pgvector_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" + test: + needs: load-versions runs-on: ${{ matrix.runner }} + timeout-minutes: 15 strategy: + fail-fast: false matrix: runner: [ubuntu-latest, ubuntu-24.04-arm] - pg_version: ["17.9", "16.13", "15.17", "14.22"] - postgis_version: ["3.6.2"] - pgvector_version: ["0.8.2"] + pg_version: ${{ fromJSON(needs.load-versions.outputs.pg_versions) }} + postgis_version: ${{ fromJSON(needs.load-versions.outputs.postgis_versions) }} + pgvector_version: ${{ fromJSON(needs.load-versions.outputs.pgvector_versions) }} steps: - uses: actions/checkout@v6 @@ -33,8 +50,10 @@ jobs: PG_VERSION=${{ matrix.pg_version }} POSTGIS_VERSION=${{ matrix.postgis_version }} PGVECTOR_VERSION=${{ matrix.pgvector_version }} + cache-from: type=gha,scope=${{ matrix.pg_version }}-${{ matrix.runner }} + cache-to: type=gha,mode=max,scope=${{ matrix.pg_version }}-${{ matrix.runner }} - - name: Start PostgreSQL container and test extensions + - name: Start PostgreSQL container run: | IMAGE_TAG="postgres-test:pg${{ matrix.pg_version }}" echo "Testing image: $IMAGE_TAG (runner: ${{ matrix.runner }})" @@ -43,26 +62,60 @@ jobs: -e POSTGRES_PASSWORD=test \ -e POSTGRES_USER=test \ -e POSTGRES_DB=test \ + --health-cmd="pg_isready -U test -d test" \ + --health-interval=2s \ + --health-timeout=5s \ + --health-retries=30 \ $IMAGE_TAG \ postgres -c shared_preload_libraries=vector - echo "Waiting for PostgreSQL to start..." - sleep 15 - - echo "PostgreSQL logs:" - docker logs test-db + echo "Waiting for PostgreSQL to become healthy..." + until [ "$(docker inspect -f '{{.State.Health.Status}}' test-db)" = "healthy" ]; do + if [ "$(docker inspect -f '{{.State.Status}}' test-db)" != "running" ]; then + echo "Container exited unexpectedly!" + docker logs test-db + exit 1 + fi + sleep 1 + done + echo "PostgreSQL is ready." + - name: Test PostGIS extension + run: | echo "Checking PostGIS version..." docker exec test-db psql -U test -d test -c "SELECT postgis_full_version();" + echo "Testing PostGIS spatial functionality..." + docker exec test-db psql -U test -d test -c "SELECT ST_AsText(ST_Point(1, 2));" + + - name: Test pgvector extension + run: | echo "Checking pgvector extension version..." docker exec test-db psql -U test -d test -c "CREATE EXTENSION IF NOT EXISTS vector; SELECT extversion FROM pg_extension WHERE extname = 'vector';" - echo "Attempting to create a table with a vector column and insert data..." + echo "Testing vector operations..." docker exec test-db psql -U test -d test -c "CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3)); INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]'); SELECT COUNT(*) FROM items;" + - name: Verify installed versions match requested + run: | + EXPECTED_PGVECTOR="${{ matrix.pgvector_version }}" + ACTUAL_PGVECTOR=$(docker exec test-db psql -U test -d test -tAc \ + "SELECT extversion FROM pg_extension WHERE extname = 'vector';") + if [ "$ACTUAL_PGVECTOR" != "$EXPECTED_PGVECTOR" ]; then + echo "FAIL: expected pgvector $EXPECTED_PGVECTOR, got $ACTUAL_PGVECTOR" + exit 1 + fi + echo "pgvector version OK: $ACTUAL_PGVECTOR" + + EXPECTED_POSTGIS="${{ matrix.postgis_version }}" + ACTUAL_POSTGIS=$(docker exec test-db psql -U test -d test -tAc \ + "SELECT extversion FROM pg_extension WHERE extname = 'postgis';") + if [ "$ACTUAL_POSTGIS" != "$EXPECTED_POSTGIS" ]; then + echo "FAIL: expected PostGIS $EXPECTED_POSTGIS, got $ACTUAL_POSTGIS" + exit 1 + fi + echo "PostGIS version OK: $ACTUAL_POSTGIS" + - name: Stop and remove container if: always() - run: | - docker stop test-db || true - docker rm test-db || true + run: docker rm -f test-db || true diff --git a/Dockerfile b/Dockerfile index f04061f..9d235bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,10 +40,7 @@ RUN POSTGIS_MAJOR=$(echo "${POSTGIS_VERSION}" | cut -d. -f1) \ "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}-scripts=${POSTGIS_VERSION}+dfsg*" # Build and install pinned pgvector version from source -RUN apt-get update \ - && apt-get install -y --no-install-recommends git make gcc "postgresql-server-dev-${PG_MAJOR}" \ - && mkdir -p /usr/src/pgvector \ - && git clone --branch "v${PGVECTOR_VERSION}" https://github.com/pgvector/pgvector.git /usr/src/pgvector \ +RUN git clone --branch "v${PGVECTOR_VERSION}" --depth 1 https://github.com/pgvector/pgvector.git /usr/src/pgvector \ && cd /usr/src/pgvector \ && make \ && make install @@ -56,6 +53,8 @@ RUN apt-get purge -y --auto-remove \ gcc \ "postgresql-server-dev-${PG_MAJOR}" \ wget \ + lsb-release \ + gnupg \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /usr/src/pgvector diff --git a/versions.json b/versions.json new file mode 100644 index 0000000..c88ac01 --- /dev/null +++ b/versions.json @@ -0,0 +1,26 @@ +[ + { + "pg_version": "17.9", + "postgis_version": "3.6.2", + "pgvector_version": "0.8.2", + "latest": true + }, + { + "pg_version": "16.13", + "postgis_version": "3.6.2", + "pgvector_version": "0.8.2", + "latest": false + }, + { + "pg_version": "15.17", + "postgis_version": "3.6.2", + "pgvector_version": "0.8.2", + "latest": false + }, + { + "pg_version": "14.22", + "postgis_version": "3.6.2", + "pgvector_version": "0.8.2", + "latest": false + } +] From b4fc226003cf595ca37f215c82ddd4f37f0fa25e Mon Sep 17 00:00:00 2001 From: naorpeled Date: Mon, 23 Mar 2026 00:55:56 +0200 Subject: [PATCH 6/7] fix: address all review findings across workflows and Dockerfile Matrix correctness: - Fix broken include for runner-to-platform mapping by pre-building the full matrix in load-versions job using jq - Iterate versions.json entries directly instead of cross-product of unique arrays (prevents invalid combos if versions diverge) Dockerfile: - Collapse 4 RUN layers into single layer (~150-300MB image savings) - Preserve ca-certificates at runtime via apt-mark manual - Add HEALTHCHECK instruction - Add .dockerignore to reduce build context Security: - Add top-level permissions: contents: read to test.yml - Pass all ${{ }} values through env vars in run blocks - Add provenance and SBOM attestations to published images - Remove cache-to from publish (test workflow warms the cache) --- .dockerignore | 9 +++++ .github/workflows/publish.yml | 41 +++++++++------------ .github/workflows/test.yml | 32 ++++++++--------- Dockerfile | 67 +++++++++++++++++------------------ 4 files changed, 74 insertions(+), 75 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9a05080 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.github +.dockerignore +.gitignore +.env* +*.md +LICENSE +docker-compose*.yml +versions.json diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6d05494..5d084f0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,18 +13,16 @@ jobs: load-versions: runs-on: ubuntu-latest outputs: - pg_versions: ${{ steps.set-matrix.outputs.pg_versions }} - postgis_versions: ${{ steps.set-matrix.outputs.postgis_versions }} - pgvector_versions: ${{ steps.set-matrix.outputs.pgvector_versions }} - versions: ${{ steps.set-matrix.outputs.versions }} + build-matrix: ${{ steps.set-matrix.outputs.build_matrix }} + merge-matrix: ${{ steps.set-matrix.outputs.merge_matrix }} steps: - uses: actions/checkout@v6 - id: set-matrix run: | - echo "pg_versions=$(jq -c '[.[].pg_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" - echo "postgis_versions=$(jq -c '[.[].postgis_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" - echo "pgvector_versions=$(jq -c '[.[].pgvector_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" - echo "versions=$(jq -c '.' versions.json)" >> "$GITHUB_OUTPUT" + RUNNERS='[{"runner":"ubuntu-latest","platform":"linux/amd64"},{"runner":"ubuntu-24.04-arm","platform":"linux/arm64"}]' + BUILD_MATRIX=$(jq -c --argjson runners "$RUNNERS" '{include: [.[] | . as $v | $runners[] | . + $v]}' versions.json) + echo "build_matrix=$BUILD_MATRIX" >> "$GITHUB_OUTPUT" + echo "merge_matrix=$(jq -c '{include: .}' versions.json)" >> "$GITHUB_OUTPUT" build: needs: load-versions @@ -36,16 +34,7 @@ jobs: strategy: fail-fast: false - matrix: - runner: [ubuntu-latest, ubuntu-24.04-arm] - pg_version: ${{ fromJSON(needs.load-versions.outputs.pg_versions) }} - postgis_version: ${{ fromJSON(needs.load-versions.outputs.postgis_versions) }} - pgvector_version: ${{ fromJSON(needs.load-versions.outputs.pgvector_versions) }} - include: - - runner: ubuntu-latest - platform: linux/amd64 - - runner: ubuntu-24.04-arm - platform: linux/arm64 + matrix: ${{ fromJSON(needs.load-versions.outputs.build-matrix) }} steps: - name: Checkout repository @@ -74,20 +63,22 @@ jobs: context: . push: true platforms: ${{ matrix.platform }} + provenance: true + sbom: true labels: ${{ steps.meta.outputs.labels }} build-args: | PG_VERSION=${{ matrix.pg_version }} POSTGIS_VERSION=${{ matrix.postgis_version }} PGVECTOR_VERSION=${{ matrix.pgvector_version }} cache-from: type=gha,scope=${{ matrix.pg_version }}-${{ matrix.runner }} - cache-to: type=gha,mode=max,scope=${{ matrix.pg_version }}-${{ matrix.runner }} outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - name: Export digest + env: + DIGEST: ${{ steps.build.outputs.digest }} run: | mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" + touch "/tmp/digests/${DIGEST#sha256:}" - name: Upload digest uses: actions/upload-artifact@v4 @@ -107,8 +98,7 @@ jobs: strategy: fail-fast: false - matrix: - include: ${{ fromJSON(needs.load-versions.outputs.versions) }} + matrix: ${{ fromJSON(needs.load-versions.outputs.merge-matrix) }} steps: - name: Download digests @@ -137,10 +127,13 @@ jobs: - name: Create multi-arch manifest and push working-directory: /tmp/digests + env: + REGISTRY: ${{ env.REGISTRY }} + IMAGE: ${{ env.IMAGE_NAME }} run: | docker buildx imagetools create \ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) + $(printf "${REGISTRY}/${IMAGE}@sha256:%s " *) - name: Verify multi-arch manifest run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9523341..1f81f80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,20 +7,20 @@ on: branches: [main] workflow_dispatch: +permissions: + contents: read + jobs: load-versions: runs-on: ubuntu-latest outputs: - pg_versions: ${{ steps.set-matrix.outputs.pg_versions }} - postgis_versions: ${{ steps.set-matrix.outputs.postgis_versions }} - pgvector_versions: ${{ steps.set-matrix.outputs.pgvector_versions }} + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v6 - id: set-matrix run: | - echo "pg_versions=$(jq -c '[.[].pg_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" - echo "postgis_versions=$(jq -c '[.[].postgis_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" - echo "pgvector_versions=$(jq -c '[.[].pgvector_version] | unique' versions.json)" >> "$GITHUB_OUTPUT" + MATRIX=$(jq -c '{include: [.[] | . as $v | {runner: "ubuntu-latest"}, {runner: "ubuntu-24.04-arm"} | . + $v]}' versions.json) + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" test: needs: load-versions @@ -28,11 +28,7 @@ jobs: timeout-minutes: 15 strategy: fail-fast: false - matrix: - runner: [ubuntu-latest, ubuntu-24.04-arm] - pg_version: ${{ fromJSON(needs.load-versions.outputs.pg_versions) }} - postgis_version: ${{ fromJSON(needs.load-versions.outputs.postgis_versions) }} - pgvector_version: ${{ fromJSON(needs.load-versions.outputs.pgvector_versions) }} + matrix: ${{ fromJSON(needs.load-versions.outputs.matrix) }} steps: - uses: actions/checkout@v6 @@ -54,9 +50,12 @@ jobs: cache-to: type=gha,mode=max,scope=${{ matrix.pg_version }}-${{ matrix.runner }} - name: Start PostgreSQL container + env: + PG_VERSION: ${{ matrix.pg_version }} + RUNNER: ${{ matrix.runner }} run: | - IMAGE_TAG="postgres-test:pg${{ matrix.pg_version }}" - echo "Testing image: $IMAGE_TAG (runner: ${{ matrix.runner }})" + IMAGE_TAG="postgres-test:pg${PG_VERSION}" + echo "Testing image: $IMAGE_TAG (runner: $RUNNER)" docker run -d --name test-db \ -e POSTGRES_PASSWORD=test \ @@ -66,7 +65,7 @@ jobs: --health-interval=2s \ --health-timeout=5s \ --health-retries=30 \ - $IMAGE_TAG \ + "$IMAGE_TAG" \ postgres -c shared_preload_libraries=vector echo "Waiting for PostgreSQL to become healthy..." @@ -97,8 +96,10 @@ jobs: docker exec test-db psql -U test -d test -c "CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3)); INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]'); SELECT COUNT(*) FROM items;" - name: Verify installed versions match requested + env: + EXPECTED_PGVECTOR: ${{ matrix.pgvector_version }} + EXPECTED_POSTGIS: ${{ matrix.postgis_version }} run: | - EXPECTED_PGVECTOR="${{ matrix.pgvector_version }}" ACTUAL_PGVECTOR=$(docker exec test-db psql -U test -d test -tAc \ "SELECT extversion FROM pg_extension WHERE extname = 'vector';") if [ "$ACTUAL_PGVECTOR" != "$EXPECTED_PGVECTOR" ]; then @@ -107,7 +108,6 @@ jobs: fi echo "pgvector version OK: $ACTUAL_PGVECTOR" - EXPECTED_POSTGIS="${{ matrix.postgis_version }}" ACTUAL_POSTGIS=$(docker exec test-db psql -U test -d test -tAc \ "SELECT extversion FROM pg_extension WHERE extname = 'postgis';") if [ "$ACTUAL_POSTGIS" != "$EXPECTED_POSTGIS" ]; then diff --git a/Dockerfile b/Dockerfile index 9d235bf..196fe0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,53 +13,50 @@ LABEL maintainer="TypeORM" LABEL description="PostgreSQL with PostGIS and pgvector extensions for TypeORM" LABEL org.opencontainers.image.source="https://github.com/typeorm/docker" -# Install base dependencies, setup PGDG repository, and install build tools +# Install PostGIS, build pgvector from source, then clean up in a single layer # Note: PG_MAJOR is provided by the official postgres base image -RUN apt-get update \ +RUN set -eux \ + && apt-get update \ && apt-get install -y --no-install-recommends \ - lsb-release \ - gnupg \ - ca-certificates \ - wget \ + lsb-release \ + gnupg \ + ca-certificates \ + wget \ && wget --quiet -O /usr/share/keyrings/postgresql-archive-keyring.gpg https://www.postgresql.org/media/keys/ACCC4CF8.asc \ && sh -c 'echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \ && apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - git \ - make \ - gcc \ - "postgresql-server-dev-${PG_MAJOR}" - -# Install pinned PostGIS version (apt packages use major version in name) -RUN POSTGIS_MAJOR=$(echo "${POSTGIS_VERSION}" | cut -d. -f1) \ - && apt-get update \ + build-essential \ + git \ + make \ + gcc \ + "postgresql-server-dev-${PG_MAJOR}" \ + && POSTGIS_MAJOR=$(echo "${POSTGIS_VERSION}" | cut -d. -f1) \ && apt-get install -y --no-install-recommends \ - "postgis=${POSTGIS_VERSION}+dfsg*" \ - "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}=${POSTGIS_VERSION}+dfsg*" \ - "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}-scripts=${POSTGIS_VERSION}+dfsg*" - -# Build and install pinned pgvector version from source -RUN git clone --branch "v${PGVECTOR_VERSION}" --depth 1 https://github.com/pgvector/pgvector.git /usr/src/pgvector \ + "postgis=${POSTGIS_VERSION}+dfsg*" \ + "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}=${POSTGIS_VERSION}+dfsg*" \ + "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}-scripts=${POSTGIS_VERSION}+dfsg*" \ + && git clone --branch "v${PGVECTOR_VERSION}" --depth 1 https://github.com/pgvector/pgvector.git /usr/src/pgvector \ && cd /usr/src/pgvector \ && make \ - && make install - -# Cleanup build dependencies -RUN apt-get purge -y --auto-remove \ - build-essential \ - git \ - make \ - gcc \ - "postgresql-server-dev-${PG_MAJOR}" \ - wget \ - lsb-release \ - gnupg \ + && make install \ + && apt-mark manual ca-certificates \ + && apt-get purge -y --auto-remove \ + build-essential \ + git \ + make \ + gcc \ + "postgresql-server-dev-${PG_MAJOR}" \ + wget \ + lsb-release \ + gnupg \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /usr/src/pgvector + && rm -rf /var/lib/apt/lists/* /usr/src/pgvector # Copy initialization scripts COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD pg_isready -U postgres || exit 1 + EXPOSE 5432 From d1d18ad00d70643904fc0d685490a4ddc15bc22b Mon Sep 17 00:00:00 2001 From: naorpeled Date: Mon, 23 Mar 2026 00:58:35 +0200 Subject: [PATCH 7/7] fix: remove provenance/sbom from per-platform builds, clean up Dockerfile publish.yml: - Set provenance: false and sbom: false in per-platform build jobs (provenance with push-by-digest produces index digests that break the multi-arch manifest merge) - Remove redundant push: true (already set in outputs) Dockerfile: - Remove redundant apt-mark manual ca-certificates (already manual from explicit install) - Clean up PGDG source list and keyring after build - Remove redundant || exit 1 from HEALTHCHECK --- .github/workflows/publish.yml | 5 ++--- Dockerfile | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5d084f0..7f7b15d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -61,10 +61,9 @@ jobs: uses: docker/build-push-action@v7 with: context: . - push: true platforms: ${{ matrix.platform }} - provenance: true - sbom: true + provenance: false + sbom: false labels: ${{ steps.meta.outputs.labels }} build-args: | PG_VERSION=${{ matrix.pg_version }} diff --git a/Dockerfile b/Dockerfile index 196fe0b..8937f75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,6 @@ RUN set -eux \ && cd /usr/src/pgvector \ && make \ && make install \ - && apt-mark manual ca-certificates \ && apt-get purge -y --auto-remove \ build-essential \ git \ @@ -51,12 +50,14 @@ RUN set -eux \ lsb-release \ gnupg \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /usr/src/pgvector + && rm -rf /var/lib/apt/lists/* /usr/src/pgvector \ + /etc/apt/sources.list.d/pgdg.list \ + /usr/share/keyrings/postgresql-archive-keyring.gpg # Copy initialization scripts COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ - CMD pg_isready -U postgres || exit 1 + CMD pg_isready -U postgres EXPOSE 5432