From 70c9a33d085054ecb72994be50aaf998561266ba Mon Sep 17 00:00:00 2001 From: Will Faithfull Date: Mon, 2 Sep 2024 20:19:55 +0200 Subject: [PATCH 01/14] feat: Enable multi-platform builds for board server --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d801a7ce44b..a6c2fdf4820 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -109,6 +109,7 @@ jobs: cache-to: type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} push: true provenance: true + platforms: linux/amd64,linux/arm64,darwin/arm64 tags: > ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ github.sha }}, ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ needs.setup.outputs.short_hash }}, @@ -167,6 +168,7 @@ jobs: type=registry,mode=max,ref=${{ needs.setup.outputs.board_server_firestore_image_path }} push: true provenance: true + platforms: linux/amd64,linux/arm64,darwin/arm64 tags: > ${{ needs.setup.outputs.board_server_firestore_image_path }}:${{ github.sha }}, ${{ needs.setup.outputs.board_server_firestore_image_path }}:${{ needs.setup.outputs.short_hash }}, @@ -183,4 +185,4 @@ jobs: with: subject-name: ${{ needs.setup.outputs.board_server_firestore_image_path }} subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true \ No newline at end of file + push-to-registry: true From 6918f3f65d72f13a6581b3e5b35ca6422ca8a012 Mon Sep 17 00:00:00 2001 From: Will Date: Mon, 2 Sep 2024 20:32:42 +0200 Subject: [PATCH 02/14] fix: remove sha256 tagging while we figure out what is wrong --- .github/workflows/docker.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a6c2fdf4820..7495b0bbe88 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -111,8 +111,6 @@ jobs: provenance: true platforms: linux/amd64,linux/arm64,darwin/arm64 tags: > - ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ github.sha }}, - ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ needs.setup.outputs.short_hash }}, ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ needs.setup.outputs.package_version }}, ${{ needs.setup.outputs.board_server_sqlite_image_path }}:latest build-args: | @@ -170,8 +168,6 @@ jobs: provenance: true platforms: linux/amd64,linux/arm64,darwin/arm64 tags: > - ${{ needs.setup.outputs.board_server_firestore_image_path }}:${{ github.sha }}, - ${{ needs.setup.outputs.board_server_firestore_image_path }}:${{ needs.setup.outputs.short_hash }}, ${{ needs.setup.outputs.board_server_firestore_image_path }}:${{ needs.setup.outputs.package_version }}, ${{ needs.setup.outputs.board_server_firestore_image_path }}:latest build-args: | From 6d4e64afe11a5583beb3aeaf2c250b18de29a52a Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 16 Sep 2024 15:20:30 +0100 Subject: [PATCH 03/14] Add support for pull requests in Docker workflow and conditionally push images - resolves: #3181 --- .github/workflows/docker.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7495b0bbe88..f1b276727d7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,9 @@ on: push: branches: - main + pull_request: + branches: + - main env: REGISTRY: ghcr.io @@ -107,7 +110,7 @@ jobs: type=gha,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} type=registry,ref=${{ needs.setup.outputs.board_server_sqlite_image_path }} cache-to: type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} - push: true + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} provenance: true platforms: linux/amd64,linux/arm64,darwin/arm64 tags: > @@ -124,7 +127,7 @@ jobs: with: subject-name: ${{ needs.setup.outputs.board_server_sqlite_image_path }} subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true + push-to-registry: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} board-server-firestore-image: name: Build board server firestore image From 7ec4fcb4506e62ec7a7c136dabeb27b3f3750c97 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 16 Sep 2024 15:24:51 +0100 Subject: [PATCH 04/14] Update Dockerfile to use node:20 base image --- packages/board-server/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/board-server/Dockerfile b/packages/board-server/Dockerfile index 3adbb66617d..7786bd62a46 100644 --- a/packages/board-server/Dockerfile +++ b/packages/board-server/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM node:20-slim AS build +FROM node:20 AS build WORKDIR /build # Copy the entire monorepo @@ -35,4 +35,4 @@ RUN npm install --only=production && \ npm install -g tsx EXPOSE 3000 -CMD ["node", "dist/server/index.js", "--host=0.0.0.0" ] \ No newline at end of file +CMD ["node", "dist/server/index.js", "--host=0.0.0.0" ] From a40ce1344fefec008e8802b9f0695981a7a8f8a9 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 16 Sep 2024 15:27:42 +0100 Subject: [PATCH 05/14] Update Dockerfile to use node:20 instead of node:20-slim --- packages/board-server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/board-server/Dockerfile b/packages/board-server/Dockerfile index 7786bd62a46..99ba5291bb0 100644 --- a/packages/board-server/Dockerfile +++ b/packages/board-server/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /build/packages/board-server RUN npm run build # Production stage -FROM node:20-slim +FROM node:20 ARG STORAGE_BACKEND ARG ALLOWED_ORIGINS="" From 904b058ec4868b94647fda1448d7521245a0c7ba Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 16 Sep 2024 15:34:26 +0100 Subject: [PATCH 06/14] Remove darwin/arm64 platform from Docker workflow configuration --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f1b276727d7..85f61ad03a3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -112,7 +112,7 @@ jobs: cache-to: type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} provenance: true - platforms: linux/amd64,linux/arm64,darwin/arm64 + platforms: linux/amd64,linux/arm64 tags: > ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ needs.setup.outputs.package_version }}, ${{ needs.setup.outputs.board_server_sqlite_image_path }}:latest From 9a70be35f221eddd402b323e198bab8fc570b986 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 16 Sep 2024 15:52:55 +0100 Subject: [PATCH 07/14] Add QEMU setup to Docker workflow for multi-platform builds --- .github/workflows/docker.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 85f61ad03a3..449d181a78f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -90,6 +90,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -145,6 +150,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 8483478f10938905a6b51128b6cd2e2ad183e237 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 16 Sep 2024 15:53:45 +0100 Subject: [PATCH 08/14] Add darwin/amd64 support to Docker workflow platforms --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 449d181a78f..92b5105b140 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -117,7 +117,7 @@ jobs: cache-to: type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} provenance: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64,darwin/amd64,darwin/arm64 tags: > ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ needs.setup.outputs.package_version }}, ${{ needs.setup.outputs.board_server_sqlite_image_path }}:latest @@ -179,7 +179,7 @@ jobs: type=registry,mode=max,ref=${{ needs.setup.outputs.board_server_firestore_image_path }} push: true provenance: true - platforms: linux/amd64,linux/arm64,darwin/arm64 + platforms: linux/amd64,linux/arm64,darwin/amd64,darwin/arm64 tags: > ${{ needs.setup.outputs.board_server_firestore_image_path }}:${{ needs.setup.outputs.package_version }}, ${{ needs.setup.outputs.board_server_firestore_image_path }}:latest From 12b5df7d92a4eae653430ef56dc5a777b7792a75 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 16 Sep 2024 15:54:29 +0100 Subject: [PATCH 09/14] Use multi-platform support in Dockerfile for build and runtime stages --- packages/board-server/Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/board-server/Dockerfile b/packages/board-server/Dockerfile index 99ba5291bb0..ed2f697fd0a 100644 --- a/packages/board-server/Dockerfile +++ b/packages/board-server/Dockerfile @@ -1,5 +1,6 @@ -# Build stage -FROM node:20 AS build +# Use a base image that supports multiple platforms for the build stage +FROM --platform=$BUILDPLATFORM node:20 AS build + WORKDIR /build # Copy the entire monorepo @@ -12,8 +13,8 @@ RUN npm ci WORKDIR /build/packages/board-server RUN npm run build -# Production stage -FROM node:20 +# Use a lightweight base image for the runtime stage +FROM --platform=$TARGETPLATFORM node:20 ARG STORAGE_BACKEND ARG ALLOWED_ORIGINS="" @@ -35,4 +36,4 @@ RUN npm install --only=production && \ npm install -g tsx EXPOSE 3000 -CMD ["node", "dist/server/index.js", "--host=0.0.0.0" ] +CMD ["node", "dist/server/index.js", "--host=0.0.0.0"] From 9d95d9076953c0e4bfea1f190129ecb3c7ead7e4 Mon Sep 17 00:00:00 2001 From: Will Date: Wed, 18 Sep 2024 13:16:38 +0100 Subject: [PATCH 10/14] fix: remove darwin --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 92b5105b140..f3bc94088af 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -117,7 +117,7 @@ jobs: cache-to: type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} provenance: true - platforms: linux/amd64,linux/arm64,darwin/amd64,darwin/arm64 + platforms: linux/amd64,linux/arm64 tags: > ${{ needs.setup.outputs.board_server_sqlite_image_path }}:${{ needs.setup.outputs.package_version }}, ${{ needs.setup.outputs.board_server_sqlite_image_path }}:latest From bbbe9512b8fc433f24902f63234afd984fb917f9 Mon Sep 17 00:00:00 2001 From: Will Date: Wed, 18 Sep 2024 13:25:45 +0100 Subject: [PATCH 11/14] fix: disable attestation on PR builds --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f3bc94088af..743b25ed128 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -128,11 +128,12 @@ jobs: breadboard=. - name: Generate artifact attestation + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork uses: actions/attest-build-provenance@v1 with: subject-name: ${{ needs.setup.outputs.board_server_sqlite_image_path }} subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} + push-to-registry: true board-server-firestore-image: name: Build board server firestore image @@ -190,6 +191,7 @@ jobs: breadboard=. - name: Generate artifact attestation + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork uses: actions/attest-build-provenance@v1 with: subject-name: ${{ needs.setup.outputs.board_server_firestore_image_path }} From abd22b3f8e9b9a4a920a3f76839d96f99bd79d6c Mon Sep 17 00:00:00 2001 From: Will Date: Wed, 18 Sep 2024 13:41:54 +0100 Subject: [PATCH 12/14] fix: remove darwin builds for firestore --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 743b25ed128..7903af139fe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -180,7 +180,7 @@ jobs: type=registry,mode=max,ref=${{ needs.setup.outputs.board_server_firestore_image_path }} push: true provenance: true - platforms: linux/amd64,linux/arm64,darwin/amd64,darwin/arm64 + platforms: linux/amd64,linux/arm64 tags: > ${{ needs.setup.outputs.board_server_firestore_image_path }}:${{ needs.setup.outputs.package_version }}, ${{ needs.setup.outputs.board_server_firestore_image_path }}:latest From a1ef24cc9bc313aa83a7c3310dff76004aded457 Mon Sep 17 00:00:00 2001 From: Will Date: Wed, 18 Sep 2024 13:53:19 +0100 Subject: [PATCH 13/14] fix: only push firestore on main --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7903af139fe..5b6ae858469 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -178,7 +178,7 @@ jobs: cache-to: | type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_firestore_image_name }} type=registry,mode=max,ref=${{ needs.setup.outputs.board_server_firestore_image_path }} - push: true + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} provenance: true platforms: linux/amd64,linux/arm64 tags: > From 3eab1a6fd8bf529bb35524bbb29bcb63af94c215 Mon Sep 17 00:00:00 2001 From: Will Date: Wed, 18 Sep 2024 15:22:40 +0100 Subject: [PATCH 14/14] feat: make caching more intellgent, no registry caching on PRs --- .github/workflows/docker.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5b6ae858469..28c0e1d9eff 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -112,9 +112,9 @@ jobs: context: . file: ./packages/board-server/Dockerfile cache-from: | - type=gha,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} - type=registry,ref=${{ needs.setup.outputs.board_server_sqlite_image_path }} - cache-to: type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_sqlite_image_name }} + ${{ github.event_name != 'pull_request' && format('type=gha,ref={0}', needs.setup.outputs.board_server_sqlite_image_name) || 'type=gha,scope=pr' }} + ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}', needs.setup.outputs.board_server_sqlite_image_path) || '' }} + cache-to: ${{ github.event_name != 'pull_request' && format('type=gha,mode=max,ref={0}', needs.setup.outputs.board_server_sqlite_image_name) || 'type=gha,mode=max,scope=pr' }} push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} provenance: true platforms: linux/amd64,linux/arm64 @@ -173,11 +173,9 @@ jobs: context: . file: ./packages/board-server/Dockerfile cache-from: | - type=gha,ref=${{ needs.setup.outputs.board_server_firestore_image_name }} - type=registry,ref=${{ needs.setup.outputs.board_server_firestore_image_path }} - cache-to: | - type=gha,mode=max,ref=${{ needs.setup.outputs.board_server_firestore_image_name }} - type=registry,mode=max,ref=${{ needs.setup.outputs.board_server_firestore_image_path }} + ${{ github.event_name != 'pull_request' && format('type=gha,ref={0}', needs.setup.outputs.board_server_firestore_image_name) || 'type=gha,scope=pr' }} + ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}', needs.setup.outputs.board_server_firestore_image_path) || '' }} + cache-to: ${{ github.event_name != 'pull_request' && format('type=gha,mode=max,ref={0}|type=registry,mode=max,ref={1}', needs.setup.outputs.board_server_firestore_image_name, needs.setup.outputs.board_server_firestore_image_path) || 'type=gha,mode=max,scope=pr' }} push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork }} provenance: true platforms: linux/amd64,linux/arm64