From e52ae7a21de4fc3bc48ff602632d8acd143d13f4 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 9 Jan 2024 16:58:28 -0500 Subject: [PATCH] Publish arm docker base image (#1045) Also builds images for the upcoming Ubuntu 24.04 LTS --- .github/workflows/vcpkg_docker.yml | 169 +++++++++++++++++++++++ .github/workflows/vcpkg_docker_amd64.yml | 75 ---------- docker/Dockerfile.ubuntu.vcpkg | 29 ++-- docker/build.sh | 19 ++- 4 files changed, 189 insertions(+), 103 deletions(-) create mode 100644 .github/workflows/vcpkg_docker.yml delete mode 100644 .github/workflows/vcpkg_docker_amd64.yml diff --git a/.github/workflows/vcpkg_docker.yml b/.github/workflows/vcpkg_docker.yml new file mode 100644 index 00000000..cba23023 --- /dev/null +++ b/.github/workflows/vcpkg_docker.yml @@ -0,0 +1,169 @@ +name: Docker Build Images + +# Based on https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners + +on: + schedule: + # Once every Wednesday at 00:00 + - cron: '0 0 * * 3' + push: + branches: + - master + paths: + - 'docker/**' + - '.github/workflows/vcpkg_docker.yml' + pull_request: + paths: + - 'docker/**' + - '.github/workflows/vcpkg_docker.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + env: + # This needs to be the same as in the `merge` job + # Also remember to change the 'docker/build.sh' script + REGISTRY_IMAGE: ghcr.io/lifting-bits/cxx-common/vcpkg-builder-ubuntu-v2 + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + ubuntu_version: + - 22.04 + - 24.04 + steps: + - name: Prepare + run: | + platform="${{ matrix.platform }}" + echo "PLATFORM_PAIR=${platform//\//-}" >> "${GITHUB_ENV}" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # BEGIN Copied to the next job + - name: Generate Tag + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: | + test_tag="" + if [[ "${GITHUB_REF}" != "refs/heads/${{ github.event.repository.default_branch }}" ]] ; then + test_tag="test-${BRANCH_NAME////_}-" + fi + echo "TEST_TAG=${test_tag}" >> "${GITHUB_ENV}" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTERY_IMAGE }} + flavor: | + latest=false + tags: | + type=raw,value=${{ env.TEST_TAG }}${{ matrix.ubuntu_version }} + # END Copied to the next job + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5 + with: + context: docker + file: docker/Dockerfile.ubuntu.vcpkg + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.ubuntu_version }}-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: + - build + env: + # This needs to be the same as in the `build` job + REGISTRY_IMAGE: ghcr.io/lifting-bits/cxx-common/vcpkg-builder-ubuntu-v2 + strategy: + fail-fast: false + matrix: + ubuntu_version: + - 22.04 + - 24.04 + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-${{ matrix.ubuntu_version }}-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # BEGIN Copied to the previous job + - name: Generate Tag + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: | + test_tag="" + if [[ "${GITHUB_REF}" != "refs/heads/${{ github.event.repository.default_branch }}" ]] ; then + test_tag="test-${BRANCH_NAME////_}-" + fi + echo "TEST_TAG=${test_tag}" >> "${GITHUB_ENV}" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + flavor: | + latest=false + tags: | + type=raw,value=${{ env.TEST_TAG }}${{ matrix.ubuntu_version }} + # END Copied from the previous job + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/vcpkg_docker_amd64.yml b/.github/workflows/vcpkg_docker_amd64.yml deleted file mode 100644 index 51604772..00000000 --- a/.github/workflows/vcpkg_docker_amd64.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Docker Build Images (AMD64) - -env: - # Also remember to change the 'docker/build.sh' script - IMAGE_VER: "v2" - -on: - schedule: - # Once every Wednesday at 00:00 - - cron: '0 0 * * 3' - push: - branches: - - master - paths: - - 'docker/**' - - '.github/workflows/vcpkg_docker_amd64.yml' - pull_request: - paths: - - 'docker/**' - - '.github/workflows/vcpkg_docker_amd64.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - strategy: - fail-fast: false - matrix: - # arch is to tag docker images for uniqueness - host: - - { name: 'ubuntu-22.04', arch: '' } - container: - - { version: '22.04', codename: 'jammy' } - - runs-on: ${{ matrix.host.name }} - - permissions: - packages: write - contents: read - - steps: - - name: Cleanup working directory with container root - run: | - docker run --rm -v $(pwd):/workspace -w /workspace ubuntu:latest find . -name . -o -prune -exec rm -rf -- {} + || true - - uses: actions/checkout@v4 - - name: Generate Image Name - env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - run: | - test_name="" - if [[ "${GITHUB_REF}" != "refs/heads/master" ]] ; then - test_name="test-${BRANCH_NAME////_}-" - fi - echo "IMAGE_NAME=ghcr.io/lifting-bits/cxx-common/${test_name}vcpkg-builder-ubuntu-${{ env.IMAGE_VER }}:${{ matrix.container.version }}${{ matrix.host.arch }}" >> ${GITHUB_ENV} - - - name: Build image - working-directory: docker - run: | - # Pull freshest ubuntu Docker image - docker pull ubuntu:${{ matrix.container.version}} - - docker build -f Dockerfile.ubuntu.vcpkg \ - --no-cache \ - --target caching \ - --build-arg "DISTRO_VERSION=${{ matrix.container.codename }}" \ - -t "${IMAGE_NAME}" \ - . - - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Push image - run: docker push "${IMAGE_NAME}" diff --git a/docker/Dockerfile.ubuntu.vcpkg b/docker/Dockerfile.ubuntu.vcpkg index b16f82c7..2c406b74 100644 --- a/docker/Dockerfile.ubuntu.vcpkg +++ b/docker/Dockerfile.ubuntu.vcpkg @@ -1,9 +1,8 @@ -ARG DISTRO_VERSION=focal +ARG UBUNTU_VERSION=22.04 ARG LLVM_VERSION=16 -ARG BUILD_BASE=ubuntu:${DISTRO_VERSION} -FROM ${BUILD_BASE} as base -ARG DISTRO_VERSION +FROM ubuntu:${UBUNTU_VERSION} as base +ARG UBUNTU_VERSION ARG LLVM_VERSION # All build dependencies for vcpkg packages @@ -15,16 +14,18 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get install --yes apt-utils && apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends apt-transport-https software-properties-common gnupg ca-certificates wget && \ apt-add-repository ppa:git-core/ppa --yes && \ - wget "https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-linux-$(uname -m).sh" && \ + wget "https://github.com/Kitware/CMake/releases/download/v3.28.1/cmake-3.28.1-linux-$(uname -m).sh" && \ /bin/bash cmake-*.sh --skip-license --prefix=/usr/local && rm cmake-*.sh && \ \ - wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - ./llvm.sh ${LLVM_VERSION} clang lld && \ + if [ "${UBUNTU_VERSION}" != "24.04" ] ; then \ + wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} clang lld ; \ + fi && \ \ apt-get update && apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends \ - libtinfo-dev libzstd-dev python3-pip python3-setuptools python-setuptools \ + libtinfo-dev libzstd-dev python3-pip python3-setuptools \ build-essential binutils-multiarch g++ gcc clang lld clang-${LLVM_VERSION} lld-${LLVM_VERSION} ninja-build \ pixz xz-utils make rpm curl unzip tar git zip python3 pkg-config && \ apt-get install --yes --no-install-recommends \ @@ -37,21 +38,13 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ cd ~ && mkdir build && cd build && \ curl -s https://api.github.com/repos/ccache/ccache/releases/latest | grep tarball_url | cut -d '"' -f 4 | wget -i- -O - | tar -xz && \ cd ccache-ccache-* && \ - cmake -S . -B build -G Ninja -DREDIS_STORAGE_BACKEND=OFF -DCMAKE_BUILD_TYPE=Release && \ + cmake -S . -B build -G Ninja -DREDIS_STORAGE_BACKEND=OFF -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release && \ cmake --build build --target install && \ cd .. && rm -rf ccache-ccache-* # Much heavier installation due to mono dependency for NuGet FROM base as caching -ARG DISTRO_VERSION RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - \ - if [ ${DISTRO_VERSION} != "jammy" ] ; then \ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \ - echo "deb https://download.mono-project.com/repo/ubuntu stable-${DISTRO_VERSION} main" | tee /etc/apt/sources.list.d/mono-official-stable.list ; \ - fi && \ - \ apt-get update && \ apt-get install --yes mono-devel && \ apt-get clean --yes && \ diff --git a/docker/build.sh b/docker/build.sh index f5fdda02..a5fa45e8 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -5,24 +5,23 @@ set -euo pipefail # Builds base images with all required dependencies to bootstrap vcpkg and # build vcpkg libraries as well as all lifting-bits tools -# Also remember to change the '.github/workflows/vcpkg_docker_amd64.yml' variable -IMAGE_VER=v2 - # Ubuntu versions to build -UBUNTU_VERSION_MATRIX=( "focal" "jammy" ) +UBUNTU_VERSION_MATRIX=( "22.04" "24.04" ) -for version in "${UBUNTU_VERSION_MATRIX[@]}"; do +for ubuntu_version in "${UBUNTU_VERSION_MATRIX[@]}"; do # Always pull from upstream - docker pull "ubuntu:${version}" + docker pull "ubuntu:${ubuntu_version}" - # Image identification - tag="vcpkg-builder-ubuntu-${IMAGE_VER}:${version}" + # Image identification. "v2" Image version is to identify big changes to the + # build toolchain like LLVM version + # Also remember to change the '.github/workflows/vcpkg_docker.yml' variable + image="vcpkg-builder-ubuntu-v2" # Build docker build \ -f Dockerfile.ubuntu.vcpkg \ --no-cache \ - --build-arg "DISTRO_VERSION=${version}" \ - -t "${tag}" \ + --build-arg "UBUNTU_VERSION=${ubuntu_version}" \ + -t "${image}:${ubuntu_version}" \ . done