From cdee0e6a0089fd13ae1c17ea437de7882c23f192 Mon Sep 17 00:00:00 2001 From: Sergey Zabolotny Date: Thu, 4 Jun 2020 16:16:06 +0300 Subject: [PATCH 01/18] Install minio specific version --- base/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/Dockerfile b/base/Dockerfile index a30b6f3..0acefee 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -17,6 +17,7 @@ RUN set -xe; \ ARG DOCKER_VERSION=18.09.2 ARG DOCKER_COMPOSE_VERSION=1.23.2 +ARG MINIO_VERSION="RELEASE.2020-01-13T22-49-03Z" RUN set -xe; \ # Install docker cli curl -fsSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz"; \ @@ -26,7 +27,7 @@ RUN set -xe; \ pip install "docker-compose==${DOCKER_COMPOSE_VERSION}" >/dev/null; \ docker-compose --version; \ # Install minio client (mc) - curl -fsSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc; \ + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.${MINIO_VERSION} -o /usr/local/bin/mc; \ chmod +x /usr/local/bin/mc ENV AGENT_USER=agent From 72322f4458cfc55e2d954e091679e9b2fd252620 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Jun 2020 11:00:21 -0700 Subject: [PATCH 02/18] Updated mc policy command New minio versions changed the parameters slightly. --- base/bin/build-acp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index fbbc3aa..0434710 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -54,7 +54,7 @@ mc mb "s3/${ARTIFACTS_BUCKET_NAME}" mc mirror --overwrite --remove ${source} ${destination} # Set read-only permissions on BUCKET_PATH # Access is possible only if BUCKET_PATH is known (parent bucket stays private => XML listing of bucket objects disabled) -mc policy download "s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" +mc policy set download "s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" echo "Build artifacts upload completed:" echo "${ARTIFACTS_URL}" From 9b37d53121c6aff2135a87b1731df237a2016d40 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Jun 2020 11:01:39 -0700 Subject: [PATCH 03/18] Print minio commands for easy of debugging Print all but the config command, otherwise we'd be exposing the AWS credentials in build logs. --- base/bin/build-acp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index 0434710..2b54825 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -46,16 +46,20 @@ fi # AWS API keys must be set in the build settings mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} +set -x # Debug ON + # Create bucket if it does not exist mc mb "s3/${ARTIFACTS_BUCKET_NAME}" # Upload artifact into the bucket # TODO: figure out how to skip timestemp checking. Maybe use aws-cli instead on the minio client? # Looks like mirror does not have any benefit here over cp. Files in build have newer timestamps and are always copied. mc mirror --overwrite --remove ${source} ${destination} -# Set read-only permissions on BUCKET_PATH +# Set download (read-only) policy on BUCKET_PATH # Access is possible only if BUCKET_PATH is known (parent bucket stays private => XML listing of bucket objects disabled) mc policy set download "s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" +set +x # Debug OFF + echo "Build artifacts upload completed:" echo "${ARTIFACTS_URL}" echo "Files:" From 1fa74d42f81e39e14903c8ae62817dcaafe22314 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Jun 2020 12:09:35 -0700 Subject: [PATCH 04/18] Updated minio client to RELEASE.2020-05-28T23-43-36Z --- base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/Dockerfile b/base/Dockerfile index 0acefee..77a4d47 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -17,7 +17,7 @@ RUN set -xe; \ ARG DOCKER_VERSION=18.09.2 ARG DOCKER_COMPOSE_VERSION=1.23.2 -ARG MINIO_VERSION="RELEASE.2020-01-13T22-49-03Z" +ARG MINIO_VERSION="RELEASE.2020-05-28T23-43-36Z" RUN set -xe; \ # Install docker cli curl -fsSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz"; \ From 56d7669ea2a336f9777fe0b0a780961f20539931 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Jun 2020 12:11:50 -0700 Subject: [PATCH 05/18] Normalized trailing slashes in paths - Necessary to produce correct S3 policy rules - The latest minio client requires a trailing slash in the destination path --- base/bin/build-acp | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index 2b54825..0c14cff 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -3,39 +3,46 @@ # Build Artifact Copy (build-acp) # Copy build artifacts into S3 # -# Usage: build-acp +# Usage: build-acp [] set -e # Abort if anything fails #set -x # Echo commands +## Enable extended globbing features +shopt -s extglob + if [[ "${AWS_ACCESS_KEY_ID}" == "" ]] || [[ "${AWS_SECRET_ACCESS_KEY}" == "" ]] || [[ "${ARTIFACTS_BUCKET_NAME}" == "" ]]; then echo "Cannot upload artifacts." echo "Please make sure AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and ARTIFACTS_BUCKET_NAME environment variables are configured." exit 1 fi -# Configure ARTIFACTS_BASE_URL, unless it has been already defined -ARTIFACTS_BASE_URL=${ARTIFACTS_BASE_URL:-"https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com"} -# Configure ARTIFACTS_BUCKET_PATH, unless it has been already defined -ARTIFACTS_BUCKET_PATH=${ARTIFACTS_BUCKET_PATH:-"${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}"} +# Configure ARTIFACTS_BASE_URL, unless it has been already defined (trim any trailing slashes) +ARTIFACTS_BASE_URL=${ARTIFACTS_BASE_URL%%+(/):-"https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com"} +# Configure ARTIFACTS_BUCKET_PATH, unless it has been already defined (trim any trailing slashes) +ARTIFACTS_BUCKET_PATH=${ARTIFACTS_BUCKET_PATH%%+(/):-"${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}"} # Configure ARTIFACTS_URL, unless it has been already defined # By default expecting index.html as the default entry point into the bucket/path ARTIFACTS_URL=${ARTIFACTS_URL:-"${ARTIFACTS_BASE_URL}/${ARTIFACTS_BUCKET_PATH}/index.html"} -# Sync source locally +# Sync source (local) source=${1} -# Sync destination on S3 +# Sync destination (S3) destination="s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" -# Allow specifying a subpath within the destination path -if [[ "${2}" != "" ]]; then destination="${destination}/${2}"; fi +# Allow specifying a subpath within destination (trim any trailing slashes) +if [[ "${2}" != "" ]]; then subpath="${2%%+(/)}/"; fi + +# Ensure paths ends with a single trailing slash +# ${VAR%%+(/)} strips out all trailing slashes (requires extglob enabled), then we add a single trailing slash back +ARTIFACTS_BUCKET_PATH="${ARTIFACTS_BUCKET_PATH%%+(/)}/" +destination="${destination%%+(/)}/" # Print configuration for debugging purposes -echo "Configuration: " +echo "Configuration (normalized): " echo "source='${source}'" echo "destination='${destination}'" -echo "ARTIFACTS_BASE_URL='${ARTIFACTS_BASE_URL}'" -echo "ARTIFACTS_BUCKET_PATH='${ARTIFACTS_BUCKET_PATH}'" -echo "ARTIFACTS_URL='${ARTIFACTS_URL}'" +echo "subpath='${subpath}'" +echo "url='${ARTIFACTS_URL}'" if [[ "${source}" == "" ]]; then echo "Error: Source path missing." @@ -53,10 +60,12 @@ mc mb "s3/${ARTIFACTS_BUCKET_NAME}" # Upload artifact into the bucket # TODO: figure out how to skip timestemp checking. Maybe use aws-cli instead on the minio client? # Looks like mirror does not have any benefit here over cp. Files in build have newer timestamps and are always copied. -mc mirror --overwrite --remove ${source} ${destination} +mc mirror --overwrite --remove ${source} ${destination}${subpath} # Set download (read-only) policy on BUCKET_PATH # Access is possible only if BUCKET_PATH is known (parent bucket stays private => XML listing of bucket objects disabled) -mc policy set download "s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" +# Note: ${destination} is used here instead of ${destination}${subpath} to minimize the number of S3 policy rules when +# multiple subpaths are used within the same destination. +mc policy set download ${destination} set +x # Debug OFF From d0afcd27622cd98505e6ef0bf6fe4c85bca1d281 Mon Sep 17 00:00:00 2001 From: Les Peabody Date: Fri, 17 Jul 2020 12:33:41 -0400 Subject: [PATCH 06/18] Moved REMOTE_BUILD_DIR_CLEANUP to build-init (#63) Move REMOTE_BUILD_DIR_CLEANUP into build-init since that is where its used. Co-authored-by: Les Peabody (BOS-GEN) --- base/bin/build-env | 1 - base/bin/build-init | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/base/bin/build-env b/base/bin/build-env index 502e0f5..d755bae 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -18,7 +18,6 @@ DEBUG=${DEBUG:-0} # `DEBUG=1 build-env` to run with debugging turned ON DOCKER_HOST_TUNNEL=localhost:2374 GIT_USER_EMAIL=${GIT_USER_EMAIL:-ci@docksal.io} GIT_USER_NAME=${GIT_USER_NAME:-Docksal CI} -REMOTE_BUILD_DIR_CLEANUP=${REMOTE_BUILD_DIR_CLEANUP:-1} # Default to re-initializing environment. # These are used to generate the sandbox sub-domain (branch-project.example.com) # There is a limit of 63 characters for any part of the domain name. diff --git a/base/bin/build-init b/base/bin/build-init index e0315ba..978a0ee 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -4,6 +4,7 @@ # Set script-specific variables. BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT:-local} +REMOTE_BUILD_DIR_CLEANUP=${REMOTE_BUILD_DIR_CLEANUP:-1} # Default to re-initializing environment. # Exit if using an invalid codebase method. if [[ "${REMOTE_CODEBASE_METHOD}" != "rsync" ]] && [[ "${REMOTE_CODEBASE_METHOD}" != "git" ]]; then From 8c514c5050b18c96b10f1db77718cdf38270769e Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 9 Apr 2021 17:22:15 -0700 Subject: [PATCH 07/18] Generate SSH public keys in build-env --- base/bin/build-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/bin/build-env b/base/bin/build-env index 502e0f5..9392349 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -229,12 +229,14 @@ ssh_init () if [[ "$CI_SSH_KEY" != "" ]]; then echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa chmod 0600 $HOME/.ssh/id_rsa + ssh-keygen -y -f $HOME/.ssh/id_rsa > $HOME/.ssh/id_rsa.pub fi # Docksal Sandbox server key if [[ "$DOCKSAL_HOST_SSH_KEY" != "" ]]; then echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa chmod 0600 $HOME/.ssh/docksal_host_id_rsa + ssh-keygen -y -f $HOME/.ssh/docksal_host_id_rsa > $HOME/.ssh/docksal_host_id_rsa.pub fi # Initialize ssh-agent and load the default key ($HOME/.ssh/id_rsa) From c1f942a4db3d425b58f7b46aa560c8c7d039e9c8 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 08:42:28 -0700 Subject: [PATCH 08/18] GitHub Actions configuration --- .github/scripts/docker-tags.sh | 50 ++++++++++ .github/workflows/default.yaml | 164 +++++++++++++++++++++++++++++++++ Makefile | 25 ++--- php/Dockerfile | 2 +- 4 files changed, 229 insertions(+), 12 deletions(-) create mode 100755 .github/scripts/docker-tags.sh create mode 100644 .github/workflows/default.yaml diff --git a/.github/scripts/docker-tags.sh b/.github/scripts/docker-tags.sh new file mode 100755 index 0000000..ae49ee1 --- /dev/null +++ b/.github/scripts/docker-tags.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Generates docker images tags for the docker/build-push-action@v2 action depending on the branch/tag. + +declare -a registryArr +registryArr+=("docker.io") # Docker Hub +registryArr+=("ghcr.io") # GitHub Container Registry + +declare -a imageTagArr + +# feature/* => sha-xxxxxxx +# Note: disabled +#if [[ "${GITHUB_REF}" =~ "refs/heads/feature/" ]]; then +# GIT_SHA7=$(echo ${GITHUB_SHA} | cut -c1-7) # Short SHA (7 characters) +# imageTagArr+=("${IMAGE}:${VERSION}-sha-${GIT_SHA7}") +#fi + +# develop => version-edge +if [[ "${GITHUB_REF}" == "refs/heads/develop" ]]; then + imageTagArr+=("${IMAGE}:${VERSION}-edge") +fi + +# master => version +if [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then + imageTagArr+=("${IMAGE}:${VERSION}") +fi + +# tags/v1.0.0 => 1.0 +if [[ "${GITHUB_REF}" =~ "refs/tags/" ]]; then + # Extract version parts from release tag + IFS='.' read -a release_arr <<< "${GITHUB_REF#refs/tags/}" + releaseMajor=${release_arr[0]#v*} # 2.7.0 => "2" + releaseMinor=${release_arr[1]} # "2.7.0" => "7" + imageTagArr+=("${IMAGE}:${VERSION}") + imageTagArr+=("${IMAGE}:${VERSION}-${releaseMajor}") + imageTagArr+=("${IMAGE}:${VERSION}-${releaseMajor}.${releaseMinor}") +fi + +# Build an array of registry/image:tag values +declare -a repoImageTagArr +for registry in ${registryArr[@]}; do + for imageTag in ${imageTagArr[@]}; do + repoImageTagArr+=("${registry}/${imageTag}") + done +done + +# Print with new lines for output in build logs +(IFS=$'\n'; echo "${repoImageTagArr[*]}") +# Using newlines in outputs variables does not seem to work, so we'll use comas +(IFS=$','; echo "::set-output name=tags::${repoImageTagArr[*]}") diff --git a/.github/workflows/default.yaml b/.github/workflows/default.yaml new file mode 100644 index 0000000..ff0ffe1 --- /dev/null +++ b/.github/workflows/default.yaml @@ -0,0 +1,164 @@ +name: Docker Build and Push + +on: + schedule: + - cron: '0 10 * * 0' # everyday sunday at 10am + push: + branches: + - master + - develop + - feature/* + tags: + - 'v*.*.*' + +defaults: + run: + shell: bash + +jobs: + build-test-push: + name: Build, Test, Push + runs-on: ubuntu-20.04 + + env: + IMAGE: docksal/ci-agent + + steps: + - + name: Install prerequisites for tests + run: | + set -xeuo pipefail + # Install bats for tests + git clone https://github.com/bats-core/bats-core.git + cd bats-core + sudo ./install.sh /usr/local + bats -v + - + name: Checkout + uses: actions/checkout@v2 +# - +# name: Set up QEMU +# uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Check Docker + run: | + docker version + docker info + - + name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + # Calculates docker image tags for the given build context + # The output is used in build and push step as `tags: ${{ steps.docker_meta.outputs.tags }}` + # See https://github.com/crazy-max/ghaction-docker-meta + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + # List of Docker images to use as base name for tags + images: | + ${{ env.IMAGE }} + ghcr.io/${{ env.IMAGE }} + tag-sha: true # add git short SHA as Docker tag + - + # Generate image meta information + name: Docker image tags + id: docker_tags + run: make tags + - + # Build for local use + name: Build image (base) + run: make build VERSION=base + - + # Build for local use + name: Build image (php) + run: make build VERSION=php + - + # Print image info + name: Docker image info + run: | + set -xeuo pipefail + docker image ls | grep "${{ env.IMAGE }}" + docker image inspect "${{ env.IMAGE }}:base-build" + docker image inspect "${{ env.IMAGE }}:php-build" + + # Cache image layers in the registry + - + name: Push image cache (base) + uses: docker/build-push-action@v2 + env: + IMAGE_CACHE: ghcr.io/${{ env.IMAGE }}:base-build + with: + context: base + file: base/Dockerfile + #platforms: linux/amd64,linux/arm64 + tags: ${{ env.IMAGE_CACHE }} # Build cache tag in ghcr.io + push: ${{ github.event_name != 'pull_request' }} # Don't push for PRs + cache-to: type=inline # Write the cache metadata into the image configuration + - + name: Push image cache (php) + uses: docker/build-push-action@v2 + env: + IMAGE_CACHE: ghcr.io/${{ env.IMAGE }}:php-build + with: + context: php + file: php/Dockerfile + #platforms: linux/amd64,linux/arm64 + tags: ${{ env.IMAGE_CACHE }} # Build cache tag in ghcr.io + push: ${{ github.event_name != 'pull_request' }} # Don't push for PRs + cache-to: type=inline # Write the cache metadata into the image configuration + + # Tests + - + name: Test image (base) + run: make test VERSION=base + - + name: Test image (php) + run: make test VERSION=php + + - + # Push final image to the registry + # This will pick-up the build cache from the local build step + name: Push image (base) + # Don't run if the list of tags is empty + # Note: using tags from docker_tags (custom) + if: ${{ steps.docker_tags.outputs.tags != '' }} + uses: docker/build-push-action@v2 + with: + context: base + file: base/Dockerfile + #platforms: linux/amd64,linux/arm64 + tags: ${{ steps.docker_tags.outputs.tags }} # Note: using tags from docker_tags (custom script) + labels: ${{ steps.docker_meta.outputs.labels }} # Note: using lables from docker_meta + push: ${{ github.event_name != 'pull_request' }} # Don't push for PRs + cache-to: type=inline # Write the cache metadata into the image configuration + + - + # Push final image to the registry + # This will pick-up the build cache from the local build step + name: Push image (php) + # Don't run if the list of tags is empty + # Note: using tags from docker_tags (custom) + if: ${{ steps.docker_tags.outputs.tags != '' }} + uses: docker/build-push-action@v2 + with: + context: php + file: php/Dockerfile + #platforms: linux/amd64,linux/arm64 + tags: ${{ steps.docker_tags.outputs.tags }} # Note: using tags from docker_tags (custom script) + labels: ${{ steps.docker_meta.outputs.labels }} # Note: using lables from docker_meta + push: ${{ github.event_name != 'pull_request' }} # Don't push for PRs + cache-to: type=inline # Write the cache metadata into the image configuration diff --git a/Makefile b/Makefile index 43da3bb..0bdf1e6 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,36 @@ -include env_make +IMAGE ?= docksal/ci-agent VERSION ?= base +BUILD_TAG ?= $(VERSION)-build -REPO = docksal/ci-agent -NAME = ci-agent +NAME = docksal-ci-agent-$(VERSION) + +.EXPORT_ALL_VARIABLES: .PHONY: build test push shell run start stop logs clean release build: - docker build -t $(REPO):$(VERSION) $(VERSION) + docker build -t $(IMAGE):$(BUILD_TAG) ./$(VERSION) test: - IMAGE=$(REPO):$(VERSION) NAME=$(NAME) tests/$(VERSION).bats + IMAGE=$(IMAGE) BUILD_TAG=$(BUILD_TAG) NAME=$(NAME) ./tests/$(VERSION).bats push: - docker push $(REPO):$(VERSION) + docker push $(IMAGE):$(BUILD_TAG) shell: clean - docker run --rm --name $(NAME) -it $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) /bin/bash -oe pipefail + docker run --rm --name $(NAME) -it $(PORTS) $(VOLUMES) $(ENV) $(IMAGE):$(BUILD_TAG) /bin/bash exec: # Note: variables defined inside COMMAND get interpreted on the host, unless escaped, e.g. \$${CI_SSH_KEY}. docker exec $(NAME) /bin/bash -oe pipefail -c "$(COMMAND)" run: clean - docker run --rm --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) + docker run --rm --name $(NAME) -it $(PORTS) $(VOLUMES) $(ENV) $(IMAGE):$(BUILD_TAG) start: clean - docker run -d --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) top -b + docker run -d --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(IMAGE):$(BUILD_TAG) top -b stop: docker stop $(NAME) @@ -36,9 +39,9 @@ logs: docker logs $(NAME) clean: - docker rm -f $(NAME) || true + docker rm -f $(NAME) >/dev/null 2>&1 || true -release: build - make push -e VERSION=$(VERSION) +tags: + @.github/scripts/docker-tags.sh default: build diff --git a/php/Dockerfile b/php/Dockerfile index c3d11cf..bb14b85 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -1,4 +1,4 @@ -FROM docksal/ci-agent:base +FROM docksal/ci-agent:base-build # Switch to root to install some system-wide stuff USER root From c08474e1dec2ac5bf3f8cf67cebba4e8c2fae8cd Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 15:58:44 -0700 Subject: [PATCH 09/18] Disabled docker/setup-buildx-action buildx has some glitches with local upstream (FROM) images, which breaks the php image flavor build --- .github/workflows/default.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/default.yaml b/.github/workflows/default.yaml index ff0ffe1..0a9fe15 100644 --- a/.github/workflows/default.yaml +++ b/.github/workflows/default.yaml @@ -36,12 +36,13 @@ jobs: - name: Checkout uses: actions/checkout@v2 -# - -# name: Set up QEMU -# uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + #- + # name: Set up QEMU + # uses: docker/setup-qemu-action@v1 + # buildx has some glitches with local upstream (FROM) images. Disabled. + #- + # name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v1 - name: Check Docker run: | From 7fe81a5a10cc8427da1c7ce22fa5322055ba2e52 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 16:03:15 -0700 Subject: [PATCH 10/18] Removed Travis config and scripts --- .travis.yml | 27 --------------------------- scripts/push-image.sh | 22 ---------------------- 2 files changed, 49 deletions(-) delete mode 100644 .travis.yml delete mode 100755 scripts/push-image.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 990cabc..0000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -dist: bionic - -language: minimal - -env: - global: - - REPO=docksal/ci-agent - -install: - - curl -fsSL get.docksal.io | bash - - fin version - - fin sysinfo - -script: - # Build base first, then php - - make VERSION=base && make test VERSION=base - - make VERSION=php && make test VERSION=php - -after_success: | - # Push images to Docker Hub - if [[ "${TRAVIS_PULL_REQUEST}" == "false" ]]; then - scripts/push-image.sh base - scripts/push-image.sh php - fi - -after_failure: - - make logs diff --git a/scripts/push-image.sh b/scripts/push-image.sh deleted file mode 100755 index b2fca72..0000000 --- a/scripts/push-image.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Pushes an image to Docker Hub - -VERSION=$1 - -[[ "${TRAVIS_BRANCH}" == "develop" ]] && TAG="edge-${VERSION}" -[[ "${TRAVIS_BRANCH}" == "master" ]] && TAG="${VERSION}" -[[ "${TRAVIS_TAG}" != "" ]] && TAG="${TRAVIS_TAG:1:3}-${VERSION}" - -if [[ "$TAG" != "" ]]; then - docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" - # Push edge, stable and release tags - docker tag ${REPO}:${VERSION} ${REPO}:${TAG} - docker push ${REPO}:${TAG} - - # Push "latest" tag - if [[ "${TRAVIS_BRANCH}" == "master" ]] && [[ "${VERSION}" == "base" ]]; then - docker tag ${REPO}:${VERSION} ${REPO}:latest - docker push ${REPO}:latest - fi -fi From 8355aef5c89591fa4c4920dfa48e3b59d4752017 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 9 Apr 2021 17:25:30 -0700 Subject: [PATCH 11/18] Version bumps (base) - Alpine 3.13 - Docker 20.10.5 - Docker Compose 1.28.5 - Minio RELEASE.2021-04-06T23-11-00Z (@minio why not use semver for releases??) --- base/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 77a4d47..c120cf2 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.9 +FROM alpine:3.13 RUN set -xe; \ apk add --update --no-cache \ @@ -15,9 +15,9 @@ RUN set -xe; \ ; \ rm -rf /var/cache/apk/*; -ARG DOCKER_VERSION=18.09.2 -ARG DOCKER_COMPOSE_VERSION=1.23.2 -ARG MINIO_VERSION="RELEASE.2020-05-28T23-43-36Z" +ARG DOCKER_VERSION=20.10.5 +ARG DOCKER_COMPOSE_VERSION=1.28.5 +ARG MINIO_VERSION="RELEASE.2021-04-06T23-11-00Z" RUN set -xe; \ # Install docker cli curl -fsSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz"; \ From a807c404d0790d2eb6717be144b9082eba9a92b9 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 16:20:43 -0700 Subject: [PATCH 12/18] Version bumps (php) - Composer 1.10.21 - Drush 8.4.8 - Drupal Console Launcher 1.9.7 - wp-cli 2.4.0 --- php/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/php/Dockerfile b/php/Dockerfile index bb14b85..7a4141a 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -25,10 +25,10 @@ RUN set -xe; \ php7-zlib; \ rm -rf /var/cache/apk/*; -ENV COMPOSER_VERSION=1.6.3 -ENV DRUSH_VERSION 8.1.16 -ENV DRUPAL_CONSOLE_VERSION 1.7.0 -ENV WPCLI_VERSION 1.5.0 +ENV COMPOSER_VERSION=1.10.21 \ + DRUSH_VERSION=8.4.8 \ + DRUPAL_CONSOLE_LAUNCHER_VERSION=1.9.7 \ + WPCLI_VERSION=2.4.0 RUN set -xe; \ # Composer curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer; \ From 16631542c5ab6e11ae1217e45e93fdbeb3c4e671 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 16:22:51 -0700 Subject: [PATCH 13/18] Misc php build improvements - Adjusted curl flags for failsafe downloads - Adjusted the way binaries are marked executable - chmod +x on /usr/local/bin bloats the image as it touches all executables in the directory --- php/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/php/Dockerfile b/php/Dockerfile index 7a4141a..4ffffa9 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -31,15 +31,15 @@ ENV COMPOSER_VERSION=1.10.21 \ WPCLI_VERSION=2.4.0 RUN set -xe; \ # Composer - curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer; \ + curl -fsSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer; \ # Drush 8 (default) - curl -sSL "https://github.com/drush-ops/drush/releases/download/${DRUSH_VERSION}/drush.phar" -o /usr/local/bin/drush; \ + curl -fsSL "https://github.com/drush-ops/drush/releases/download/${DRUSH_VERSION}/drush.phar" -o /usr/local/bin/drush; \ # Drupal Console - curl -sSL "https://github.com/hechoendrupal/drupal-console-launcher/releases/download/${DRUPAL_CONSOLE_VERSION}/drupal.phar" -o /usr/local/bin/drupal; \ + curl -fsSL "https://github.com/hechoendrupal/drupal-console-launcher/releases/download/${DRUPAL_CONSOLE_LAUNCHER_VERSION}/drupal.phar" -o /usr/local/bin/drupal; \ # Install wp-cli - curl -sSL "https://github.com/wp-cli/wp-cli/releases/download/v${WPCLI_VERSION}/wp-cli-${WPCLI_VERSION}.phar" -o /usr/local/bin/wp; \ - # Make all binaries executable - chmod +x /usr/local/bin/*; + curl -fsSL "https://github.com/wp-cli/wp-cli/releases/download/v${WPCLI_VERSION}/wp-cli-${WPCLI_VERSION}.phar" -o /usr/local/bin/wp; \ + # Make all downloaded binaries executable in one shot + (cd /usr/local/bin && chmod +x composer drush drupal wp); # Switch back to user USER $AGENT_USER From 4366d40f97b8a9f192935bba06a4f7e413351d70 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 17:15:04 -0700 Subject: [PATCH 14/18] Install docker and docker-compose from Alpine repos Simplified and faster install, smaller image, less control over versioning (but that's fine). --- base/Dockerfile | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index c120cf2..e75f221 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,5 +1,6 @@ FROM alpine:3.13 +# Install basic pacakges RUN set -xe; \ apk add --update --no-cache \ bash \ @@ -8,25 +9,27 @@ RUN set -xe; \ jq \ make \ openssh \ - py2-pip \ rsync \ sudo \ patch \ ; \ rm -rf /var/cache/apk/*; -ARG DOCKER_VERSION=20.10.5 -ARG DOCKER_COMPOSE_VERSION=1.28.5 +# Install docker packages +# Lookup available version for Alpine at +# https://pkgs.alpinelinux.org/packages?name=docker*&branch=v3.13&arch=x86_64 +ARG DOCKER_VERSION=20.10.3-r1 +ARG DOCKER_COMPOSE_VERSION=1.27.4-r0 +RUN set -xe; \ + apk add --update --no-cache \ + docker-cli=${DOCKER_VERSION} \ + docker-compose=${DOCKER_COMPOSE_VERSION} \ + ; \ + rm -rf /var/cache/apk/*; + +# Install minio client (mc) ARG MINIO_VERSION="RELEASE.2021-04-06T23-11-00Z" RUN set -xe; \ - # Install docker cli - curl -fsSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz"; \ - tar zxf docker-${DOCKER_VERSION}.tgz && mv docker/docker /usr/local/bin && rm -rf docker-${DOCKER_VERSION}*; \ - docker --version; \ - # Install docker-compose cli (has to be installed via pip on Alpine) - pip install "docker-compose==${DOCKER_COMPOSE_VERSION}" >/dev/null; \ - docker-compose --version; \ - # Install minio client (mc) curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.${MINIO_VERSION} -o /usr/local/bin/mc; \ chmod +x /usr/local/bin/mc From 81bd9ccb982b30039bbe0ece27daf871f608ddb8 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 17:15:25 -0700 Subject: [PATCH 15/18] Fixed minio release version --- base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/Dockerfile b/base/Dockerfile index e75f221..5704103 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -28,7 +28,7 @@ RUN set -xe; \ rm -rf /var/cache/apk/*; # Install minio client (mc) -ARG MINIO_VERSION="RELEASE.2021-04-06T23-11-00Z" +ARG MINIO_VERSION="RELEASE.2021-03-23T05-46-11Z" RUN set -xe; \ curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.${MINIO_VERSION} -o /usr/local/bin/mc; \ chmod +x /usr/local/bin/mc From bd33a690372b4e571eb8636dbf73e164dc14462f Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 17:44:26 -0700 Subject: [PATCH 16/18] Adjusted docker tags for image flavors --- .github/scripts/docker-tags.sh | 23 ++++++++++++++++++----- Makefile | 10 +++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/scripts/docker-tags.sh b/.github/scripts/docker-tags.sh index ae49ee1..c2a102a 100755 --- a/.github/scripts/docker-tags.sh +++ b/.github/scripts/docker-tags.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash # Generates docker images tags for the docker/build-push-action@v2 action depending on the branch/tag. +# Image tag format: +# develop => image:[version-]edge[-flavor] +# master => image:[version][-][flavor] +# semver tag => image:[version-]major.minor[-flavor] declare -a registryArr registryArr+=("docker.io") # Docker Hub @@ -8,6 +12,13 @@ registryArr+=("ghcr.io") # GitHub Container Registry declare -a imageTagArr +# Join arguments with hyphen (-) as a delimiter +# Usage: join [] +join() { + local IFS='-' # join delimiter + echo "$*" +} + # feature/* => sha-xxxxxxx # Note: disabled #if [[ "${GITHUB_REF}" =~ "refs/heads/feature/" ]]; then @@ -17,12 +28,14 @@ declare -a imageTagArr # develop => version-edge if [[ "${GITHUB_REF}" == "refs/heads/develop" ]]; then - imageTagArr+=("${IMAGE}:${VERSION}-edge") + tag=$(join ${VERSION} edge ${FLAVOR}) + imageTagArr+=("${IMAGE}:${tag}") fi # master => version if [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then - imageTagArr+=("${IMAGE}:${VERSION}") + tag=$(join ${VERSION} ${FLAVOR}) + imageTagArr+=("${IMAGE}:${tag}") fi # tags/v1.0.0 => 1.0 @@ -31,9 +44,9 @@ if [[ "${GITHUB_REF}" =~ "refs/tags/" ]]; then IFS='.' read -a release_arr <<< "${GITHUB_REF#refs/tags/}" releaseMajor=${release_arr[0]#v*} # 2.7.0 => "2" releaseMinor=${release_arr[1]} # "2.7.0" => "7" - imageTagArr+=("${IMAGE}:${VERSION}") - imageTagArr+=("${IMAGE}:${VERSION}-${releaseMajor}") - imageTagArr+=("${IMAGE}:${VERSION}-${releaseMajor}.${releaseMinor}") + imageTagArr+=("${IMAGE}:$(join ${VERSION} ${FLAVOR})") + imageTagArr+=("${IMAGE}:$(join ${VERSION} ${releaseMajor} ${FLAVOR})") + imageTagArr+=("${IMAGE}:$(join ${VERSION} ${releaseMajor}.${releaseMinor} ${FLAVOR})") fi # Build an array of registry/image:tag values diff --git a/Makefile b/Makefile index 0bdf1e6..e68d369 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,20 @@ -include env_make IMAGE ?= docksal/ci-agent -VERSION ?= base -BUILD_TAG ?= $(VERSION)-build +FLAVOR ?= base +BUILD_TAG ?= $(FLAVOR)-build -NAME = docksal-ci-agent-$(VERSION) +NAME = docksal-ci-agent-$(FLAVOR) .EXPORT_ALL_VARIABLES: .PHONY: build test push shell run start stop logs clean release build: - docker build -t $(IMAGE):$(BUILD_TAG) ./$(VERSION) + docker build -t $(IMAGE):$(BUILD_TAG) ./$(FLAVOR) test: - IMAGE=$(IMAGE) BUILD_TAG=$(BUILD_TAG) NAME=$(NAME) ./tests/$(VERSION).bats + IMAGE=$(IMAGE) BUILD_TAG=$(BUILD_TAG) NAME=$(NAME) ./tests/$(FLAVOR).bats push: docker push $(IMAGE):$(BUILD_TAG) From 280628e72678ffde1d40d835e2b27c2b803a6ad0 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 20:46:03 -0700 Subject: [PATCH 17/18] Fix build variable --- .github/workflows/default.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/default.yaml b/.github/workflows/default.yaml index 0a9fe15..6e08f78 100644 --- a/.github/workflows/default.yaml +++ b/.github/workflows/default.yaml @@ -82,11 +82,11 @@ jobs: - # Build for local use name: Build image (base) - run: make build VERSION=base + run: make build FLAVOR=base - # Build for local use name: Build image (php) - run: make build VERSION=php + run: make build FLAVOR=php - # Print image info name: Docker image info @@ -125,10 +125,10 @@ jobs: # Tests - name: Test image (base) - run: make test VERSION=base + run: make test FLAVOR=base - name: Test image (php) - run: make test VERSION=php + run: make test FLAVOR=php - # Push final image to the registry From 0af235f6d67bdeecf8be901d35482c5c1d707b1b Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 12 Apr 2021 21:02:24 -0700 Subject: [PATCH 18/18] Fix image tags --- .github/workflows/default.yaml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/default.yaml b/.github/workflows/default.yaml index 6e08f78..93d8d92 100644 --- a/.github/workflows/default.yaml +++ b/.github/workflows/default.yaml @@ -74,11 +74,6 @@ jobs: ${{ env.IMAGE }} ghcr.io/${{ env.IMAGE }} tag-sha: true # add git short SHA as Docker tag - - - # Generate image meta information - name: Docker image tags - id: docker_tags - run: make tags - # Build for local use name: Build image (base) @@ -130,19 +125,29 @@ jobs: name: Test image (php) run: make test FLAVOR=php + - + # Generate image meta information + name: Docker image tags (base) + id: docker_tags_base + run: make tags FLAVOR=base + - + # Generate image meta information + name: Docker image tags (php) + id: docker_tags_php + run: make tags FLAVOR=php - # Push final image to the registry # This will pick-up the build cache from the local build step name: Push image (base) # Don't run if the list of tags is empty # Note: using tags from docker_tags (custom) - if: ${{ steps.docker_tags.outputs.tags != '' }} + if: ${{ steps.docker_tags_base.outputs.tags != '' }} uses: docker/build-push-action@v2 with: context: base file: base/Dockerfile #platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_tags.outputs.tags }} # Note: using tags from docker_tags (custom script) + tags: ${{ steps.docker_tags_base.outputs.tags }} # Note: using tags from docker_tags (custom script) labels: ${{ steps.docker_meta.outputs.labels }} # Note: using lables from docker_meta push: ${{ github.event_name != 'pull_request' }} # Don't push for PRs cache-to: type=inline # Write the cache metadata into the image configuration @@ -153,13 +158,13 @@ jobs: name: Push image (php) # Don't run if the list of tags is empty # Note: using tags from docker_tags (custom) - if: ${{ steps.docker_tags.outputs.tags != '' }} + if: ${{ steps.docker_tags_php.outputs.tags != '' }} uses: docker/build-push-action@v2 with: context: php file: php/Dockerfile #platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_tags.outputs.tags }} # Note: using tags from docker_tags (custom script) + tags: ${{ steps.docker_tags_php.outputs.tags }} # Note: using tags from docker_tags (custom script) labels: ${{ steps.docker_meta.outputs.labels }} # Note: using lables from docker_meta push: ${{ github.event_name != 'pull_request' }} # Don't push for PRs cache-to: type=inline # Write the cache metadata into the image configuration