From 00bfa66c00eca80d866e707cdb32c3a81ef4fd8c Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 17 Sep 2018 21:15:33 -0700 Subject: [PATCH 01/19] Switch to Alpine 3.8 --- base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/Dockerfile b/base/Dockerfile index 93554c4..e762c00 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.7 +FROM alpine:3.8 RUN apk add --update --no-cache \ bash \ From 3afa0d67a738c1526c557a2e37fecd4b5cc52894 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 19 Sep 2018 08:50:01 -0700 Subject: [PATCH 02/19] Properly fail image builds on errors in RUN executions --- base/Dockerfile | 3 ++- php/Dockerfile | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index e762c00..16184df 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,6 +1,7 @@ FROM alpine:3.8 -RUN apk add --update --no-cache \ +RUN set -xe; \ + apk add --update --no-cache \ bash \ curl \ git \ diff --git a/php/Dockerfile b/php/Dockerfile index cd22a6d..542bc31 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -3,7 +3,8 @@ FROM docksal/ci-agent:base # Switch to root to install some system-wide stuff USER root -RUN apk add --update --no-cache \ +RUN set -xe; \ + apk add --update --no-cache \ php7 \ php7-ctype \ php7-curl \ @@ -25,7 +26,7 @@ ENV COMPOSER_VERSION=1.6.3 ENV DRUSH_VERSION 8.1.16 ENV DRUPAL_CONSOLE_VERSION 1.7.0 ENV WPCLI_VERSION 1.5.0 -RUN \ +RUN set -xe; \ # Composer curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer; \ # Drush 8 (default) @@ -41,7 +42,7 @@ RUN \ USER $AGENT_USER ENV PATH $PATH:$AGENT_HOME/.composer/vendor/bin -RUN \ +RUN set -xe; \ # Add composer bin directory to PATH echo "\n"'PATH="$PATH:$AGENT_HOME/.composer/vendor/bin"' >> $AGENT_HOME/.profile; \ # Drush modules From d8b0e30c5e97acf3ec46cc63b229a5fd68968aec Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 19 Sep 2018 09:02:53 -0700 Subject: [PATCH 03/19] Added a test for phpcs --- tests/php.bats | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/php.bats b/tests/php.bats index 5dc1e76..660fc44 100755 --- a/tests/php.bats +++ b/tests/php.bats @@ -54,6 +54,11 @@ teardown() { echo "$output" | grep "WP-CLI" unset output + run make exec COMMAND="phpcs --version" + [[ "$status" == 0 ]] + echo "$output" | grep "PHP_CodeSniffer" + unset output + ### Cleanup ### make clean } From bcead245bb5c83fd56894d8a45200f3e432a3d87 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Oct 2018 15:20:17 -0700 Subject: [PATCH 04/19] Reset "set" settings at the end of build-env Since build-env is sourced in all build steps, the "set -e" setting was propagated everywhere causing build steps to fail on any error. This is especially nasty when npm packages installed from source fail to compile without obvious reasons. --- base/bin/build-env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/base/bin/build-env b/base/bin/build-env index e415f0b..60f5f92 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -5,6 +5,8 @@ # # Usage source build-env +# Note: "set -e" MUST be restored back ("set +e") at the end of the script. +# This script is sourced, so settings will propagate to all build steps. set -e # Abort if anything fails #set -x # Echo commands @@ -197,3 +199,6 @@ git_env # Sandbox server settings echo-debug "Configuring sandbox server settings..." sandbox_server_env + +# IMPORTANT! Reverting settings set at the beginning on the script +set +e From 6e1050e4156604a8e882318c47fbde61366d6f79 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Oct 2018 15:22:26 -0700 Subject: [PATCH 05/19] Updated CircleCI basic example Doing "echo 'source build-env' >> $BASH_ENV" as the first step is important for CircleCI builds. Each run statement runs in its own isolated shell (exported variables are not preserved). $BASH_ENV can be used to pass environment variables between run statements. --- README.md | 7 ++++++- base/Dockerfile | 3 ++- examples/.circleci/config.yml | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c2b2a56..5a40b5d 100644 --- a/README.md +++ b/README.md @@ -156,8 +156,13 @@ jobs: docker: - image: docksal/ci-agent:php steps: + - run: + name: Configure agent environment + command: echo 'source build-env' >> $BASH_ENV - checkout - - run: source build-env && sandbox-init + - run: + name: Build sandbox + command: sandbox-init ``` For a more advanced example see [config.yml](examples/.circleci/config.yml). diff --git a/base/Dockerfile b/base/Dockerfile index 16184df..8bebfe2 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -10,7 +10,8 @@ RUN set -xe; \ py2-pip \ rsync \ sudo \ - patch; \ + patch \ + ; \ rm -rf /var/cache/apk/*; ARG DOCKER_VERSION=18.06.0-ce diff --git a/examples/.circleci/config.yml b/examples/.circleci/config.yml index 3112cbc..82c0c96 100644 --- a/examples/.circleci/config.yml +++ b/examples/.circleci/config.yml @@ -7,17 +7,17 @@ jobs: docker: - image: docksal/ci-agent:php steps: - # Code checkout in the build agent - - checkout # Inject build environment variables. # Each run statement runs in its own isolated shell (exported variables are not preserved). # $BASH_ENV can be used to pass environment variables between run statements. - run: name: Configure agent environment command: echo 'source build-env' > $BASH_ENV + # Code checkout in the build agent + - checkout # Launch a sandbox on the sandbox server - run: - name: Provision sandbox + name: Build sandbox command: sandbox-init # Run other commands - run: From 2e6cb1c1f9691540577bb06e4549493511f96af1 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Oct 2018 16:41:57 -0700 Subject: [PATCH 06/19] Improved git_env to only write config when necessary --- base/bin/build-env | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 60f5f92..07f70db 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -125,10 +125,16 @@ ssh_init () (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) } +# Configured preferred git settings git_env () { - git config --global user.email "$GIT_USER_EMAIL" - git config --global user.name "$GIT_USER_NAME" + # Only set these if not already configured + if [[ "$(git config --global user.email)" == "" ]] && [[ "$GIT_USER_EMAIL" != "" ]]; then + git config --global user.email "$GIT_USER_EMAIL" + fi + if [[ "$(git config --global user.name)" == "" ]] && [[ "$GIT_USER_NAME" != "" ]]; then + git config --global user.email "$GIT_USER_NAME" + fi } ssh_tunnel_init () From 469a11b59274ca088266e2f34ce9b50068d4deaf Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Oct 2018 16:44:16 -0700 Subject: [PATCH 07/19] Refactored build-env Removed sandbox_server_env (move into build_env) --- base/bin/build-env | 69 ++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 07f70db..7123cf9 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -117,6 +117,32 @@ build_env () export REPO_NAME_SAFE="$(safe_string ${GIT_REPO_NAME:0:${REPO_NAME_LENGTH_LIMIT}})" # Short version of GIT_COMMIT_HASH export COMMIT_HASH_SHORT="${GIT_COMMIT_HASH:0:7}" + + # Sandbox settings + export REMOTE_BUILD_BASE=${REMOTE_BUILD_BASE:-/home/ubuntu/builds} + export REMOTE_BUILD_DIR="${REMOTE_BUILD_BASE}/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" + export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" + export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" + + # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with nip.io + export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.nip.io}" + sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config + + # Use ubuntu as the user by default + export DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" + sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config + + # Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set. + # This is useful when working with CDNs/ELBs/WAFs/etc (when DOCKSAL_DOMAIN is different from the DOCKSAL_HOST). + # Make sure domain name is lowercase + export DOCKSAL_DOMAIN="$(echo -n ${DOCKSAL_DOMAIN:-$DOCKSAL_HOST} | awk '{print tolower($0)}')" + + # Use "flat" sub-domains (e.g. branch-project.example.com) and not multi-sub-domains (e.g. branch.project.example.com) + # This allows using a single wildcard cert for the entire sandbox server. + # Note: A wildcard cert for "*.example.com", will only cover "sub-domain.example.dom", but not + # "www.sub-domain.example.com". + # NOTE: The length of any one label (sub-domain) in the domain name is limited to 63 octets (characters). + export DOMAIN="${BRANCH_NAME_SAFE}-${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}" } ssh_init () @@ -137,6 +163,7 @@ git_env () fi } +# Support running docker commands (locally to the agent) on the sandbox server (remote Docker engine) ssh_tunnel_init () { # Check if the tunnel is already active and return if so @@ -155,41 +182,6 @@ ssh_tunnel_init () return $? } -sandbox_server_env () -{ - export REMOTE_BUILD_BASE=${REMOTE_BUILD_BASE:-/home/ubuntu/builds} - export REMOTE_BUILD_DIR="${REMOTE_BUILD_BASE}/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" - export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" - export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" - - # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with nip.io - export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.nip.io}" - sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config - - # Use ubuntu as the user by default - export DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" - sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config - - # Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set. - # This is useful when working with CDNs/ELBs/WAFs/etc (when DOCKSAL_DOMAIN is different from the DOCKSAL_HOST). - # Make sure domain name is lowercase - export DOCKSAL_DOMAIN="$(echo -n ${DOCKSAL_DOMAIN:-$DOCKSAL_HOST} | awk '{print tolower($0)}')" - - # Use "flat" sub-domains (e.g. branch-project.example.com) and not multi-sub-domains (e.g. branch.project.example.com) - # This allows using a single wildcard cert for the entire sandbox server. - # Note: A wildcard cert for "*.example.com", will only cover "sub-domain.example.dom", but not - # "www.sub-domain.example.com". - # NOTE: The length of any one label (sub-domain) in the domain name is limited to 63 octets (characters). - export DOMAIN="${BRANCH_NAME_SAFE}-${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}" - - # Initialize a tunnel to the Docker Engine on DOCKSAL_HOST - # Export local tunnel connection settings if it works - # Using full if form instead of the short one here, otherwise builds will fail, when the condition below is false - if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then - ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} - fi -} - # -------------------- Runtime -------------------- # echo-debug "Configuring build settings..." @@ -204,7 +196,12 @@ git_env # Sandbox server settings echo-debug "Configuring sandbox server settings..." -sandbox_server_env +# Initialize a tunnel to the Docker Engine on DOCKSAL_HOST +# Export local tunnel connection settings if it works +# Using full "if" form instead of the short one here, otherwise build fails, when the condition below is false +if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then + ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} +fi # IMPORTANT! Reverting settings set at the beginning on the script set +e From 01b05c885ced620bc86bcd81d3336df612dff912 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Oct 2018 16:45:23 -0700 Subject: [PATCH 08/19] Ensure ssh keys permissions are correct --- base/bin/build-env | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 7123cf9..bf8cb76 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -145,10 +145,13 @@ build_env () export DOMAIN="${BRANCH_NAME_SAFE}-${REPO_NAME_SAFE}.${DOCKSAL_DOMAIN}" } +# Configure SSH keys +# Note: CircleCI injects it's own key during checkout. +# Since this scripts is supposed to be sourced for every run command, the keys will be reset back to our values. ssh_init () { - (umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa) - (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) + (umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa; chmod 0600 $HOME/.ssh/id_rsa) + (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa; chmod 0600 $HOME/.ssh/docksal_host_id_rsa) } # Configured preferred git settings From c2b478cf12b6958c9500e2620d6c4f00db4956fe Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 4 Oct 2018 17:03:16 -0700 Subject: [PATCH 09/19] Fix typo in git_env --- base/bin/build-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/bin/build-env b/base/bin/build-env index bf8cb76..fbe7ef6 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -162,7 +162,7 @@ git_env () git config --global user.email "$GIT_USER_EMAIL" fi if [[ "$(git config --global user.name)" == "" ]] && [[ "$GIT_USER_NAME" != "" ]]; then - git config --global user.email "$GIT_USER_NAME" + git config --global user.name "$GIT_USER_NAME" fi } From 8832ed5224781dff62f459369583e36ad968a8c6 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Oct 2018 09:41:13 -0700 Subject: [PATCH 10/19] Improved ssh_init Only write keys if SSH key variables are not empty --- base/bin/build-env | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index fbe7ef6..b5d75b2 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -150,8 +150,19 @@ build_env () # Since this scripts is supposed to be sourced for every run command, the keys will be reset back to our values. ssh_init () { - (umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa; chmod 0600 $HOME/.ssh/id_rsa) - (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa; chmod 0600 $HOME/.ssh/docksal_host_id_rsa) + [[ "$CI_SSH_KEY" != "" ]] && + ( + umask 077 + echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa + chmod 0600 $HOME/.ssh/id_rsa + ) + + [[ "$DOCKSAL_HOST_SSH_KEY" != "" ]] && + ( + umask 077 + echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa + chmod 0600 $HOME/.ssh/docksal_host_id_rsa + ) } # Configured preferred git settings From f15c672980c7219a4d7dea227da128d070d6c42a Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Oct 2018 09:56:52 -0700 Subject: [PATCH 11/19] Force using the key set via CI_SSH_KEY for all hosts. Fixes #26 --- base/config/.ssh/config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/base/config/.ssh/config b/base/config/.ssh/config index 932b797..b068703 100644 --- a/base/config/.ssh/config +++ b/base/config/.ssh/config @@ -1,12 +1,19 @@ -# Disable the host key check +# All hosts Host * + # Disable the host key check StrictHostKeyChecking no UserKnownHostsFile=/dev/null LogLevel ERROR + # Force using the key set via CI_SSH_KEY for all hosts + IdentityFile ~/.ssh/id_rsa + IdentitiesOnly yes +# Docksal Sandbox Server +# TODO: rename to dss instead of docker-host in 2.0 Host docker-host HostName DOCKSAL_HOST User DOCKSAL_HOST_USER + # Disable the host key check StrictHostKeyChecking no UserKnownHostsFile=/dev/null LogLevel ERROR From b62e55a4589942707e18d9d5d77b10fb0c5d6dbf Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Oct 2018 12:12:05 -0700 Subject: [PATCH 12/19] Renamed BITBUCKETCI to BITBUCKET_CI This variable is used internally and is set to "true" when a build is running on Bitbucket Pipelines --- base/bin/build-acp | 2 +- base/bin/build-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index 42f6e1a..fb54c4b 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -60,7 +60,7 @@ echo "Files:" mc ls ${destination} # Post artifacts to Bitbucket build status API -if [[ "${BITBUCKETCI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then +if [[ "${BITBUCKET_CI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then echo "Posting artifacts URL to Bitbucket..." BUILD_STATUS_URL="${ARTIFACTS_URL}" diff --git a/base/bin/build-env b/base/bin/build-env index b5d75b2..a845d0a 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -46,7 +46,7 @@ build_env () # Support for Bitbucket Pipelines if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then echo-debug "Detected Bitbucket Pipelines build environment" - export BITBUCKETCI="true" + export BITBUCKET_CI="true" export GIT_REPO_SERVICE="bitbucket" export GIT_REPO_OWNER="$BITBUCKET_REPO_OWNER" export GIT_REPO_NAME="$BITBUCKET_REPO_SLUG" From 8b66182ce8152268286931e2f52168ad22b374e9 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Oct 2018 12:20:36 -0700 Subject: [PATCH 13/19] Added BUILD_ID variable Used as the unique identifier for a build --- README.md | 1 + base/bin/build-env | 3 +++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 5a40b5d..fc58d1b 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ The following variables are derived from the respective Bitbucket Pipelines, Cir - `GIT_COMMIT_HASH` - git commit hash - `GIT_PR_NUMBER` - git pull request / merge request number - `GIT_REPO_SERVICE` - `github`, `bitbucket` or `gitlab` (makes sense mostly for CircleCI) +- `BUILD_ID` - The unique identifier for a build - `BUILD_DIR` - The full path where the repository is cloned and where the job is run in the agent container `REMOTE_BUILD_DIR` diff --git a/base/bin/build-env b/base/bin/build-env index a845d0a..faed49b 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -56,6 +56,7 @@ build_env () # Bitbucket Pipelines does not work with PRs #GIT_PR_NUMBER=? + export BUILD_ID="$BITBUCKET_BUILD_NUMBER" export BUILD_DIR="$BITBUCKET_CLONE_DIR" fi @@ -81,6 +82,7 @@ build_env () export GIT_PR_NUMBER=${CIRCLE_PULL_REQUEST##*/} fi + export BUILD_ID="$CIRCLE_BUILD_NUM" export BUILD_DIR="$CIRCLE_WORKING_DIRECTORY" fi @@ -95,6 +97,7 @@ build_env () export GIT_COMMIT_HASH="$CI_COMMIT_SHA" export GIT_PR_NUMBER="$CI_MERGE_REQUEST_ID" + export BUILD_ID="$CI_JOB_ID" export BUILD_DIR="$CI_PROJECT_DIR" fi From 7498763c7d123f63708225f0caa6cb2ce2d6f5e8 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Oct 2018 13:58:26 -0700 Subject: [PATCH 14/19] Added make --- base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/base/Dockerfile b/base/Dockerfile index 8bebfe2..9bcb21c 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -6,6 +6,7 @@ RUN set -xe; \ curl \ git \ jq \ + make \ openssh \ py2-pip \ rsync \ From 2957695da8e21c9cf65b471d453671fcd047b52d Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 9 Oct 2018 13:53:18 -0700 Subject: [PATCH 15/19] Fix ssh_init non-zero exit code --- base/bin/build-env | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index faed49b..1823e38 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -153,19 +153,17 @@ build_env () # Since this scripts is supposed to be sourced for every run command, the keys will be reset back to our values. ssh_init () { - [[ "$CI_SSH_KEY" != "" ]] && - ( + if [[ "$CI_SSH_KEY" != "" ]]; then umask 077 echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa chmod 0600 $HOME/.ssh/id_rsa - ) + fi - [[ "$DOCKSAL_HOST_SSH_KEY" != "" ]] && - ( + if [[ "$DOCKSAL_HOST_SSH_KEY" != "" ]]; then umask 077 echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa chmod 0600 $HOME/.ssh/docksal_host_id_rsa - ) + fi } # Configured preferred git settings From 705a819aff6a9339c485f7af2b7d415d6a732d91 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 9 Oct 2018 13:55:07 -0700 Subject: [PATCH 16/19] Added a trap message on errors --- base/bin/build-env | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 1823e38..bb8bb84 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -5,10 +5,13 @@ # # Usage source build-env -# Note: "set -e" MUST be restored back ("set +e") at the end of the script. -# This script is sourced, so settings will propagate to all build steps. -set -e # Abort if anything fails -#set -x # Echo commands +# Note: bash option set with "set" MUST be restored back at the end of the script. +# This script is sourced, so these settings will propagate to all build steps. +# Abort if anything fails +set -eE # same as: `set -o errexit -o errtrace` +trap "echo 'Build environment initialization failed!'" ERR +# Echo commands +#set -x # -------------------- Constants -------------------- # @@ -219,4 +222,4 @@ if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then fi # IMPORTANT! Reverting settings set at the beginning on the script -set +e +set +eE From cbd35d790166651ca6e4caf20d121973c0348cdd Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 9 Oct 2018 14:13:40 -0700 Subject: [PATCH 17/19] Get rid of the set settings in build-env Note: instead of "set -e" use "exit 1" where necessary --- base/bin/build-env | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index bb8bb84..05f5876 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -5,13 +5,10 @@ # # Usage source build-env -# Note: bash option set with "set" MUST be restored back at the end of the script. -# This script is sourced, so these settings will propagate to all build steps. -# Abort if anything fails -set -eE # same as: `set -o errexit -o errtrace` -trap "echo 'Build environment initialization failed!'" ERR -# Echo commands -#set -x +# IMPORTANT: This script is sourced in the build environment. +# Any settings set here using set/trap/etc. will propagate to all build steps. +# As such, it's best not to make any adjustment or make sure they are reverted at the end of the script. +# E.g., instead of "set -e" use "exit 1" where necessary. # -------------------- Constants -------------------- # @@ -220,6 +217,3 @@ echo-debug "Configuring sandbox server settings..." if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} fi - -# IMPORTANT! Reverting settings set at the beginning on the script -set +eE From bcd9a3d9c08509e013634b8c791bba5976eccd9d Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 10 Oct 2018 09:56:45 -0700 Subject: [PATCH 18/19] Updated defaults for the sandbox user and builds directory User: build-agent Builds directory: /home/build-agent/builds These can be overridden via DOCKSAL_HOST_USER and REMOTE_BUILD_BASE respectively --- base/bin/build-env | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 05f5876..fb6b2f7 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -122,7 +122,9 @@ build_env () export COMMIT_HASH_SHORT="${GIT_COMMIT_HASH:0:7}" # Sandbox settings - export REMOTE_BUILD_BASE=${REMOTE_BUILD_BASE:-/home/ubuntu/builds} + # Defaults for the sandbox user and builds directory: "build-agent" and "/home/build-agent/builds" respectively. + export DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-build-agent}" + export REMOTE_BUILD_BASE=${REMOTE_BUILD_BASE:-/home/${DOCKSAL_HOST_USER}/builds} export REMOTE_BUILD_DIR="${REMOTE_BUILD_BASE}/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" @@ -131,8 +133,7 @@ build_env () export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.nip.io}" sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config - # Use ubuntu as the user by default - export DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" + # Set the sandbox user name in agent's SSH config sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config # Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set. @@ -166,7 +167,7 @@ ssh_init () fi } -# Configured preferred git settings +# Configure preferred git settings git_env () { # Only set these if not already configured From e21b3c66332ec42bd75fa06194e818aa9fd09422 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 10 Oct 2018 11:13:46 -0700 Subject: [PATCH 19/19] Updated docs on configuring SSH keys [ci skip] --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fc58d1b..fab7ed5 100644 --- a/README.md +++ b/README.md @@ -57,16 +57,14 @@ If using `DOCKSAL_HOST_IP`, the agent will use `nip.io` for dynamic wildcard dom `DOCKSAL_HOST_SSH_KEY` -A base64 encoded private SSH key used to access the remote Docksal host. -See [Access remote hosts via SSH](https://confluence.atlassian.com/bitbucket/access-remote-hosts-via-ssh-847452940.html) -tutorial for details. +A base64 encoded private SSH key, used to access the remote Docksal host. `CI_SSH_KEY` -A secondary SSH key (base64 encoded as well), which can be used for deployments and other remote operations run directly -on the agent. -E.g. cloning/pushing a repo, running commands over SSH on a remote deployment environment. +A base64 encoded private SSH key, used by default for all hosts (set as `Host *` in `~/.ssh/config`). +This key will be used to clone/push to repo, run commands over SSH on a remote deployment environment, etc. +Note: `cat /path/to/ | base64` can be used to create a base64 encoded string from a private SSH key. ### Optional