Install publish tools in builder images (#29)

README.md

@@ -1,5 +1,29 @@
 ### Building & Publishing Dockerfiles
 
+#### Logging in to registry.redhat.io
+
+1. Go to https://access.redhat.com/terms-based-registry/ and create an account.
+2. Login and create a service account.
+3. After receiving a token, run:
+
+``` shell
+docker login registry.redhat.io
+```
+
+#### Building images for testing
+
+``` shell
+export IMAGE_NAME="image_name" # use one of the directory names in the dockerfiles directory (e.g. linux-builder)
+export DOCKERHUB_USERNAME="your_dockerhub_username_here"
+export TAG_NAME="${IMAGE_NAME}-${DOCKERHUB_USERNAME}"
+docker build . -f dockerfiles/linux-publisher/Dockerfile -t sensu/sensu-release:${TAG_NAME}
+docker push sensu/sensu-release:${TAG_NAME}
+```
+
+#### Building images for release
+
 ``` shell
-for i in `ls dockerfiles`; do docker build . -f dockerfiles/$i/Dockerfile -t sensu/sensu-release:$i; docker push sensu/sensu-release:$i; done
+export IMAGE_NAME="image_name" # use one of the directory names in the dockerfiles directory (e.g. linux-builder)
+docker build . -f dockerfiles/linux-publisher/Dockerfile -t sensu/sensu-release:${IMAGE_NAME}
+docker push sensu/sensu-release:${IMAGE_NAME}
 ```

cmd/circleci-logs/fetch.go

@@ -76,9 +76,19 @@ func newFetchCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "fetch",
 		Short: "fetch logs for a given job name",
-		RunE:  executeFetchLogs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			viper.BindPFlag(flagAuthToken, cmd.Flags().Lookup(flagAuthToken))
+			viper.BindPFlag(flagWorkflowID, cmd.Flags().Lookup(flagWorkflowID))
+			viper.BindPFlag(flagProjectType, cmd.Flags().Lookup(flagProjectType))
+			viper.BindPFlag(flagProjectUsername, cmd.Flags().Lookup(flagProjectUsername))
+			viper.BindPFlag(flagProjectRepositoryName, cmd.Flags().Lookup(flagProjectRepositoryName))
+			viper.BindPFlag(flagJobName, cmd.Flags().Lookup(flagJobName))
+			viper.BindPFlag(flagOutputPath, cmd.Flags().Lookup(flagOutputPath))
+		},
+		RunE: executeFetchLogs,
 	}
 
+	cmd.Flags().StringP(flagAuthToken, "t", "", "circleci api token")
 	cmd.Flags().StringP(flagWorkflowID, "w", "", "id of the workflow that the job belongs to")
 	cmd.Flags().StringP(flagProjectType, "p", defaultProjectType, "type of the project that the organization exists within (github or bitbucket)")
 	cmd.Flags().StringP(flagProjectUsername, "u", "", "github or bitbucket username of the project")
@@ -100,13 +110,5 @@ func newFetchCmd() *cobra.Command {
 	viper.BindEnv(flagJobName, "CIRCLE_JOB")
 	viper.BindEnv(flagOutputPath, "LOG_OUTPUT_PATH")
 
-	viper.BindPFlag(flagAuthToken, cmd.PersistentFlags().Lookup(flagAuthToken))
-	viper.BindPFlag(flagWorkflowID, cmd.Flags().Lookup(flagWorkflowID))
-	viper.BindPFlag(flagProjectType, cmd.Flags().Lookup(flagProjectType))
-	viper.BindPFlag(flagProjectUsername, cmd.Flags().Lookup(flagProjectUsername))
-	viper.BindPFlag(flagProjectRepositoryName, cmd.Flags().Lookup(flagProjectRepositoryName))
-	viper.BindPFlag(flagJobName, cmd.Flags().Lookup(flagJobName))
-	viper.BindPFlag(flagOutputPath, cmd.Flags().Lookup(flagOutputPath))
-
 	return cmd
 }

cmd/circleci-logs/fetchall.go

@@ -70,9 +70,18 @@ func newFetchAllCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "fetch-all",
 		Short: "fetch logs for all jobs for a given workflow",
-		RunE:  executeFetchAllLogs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			viper.BindPFlag(flagAuthToken, cmd.Flags().Lookup(flagAuthToken))
+			viper.BindPFlag(flagWorkflowID, cmd.Flags().Lookup(flagWorkflowID))
+			viper.BindPFlag(flagProjectType, cmd.Flags().Lookup(flagProjectType))
+			viper.BindPFlag(flagProjectUsername, cmd.Flags().Lookup(flagProjectUsername))
+			viper.BindPFlag(flagProjectRepositoryName, cmd.Flags().Lookup(flagProjectRepositoryName))
+			viper.BindPFlag(flagOutputDir, cmd.Flags().Lookup(flagOutputDir))
+		},
+		RunE: executeFetchAllLogs,
 	}
 
+	cmd.Flags().StringP(flagAuthToken, "t", "", "circleci api token")
 	cmd.Flags().StringP(flagWorkflowID, "w", "", "id of the workflow that the job belongs to")
 	cmd.Flags().StringP(flagProjectType, "p", defaultProjectType, "type of the project that the organization exists within (github or bitbucket)")
 	cmd.Flags().StringP(flagProjectUsername, "u", "", "github or bitbucket username of the project")
@@ -91,12 +100,5 @@ func newFetchAllCmd() *cobra.Command {
 	viper.BindEnv(flagProjectRepositoryName, "CIRCLE_PROJECT_REPONAME")
 	viper.BindEnv(flagOutputPath, "LOG_OUTPUT_DIR")
 
-	viper.BindPFlag(flagAuthToken, cmd.PersistentFlags().Lookup(flagAuthToken))
-	viper.BindPFlag(flagWorkflowID, cmd.Flags().Lookup(flagWorkflowID))
-	viper.BindPFlag(flagProjectType, cmd.Flags().Lookup(flagProjectType))
-	viper.BindPFlag(flagProjectUsername, cmd.Flags().Lookup(flagProjectUsername))
-	viper.BindPFlag(flagProjectRepositoryName, cmd.Flags().Lookup(flagProjectRepositoryName))
-	viper.BindPFlag(flagOutputDir, cmd.Flags().Lookup(flagOutputDir))
-
 	return cmd
 }

cmd/circleci-logs/main.go

@@ -123,6 +123,10 @@ func (f Fetcher) fetchURL(url string) ([]byte, error) {
 		return nil, err
 	}
 
+	if resp.StatusCode >= 400 {
+		return nil, fmt.Errorf("received http status %d with message: %s", resp.StatusCode, string(body))
+	}
+
 	return body, nil
 }
 
@@ -177,7 +181,6 @@ func newRootCmd() *cobra.Command {
 		Short: "fetch logs for circleci jobs",
 	}
 
-	cmd.PersistentFlags().StringP(flagAuthToken, "t", "", "circleci api token")
 	cmd.AddCommand(newFetchCmd())
 	cmd.AddCommand(newFetchAllCmd())
 

dockerfiles/linux-builder/Dockerfile

@@ -1,4 +1,5 @@
 FROM golang:1.13.12-alpine3.12 as golang
+FROM sensu/sensu-release@sha256:bcd0e081ae40111c305a8e401dbafac64abad40b52bbbfefe06d4f419ea8ccd9 as publishing-tools
 FROM alpine:3.12.0
 
 LABEL name="sensu/sensu-release" \
@@ -9,12 +10,16 @@ ENV WORKSPACE_PATH="/workspace"
 ENV ARTIFACTS_PATH="${WORKSPACE_PATH}/artifacts"
 ENV VAR_TMP_PATH="/var/tmp"
 ENV RPM_LIB_PATH="/var/lib/rpm"
+ENV LOGS_PATH="${ARTIFACTS_PATH}/logs"
+ENV TOOLS_PATH="/tools"
 
 RUN mkdir -p $OBJ_PATH
 RUN mkdir -p $WORKSPACE_PATH
 RUN mkdir -p $ARTIFACTS_PATH
 RUN mkdir -p $VAR_TMP_PATH
 RUN mkdir -p $RPM_LIB_PATH
+RUN mkdir -p $LOGS_PATH
+RUN mkdir -p $TOOLS_PATH
 
 # Install packages
 ENV BUILD_TIME_PKGS="tar gcc g++ py3-setuptools"
@@ -45,6 +50,14 @@ RUN mv /tmp/packagecloud /usr/bin/
 # Symlink python to python3
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
+# Install publishing tools
+COPY --from=publishing-tools /usr/lib/python3.8 /usr/lib/python3.8
+COPY --from=publishing-tools /usr/bin/python3.8 /usr/bin/python3.8
+COPY --from=publishing-tools /usr/bin/python3 /usr/bin/python3
+COPY --from=publishing-tools $TOOLS_PATH/ci-common-functions.sh $TOOLS_PATH/ci-common-functions.sh
+COPY --from=publishing-tools $TOOLS_PATH/circleci-logs $TOOLS_PATH/circleci-logs
+COPY --from=publishing-tools $TOOLS_PATH/post-packages-s3.sh $TOOLS_PATH/post-packages-s3.sh
+
 # Cleanup
 RUN rm -rf /tmp/*
 RUN apk del $BUILD_TIME_PKGS
@@ -61,6 +74,8 @@ RUN chown -R circleci:circleci $ARTIFACTS_PATH
 RUN chown -R circleci:circleci $OBJ_PATH
 RUN chown -R circleci:circleci $VAR_TMP_PATH
 RUN chown -R circleci:circleci $RPM_LIB_PATH
+RUN chown -R circleci:circleci $LOGS_PATH
+RUN chown -R circleci:circleci $TOOLS_PATH
 
 USER circleci
 ENV HOME /home/circleci

dockerfiles/rhel7-builder/Dockerfile

@@ -1,32 +1,69 @@
+FROM registry.access.redhat.com/ubi7/python-38:1 as python
 FROM registry.access.redhat.com/ubi7/go-toolset:1.13.4
 
 LABEL name="sensu/sensu-release" \
       maintainer="[email protected]"
 
 USER root
 
-ENV WORKSPACE_PATH="/workspace"
-ENV ARTIFACTS_PATH="${WORKSPACE_PATH}/artifacts"
-
-RUN mkdir -p $WORKSPACE_PATH
-RUN mkdir -p $ARTIFACTS_PATH
+# Install python3.8
+COPY --from=python /opt/app-root/bin/python3.8 /opt/app-root/bin/python3.8
+COPY --from=python /opt/app-root/bin/python3.8 /opt/app-root/bin/python3
+COPY --from=python /opt/app-root/bin/python3.8 /opt/app-root/bin/python
+COPY --from=python /opt/app-root/lib/python3.8 /opt/app-root/lib/python3
+COPY --from=python /opt/app-root/lib64/python3.8 /opt/app-root/lib64/python3
+COPY --from=python /opt/app-root/bin/pip /opt/app-root/bin/pip
+COPY --from=python /opt/app-root/bin/pip /opt/app-root/bin/pip3
+COPY --from=python /opt/app-root/bin/pip /opt/app-root/bin/pip3.8
 
 # Install packages
 ENV SYSTEM_PKGS="openssl sudo"
-ENV PYTHON_PKGS="rh-python36"
 
-RUN INSTALL_PKGS="${SYSTEM_PKGS} ${PYTHON_PKGS}" && \
+RUN INSTALL_PKGS="${SYSTEM_PKGS}" && \
   yum install -y yum-utils && \
   yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
-  rpm -V $INSTALL_PKGS && \
-  yum -y clean all --enablerepo='*'
+  rpm -V $INSTALL_PKGS
 
-ENV GO111MODULE="on"
-ENV GOPROXY="https://proxy.golang.org"
+# Uninstall git and unstall newer git
+RUN yum -y --setopt=tsflags=nodocs install http://mirror.centos.org/centos/7/os/x86_64/Packages/pcre2-10.23-2.el7.x86_64.rpm
+RUN yum -y --setopt=tsflags=nodocs install http://mirror.centos.org/centos/7/os/x86_64/Packages/emacs-filesystem-24.3-23.el7.noarch.rpm
+RUN yum -y --setopt=tsflags=nodocs install http://mirror.centos.org/centos/7/os/x86_64/Packages/libsecret-0.18.6-1.el7.x86_64.rpm
+RUN yum -y --setopt=tsflags=nodocs install https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm
+RUN yum -y --setopt=tsflags=nodocs install git
+RUN yum -y clean all --enablerepo='*'
+
+# Install aws-cli
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip
+RUN ./aws/install
 
 # Install goreleaser
 RUN rpm -Uvh "https://github.com/goreleaser/goreleaser/releases/download/v0.136.0/goreleaser_amd64.rpm"
 
+# Set envvars & make dirs
+ENV WORKSPACE_PATH="/workspace"
+ENV ARTIFACTS_PATH="${WORKSPACE_PATH}/artifacts"
+ENV LOGS_PATH="${ARTIFACTS_PATH}/logs"
+ENV TOOLS_PATH="/tools"
+
+RUN mkdir -p $WORKSPACE_PATH
+RUN mkdir -p $ARTIFACTS_PATH
+RUN mkdir -p $LOGS_PATH
+RUN mkdir -p $TOOLS_PATH
+
+# Copy sensu-release tools to /tools
+COPY ci-common-functions.sh $TOOLS_PATH
+COPY post-packages-s3.sh $TOOLS_PATH
+COPY go.mod $TOOLS_PATH
+COPY go.sum $TOOLS_PATH
+ADD cmd $TOOLS_PATH/cmd
+
+# Build tools
+ENV GO111MODULE="on"
+ENV GOPROXY="https://proxy.golang.org"
+RUN export PATH=$PATH:$(dirname $(find /opt -name go -type f)) && \
+  cd $TOOLS_PATH && go build ./cmd/circleci-logs
+
 # Add circleci user
 RUN useradd --uid=3434 --user-group --create-home circleci
 RUN echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci
@@ -37,11 +74,14 @@ RUN sudo -u circleci mkdir /home/circleci/project
 RUN chown -R circleci:circleci $WORKSPACE_PATH
 RUN chown -R circleci:circleci $ARTIFACTS_PATH
 RUN chown -R circleci:circleci $APP_ROOT
+RUN chown -R circleci:circleci $LOGS_PATH
+RUN chown -R circleci:circleci $TOOLS_PATH
 RUN chmod +x $APP_ROOT/etc/scl_enable
 
 # Fix BASH_ENV
 RUN sed -e '/unset BASH_ENV PROMPT_COMMAND ENV/ s/^#*/#/' -i $APP_ROOT/etc/scl_enable
 
 USER circleci
 ENV HOME /home/circleci
+ENV TERM xterm-color
 WORKDIR $HOME/project

dockerfiles/rhel8-builder/Dockerfile

@@ -1,31 +1,60 @@
+FROM registry.access.redhat.com/ubi8/python-38:1 as python
 FROM registry.access.redhat.com/ubi8/go-toolset:1.13.4
 
 LABEL name="sensu/sensu-release" \
       maintainer="[email protected]"
 
 USER root
 
-ENV WORKSPACE_PATH="/workspace"
-ENV ARTIFACTS_PATH="${WORKSPACE_PATH}/artifacts"
-
-RUN mkdir -p $WORKSPACE_PATH
-RUN mkdir -p $ARTIFACTS_PATH
+# Install python3.8
+COPY --from=python /opt/app-root/bin/python3.8 /opt/app-root/bin/python3.8
+COPY --from=python /opt/app-root/bin/python3.8 /opt/app-root/bin/python3
+COPY --from=python /opt/app-root/bin/python3.8 /opt/app-root/bin/python
+COPY --from=python /opt/app-root/lib/python3.8 /opt/app-root/lib/python3
+COPY --from=python /opt/app-root/lib64/python3.8 /opt/app-root/lib64/python3
+COPY --from=python /opt/app-root/bin/pip /opt/app-root/bin/pip
+COPY --from=python /opt/app-root/bin/pip /opt/app-root/bin/pip3
+COPY --from=python /opt/app-root/bin/pip /opt/app-root/bin/pip3.8
 
 # Install packages
 ENV SYSTEM_PKGS="openssl sudo"
-ENV PYTHON_PKGS="python38"
 
-RUN INSTALL_PKGS="${SYSTEM_PKGS} ${PYTHON_PKGS}" && \
+RUN INSTALL_PKGS="${SYSTEM_PKGS}" && \
   yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
   rpm -V $INSTALL_PKGS && \
   yum -y clean all --enablerepo='*'
 
-ENV GO111MODULE="on"
-ENV GOPROXY="https://proxy.golang.org"
+# Install aws-cli
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip
+RUN ./aws/install
 
 # Install goreleaser
 RUN rpm -Uvh "https://github.com/goreleaser/goreleaser/releases/download/v0.136.0/goreleaser_amd64.rpm"
 
+# Set envvars & make dirs
+ENV WORKSPACE_PATH="/workspace"
+ENV ARTIFACTS_PATH="${WORKSPACE_PATH}/artifacts"
+ENV LOGS_PATH="${ARTIFACTS_PATH}/logs"
+ENV TOOLS_PATH="/tools"
+
+RUN mkdir -p $WORKSPACE_PATH
+RUN mkdir -p $ARTIFACTS_PATH
+RUN mkdir -p $LOGS_PATH
+RUN mkdir -p $TOOLS_PATH
+
+# Copy sensu-release tools to /tools
+COPY ci-common-functions.sh $TOOLS_PATH
+COPY post-packages-s3.sh $TOOLS_PATH
+COPY go.mod $TOOLS_PATH
+COPY go.sum $TOOLS_PATH
+ADD cmd $TOOLS_PATH/cmd
+
+# Build tools
+ENV GO111MODULE="on"
+ENV GOPROXY="https://proxy.golang.org"
+RUN cd $TOOLS_PATH && go build ./cmd/circleci-logs
+
 # Add circleci user
 RUN useradd --uid=3434 --user-group --create-home circleci
 RUN echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci
@@ -35,10 +64,13 @@ RUN sudo -u circleci mkdir /home/circleci/project
 # Fix ownership
 RUN chown -R circleci:circleci $WORKSPACE_PATH
 RUN chown -R circleci:circleci $ARTIFACTS_PATH
+RUN chown -R circleci:circleci $LOGS_PATH
+RUN chown -R circleci:circleci $TOOLS_PATH
 
 # Fix BASH_ENV
 RUN sed -e '/unset BASH_ENV PROMPT_COMMAND ENV/ s/^#*/#/' -i $APP_ROOT/etc/scl_enable
 
 USER circleci
 ENV HOME /home/circleci
+ENV TERM xterm-color
 WORKDIR $HOME/project

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/avast/retry-go v2.4.2+incompatible
 	github.com/go-resty/resty/v2 v2.1.0
+	github.com/koblas/s3-cli v0.0.0-20190423184604-d3bcafa5aff3 // indirect
 	github.com/peterhellberg/link v1.1.0
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v1.0.0