Auto generated from templates by gromit (#848) #2584
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Generated by: gromit policy | |
# Distribution channels covered by this workflow | |
# - Ubuntu and Debian | |
# - RHEL and AL | |
# - docker hub | |
# - devenv ECR | |
# - Cloudsmith | |
name: Release | |
on: | |
# Trigger release every monday at midnight for master CI images | |
schedule: | |
- cron: "0 0 * * 1" | |
pull_request: | |
push: | |
branches: | |
- master | |
- release-** | |
tags: | |
- 'v*' | |
env: | |
GOPRIVATE: github.com/TykTechnologies | |
VARIATION: prod | |
DOCKER_BUILD_SUMMARY: false | |
DOCKER_BUILD_RECORD_UPLOAD: false | |
# startsWith covers pull_request_target too | |
BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}} | |
jobs: | |
goreleaser: | |
name: '${{ matrix.golang_cross }}' | |
runs-on: ubuntu-latest-m | |
permissions: | |
id-token: write # AWS OIDC JWT | |
contents: read # actions/checkout | |
strategy: | |
fail-fast: false | |
matrix: | |
golang_cross: | |
- 1.21-bookworm | |
include: | |
- golang_cross: 1.21-bookworm | |
goreleaser: 'ci/goreleaser/goreleaser.yml' | |
cgo: 0 | |
rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023' | |
debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm' | |
outputs: | |
tags: ${{ steps.ci_metadata.outputs.tags }} | |
commit_author: ${{ steps.set_outputs.outputs.commit_author}} | |
steps: | |
- name: Checkout of tyk-pump | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 1 | |
- name: Set some outputs for later | |
id: set_outputs | |
shell: bash | |
env: | |
HEAD_REF: ${{github.head_ref}} | |
run: | | |
echo "commit_author=$(git show -s --format='%ae' HEAD)" >> $GITHUB_OUTPUT | |
echo "branch=${HEAD_REF##*/}" >> $GITHUB_OUTPUT | |
- uses: docker/setup-qemu-action@v3 | |
- uses: docker/setup-buildx-action@v3 | |
- name: Login to DockerHub | |
if: startsWith(github.ref, 'refs/tags') | |
uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.DOCKER_USERNAME }} | |
password: ${{ secrets.DOCKER_PASSWORD }} | |
- name: Login to Cloudsmith | |
if: startsWith(github.ref, 'refs/tags') | |
uses: docker/login-action@v3 | |
with: | |
registry: docker.tyk.io | |
username: ${{ secrets.CLOUDSMITH_USERNAME }} | |
password: ${{ secrets.CLOUDSMITH_API_KEY }} | |
- uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache/go-build | |
~/go/pkg/mod | |
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} | |
restore-keys: | | |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} | |
- name: Build | |
env: | |
NFPM_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }} | |
PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} | |
run: | | |
echo '#!/bin/sh | |
ci/bin/unlock-agent.sh | |
git config --global url."https://${{ secrets.ORG_GH_TOKEN }}@github.com".insteadOf "https://github.com" | |
git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk-pump | |
goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot --skip=sign' || '' }}' | tee /tmp/build.sh | |
chmod +x /tmp/build.sh | |
docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \ | |
-e GOPRIVATE=github.com/TykTechnologies \ | |
-e DEBVERS='${{ matrix.debvers }}' \ | |
-e RPMVERS='${{ matrix.rpmvers }}' \ | |
-e CGO_ENABLED=${{ matrix.cgo }} \ | |
-e NFPM_PASSPHRASE="$NFPM_PASSPHRASE" \ | |
-e GPG_FINGERPRINT=12B5D62C28F57592D1575BD51ED14C59E37DAC20 \ | |
-e PKG_SIGNING_KEY="$PKG_SIGNING_KEY" \ | |
-e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN \ | |
-v ${{github.workspace}}:/go/src/github.com/TykTechnologies/tyk-pump \ | |
-v /var/run/docker.sock:/var/run/docker.sock \ | |
-v ~/.docker/config.json:/root/.docker/config.json \ | |
-e GOCACHE=/cache/go-build \ | |
-e GOMODCACHE=/go/pkg/mod \ | |
-v ~/go/pkg/mod:/go/pkg/mod \ | |
-v ~/.cache/go-build:/cache/go-build \ | |
-v /tmp/build.sh:/tmp/build.sh \ | |
-w /go/src/github.com/TykTechnologies/tyk-pump \ | |
tykio/golang-cross:${{ matrix.golang_cross }} /tmp/build.sh | |
- uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk | |
role-session-name: cipush | |
aws-region: eu-central-1 | |
# Don't mask to pass it across job boundaries | |
mask-aws-account-id: false | |
- uses: aws-actions/amazon-ecr-login@v2 | |
id: ecr | |
if: ${{ matrix.golang_cross == '1.21-bookworm' }} | |
with: | |
mask-password: 'true' | |
- name: Docker metadata for CI | |
id: ci_metadata | |
if: ${{ matrix.golang_cross == '1.21-bookworm' }} | |
uses: docker/metadata-action@v5 | |
with: | |
images: ${{ steps.ecr.outputs.registry }}/tyk-pump | |
flavor: | | |
latest=false | |
tags: | | |
type=ref,event=branch | |
type=ref,event=pr | |
type=sha,format=long | |
type=semver,pattern={{major}},prefix=v | |
type=semver,pattern={{major}}.{{minor}},prefix=v | |
type=semver,pattern={{version}},prefix=v | |
- name: push image to CI | |
if: ${{ matrix.golang_cross == '1.21-bookworm' }} | |
uses: docker/build-push-action@v6 | |
with: | |
context: "dist" | |
platforms: linux/amd64,linux/arm64 | |
file: ci/Dockerfile.distroless | |
provenance: mode=max | |
sbom: true | |
push: true | |
cache-from: type=gha | |
cache-to: type=gha,mode=max | |
tags: ${{ steps.ci_metadata.outputs.tags }} | |
labels: ${{ steps.tag_metadata.outputs.labels }} | |
- name: Docker metadata for tag push | |
id: tag_metadata | |
uses: docker/metadata-action@v5 | |
with: | |
images: | | |
tykio/tyk-pump-docker-pub | |
docker.tyk.io/tyk-pump/tyk-pump | |
flavor: | | |
latest=false | |
prefix=v | |
tags: | | |
type=semver,pattern={{major}} | |
type=semver,pattern={{major}}.{{minor}} | |
type=semver,pattern={{version}} | |
labels: "org.opencontainers.image.title=tyk-pump (distroless) \norg.opencontainers.image.description=Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once).\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n" | |
- name: push image to prod | |
if: ${{ matrix.golang_cross == '1.21-bookworm' }} | |
uses: docker/build-push-action@v6 | |
with: | |
context: "dist" | |
platforms: linux/amd64,linux/arm64 | |
file: ci/Dockerfile.distroless | |
provenance: mode=max | |
sbom: true | |
cache-from: type=gha | |
cache-to: type=gha,mode=max | |
push: ${{ startsWith(github.ref, 'refs/tags') }} | |
tags: ${{ steps.tag_metadata.outputs.tags }} | |
labels: ${{ steps.tag_metadata.outputs.labels }} | |
- name: save deb | |
uses: actions/upload-artifact@v4 | |
if: ${{ matrix.golang_cross == '1.21-bookworm' }} | |
with: | |
name: deb | |
retention-days: 1 | |
path: | | |
dist/*.deb | |
!dist/*PAYG*.deb | |
!dist/*fips*.deb | |
- name: save rpm | |
uses: actions/upload-artifact@v4 | |
if: ${{ matrix.golang_cross == '1.21-bookworm' }} | |
with: | |
name: rpm | |
retention-days: 1 | |
path: | | |
dist/*.rpm | |
!dist/*PAYG*.rpm | |
!dist/*fips*.rpm | |
test-controller-api: | |
needs: | |
- goreleaser | |
runs-on: ubuntu-latest | |
outputs: | |
envfiles: ${{ steps.params.outputs.envfiles }} | |
pump: ${{ steps.params.outputs.pump }} | |
sink: ${{ steps.params.outputs.sink }} | |
steps: | |
- name: set params | |
id: params | |
shell: bash | |
run: | | |
set -eo pipefail | |
curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api.gho" | tee -a "$GITHUB_OUTPUT" | |
if ! [[ $VARIATION =~ prod ]] ;then | |
echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using non-prod variation" | |
echo "### :warning: You are using VARIATION=${VARIATION} in test-controller-api" >> $GITHUB_STEP_SUMMARY | |
fi | |
api-tests: | |
needs: | |
- test-controller-api | |
- goreleaser | |
runs-on: ubuntu-latest-m-2 | |
env: | |
XUNIT_REPORT_PATH: ${{ github.workspace}}/test-results.xml | |
permissions: | |
id-token: write # This is required for requesting the Github JWT | |
contents: read # This is required for actions/checkout | |
strategy: | |
fail-fast: false | |
matrix: | |
envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }} | |
sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }} | |
steps: | |
- uses: aws-actions/configure-aws-credentials@v4 | |
with: | |
role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk | |
role-session-name: cipush | |
aws-region: eu-central-1 | |
- id: ecr | |
uses: aws-actions/amazon-ecr-login@v2 | |
with: | |
mask-password: 'true' | |
- name: Setup tmate session only in debug mode | |
uses: mxschmitt/action-tmate@v3 | |
if: runner.debug == '1' | |
with: | |
detached: true | |
limit-access-to-actor: true | |
# Only ${{ github.actor }} has access | |
# See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys | |
- name: fetch env from tyk-pro | |
env: | |
GH_TOKEN: ${{ github.token }} | |
run: | | |
gh release download --repo github.com/tyklabs/tyk-pro --archive tar.gz -O env.tgz | |
mkdir auto && tar --strip-components=1 -C auto -xzvf env.tgz | |
- name: env up | |
shell: bash | |
working-directory: auto | |
id: env_up | |
env: | |
pull_policy: 'if_not_present' | |
GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} | |
TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }} | |
TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }} | |
run: | | |
match_tag=${{steps.ecr.outputs.registry}}/tyk-pump:$BASE_REF | |
tags=(${{ needs.goreleaser.outputs.tags }}) | |
set -eaxo pipefail | |
docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env | |
echo '# alfa and beta have to come after the override | |
tyk_alfa_image=$tyk_image | |
tyk_beta_image=$tyk_image | |
ECR=${{steps.ecr.outputs.registry}} | |
tyk_sink_image=${{matrix.sink}} | |
confs_dir=./pro-ha | |
env_file=local.env' >> versions.env | |
cat ./confs/${{ matrix.envfiles.config }}.env local-${{ matrix.envfiles.db }}.env > local.env | |
echo "::group::versions" | |
cat versions.env local.env | |
echo "::endgroup::" | |
# bring up env, the project name is important | |
docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d | |
./dash-bootstrap.sh http://localhost:3000 | |
docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d | |
echo "$(cat pytest.env | grep USER_API_SECRET)" >> $GITHUB_OUTPUT | |
- uses: actions/checkout@v4 | |
with: | |
repository: TykTechnologies/tyk-analytics | |
path: tyk-analytics | |
token: ${{ secrets.ORG_GH_TOKEN }} | |
fetch-depth: 1 | |
sparse-checkout: tests/api | |
- name: Branch for test code | |
id: timestamp | |
working-directory: tyk-analytics/tests/api | |
run: | | |
# Generate report id | |
git fetch --no-tags --depth 1 origin "refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" | |
git switch $BASE_REF | |
echo "ts=$(date +%s%N)" >> $GITHUB_OUTPUT | |
- uses: actions/setup-python@v5 | |
with: | |
cache: 'pip' | |
python-version: '3.10' | |
- name: Run API tests | |
id: test_execution | |
working-directory: tyk-analytics/tests/api | |
run: | | |
pytest="pytest --ci --random-order --force-flaky --no-success-flaky-report --maxfail=3 --junitxml=${XUNIT_REPORT_PATH} --cache-clear --ignore=./tests/mdcb -v --log-cli-level=ERROR" | |
pip install -r requirements.txt | |
cat >pytest.env <<-EOF | |
TYK_TEST_BASE_URL=http://localhost:3000/ | |
TYK_TEST_GW_URL=https://localhost:8080/ | |
TYK_TEST_GW_1_ALFA_URL=https://localhost:8181/ | |
TYK_TEST_GW_1_BETA_URL=https://localhost:8182/ | |
TYK_TEST_GW_2_ALFA_URL=https://localhost:8281/ | |
TYK_TEST_GW_2_BETA_URL=https://localhost:8282/ | |
TYK_TEST_MONGODB=localhost:27017 | |
TYK_TEST_REDIS=localhost | |
TYK_TEST_DB_ADMIN=12345 | |
TYK_TEST_GW_SECRET=352d20ee67be67f6340b4c0605b044b7 | |
TYK_TEST_DB_NAME=tyk_analytics | |
TYK_TEST_FEDERATION_HOST=federation | |
TYK_TEST_GRAPHQL_FAKER_HOST=graphql-faker | |
GATEWAY_CONTAINER_NAME=tyk | |
USER_API_SECRET=${{ steps.env_up.outputs.USER_API_SECRET }} | |
EOF | |
env $(cat pytest.env | xargs) $pytest -m "${{ matrix.envfiles.apimarkers }}" | |
- name: Generate metadata and upload test reports | |
id: metadata_report | |
if: always() && (steps.test_execution.conclusion != 'skipped') | |
env: | |
REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.timestamp.outputs.ts}} | |
METADATA_REPORT_PATH: metadata.toml | |
run: | | |
# Generate metadata report | |
set -eo pipefail | |
echo "[metadata] | |
repo = ${{ github.repository }} | |
branch = ${{ github.ref }} | |
commit = ${{ github.sha }} | |
test_suite_version = $BASE_REF | |
test_suite_name = ${{ github.job }} | |
test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }} | |
db = ${{ matrix.envfiles.db }} | |
conf = ${{ matrix.envfiles.config }} | |
cache = ${{ matrix.envfiles.cache }} | |
pump_compatibility = ${{ matrix.pump }} | |
sink_compatibility = ${{ matrix.sink }} | |
" | tee ${METADATA_REPORT_PATH} | |
aws s3 cp ${XUNIT_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml | |
aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml | |
- name: Docker logs for all components | |
if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success') | |
working-directory: auto | |
env: | |
pull_policy: 'if_not_present' | |
GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} | |
TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }} | |
TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }} | |
ECR: ${{ steps.ecr.outputs.registry }} | |
run: | | |
docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log | |
echo "::group::DockerLogs" | |
cat ${{ github.workspace }}/docker-compose.log | |
echo "::endgroup::" | |
- name: Upload compose logs | |
uses: actions/upload-artifact@v4 | |
if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success') | |
with: | |
name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }} | |
path: ${{ github.workspace }}/docker-compose.log | |
retention-days: 3 | |
overwrite: true | |
test-controller-distros: | |
needs: | |
- goreleaser | |
runs-on: ubuntu-latest | |
outputs: | |
deb: ${{ steps.params.outputs.deb }} | |
rpm: ${{ steps.params.outputs.rpm }} | |
steps: | |
- name: set params | |
id: params | |
shell: bash | |
env: | |
# startsWith covers pull_request_target too | |
BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}} | |
run: | | |
set -eo pipefail | |
curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api/Distros.gho" | tee -a "$GITHUB_OUTPUT" | |
if ! [[ $VARIATION =~ prod ]];then | |
echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using test variation" | |
fi | |
upgrade-deb: | |
services: | |
httpbin.org: | |
image: kennethreitz/httpbin | |
runs-on: ubuntu-latest | |
needs: | |
- test-controller-distros | |
strategy: | |
fail-fast: true | |
matrix: | |
arch: | |
- amd64 | |
- arm64 | |
distro: ${{ fromJson(needs.test-controller-distros.outputs.deb) }} | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
fetch-depth: 1 | |
sparse-checkout: ci | |
- uses: actions/download-artifact@v4 | |
with: | |
name: deb | |
- uses: docker/setup-qemu-action@v3 | |
- uses: docker/setup-buildx-action@v3 | |
- name: generate dockerfile | |
run: | | |
echo 'FROM ${{ matrix.distro }} | |
ARG TARGETARCH | |
COPY tyk-pump*_${TARGETARCH}.deb /tyk-pump.deb | |
RUN apt-get update && apt-get install -y curl | |
RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash && apt-get install -y tyk-pump=1.6.0 | |
RUN dpkg -i /tyk-pump.deb | |
' | tee Dockerfile | |
- name: install on ${{ matrix.distro }} | |
uses: docker/build-push-action@v6 | |
with: | |
context: "." | |
platforms: linux/${{ matrix.arch }} | |
build-args: | | |
RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }} | |
cache-from: type=gha | |
cache-to: type=gha,mode=max | |
file: Dockerfile | |
push: false | |
upgrade-rpm: | |
services: | |
httpbin.org: | |
image: kennethreitz/httpbin | |
runs-on: ubuntu-latest | |
needs: | |
- test-controller-distros | |
strategy: | |
fail-fast: true | |
matrix: | |
arch: | |
- amd64 | |
- arm64 | |
distro: ${{ fromJson(needs.test-controller-distros.outputs.rpm) }} | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
fetch-depth: 1 | |
sparse-checkout: ci | |
- uses: actions/download-artifact@v4 | |
with: | |
name: rpm | |
- uses: docker/setup-qemu-action@v3 | |
- uses: docker/setup-buildx-action@v3 | |
- name: generate dockerfile | |
run: | | |
echo 'FROM ${{ matrix.distro }} | |
ARG RHELARCH | |
COPY tyk-pump*.${RHELARCH}.rpm /tyk-pump.rpm | |
RUN command -v curl || yum install -y curl | |
RUN command -v useradd || yum install -y shadow-utils | |
RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1 | |
RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key | |
RUN rpm --checksig /tyk-pump.rpm | |
RUN rpm -Uvh --force /tyk-pump.rpm | |
' | tee Dockerfile | |
- name: install on ${{ matrix.distro }} | |
uses: docker/build-push-action@v6 | |
with: | |
context: "." | |
platforms: linux/${{ matrix.arch }} | |
build-args: | | |
RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }} | |
cache-from: type=gha | |
cache-to: type=gha,mode=max | |
file: Dockerfile | |
push: false | |
sbom: | |
needs: goreleaser | |
uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main | |
secrets: | |
DEPDASH_URL: ${{ secrets.DEPDASH_URL }} | |
DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }} | |
ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} |