release.yml

# yamllint disable rule:line-length rule:truthy
name: Release
# Generated by: gromit policy

# Distribution channels covered by this workflow
# - Ubuntu and Debian
# - RHEL and AL
# - docker hub
# - devenv ECR
# - Cloudsmith
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
on:
  # Trigger release every monday at midnight for master CI images
  schedule:
    - cron: "0 0 * * 1"
  pull_request:
  push:
    branches:
      - master
      - release-**
    tags:
      - 'v*'
env:
  GOPRIVATE: github.com/TykTechnologies
  VARIATION: inverted
  DOCKER_BUILD_SUMMARY: false
  DOCKER_BUILD_RECORD_UPLOAD: false
  # startsWith covers pull_request_target too
  BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
jobs:
  goreleaser:
    if: github.event.pull_request.draft == false
    name: '${{ matrix.golang_cross }}'
    runs-on: ubuntu-latest-m
    permissions:
      id-token: write # AWS OIDC JWT
      contents: read # actions/checkout
    strategy:
      fail-fast: false
      matrix:
        golang_cross:
          - 1.21-bookworm
        include:
          - golang_cross: 1.21-bookworm
            goreleaser: 'ci/goreleaser/goreleaser.yml'
            cgo: 0
            rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
            debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm'
    outputs:
      tags: ${{ steps.ci_metadata.outputs.tags }}
      commit_author: ${{ steps.set_outputs.outputs.commit_author}}
    steps:
      - name: Checkout of tyk-pump
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: Set some outputs for later
        id: set_outputs
        shell: bash
        env:
          HEAD_REF: ${{github.head_ref}}
        run: |
          echo "commit_author=$(git show -s --format='%ae' HEAD)" >> $GITHUB_OUTPUT
          echo "branch=${HEAD_REF##*/}" >> $GITHUB_OUTPUT
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to Cloudsmith
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@v3
        with:
          registry: docker.tyk.io
          username: ${{ secrets.CLOUDSMITH_USERNAME }}
          password: ${{ secrets.CLOUDSMITH_API_KEY }}
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
      - name: Build
        env:
          NFPM_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }}
          PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
        run: |
          echo '#!/bin/sh
          ci/bin/unlock-agent.sh
          git config --global url."https://${{ secrets.ORG_GH_TOKEN }}@github.com".insteadOf "https://github.com"
          git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk-pump
          goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot --skip=sign' || '' }}' | tee /tmp/build.sh
          chmod +x /tmp/build.sh
          docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \
          -e GOPRIVATE=github.com/TykTechnologies                                \
          -e DEBVERS='${{ matrix.debvers }}'                               \
          -e RPMVERS='${{ matrix.rpmvers }}'                               \
          -e CGO_ENABLED=${{ matrix.cgo }}                                 \
          -e NFPM_PASSPHRASE="$NFPM_PASSPHRASE"                          \
          -e GPG_FINGERPRINT=12B5D62C28F57592D1575BD51ED14C59E37DAC20            \
          -e PKG_SIGNING_KEY="$PKG_SIGNING_KEY"                                  \
          -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN                              \
          -v ${{github.workspace}}:/go/src/github.com/TykTechnologies/tyk-pump \
          -v /var/run/docker.sock:/var/run/docker.sock                           \
          -v ~/.docker/config.json:/root/.docker/config.json                     \
          -e GOCACHE=/cache/go-build                                             \
          -e GOMODCACHE=/go/pkg/mod                                              \
          -v ~/go/pkg/mod:/go/pkg/mod                                            \
          -v ~/.cache/go-build:/cache/go-build                                   \
          -v /tmp/build.sh:/tmp/build.sh                                         \
          -w /go/src/github.com/TykTechnologies/tyk-pump                      \
          tykio/golang-cross:${{ matrix.golang_cross }} /tmp/build.sh
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
          # Don't mask to pass it across job boundaries
          mask-aws-account-id: false
      - uses: aws-actions/amazon-ecr-login@v2
        id: ecr
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        with:
          mask-password: 'true'
      - name: Docker metadata for CI
        id: ci_metadata
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        uses: docker/metadata-action@v5
        with:
          images: ${{ steps.ecr.outputs.registry }}/tyk-pump
          flavor: |
            latest=false
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=sha,format=long
            type=semver,pattern={{major}},prefix=v
            type=semver,pattern={{major}}.{{minor}},prefix=v
            type=semver,pattern={{version}},prefix=v
      - name: push image to CI
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        uses: docker/build-push-action@v6
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          push: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          tags: ${{ steps.ci_metadata.outputs.tags }}
          labels: ${{ steps.tag_metadata.outputs.labels }}
      - name: Docker metadata for tag push
        id: tag_metadata
        uses: docker/metadata-action@v5
        with:
          images: |
            tykio/tyk-pump-docker-pub
            docker.tyk.io/tyk-pump/tyk-pump
          flavor: |
            latest=false
            prefix=v
          tags: |
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{version}}
          labels: "org.opencontainers.image.title=tyk-pump  (distroless) \norg.opencontainers.image.description=Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once).\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n"
      - name: push image to prod
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        uses: docker/build-push-action@v6
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: ${{ startsWith(github.ref, 'refs/tags') }}
          tags: ${{ steps.tag_metadata.outputs.tags }}
          labels: ${{ steps.tag_metadata.outputs.labels }}
      - name: save deb
        uses: actions/upload-artifact@v4
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        with:
          name: deb
          retention-days: 1
          path: |
            dist/*.deb
            !dist/*PAYG*.deb
            !dist/*fips*.deb
      - name: save rpm
        uses: actions/upload-artifact@v4
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        with:
          name: rpm
          retention-days: 1
          path: |
            dist/*.rpm
            !dist/*PAYG*.rpm
            !dist/*fips*.rpm
  test-controller-api:
    if: github.event.pull_request.draft == false
    needs:
      - goreleaser
    runs-on: ubuntu-latest
    outputs:
      envfiles: ${{ steps.params.outputs.envfiles }}
      pump: ${{ steps.params.outputs.pump }}
      sink: ${{ steps.params.outputs.sink }}
    steps:
      - name: set params
        id: params
        shell: bash
        run: |
          set -eo pipefail
          curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api.gho" | tee -a "$GITHUB_OUTPUT"
          if ! [[ $VARIATION =~ prod ]] ;then
              echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using non-prod variation"
              echo "### :warning: You are using VARIATION=${VARIATION} in test-controller-api" >> $GITHUB_STEP_SUMMARY
          fi
  api-tests:
    needs:
      - test-controller-api
      - goreleaser
    runs-on: ubuntu-latest-m-2
    env:
      XUNIT_REPORT_PATH: ${{ github.workspace}}/test-results.xml
    permissions:
      id-token: write # This is required for requesting the Github JWT
      contents: read # This is required for actions/checkout
    strategy:
      fail-fast: false
      matrix:
        envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }}
        sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
      - id: ecr
        uses: aws-actions/amazon-ecr-login@v2
        with:
          mask-password: 'true'
      - name: Setup tmate session only in debug mode
        uses: mxschmitt/action-tmate@v3
        if: runner.debug == '1'
        with:
          detached: true
          limit-access-to-actor: true
          # Only ${{ github.actor }} has access
          # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
      - name: fetch env from tyk-pro
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh release download --repo github.com/tyklabs/tyk-pro --archive tar.gz -O env.tgz
          mkdir auto && tar --strip-components=1 -C auto -xzvf env.tgz
      - name: env up
        shell: bash
        working-directory: auto
        id: env_up
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
        run: |
          match_tag=${{steps.ecr.outputs.registry}}/tyk-pump:$BASE_REF
          tags=(${{ needs.goreleaser.outputs.tags }})
          set -eaxo pipefail
          docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env
          echo '# alfa and beta have to come after the override
          tyk_alfa_image=$tyk_image
          tyk_beta_image=$tyk_image
          ECR=${{steps.ecr.outputs.registry}}
          tyk_sink_image=${{matrix.sink}}
          confs_dir=./pro-ha
          env_file=local.env' >> versions.env
          cat ./confs/${{ matrix.envfiles.config }}.env local-${{ matrix.envfiles.db }}.env > local.env
          echo "::group::versions"
          cat versions.env local.env
          echo "::endgroup::"
          # bring up env, the project name is important
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d
          ./dash-bootstrap.sh http://localhost:3000
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d
          echo "$(cat pytest.env | grep USER_API_SECRET)" >> $GITHUB_OUTPUT
          echo "ts=$(date +%s%N)" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v4
        with:
          repository: TykTechnologies/tyk-analytics
          path: tyk-analytics
          token: ${{ secrets.ORG_GH_TOKEN }}
          fetch-depth: 1
          sparse-checkout: tests/api
      - name: Choosing test code branch
        working-directory: tyk-analytics/tests/api
        run: |
          if [[ ${{ github.event_name }} == "pull_request" ]]; then
            PR_BRANCH=${{ github.event.pull_request.head.ref }}
            TARGET_BRANCH=${{ github.event.pull_request.base.ref }}
            if git rev-parse "origin/$PR_BRANCH" >/dev/null 2>&1; then
              echo "PR branch $PR_BRANCH exists. Checking out..."
              git checkout "$PR_BRANCH"
            elif git rev-parse "origin/$TARGET_BRANCH" >/dev/null 2>&1; then
              echo "Target branch $TARGET_BRANCH exists. Checking out..."
              git checkout "$TARGET_BRANCH"
            fi
          fi
          if [[ ${{ github.event_name }} == "push" ]]; then
            PUSH_BRANCH=${{ github.ref_name }}
            if git rev-parse "origin/$PUSH_BRANCH" >/dev/null 2>&1; then
              echo "Push branch $PUSH_BRANCH exists. Checking out..."
              git checkout "$PUSH_BRANCH"
            fi
          fi
          echo "Current commit: $(git rev-parse HEAD)"
      - uses: actions/setup-python@v5
        with:
          cache: 'pip'
          python-version: '3.10'
      - name: Run API tests
        id: test_execution
        working-directory: tyk-analytics/tests/api
        run: |
          pytest="pytest --ci --random-order --force-flaky --no-success-flaky-report --maxfail=3 --junitxml=${XUNIT_REPORT_PATH} --cache-clear --ignore=./tests/mdcb -v --log-cli-level=ERROR"
          pip install -r requirements.txt
          cat >pytest.env <<-EOF
          TYK_TEST_BASE_URL=http://localhost:3000/
          TYK_TEST_GW_URL=https://localhost:8080/
          TYK_TEST_GW_1_ALFA_URL=https://localhost:8181/
          TYK_TEST_GW_1_BETA_URL=https://localhost:8182/
          TYK_TEST_GW_2_ALFA_URL=https://localhost:8281/
          TYK_TEST_GW_2_BETA_URL=https://localhost:8282/
          TYK_TEST_MONGODB=localhost:27017
          TYK_TEST_REDIS=localhost
          TYK_TEST_DB_ADMIN=12345
          TYK_TEST_GW_SECRET=352d20ee67be67f6340b4c0605b044b7
          TYK_TEST_DB_NAME=tyk_analytics
          TYK_TEST_FEDERATION_HOST=federation
          TYK_TEST_GRAPHQL_FAKER_HOST=graphql-faker
          GATEWAY_CONTAINER_NAME=tyk
          USER_API_SECRET=${{ steps.env_up.outputs.USER_API_SECRET }}
          EOF
          env $(cat pytest.env | xargs) $pytest -m "${{ matrix.envfiles.apimarkers }}"
      - name: Generate metadata and upload test reports
        id: metadata_report
        if: always() && (steps.test_execution.conclusion != 'skipped')
        env:
          REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.env_up.outputs.ts}}
          METADATA_REPORT_PATH: metadata.toml
        run: |
          # Generate metadata report
          set -eo pipefail
          echo "[metadata]
          repo = ${{ github.repository }}
          branch = ${{ github.ref }}
          commit = ${{ github.sha }}
          test_suite_version = $BASE_REF
          test_suite_name = ${{ github.job }}
          test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }}
          db = ${{ matrix.envfiles.db }}
          conf = ${{ matrix.envfiles.config }}
          cache = ${{ matrix.envfiles.cache }}
          pump_compatibility = ${{ matrix.pump }}
          sink_compatibility = ${{ matrix.sink }}
          " | tee ${METADATA_REPORT_PATH}
          aws s3 cp ${XUNIT_REPORT_PATH}  s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
          aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
      - name: Docker logs for all components
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        working-directory: auto
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          ECR: ${{ steps.ecr.outputs.registry }}
        run: |
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
          echo "::group::DockerLogs"
          cat ${{ github.workspace }}/docker-compose.log
          echo "::endgroup::"
      - name: Upload compose logs
        uses: actions/upload-artifact@v4
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        with:
          name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          path: ${{ github.workspace }}/docker-compose.log
          retention-days: 3
          overwrite: true
  test-controller-distros:
    if: github.event.pull_request.draft == false
    needs:
      - goreleaser
    runs-on: ubuntu-latest
    outputs:
      deb: ${{ steps.params.outputs.deb }}
      rpm: ${{ steps.params.outputs.rpm }}
    steps:
      - name: set params
        id: params
        shell: bash
        env:
          # startsWith covers pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
        run: |
          set -eo pipefail
          curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api/Distros.gho" | tee -a "$GITHUB_OUTPUT"
          if ! [[ $VARIATION =~ prod ]];then
            echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using test variation"
          fi
  upgrade-deb:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    runs-on: ubuntu-latest
    needs:
      - test-controller-distros
    strategy:
      fail-fast: true
      matrix:
        arch:
          - amd64
          - arm64
        distro: ${{ fromJson(needs.test-controller-distros.outputs.deb) }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          sparse-checkout: ci
      - uses: actions/download-artifact@v4
        with:
          name: deb
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          ARG TARGETARCH
          COPY tyk-pump*_${TARGETARCH}.deb /tyk-pump.deb
          RUN apt-get update && apt-get install -y curl
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash && apt-get install -y tyk-pump=1.6.0
          RUN dpkg -i /tyk-pump.deb

          ' | tee Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@v6
        with:
          context: "."
          platforms: linux/${{ matrix.arch }}
          build-args: |
            RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: Dockerfile
          push: false
  upgrade-rpm:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    runs-on: ubuntu-latest
    needs:
      - test-controller-distros
    strategy:
      fail-fast: true
      matrix:
        arch:
          - amd64
          - arm64
        distro: ${{ fromJson(needs.test-controller-distros.outputs.rpm) }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          sparse-checkout: ci
      - uses: actions/download-artifact@v4
        with:
          name: rpm
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          ARG RHELARCH
          COPY tyk-pump*.${RHELARCH}.rpm /tyk-pump.rpm
          RUN command -v curl || yum install -y curl
          RUN command -v useradd || yum install -y shadow-utils
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1
          RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key
          RUN rpm --checksig /tyk-pump.rpm
          RUN rpm -Uvh --force /tyk-pump.rpm

          ' | tee Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@v6
        with:
          context: "."
          platforms: linux/${{ matrix.arch }}
          build-args: |
            RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: Dockerfile
          push: false
  sbom:
    needs: goreleaser
    uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main
    secrets:
      DEPDASH_URL: ${{ secrets.DEPDASH_URL }}
      DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }}
      ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}