Merge branch 'master' into fix-jwk-fetch-no-proxy

.github/workflows/ci-tests.yml

@@ -17,55 +17,109 @@ on:
       - master
       - release-**
 
-# Only have one runner per PR, and per merged commit.
-#
-# - As a PR gets new commits, any old run jobs get cancelled (PR number)
-# - As a commit gets merged, it doesn't cancel previous running PR's (github.sha)
 concurrency:
-  group: ${{ github.event.pull_request.number || github.sha }}-ci-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   PYTHON_VERSION: "3.11"
   PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
 
 jobs:
+  golangci-lint:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    steps:
+      - name: "Checkout PR"
+        uses: TykTechnologies/github-actions/.github/actions/checkout-pr@main
+        with:
+          token: ${{ secrets.ORG_GH_TOKEN }}
+
+      - name: "Get base ref"
+        run: |
+          git fetch origin ${{ github.base_ref }}
+          git rev-parse origin/${{ github.base_ref }}
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/golangci-lint
+          key: 'golangci-lint-${{ runner.os }}-${{ hashFiles(''**/go.sum'') }}'
+          restore-keys: |
+            golangci-lint-${{ runner.os }}-
+
+      - name: Setup CI Tooling
+        uses: shrink/actions-docker-extract@v3
+        with:
+          image: tykio/ci-tools:latest
+          path: /usr/local/bin/golangci-lint
+          destination: /usr/local/bin
+
+      - run: golangci-lint version && golangci-lint cache status
+
+      - name: golangci-lint
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          golangci-lint run --out-format colored-line-number,checkstyle:golangci-lint-report.json --issues-exit-code=0 --new-from-rev=origin/${{ github.base_ref }} -v ./...
+
+      - name: golangci-lint-on-push
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          golangci-lint run --out-format checkstyle:golangci-lint-report.json --issues-exit-code=0 -v ./...
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: golangcilint
+          retention-days: 1
+          path: |
+            golangci-lint-report.json
+
   test:
     name: Go ${{ matrix.go-version }} Redis ${{ matrix.redis-version }}
-    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    needs: golangci-lint
+    # Runs on is pinned to a version that provides python 3.10.
+    # See: https://github.com/actions/runner-images?tab=readme-ov-file#available-images
+    # Avoid using ubuntu-latest as it would upgrade python unattended.
+    runs-on: ubuntu-22.04
     strategy:
-      fail-fast: true
-      # This workflow isn't designed to be run as a pipeline, several issues:
-      #
-      # - contains golangci-lint jobs, sonarcloud (would duplicate)
-      # - cache config not suitable for multiple pipelines
-      # - python tests should be separate job, or no job
-      #
-      # Keep it to a single job run from the matrix as configured
-      # until we get a chance to redesign the pipeline properly.
+      fail-fast: false
       matrix:
         redis-version: [7]
-        python-version: ["3.11"]
         go-version: [1.22.x]
 
     env:
       REDIS_IMAGE: redis:${{ matrix.redis-version }}
 
     steps:
+      - name: "Reclaim some runner space"
+        run: sudo rm -rf /usr/local/bin/* /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
       - name: Checkout Tyk
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.ref }}
 
-      - name: Setup Golang
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
-
+      # Regardless that the base image provides a python release, we need
+      # setup-python so it properly configures the python3-venv.
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Print runtime python version
+        run: python3 -c 'import sys; print("%d.%d" % (sys.version_info[0], sys.version_info[1]))'
+
+      - name: Print runtime pip version
+        run: pip -V && pip3 -V
 
       - name: Setup CI Tooling
         uses: shrink/actions-docker-extract@v3
@@ -74,7 +128,13 @@ jobs:
           path: /usr/local/bin/.
           destination: /usr/local/bin
 
-      - name: Cache
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Restore Golang Cache
         uses: actions/cache@v4
         with:
           path: |
@@ -113,33 +173,63 @@ jobs:
             exit 1
           fi
 
-      - name: Fetch base branch
-        if: ${{ github.event_name == 'pull_request' }}
-        run: git fetch origin ${{ github.base_ref }}
-
-      - name: Print CPU info
-        run: grep '^model name' /proc/cpuinfo
+      - name: Bring up test services
+        run: task services:up
 
-      - name: Print Go env
-        run: go env
+      - name: Preflight Python tests
+        if: runner.debug == '1'
+        run: TYK_LOGLEVEL=debug go test -p 1 -parallel 1 -race -v ./dlpython ./coprocess/...
 
       - name: Run Gateway Tests
         id: ci-tests
         run: |
           task test:e2e-combined args="-race -timeout=15m"
           task test:coverage
 
-      # golangci-lint actions *require* issues-exit-code=0 to pass data along to sonarcloud
-      # rather than erroring out on github issues directly with out-format github.
-      - name: golangci-lint
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          golangci-lint run --out-format checkstyle --issues-exit-code=0 --new-from-rev=origin/${{ github.base_ref }} ./... > golanglint.xml
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: coverage
+          retention-days: 1
+          path: coverage/gateway-all.cov
 
-      - name: golangci-lint-on-push
-        if: ${{ github.event_name == 'push' }}
-        run: |
-          golangci-lint run --out-format checkstyle --issues-exit-code=0 ./... > golanglint.xml
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: testjson
+          retention-days: 1
+          path: coverage/gateway-all.json
+
+  sonar-cloud-analysis:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    needs: [test, golangci-lint]
+    steps:
+      - name: "Checkout repository"
+        uses: TykTechnologies/github-actions/.github/actions/checkout-pr@main
+        with:
+          token: ${{ secrets.ORG_GH_TOKEN }}
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage
+
+      - name: Download golangcilint artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: golangcilint
+
+      - name: Check reports existence
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: 'coverage/gateway-all.cov, golangci-lint-report.json'
+
+      - name: Install Dependencies
+        env:
+          TOKEN: '${{ secrets.ORG_GH_TOKEN }}'
+        run: git config --global url."https://${TOKEN}@github.com".insteadOf "https://github.com"
 
       - name: SonarCloud Scan
         if: always()
@@ -149,12 +239,12 @@ jobs:
             -Dsonar.organization=tyktechnologies
             -Dsonar.projectKey=TykTechnologies_tyk
             -Dsonar.sources=.
-            -Dsonar.exclusions=**/testdata/*,test/**,coprocess/**/*,ci/**,smoke-tests/**,apidef/oas/schema/schema.gen.go
+            -Dsonar.exclusions=**/testdata/*,test/**,coprocess/**/*,ci/**,smoke-tests/**,apidef/oas/schema/schema.gen.go,templates/**
             -Dsonar.coverage.exclusions=**/*_test.go,**/mock/*
             -Dsonar.test.inclusions=**/*_test.go
             -Dsonar.tests=.
             -Dsonar.go.coverage.reportPaths=coverage/gateway-all.cov
-            -Dsonar.go.golangci-lint.reportPaths=golanglint.xml
+            -Dsonar.go.golangci-lint.reportPaths=golangci-lint-report.json
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -11,9 +11,14 @@ on:
   schedule:
     - cron: '0 18 * * 4'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   analyze:
     name: Analyze
+    if: ${{ !github.event.pull_request.draft }}
     runs-on: ubuntu-latest
 
     strategy:
@@ -27,7 +32,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
@@ -39,7 +44,7 @@ jobs:
       if: ${{ github.event_name == 'pull_request' }}
 
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version-file: go.mod
 

.github/workflows/lint-swagger.yml

@@ -7,6 +7,10 @@ on:
     paths:
       - 'swagger.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   redocly_validator:
     runs-on: ubuntu-latest
@@ -41,7 +45,7 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22.x'
+          go-version: stable
 
       - name: Install dyff binary
         run: |

.github/workflows/lint.yml

@@ -0,0 +1,29 @@
+# yamllint disable rule:truthy
+---
+name: CI lint
+
+# API Changes (godoc) and JIRA validation only apply to PRs.
+# Branches are not important, supports chaining PRs.
+
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  godoc:
+    if: ${{ !github.event.pull_request.draft }}
+    uses: TykTechnologies/github-actions/.github/workflows/godoc.yml@main
+    secrets:
+      ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
+    with:
+      go-version: stable
+
+  jira:
+    if: ${{ !github.event.pull_request.draft }}
+    uses: TykTechnologies/github-actions/.github/workflows/jira-lint.yaml@main
+    secrets:
+      JIRA_TOKEN: ${{ secrets.JIRA_TOKEN }}
+      ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}