Merge branch 'extractor_regex_replace' of github.com:solo-io/gloo int…

…o extractor_regex_replace

.github/workflows/composite-actions/istio-tests/action.yaml

@@ -0,0 +1,50 @@
+name: Gloo Istio Regression Tests
+
+description: Tests which run Gloo Edge in a Kubernetes cluster with Istio
+
+runs:
+  using: "composite"
+  steps:
+    - name: Prep Go Runner
+      uses: ./.github/workflows/composite-actions/prep-go-runner
+    - name: Install kind
+      uses: helm/[email protected]
+      with:
+        install_only: true
+        version: ${{ matrix.kube-version.kind }}
+    - uses: azure/setup-kubectl@v3
+      id: kubectl
+      with:
+        version: ${{ matrix.kube-version.kubectl }}
+    - uses: azure/setup-helm@v3
+      with:
+        version: ${{ matrix.kube-version.helm }}
+    - name: Setup test env
+      shell: bash
+      env:
+        KUBE2E_TESTS: istio
+        CLUSTER_NAME: 'kind'
+        CLUSTER_NODE_VERSION: ${{ matrix.kube-version.node }}
+      run: ./ci/kind/setup-kind.sh
+    - name: Testing - kube e2e regression tests
+      env:
+        KUBE2E_TESTS: istio
+      shell: bash
+      run: |
+        if [[ "${{ matrix.kube-e2e-test-type }}" == "gloo-gateway" ]]; then
+          export GLOO_GATEWAY_SETUP=true
+        elif [[ "${{ matrix.kube-e2e-test-type }}" == "istio-auto" ]]; then
+          export ISTIO_AUTO_MTLS=true
+        fi
+        make install-test-tools run-kube-e2e-tests
+    - name: save results
+      if: ${{ failure() }}
+      shell: bash
+      run: mkdir -p _output/kube2e-artifacts && echo '{"url":"https://github.com/solo-io/gloo/actions/runs/${{github.run_id}}",
+        "name":"${{matrix.kube-e2e-test-type}}@k8s${{matrix.kube-version.kubectl}}"}' > _output/kube2e-artifacts/test-out.json
+    - uses: actions/upload-artifact@v3
+      if: ${{ failure() }}
+      with:
+        name: ${{matrix.kube-e2e-test-type}}@k8s${{matrix.kube-version.kubectl}}-kube-dump
+        path: "_output/kube2e-artifacts"
+        if-no-files-found: warn

.github/workflows/istio-tests.yaml

@@ -0,0 +1,65 @@
+name: Istio Tests
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+env:
+  VERSION: '1.0.0-ci'
+  GITHUB_TOKEN: ${{ github.token }} # necessary to pass upgrade tests
+
+jobs:
+  prepare_env:
+    name: Prepare Environment
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    outputs:
+      should-auto-succeed-istio-tests: ${{ steps.run-strategy.outputs.auto_succeed }}
+    steps:
+      - name: Cancel Previous Actions
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - id: checkout-code
+        uses: actions/checkout@v3
+        with:
+          # We require gathering the branch and tag history since we rely on a `git diff`
+          # which compares the state of two branches
+          fetch-depth: 0
+      - id: process-skip-directives
+        uses: ./.github/workflows/composite-actions/process-skip-directives
+        with:
+          base-ref: ${{ github.base_ref }}
+      - id: run-strategy
+        name: Determine Test Run Strategy
+        run: |
+          skip_kube_tests=${{ steps.process-skip-directives.outputs.skip-kube-tests }}
+          if [[ ! -z $skip_kube_tests && $skip_kube_tests = true ]] ; then
+            echo "auto_succeed=true" >> $GITHUB_OUTPUT
+          fi
+
+  # TODO(npolshak): Add multiple istio versions to test matrix
+  istio_tests:
+    name: istio integration tests
+    needs: prepare_env
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    if: ${{ !github.event.pull_request.draft }}
+    strategy:
+      fail-fast: false
+      matrix:
+        kube-e2e-test-type: [ 'istio-auto', 'gloo-gateway' ]
+        kube-version: [ { node: 'v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31', kubectl: 'v1.28.4', kind: 'v0.20.0', helm: 'v3.13.2' } ]
+        merge-to-main:
+          - ${{ github.event.pull_request.base.ref == 'main' }}
+        exclude:
+          - merge-to-main: true
+            kube-e2e-test-type: upgrade
+    steps:
+      - uses: actions/checkout@v3
+      - id: run-tests
+        if: needs.prepare_env.outputs.should-auto-succeed-istio-tests != 'true'
+        uses: ./.github/workflows/composite-actions/istio-tests
+      - id: auto-succeed-tests
+        if: needs.prepare_env.outputs.should-auto-succeed-istio-tests == 'true'
+        run: |
+          echo "Istio tests auto-succeeded"

Makefile

@@ -174,6 +174,32 @@ GINKGO_USER_FLAGS ?=
 install-test-tools: check-go-version
 	go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
 
+# proto compiler installation
+PROTOC_VERSION:=3.6.1
+PROTOC_URL:=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}
+.PHONY: install-protoc
+.SILENT: install-protoc
+install-protoc:
+	mkdir -p $(DEPSGOBIN)
+	if [ $(shell ${DEPSGOBIN}/protoc --version | grep -c ${PROTOC_VERSION}) -ne 0 ]; then \
+		echo expected protoc version ${PROTOC_VERSION} already installed ;\
+	else \
+		if [ "$(shell uname)" = "Darwin" ]; then \
+			echo "downloading protoc for osx" ;\
+			wget $(PROTOC_URL)-osx-x86_64.zip -O $(DEPSGOBIN)/protoc-${PROTOC_VERSION}.zip ;\
+		elif [ "$(shell uname -m)" = "aarch64" ]; then \
+			echo "downloading protoc for linux aarch64" ;\
+			wget $(PROTOC_URL)-linux-aarch_64.zip -O $(DEPSGOBIN)/protoc-${PROTOC_VERSION}.zip ;\
+		else \
+			echo "downloading protoc for linux x86-64" ;\
+			wget $(PROTOC_URL)-linux-x86_64.zip -O $(DEPSGOBIN)/protoc-${PROTOC_VERSION}.zip ;\
+		fi ;\
+		unzip $(DEPSGOBIN)/protoc-${PROTOC_VERSION}.zip -d $(DEPSGOBIN)/protoc-${PROTOC_VERSION} ;\
+		mv $(DEPSGOBIN)/protoc-${PROTOC_VERSION}/bin/protoc $(DEPSGOBIN)/protoc ;\
+		chmod +x $(DEPSGOBIN)/protoc ;\
+		rm -rf $(DEPSGOBIN)/protoc-${PROTOC_VERSION} $(DEPSGOBIN)/protoc-${PROTOC_VERSION}.zip ;\
+	fi
+
 .PHONY: test
 test: ## Run all tests, or only run the test package at {TEST_PKG} if it is specified
 	$(GINKGO_ENV) $(DEPSGOBIN)/ginkgo -ldflags=$(LDFLAGS) \

changelog/v1.17.0-beta13/add-istio-auto-mtls-support.yaml

@@ -0,0 +1,5 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/solo-projects/issues/5695
+    description: >-
+      Added support for `settings.gloo.istioOptions.enableAutoMtls` to implement auto mTLS via Envoy transport_socket_match.

docs/content/reference/values.txt

@@ -1270,6 +1270,7 @@
 |global.istioIntegration.enableIstioSidecarOnGateway|bool|false|Enable Istio sidecar injection on the gateway-proxy deployment. Ignored if LabelInstallNamespace is not 'true'. Ignored if disableAutoinjection is 'true'.|
 |global.istioIntegration.istioSidecarRevTag|string||Value of revision tag for Istio sidecar injection on the gateway-proxy and discovery deployments (when enabled with LabelInstallNamespace, WhitelistDiscovery or EnableIstioSidecarOnGateway). If set, applies the label 'istio.io/rev:<rev>' instead of 'sidecar.istio.io/inject' or 'istio-injection:enabled'. Ignored if disableAutoinjection is 'true'.|
 |global.istioIntegration.appendXForwardedHost|bool|true|Enable appending the X-Forwarded-Host header with the Istio-provided value. Default: true.|
+|global.istioIntegration.enableAutoMtls|bool|false|Enables Istio auto mtls configuration for Gloo Edge upstreams. Defaults to false.|
 |global.extraSpecs|bool||Add additional specs to include in the settings manifest, as defined by a helm partial. Defaults to false in open source, and true in enterprise.|
 |global.extauthCustomYaml|bool|true|Inject whatever yaml exists in .Values.global.extensions.extAuth into settings.spec.extauth, instead of structured yaml (which is enterprise only). Defaults to true in open source, and false in enterprise|
 |global.console|interface||Configuration options for the Enterprise Console (UI).|

install/helm/gloo/crds/gloo.solo.io_v1_Settings.yaml

@@ -636,6 +636,9 @@ spec:
                       appendXForwardedHost:
                         nullable: true
                         type: boolean
+                      enableAutoMtls:
+                        nullable: true
+                        type: boolean
                     type: object
                   logTransformationRequestResponseInfo:
                     nullable: true

install/helm/gloo/crds/gloo.solo.io_v1_Upstream.yaml

@@ -400,6 +400,9 @@ spec:
                       type: string
                     type: array
                 type: object
+              disableIstioAutoMtls:
+                nullable: true
+                type: boolean
               discoveryMetadata:
                 properties:
                   labels:

install/helm/gloo/generate/values.go

@@ -760,4 +760,5 @@ type IstioIntegration struct {
 	EnableIstioSidecarOnGateway *bool   `json:"enableIstioSidecarOnGateway,omitempty" desc:"Enable Istio sidecar injection on the gateway-proxy deployment. Ignored if LabelInstallNamespace is not 'true'. Ignored if disableAutoinjection is 'true'."`
 	IstioSidecarRevTag          *string `json:"istioSidecarRevTag,omitempty" desc:"Value of revision tag for Istio sidecar injection on the gateway-proxy and discovery deployments (when enabled with LabelInstallNamespace, WhitelistDiscovery or EnableIstioSidecarOnGateway). If set, applies the label 'istio.io/rev:<rev>' instead of 'sidecar.istio.io/inject' or 'istio-injection:enabled'. Ignored if disableAutoinjection is 'true'."`
 	AppendXForwardedHost        *bool   `json:"appendXForwardedHost,omitempty" desc:"Enable appending the X-Forwarded-Host header with the Istio-provided value. Default: true."`
+	EnableAutoMtls              *bool   `json:"enableAutoMtls,omitempty" desc:"Enables Istio auto mtls configuration for Gloo Edge upstreams. Defaults to false."`
 }

install/helm/gloo/templates/1-gloo-deployment.yaml

@@ -235,8 +235,12 @@ spec:
           - name: LOG_LEVEL
             value: {{ .Values.gloo.logLevel }}
         {{- end}}
+        {{- if .Values.global.istioSDS.enabled }}
+          - name: GG_EXPERIMENTAL_ISTIO_MTLS_SDS_ENABLED
+            value: "true"
+        {{- end}}
         {{- if .Values.global.istioIntegration.enableIstioSidecarOnGateway }}
-          - name: ENABLE_ISTIO_INTEGRATION
+          - name: ENABLE_ISTIO_SIDECAR_ON_GATEWAY
             value: "true"
         {{- end}}
         {{- if .Values.gloo.disableLeaderElection }}

install/helm/gloo/templates/18-settings.yaml

@@ -75,6 +75,7 @@ spec:
   {{- if .Values.global.istioIntegration }}
     istioOptions:
       appendXForwardedHost: {{ .Values.global.istioIntegration.appendXForwardedHost }}
+      enableAutoMtls: {{ .Values.global.istioIntegration.enableAutoMtls }}
   {{- end }}
 
   {{- if .Values.settings.integrations.consul }}

install/helm/gloo/values-template.yaml

@@ -257,6 +257,7 @@ global:
     disableAutoinjection: false
     enableIstioSidecarOnGateway: false
     appendXForwardedHost: true
+    enableAutoMtls: false
   # AddEnterpriseSettings and extauthCustomYaml control differences in the settings manifest between open source and enterprise
   AddEnterpriseSettings: false
   extauthCustomYaml: true

install/test/fixtures/settings/compressed_proxy_spec.yaml

@@ -28,6 +28,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/consul_config_upstream_discovery.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  consulDiscovery:
    useTlsTagging: true
    tlsTagName: tag

install/test/fixtures/settings/consul_config_values.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  consul:
    datacenter: datacenter
    username: user

install/test/fixtures/settings/disable_kubernetes_destinations.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/disable_proxy_garbage_collection.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/disabled_gateway.yaml

@@ -36,6 +36,7 @@ spec:
       invalidRouteResponseCode: 404
     istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
   kubernetesArtifactSource: {}
   kubernetesConfigSource: {}
   kubernetesSecretSource: {}

install/test/fixtures/settings/enable_default_credentials.yaml

@@ -35,6 +35,7 @@ spec:
       replaceInvalidRoutes: false
     istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
     awsOptions:
       enableCredentialsDiscovey: true
   kubernetesArtifactSource: {}

install/test/fixtures/settings/enable_rest_eds.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/enable_rest_eds_and_gloo_mtls.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/gateway_circuit_breakers.yaml

@@ -40,6 +40,7 @@ spec:
      invalidRouteResponseCode: 404
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/gateway_settings.yaml

@@ -35,6 +35,7 @@ spec:
      invalidRouteResponseCode: 404
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/gateway_validation.yaml

@@ -35,6 +35,7 @@ spec:
      invalidRouteResponseCode: 404
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/graphql_fds_disabled.yaml

@@ -37,6 +37,7 @@ spec:
       invalidRouteResponseCode: 404
     istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
   kubernetesArtifactSource: {}
   kubernetesConfigSource: {}
   kubernetesSecretSource: {}

install/test/fixtures/settings/isolate_virtual_hosts_by_ssl_config.yaml

@@ -35,6 +35,7 @@ spec:
       replaceInvalidRoutes: false
     istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
   kubernetesArtifactSource: {}
   kubernetesConfigSource: {}
   kubernetesSecretSource: {}

install/test/fixtures/settings/ratelimit_descriptors.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/ratelimit_server.yaml

@@ -35,6 +35,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/read_gateways_from_all_namespaces.yaml

@@ -27,6 +27,7 @@ spec:
      replaceInvalidRoutes: false
    istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
  kubernetesArtifactSource: {}
  kubernetesConfigSource: {}
  kubernetesSecretSource: {}

install/test/fixtures/settings/set_regex_max_program_size.yaml

@@ -35,6 +35,7 @@ spec:
       replaceInvalidRoutes: false
     istioOptions:
      appendXForwardedHost: true
+     enableAutoMtls: false
   kubernetesArtifactSource: {}
   kubernetesConfigSource: {}
   kubernetesSecretSource: {}