feat(envoy): add full-featured Envoy support (#856)

This change adds Envoy support with all the basic capabilities
used today with the NGINX Gateway and additional stubs for future
optimizations.

* Documentation has been updated to separate Envoy from NGINX Gateway
  implementations
* Traffic policies have been added for basic capabilies with the ability
  to extend and create comprehensive rate limits later.

This change is a 1:1 replacement for the NGINX Gateway API ensuing it
operates with self-signed and lets encrypt certificates, supports
inplace replacements, and makes use of all the pre-defined routes/listeners.

Nice to haves added

* X-Forwarded-For and Client-IPs are now properly tracked
* Cluster policies allow for better traffic management by default
* HPA for envoy gives the system better scale

Signed-off-by: Kevin Carter <[email protected]>

.github/workflows/kustomize-gateway-api-envoyproxy.yaml

@@ -3,7 +3,7 @@ name: Kustomize GitHub Actions for Gateway API(envoyproxy)
 on:
   pull_request:
     paths:
-      - base-kustomize/gateway/envoyproxy/**
+      - base-kustomize/gateway/envoyproxy-gateway/**
       - .github/workflows/kustomize-gateway-api-envoyproxy.yaml
 jobs:
   kustomize:
@@ -25,7 +25,7 @@ jobs:
           fi
       - name: Run Kustomize Build
         run: |
-          kustomize build base-kustomize/gateway/envoyproxy/ --enable-helm --helm-command ${{ steps.helm.outputs.helm-path }} > /tmp/rendered.yaml
+          kustomize build base-kustomize/envoyproxy-gateway/base > /tmp/rendered.yaml
       - name: Return Kustomize Build
         uses: actions/upload-artifact@v4
         with:

base-helm-configs/envoyproxy-gateway/envoy-gateway-helm-overrides.yaml

@@ -0,0 +1 @@
+---

base-kustomize/envoyproxy-gateway/base/envoy-custom-proxy-config.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: envoy-gateway
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyHpa:
+        minReplicas: 2
+        maxReplicas: 9
+        metrics:
+          - resource:
+              name: cpu
+              target:
+                averageUtilization: 60
+                type: Utilization
+            type: Resource
+          - resource:
+              name: memory
+              target:
+                type: AverageValue
+                averageValue: 500Mi
+            type: Resource

base-kustomize/envoyproxy-gateway/base/envoy-endpoint-policies.yaml

@@ -0,0 +1,92 @@
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: ClientTrafficPolicy
+metadata:
+  name: flex-gateway-client-policy
+  namespace: envoy-gateway
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: flex-gateway
+  clientIPDetection:
+    xForwardedFor:
+      numTrustedHops: 2
+  connection:
+    bufferLimit: 16384
+  timeout:
+    http:
+      idleTimeout: 5s
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: BackendTrafficPolicy
+metadata:
+  name: least-request-policy
+  namespace: envoy-gateway
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-barbican-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-cinder-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-cloudformation-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-glance-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-gnocchi-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-heat-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-keystone-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-magnum-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-metadata-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-neutron-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-nova-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-novnc-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-octavia-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-placement-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: internal-loki-gateway-route
+  loadBalancer:
+    type: LeastRequest
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: BackendTrafficPolicy
+metadata:
+  name: source-ip-policy
+  namespace: envoy-gateway
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: grafana-gateway-route
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: custom-skyline-gateway-route
+  loadBalancer:
+    type: ConsistentHash
+    consistentHash:
+      type: SourceIP

base-kustomize/envoyproxy-gateway/base/envoy-gateway-namespace.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    kubernetes.io/metadata.name: envoy-gateway
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/audit-version: latest
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/warn: privileged
+    pod-security.kubernetes.io/warn-version: latest
+  name: envoy-gateway

base-kustomize/envoyproxy-gateway/base/envoy-gateway.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: flex-gateway
+  namespace: envoy-gateway
+  annotations:
+    cert-manager.io/cluster-issuer: flex-gateway-issuer
+    acme.cert-manager.io/http01-edit-in-place: "true"
+spec:
+  gatewayClassName: eg
+  infrastructure:
+    annotations:
+      metallb.universe.tf/address-pool: gateway-api-external
+  listeners:
+    - name: cluster-http
+      port: 80
+      protocol: HTTP
+      hostname: "*.cluster.local"
+      allowedRoutes:
+        namespaces:
+          from: All
+    - name: cluster-tls
+      port: 443
+      protocol: HTTPS
+      hostname: "*.cluster.local"
+      allowedRoutes:
+        namespaces:
+          from: All
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - kind: Secret
+            name: wildcard-cluster-tls-secret

base-kustomize/envoyproxy-gateway/base/envoy-gatewayclass.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+  name: eg
+  namespace: envoy-gateway
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  parametersRef:
+    group: gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: custom-proxy-config
+    namespace: envoy-gateway

base-kustomize/envoyproxy-gateway/base/kustomization.yaml

@@ -1,13 +1,8 @@
 sortOptions:
   order: fifo
 resources:
-  - './namespace.yaml'
-  - './gatewayclass.yaml'
-namespace: envoy-gateway-system
-helmGlobals:
-  chartHome: ../../../submodules/envoyproxy-gateway/charts/
-helmCharts:
-- name: gateway-helm
-  valuesFile: values.yaml
-  includeCRDs: true
-  releaseName: envoyproxy-gateway
+  - envoy-gateway-namespace.yaml
+  - envoy-custom-proxy-config.yaml
+  - envoy-gatewayclass.yaml
+  - envoy-gateway.yaml
+  - envoy-endpoint-policies.yaml

bin/install-envoy-gateway.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# shellcheck disable=SC2124,SC2145,SC2294
+
+GLOBAL_OVERRIDES_DIR="/etc/genestack/helm-configs/global_overrides"
+SERVICE_CONFIG_DIR="/etc/genestack/helm-configs/envoyproxy-gateway"
+BASE_OVERRIDES="/opt/genestack/base-helm-configs/envoyproxy-gateway/envoyproxy-gateway-helm-overrides.yaml"
+ENVOY_VERSION="v1.3.0"
+HELM_CMD="helm upgrade --install envoyproxy-gateway oci://docker.io/envoyproxy/gateway-helm \
+                       --version ${ENVOY_VERSION} \
+                       --namespace envoyproxy-gateway-system \
+                       --create-namespace"
+
+HELM_CMD+=" -f ${BASE_OVERRIDES}"
+
+for dir in "$GLOBAL_OVERRIDES_DIR" "$SERVICE_CONFIG_DIR"; do
+    if compgen -G "${dir}/*.yaml" > /dev/null; then
+        for yaml_file in "${dir}"/*.yaml; do
+            # Avoid re-adding the base override file if present in the service directory
+            if [ "${yaml_file}" != "${BASE_OVERRIDES}" ]; then
+                HELM_CMD+=" -f ${yaml_file}"
+            fi
+        done
+    fi
+done
+
+HELM_CMD+=" $@"
+
+echo "Executing Helm command:"
+echo "${HELM_CMD}"
+eval "${HELM_CMD}"
+
+# Install egctl
+mkdir -p /opt/egctl-install
+
+pushd /opt/egctl-install || exit 1
+    wget "https://github.com/envoyproxy/gateway/releases/download/${ENVOY_VERSION}/egctl_${ENVOY_VERSION}_linux_amd64.tar.gz" -O egctl.tar.gz
+    tar -xvf egctl.tar.gz
+    sudo install -o root -g root -m 0755 bin/linux/amd64/egctl /usr/local/bin/egctl
+    /usr/local/bin/egctl completion bash > egctl.bash
+    sudo cp egctl.bash /etc/bash_completion.d/egctl
+popd || exit 1