diff --git a/.github/workflows/kustomize-gateway-api-envoyproxy.yaml b/.github/workflows/kustomize-gateway-api-envoyproxy.yaml index 1f72f28c..7ef814d3 100644 --- a/.github/workflows/kustomize-gateway-api-envoyproxy.yaml +++ b/.github/workflows/kustomize-gateway-api-envoyproxy.yaml @@ -3,7 +3,7 @@ name: Kustomize GitHub Actions for Gateway API(envoyproxy) on: pull_request: paths: - - base-kustomize/gateway/envoyproxy/** + - base-kustomize/gateway/envoyproxy-gateway/** - .github/workflows/kustomize-gateway-api-envoyproxy.yaml jobs: kustomize: @@ -25,7 +25,7 @@ jobs: fi - name: Run Kustomize Build run: | - kustomize build base-kustomize/gateway/envoyproxy/ --enable-helm --helm-command ${{ steps.helm.outputs.helm-path }} > /tmp/rendered.yaml + kustomize build base-kustomize/envoyproxy-gateway/base > /tmp/rendered.yaml - name: Return Kustomize Build uses: actions/upload-artifact@v4 with: diff --git a/base-helm-configs/envoyproxy-gateway/envoy-gateway-helm-overrides.yaml b/base-helm-configs/envoyproxy-gateway/envoy-gateway-helm-overrides.yaml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/base-helm-configs/envoyproxy-gateway/envoy-gateway-helm-overrides.yaml @@ -0,0 +1 @@ +--- diff --git a/base-kustomize/envoyproxy-gateway/base/envoy-custom-proxy-config.yaml b/base-kustomize/envoyproxy-gateway/base/envoy-custom-proxy-config.yaml new file mode 100644 index 00000000..190c67eb --- /dev/null +++ b/base-kustomize/envoyproxy-gateway/base/envoy-custom-proxy-config.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: custom-proxy-config + namespace: envoy-gateway +spec: + provider: + type: Kubernetes + kubernetes: + envoyHpa: + minReplicas: 2 + maxReplicas: 9 + metrics: + - resource: + name: cpu + target: + averageUtilization: 60 + type: Utilization + type: Resource + - resource: + name: memory + target: + type: AverageValue + averageValue: 500Mi + type: Resource diff --git a/base-kustomize/envoyproxy-gateway/base/envoy-endpoint-policies.yaml b/base-kustomize/envoyproxy-gateway/base/envoy-endpoint-policies.yaml new file mode 100644 index 00000000..50aa3dd0 --- /dev/null +++ b/base-kustomize/envoyproxy-gateway/base/envoy-endpoint-policies.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: flex-gateway-client-policy + namespace: envoy-gateway +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: flex-gateway + clientIPDetection: + xForwardedFor: + numTrustedHops: 2 + connection: + bufferLimit: 16384 + timeout: + http: + idleTimeout: 5s +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: least-request-policy + namespace: envoy-gateway +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-barbican-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-cinder-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-cloudformation-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-glance-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-gnocchi-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-heat-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-keystone-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-magnum-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-metadata-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-neutron-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-nova-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-novnc-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-octavia-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-placement-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: internal-loki-gateway-route + loadBalancer: + type: LeastRequest +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: source-ip-policy + namespace: envoy-gateway +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: grafana-gateway-route + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: custom-skyline-gateway-route + loadBalancer: + type: ConsistentHash + consistentHash: + type: SourceIP diff --git a/base-kustomize/envoyproxy-gateway/base/envoy-gateway-namespace.yaml b/base-kustomize/envoyproxy-gateway/base/envoy-gateway-namespace.yaml new file mode 100644 index 00000000..2f3d337b --- /dev/null +++ b/base-kustomize/envoyproxy-gateway/base/envoy-gateway-namespace.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: envoy-gateway + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: privileged + pod-security.kubernetes.io/warn-version: latest + name: envoy-gateway diff --git a/base-kustomize/envoyproxy-gateway/base/envoy-gateway.yaml b/base-kustomize/envoyproxy-gateway/base/envoy-gateway.yaml new file mode 100644 index 00000000..9b4fd362 --- /dev/null +++ b/base-kustomize/envoyproxy-gateway/base/envoy-gateway.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: flex-gateway + namespace: envoy-gateway + annotations: + cert-manager.io/cluster-issuer: flex-gateway-issuer + acme.cert-manager.io/http01-edit-in-place: "true" +spec: + gatewayClassName: eg + infrastructure: + annotations: + metallb.universe.tf/address-pool: gateway-api-external + listeners: + - name: cluster-http + port: 80 + protocol: HTTP + hostname: "*.cluster.local" + allowedRoutes: + namespaces: + from: All + - name: cluster-tls + port: 443 + protocol: HTTPS + hostname: "*.cluster.local" + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: wildcard-cluster-tls-secret diff --git a/base-kustomize/envoyproxy-gateway/base/envoy-gatewayclass.yaml b/base-kustomize/envoyproxy-gateway/base/envoy-gatewayclass.yaml new file mode 100644 index 00000000..ad00ef96 --- /dev/null +++ b/base-kustomize/envoyproxy-gateway/base/envoy-gatewayclass.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg + namespace: envoy-gateway +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: custom-proxy-config + namespace: envoy-gateway diff --git a/base-kustomize/envoyproxy-gateway/base/gatewayclass.yaml b/base-kustomize/envoyproxy-gateway/base/gatewayclass.yaml deleted file mode 100644 index db5b2527..00000000 --- a/base-kustomize/envoyproxy-gateway/base/gatewayclass.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1 -kind: GatewayClass -metadata: - name: envoyproxy -spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/base-kustomize/envoyproxy-gateway/base/kustomization.yaml b/base-kustomize/envoyproxy-gateway/base/kustomization.yaml index a3e01b75..5077c04b 100644 --- a/base-kustomize/envoyproxy-gateway/base/kustomization.yaml +++ b/base-kustomize/envoyproxy-gateway/base/kustomization.yaml @@ -1,13 +1,8 @@ sortOptions: order: fifo resources: - - './namespace.yaml' - - './gatewayclass.yaml' -namespace: envoy-gateway-system -helmGlobals: - chartHome: ../../../submodules/envoyproxy-gateway/charts/ -helmCharts: -- name: gateway-helm - valuesFile: values.yaml - includeCRDs: true - releaseName: envoyproxy-gateway + - envoy-gateway-namespace.yaml + - envoy-custom-proxy-config.yaml + - envoy-gatewayclass.yaml + - envoy-gateway.yaml + - envoy-endpoint-policies.yaml diff --git a/base-kustomize/envoyproxy-gateway/base/namespace.yaml b/base-kustomize/envoyproxy-gateway/base/namespace.yaml deleted file mode 100644 index b237e0d3..00000000 --- a/base-kustomize/envoyproxy-gateway/base/namespace.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - labels: - kubernetes.io/metadata.name: envoy-gateway-system - name: envoy-gateway-system - name: envoy-gateway-system diff --git a/base-kustomize/envoyproxy-gateway/base/values.yaml b/base-kustomize/envoyproxy-gateway/base/values.yaml deleted file mode 100644 index c64911d7..00000000 --- a/base-kustomize/envoyproxy-gateway/base/values.yaml +++ /dev/null @@ -1,52 +0,0 @@ -deployment: - envoyGateway: - image: - repository: docker.io/envoyproxy/gateway - tag: 'v1.0.1' - imagePullPolicy: Always - imagePullSecrets: [] - resources: - limits: - cpu: 500m - memory: 1024Mi - requests: - cpu: 100m - memory: 256Mi - ports: - - name: grpc - port: 18000 - targetPort: 18000 - - name: ratelimit - port: 18001 - targetPort: 18001 - replicas: 1 - pod: - affinity: {} - annotations: {} - labels: {} - -config: - envoyGateway: - gateway: - controllerName: gateway.envoyproxy.io/gatewayclass-controller - provider: - type: Kubernetes - logging: - level: - default: info - -envoyGatewayMetricsService: - port: 19001 - -createNamespace: false - -kubernetesClusterDomain: cluster.local - -certgen: - job: - annotations: {} - resources: {} - ttlSecondsAfterFinished: 0 - rbac: - annotations: {} - labels: {} diff --git a/base-kustomize/gateway/envoyproxy/gateway.yaml b/base-kustomize/gateway/envoyproxy/gateway.yaml deleted file mode 100644 index 97e48174..00000000 --- a/base-kustomize/gateway/envoyproxy/gateway.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: flex-gateway - namespace: envoy-gateway-system - annotations: - acme.cert-manager.io/http01-edit-in-place: "true" - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - gatewayClassName: envoyproxy - infrastructure: - annotations: - metallb.universe.tf/address-pool: openstack-external - listeners: - - name: http - port: 80 - protocol: HTTP - hostname: "*.your.domain.tld" - allowedRoutes: - namespaces: - from: All - - allowedRoutes: - namespaces: - from: All - hostname: '*.your.domain.tld' - name: https - port: 443 - protocol: HTTPS - tls: - certificateRefs: - - group: "" - kind: Secret - name: flex-endpoints - mode: Terminate diff --git a/base-kustomize/gateway/envoyproxy/kustomization.yaml b/base-kustomize/gateway/envoyproxy/kustomization.yaml deleted file mode 100644 index 86ef2192..00000000 --- a/base-kustomize/gateway/envoyproxy/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -sortOptions: - order: fifo -resources: - - './gateway.yaml' # namespace: envoy-gateway-system (common gateway) diff --git a/bin/install-envoy-gateway.sh b/bin/install-envoy-gateway.sh new file mode 100755 index 00000000..34dfa896 --- /dev/null +++ b/bin/install-envoy-gateway.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# shellcheck disable=SC2124,SC2145,SC2294 + +GLOBAL_OVERRIDES_DIR="/etc/genestack/helm-configs/global_overrides" +SERVICE_CONFIG_DIR="/etc/genestack/helm-configs/envoyproxy-gateway" +BASE_OVERRIDES="/opt/genestack/base-helm-configs/envoyproxy-gateway/envoyproxy-gateway-helm-overrides.yaml" +ENVOY_VERSION="v1.3.0" +HELM_CMD="helm upgrade --install envoyproxy-gateway oci://docker.io/envoyproxy/gateway-helm \ + --version ${ENVOY_VERSION} \ + --namespace envoyproxy-gateway-system \ + --create-namespace" + +HELM_CMD+=" -f ${BASE_OVERRIDES}" + +for dir in "$GLOBAL_OVERRIDES_DIR" "$SERVICE_CONFIG_DIR"; do + if compgen -G "${dir}/*.yaml" > /dev/null; then + for yaml_file in "${dir}"/*.yaml; do + # Avoid re-adding the base override file if present in the service directory + if [ "${yaml_file}" != "${BASE_OVERRIDES}" ]; then + HELM_CMD+=" -f ${yaml_file}" + fi + done + fi +done + +HELM_CMD+=" $@" + +echo "Executing Helm command:" +echo "${HELM_CMD}" +eval "${HELM_CMD}" + +# Install egctl +mkdir -p /opt/egctl-install + +pushd /opt/egctl-install || exit 1 + wget "https://github.com/envoyproxy/gateway/releases/download/${ENVOY_VERSION}/egctl_${ENVOY_VERSION}_linux_amd64.tar.gz" -O egctl.tar.gz + tar -xvf egctl.tar.gz + sudo install -o root -g root -m 0755 bin/linux/amd64/egctl /usr/local/bin/egctl + /usr/local/bin/egctl completion bash > egctl.bash + sudo cp egctl.bash /etc/bash_completion.d/egctl +popd || exit 1 diff --git a/bin/setup-envoy-gateway.sh b/bin/setup-envoy-gateway.sh new file mode 100755 index 00000000..d96ff1d9 --- /dev/null +++ b/bin/setup-envoy-gateway.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# shellcheck disable=SC2045,SC2124,SC2145,SC2164,SC2236,SC2294 +set -e +set -o pipefail + +if [ -z "${ACME_EMAIL}" ]; then + read -rp "Enter a valid email address for use with ACME, press enter to skip: " ACME_EMAIL +fi + +if [ -z "${GATEWAY_DOMAIN}" ]; then + echo "The domain name for the gateway is required, if you do not have a domain name press enter to use the default" + read -rp "Enter the domain name for the gateway [cluster.local]: " GATEWAY_DOMAIN + export GATEWAY_DOMAIN=${GATEWAY_DOMAIN:-cluster.local} +fi + +if [ -z "${GATEWAY_DOMAIN}" ]; then + echo "Gateway domain is required" + exit 1 +fi + +kubectl apply -k /etc/genestack/kustomize/envoyproxy-gateway/overlay + +echo "Waiting for the gateway to be programmed" +kubectl -n envoy-gateway wait --timeout=5m gateways.gateway.networking.k8s.io flex-gateway --for=condition=Programmed + +if [ ! -z "${ACME_EMAIL}" ]; then + cat < "/etc/genestack/gateway-api/routes/${route}" + sed -i 's/namespace: nginx-gateway/namespace: envoy-gateway/g' "/etc/genestack/gateway-api/routes/${route}" +done + +kubectl apply -f /etc/genestack/gateway-api/routes + +mkdir -p /etc/genestack/gateway-api/listeners +for listener in $(ls -1 /opt/genestack/etc/gateway-api/listeners); do + sed "s/your.domain.tld/${GATEWAY_DOMAIN}/g" "/opt/genestack/etc/gateway-api/listeners/${listener}" > "/etc/genestack/gateway-api/listeners/${listener}" +done + +kubectl patch -n envoy-gateway gateway flex-gateway \ + --type='json' \ + --patch="$(jq -s 'flatten | .' /etc/genestack/gateway-api/listeners/*)" + +echo "Setup Complete" diff --git a/docs/grafana.md b/docs/grafana.md index 65561940..8624cbc0 100644 --- a/docs/grafana.md +++ b/docs/grafana.md @@ -63,7 +63,7 @@ Before running the deployment script, you must set the `custom_host` value `graf ### Listeners and Routes -Listeners and Routes should have been configureed when you installed the Gateway API. If so some reason they were not created, please following the install guide here: [Gateway API](infrastructure-gateway-api-custom.md) +Listeners and Routes should have been configureed when you installed the Gateway API. If so some reason they were not created, please following the install guide here: [Gateway API](infrastructure-gateway-api.md) ### Deployment diff --git a/docs/infrastructure-envoy-gateway-api.md b/docs/infrastructure-envoy-gateway-api.md new file mode 100644 index 00000000..415ff5b5 --- /dev/null +++ b/docs/infrastructure-envoy-gateway-api.md @@ -0,0 +1,64 @@ +--- +hide: + - footer +--- + +# Envoy Gateway API + +The [Envoy Gateway](https://gateway.envoyproxy.io/) is an open-source project that provides an implementation +of the Gateway API using Envoyproxy as the data plane. The Gateway API is a set of APIs that allow users to configure +API gateways using a declarative configuration model. + +## Installation + +Run the helm command to install Envoy Gateway. + +??? example "Run the Envoy Gateway deployment Script `/opt/genestack/bin/install-envoy-gateway.sh`" + + ``` shell + --8<-- "bin/install-envoy-gateway.sh" + ``` + +The install script will deploy Envoy Gateway to the `envoy-gateway-system` namespace via Helm. + +## Setup + +??? example "Run the Envoy Gateway setup Script `/opt/genestack/bin/setup-envoy-gateway.sh`" + + ``` shell + --8<-- "bin/setup-envoy-gateway.sh" + ``` + +The setup script will ask the following questions: + +* Enter a valid email address for use with ACME, press enter to skip" +* Enter the domain name for the gateway" + +These values will be used to generate a certificate for the gateway and set the routes used within the flex-gateway, +typically for OpenStack. This script can also be fully automated by providing the required values as arguments. + +!!! example "Run the Envoy Gateway setup Script with arguments" + + ``` shell + ACME_EMAIL="username@your.domain.tld" GATEWAY_DOMAIN="your.domain.tld" /opt/genestack/bin/setup-envoy-gateway.sh + ``` + +## Validation + +At this stage, Envoy Gateway should be operational. To validate the configuration, run the following command. + +``` shell +kubectl -n openstack get httproute +``` + +``` shell +kubectl -n envoy-gateway get gateways.gateway.networking.k8s.io flex-gateway +``` + +## Troubleshooting + +If you encounter any issues, check the logs of the `envoy-gateway` deployment. + +``` shell +kubectl logs -n envoy-gateway-system deployment/envoy-gateway +``` diff --git a/docs/infrastructure-gateway-api.md b/docs/infrastructure-gateway-api.md index 2816d3d4..88e2afa0 100644 --- a/docs/infrastructure-gateway-api.md +++ b/docs/infrastructure-gateway-api.md @@ -1,3 +1,8 @@ +--- +hide: + - footer +--- + # Gateway API Gateway API is L4 and L7 layer routing project in Kubernetes. It represents next generation of k8s Ingress, LB and Service Mesh APIs. @@ -15,326 +20,25 @@ For more information on the project see: [Gateway API SIG.](https://gateway-api. External --> External_VIP_Address --> MetalLB_VIP_Address --> Gateway_Service ``` -## Move from Ingress to Gateway APIs - -Since Gateway APIs are successor to Ingress Controllers there needs to be a one time migration from Ingress to GW API resources. - -!!! tip "Learn more about migrating to the Gateway API: [Ingress Migration](https://gateway-api.sigs.k8s.io/guides/migrating-from-ingress/#migrating-from-ingress)" +The k8s Gateway API is NOT the same an API Gateway. While both sound the same, API Gateway is a more of a general +concept that defines a set of resources that exposes capabilities of a backend service but also provide other +functionalities like traffic management, rate limiting, authentication and more. It is geared towards commercial +API management and monetisation. ## Resource Models in Gateway API -There are 3 main resource models in gateway apis: - -1. GatewayClass - Mostly managed by a controller. -2. Gateway - An instance of traffic handling infra like a LB. -3. Routes - Defines HTTP-specific rules for mapping traffic from a Gateway listener to a representation of backend network endpoints. - -!!! warning "k8s Gateway API is NOT the same as API Gateways" - -While both sound the same, API Gateway is a more of a general concept that defines a set of resources that exposes capabilities of a backend service but -also provide other functionalities like traffic management, rate limiting, authentication and more. It is geared towards commercial API management and monetisation. - -From the gateway api sig: - -!!! note - - Most Gateway API implementations are API Gateways to some extent, but not all API Gateways are Gateway API implementations. - -## Controller Selection - -There are various implementations of the Gateway API. In this document, we will cover two of them: - -=== "NGINX Gateway Fabric _(Recommended)_" - - [NGINX Gateway Fabric](https://github.com/nginxinc/nginx-gateway-fabric) is an open-source project that provides an implementation of the Gateway - API using nginx as the data plane. - - ### Create the Namespace - - ``` shell - kubectl apply -f /opt/genestack/manifests/nginx-gateway/nginx-gateway-namespace.yaml - ``` - - ### Install the Gateway API Resource from Kubernetes - - === "Stable _(Recommended)_" - - ``` shell - kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.4.0" | kubectl apply -f - - ``` - - === "Experimental" - - The experimental version of the Gateway API is available in the `v1.6.1` checkout. Use with caution. - - ``` shell - kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.6.1" | kubectl apply -f - - ``` - - ### Install the NGINX Gateway Fabric controller - - !!! tip - - If attempting to perform an **upgrade** of an existing Gateway API deployment, note that the Helm install does not automatically upgrade the CRDs for - this resource. To upgrade them, refer to the process outlined by the - [Nginx upgrade documentation](https://docs.nginx.com/nginx-gateway-fabric/installation/installing-ngf/helm/#upgrade-nginx-gateway-fabric-crds). You - can safely ignore this note for new installations. - - === "Stable _(Recommended)_" - - Edit the file `/etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml`. - - !!! example "Create an empty override file" - - If no overrides are needed, create an empty file. - - ``` shell - echo "---" | tee /etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml - ``` - - ``` shell - pushd /opt/genestack/submodules/nginx-gateway-fabric/charts || exit 1 - helm upgrade --install nginx-gateway-fabric ./nginx-gateway-fabric \ - --namespace=nginx-gateway \ - --create-namespace \ - -f /opt/genestack/base-helm-configs/nginx-gateway-fabric/helm-overrides.yaml \ - -f /etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml \ - --post-renderer /etc/genestack/kustomize/kustomize.sh \ - --post-renderer-args gateway/overlay - popd || exit 1 - ``` - - === "Experimental" - - The experimental version of the Gateway API is available in the `v1.6.1` checkout. Use with caution. - - Update the submodule with the experimental version of the Gateway API. - - Edit the file `/etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml`. - - !!! example "Create the experimental override file" - - ``` yaml - --- - nginxGateway: - replicaCount: 3 - gwAPIExperimentalFeatures: - enable: true - service: - ## The externalTrafficPolicy of the service. The value Local preserves the client source IP. - externalTrafficPolicy: Cluster - ## The annotations of the NGINX Gateway Fabric service. - annotations: - "metallb.universe.tf/address-pool": "gateway-api-external" - "metallb.universe.tf/allow-shared-ip": "openstack-external-svc" - ``` - - Run the helm command to install the experimental version of the Gateway API. - - ``` shell - helm upgrade --install nginx-gateway-fabric oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ - --create-namespace \ - --namespace=nginx-gateway \ - -f /etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml \ - --post-renderer /etc/genestack/kustomize/kustomize.sh \ - --post-renderer-args gateway/overlay \ - --version 1.6.1 - ``` - - Once deployed ensure a system rollout has been completed for Cert Manager. - - ``` shell - kubectl rollout restart deployment cert-manager --namespace cert-manager - ``` - - ### Create the shared gateway resource - - ``` shell - kubectl kustomize /etc/genestack/kustomize/gateway/nginx-gateway-fabric | kubectl apply -f - - ``` - -=== "Envoyproxy" - - [Envoyproxy](https://gateway.envoyproxy.io/) is an open-source project that provides an implementation of the Gateway API using Envoyproxy as the data plane. - - ### Installation - - Update the `/etc/genestack/kustomize/envoyproxy-gateway/base/values.yaml` file according to your requirements. - - Apply the configuration using the following command: - - ``` shell - kubectl kustomize --enable-helm /etc/genestack/kustomize/envoyproxy-gateway/overlay | kubectl apply -f - - ``` - - ### After installation - - You need to create Gateway and HTTPRoute resources based on your requirements - - !!! example "exposing an application using Gateway API (Envoyproxy)" - - In this example, we will demonstrate how to expose an application through a gateway. Apply the Kustomize configuration which will create `Gateway` resource: - - ``` shell - kubectl kustomize /etc/genestack/kustomize/gateway/envoyproxy | kubectl apply -f - - ``` - - Once gateway is created, user can expose an application by creating `HTTPRoute` resource. - - ??? abstract "Sample `HTTPRoute` resource" - - ``` yaml - --8<-- "etc/gateway-api/gateway-envoy-http-routes.yaml" - ``` - - !!! example "Example modifying and apply the routes" - - ``` shell - mkdir -p /etc/genestack/gateway-api - sed 's/your.domain.tld//g' /opt/genestack/etc/gateway-api/gateway-envoy-http-routes.yaml > /etc/genestack/gateway-api/gateway-envoy-http-routes.yaml - kubectl apply -f /etc/genestack/gateway-api/gateway-envoy-http-routes.yaml - ``` - ----- - -## Deploy with Let's Encrypt Certificates - -By default, certificates are issued by an instance of the selfsigned-cluster-issuer. This section focuses on replacing that with a -Let's Encrypt issuer to ensure valid certificates are deployed in our cluster. - -[![asciicast](https://asciinema.org/a/h7npXnDjkSpn3uQtuQwWG9zju.svg)](https://asciinema.org/a/h7npXnDjkSpn3uQtuQwWG9zju) - -### Apply the Let's Encrypt Cluster Issuer - -Before we can have Cert Manager start coordinating Let's Encrypt certificate -requests for us, we need to add an ACME issuer with a valid, monitored -email (for expiration reminders and other important ACME related information). - -``` yaml -read -p "Enter a valid email address for use with ACME: " ACME_EMAIL; \ -cat </g' /opt/genestack/etc/gateway-api/listeners/$listener > /etc/genestack/gateway-api/listeners/$listener - done - ``` - -``` shell -kubectl patch -n nginx-gateway gateway flex-gateway \ - --type='json' \ - --patch="$(jq -s 'flatten | .' /etc/genestack/gateway-api/listeners/*)" -``` - -## Apply Related Gateway routes - -Another example with most of the OpenStack services is located at `/opt/genestack/etc/gateway-api/routes/http-wildcard-listener.yaml`. Similarly, you must modify -and apply them as shown below, or apply your own. - -??? abstract "Example routes file" - - ``` yaml - --8<-- "etc/gateway-api/routes/http-wildcard-listener.yaml" - ``` - -All routes can be found at `/etc/genestack/gateway-api/routes`. - -!!! example "Example modifying all available Gateway routes with `your.domain.tld`" - - ``` shell - mkdir -p /etc/genestack/gateway-api/routes - for route in $(ls -1 /opt/genestack/etc/gateway-api/routes); do - sed 's/your.domain.tld//g' /opt/genestack/etc/gateway-api/routes/$route > /etc/genestack/gateway-api/routes/$route - done - ``` - -``` shell -kubectl apply -f /etc/genestack/gateway-api/routes -``` - -## Patch Gateway with Let's Encrypt Cluster Issuer - -??? abstract "Example patch to enable LetsEncrypt `/etc/genestack/gateway-api/gateway-letsencrypt.yaml`" - - ``` yaml - --8<-- "etc/gateway-api/gateway-letsencrypt.yaml" - ``` - -``` shell -kubectl patch --namespace nginx-gateway \ - --type merge \ - --patch-file /etc/genestack/gateway-api/gateway-letsencrypt.yaml \ - gateway flex-gateway -``` - -## Example Implementation with Prometheus UI (NGINX Gateway Fabric) - -In this example we will look at how Prometheus UI is exposed through the gateway. For other services the gateway kustomization file for the service. - -First, create the shared gateway and then the httproute resource for prometheus. - -??? abstract "Example patch to enable Prometheus `/etc/genestack/gateway-api/gateway-prometheus.yaml`" - - ``` yaml - --8<-- "etc/gateway-api/gateway-prometheus.yaml" - ``` - -!!! example "Example modifying Prometheus' Gateway deployment" - - ``` shell - mkdir -p /etc/genestack/gateway-api - sed 's/your.domain.tld//g' /opt/genestack/etc/gateway-api/gateway-prometheus.yaml > /etc/genestack/gateway-api/gateway-prometheus.yaml - ``` - -``` shell -kubectl apply -f /etc/genestack/gateway-api/gateway-prometheus.yaml -``` - -At this point, flex-gateway has a listener pointed to the port 80 matching *.your.domain.tld hostname. The HTTPRoute resource configures routes -for this gateway. Here, we match all path and simply pass any request from the matching hostname to kube-prometheus-stack-prometheus backend service. +| Type | Description | +| ---- | ----------- | +| [GatewayClass](https://gateway-api.sigs.k8s.io/api-types/gatewayclass/) | Represents a class of Gateway instances. | +| [Gateway](https://gateway-api.sigs.k8s.io/api-types/gateway/) | Represents a single Gateway instance. | +| [HTTPRoute](https://gateway-api.sigs.k8s.io/api-types/httproute/) | Represents a set of HTTP-specific rules for mapping traffic to a backend. | +| [Listener](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1.Listener) | Represents a network endpoint that can accept incoming traffic. | -## Cross Namespace Routing +## Choosing a Gateway API Implementation -Gateway API has support for multi-ns and cross namespace routing. Routes can be deployed into different Namespaces and Routes can attach to Gateways across -Namespace boundaries. This allows user access control to be applied differently across Namespaces for Routes and Gateways, effectively segmenting access and -control to different parts of the cluster-wide routing configuration. +Within Genestack, multiple options are available for use as Gateway API implementations. The following table provides a comparison of the available options. -More information on cross namespace routing can be found [here](https://gateway-api.sigs.k8s.io/guides/multiple-ns/). +| Backend Options | Status |
Overview
| +| --------------- | ------ | --------------------------------------- | +| [Envoy](infrastructure-envoy-gateway-api.md) | **Recommended** | Feature rich, large community, recommended for Production environments. | +| [NGINX](infrastructure-nginx-gateway-api.md) | | Stable codebase, simple implementation | diff --git a/docs/gateway-api-ca-issuer.md b/docs/infrastructure-nginx-gateway-api-ca-issuer.md similarity index 98% rename from docs/gateway-api-ca-issuer.md rename to docs/infrastructure-nginx-gateway-api-ca-issuer.md index badf5e6b..f5e95d5f 100644 --- a/docs/gateway-api-ca-issuer.md +++ b/docs/infrastructure-nginx-gateway-api-ca-issuer.md @@ -1,4 +1,4 @@ -# Creating a CA issuer for Gateway API +# NGINX Creating a CA issuer for Gateway API By default in Genestack the selfSigned issuer is used to issue certificates to Gateway API listeners. This is a fairly simple issuer to create and requires a very simple yaml manifest. Although the main purpose of the selfSigned issuer to create a local PKI i.e bootstrap a local self-signed CA which can then be used to issue certificates as required. This is helpful for test environments. The selfSigned issuer itself doesn't represent a certificate authority by rather indicates that the certificates will sign themselves. diff --git a/docs/infrastructure-gateway-api-custom.md b/docs/infrastructure-nginx-gateway-api-custom.md similarity index 100% rename from docs/infrastructure-gateway-api-custom.md rename to docs/infrastructure-nginx-gateway-api-custom.md diff --git a/docs/infrastructure-nginx-gateway-api.md b/docs/infrastructure-nginx-gateway-api.md new file mode 100644 index 00000000..03395247 --- /dev/null +++ b/docs/infrastructure-nginx-gateway-api.md @@ -0,0 +1,258 @@ +--- +hide: + - footer +--- + +# NGINX Gateway API + +The [NGINX Gateway Fabric](https://github.com/nginxinc/nginx-gateway-fabric) is an open-source project that provides an +implementation of the Gateway API using NGINX as the data plane. + +## Install the Gateway API Resource from Kubernetes + +=== "Stable _(Recommended)_" + + ``` shell + kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.4.0" | kubectl apply -f - + ``` + +=== "Experimental" + + The experimental version of the Gateway API is available in the `v1.6.1` checkout. Use with caution. + + ``` shell + kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.6.1" | kubectl apply -f - + ``` + +## Install the NGINX Gateway Fabric controller + +The NGINX Gateway Fabric controller is a Kubernetes controller that manages the Gateway API resources. + +### Create the Namespace + +``` shell +kubectl apply -f /opt/genestack/manifests/nginx-gateway/nginx-gateway-namespace.yaml +``` + +!!! tip + + If attempting to perform an **upgrade** of an existing Gateway API deployment, note that the Helm install does not automatically upgrade the CRDs for + this resource. To upgrade them, refer to the process outlined by the + [Nginx upgrade documentation](https://docs.nginx.com/nginx-gateway-fabric/installation/installing-ngf/helm/#upgrade-nginx-gateway-fabric-crds). You + can safely ignore this note for new installations. + +=== "Stable _(Recommended)_" + + Edit the file `/etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml`. + + !!! example "Create an empty override file" + + If no overrides are needed, create an empty file. + + ``` shell + echo "---" | tee /etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml + ``` + + ``` shell + pushd /opt/genestack/submodules/nginx-gateway-fabric/charts || exit 1 + helm upgrade --install nginx-gateway-fabric ./nginx-gateway-fabric \ + --namespace=nginx-gateway \ + --create-namespace \ + -f /opt/genestack/base-helm-configs/nginx-gateway-fabric/helm-overrides.yaml \ + -f /etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml \ + --post-renderer /etc/genestack/kustomize/kustomize.sh \ + --post-renderer-args gateway/overlay + popd || exit 1 + ``` + +=== "Experimental" + + The experimental version of the Gateway API is available in the `v1.6.1` checkout. Use with caution. + + Update the submodule with the experimental version of the Gateway API. + + Edit the file `/etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml`. + + !!! example "Create the experimental override file" + + ``` yaml + --- + nginxGateway: + replicaCount: 3 + gwAPIExperimentalFeatures: + enable: true + service: + ## The externalTrafficPolicy of the service. The value Local preserves the client source IP. + externalTrafficPolicy: Cluster + ## The annotations of the NGINX Gateway Fabric service. + annotations: + "metallb.universe.tf/address-pool": "gateway-api-external" + "metallb.universe.tf/allow-shared-ip": "openstack-external-svc" + ``` + + Run the helm command to install the experimental version of the Gateway API. + + ``` shell + helm upgrade --install nginx-gateway-fabric oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ + --create-namespace \ + --namespace=nginx-gateway \ + -f /etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml \ + --post-renderer /etc/genestack/kustomize/kustomize.sh \ + --post-renderer-args gateway/overlay \ + --version 1.6.1 + ``` + +Once deployed ensure a system rollout has been completed for Cert Manager. + +``` shell +kubectl rollout restart deployment cert-manager --namespace cert-manager +``` + +## Create the shared gateway resource + +``` shell +kubectl kustomize /etc/genestack/kustomize/gateway/nginx-gateway-fabric | kubectl apply -f - +``` + +## Deploy with Let's Encrypt Certificates + +By default, certificates are issued by an instance of the selfsigned-cluster-issuer. This section focuses on replacing that with a +Let's Encrypt issuer to ensure valid certificates are deployed in our cluster. + +[![asciicast](https://asciinema.org/a/h7npXnDjkSpn3uQtuQwWG9zju.svg)](https://asciinema.org/a/h7npXnDjkSpn3uQtuQwWG9zju) + +### Apply the Let's Encrypt Cluster Issuer + +Before we can have Cert Manager start coordinating Let's Encrypt certificate +requests for us, we need to add an ACME issuer with a valid, monitored +email (for expiration reminders and other important ACME related information). + +``` yaml +read -p "Enter a valid email address for use with ACME: " ACME_EMAIL; \ +cat </g' /opt/genestack/etc/gateway-api/listeners/$listener > /etc/genestack/gateway-api/listeners/$listener + done + ``` + +``` shell +kubectl patch -n nginx-gateway gateway flex-gateway \ + --type='json' \ + --patch="$(jq -s 'flatten | .' /etc/genestack/gateway-api/listeners/*)" +``` + +## Apply Related Gateway routes + +Another example with most of the OpenStack services is located at `/opt/genestack/etc/gateway-api/routes/http-wildcard-listener.yaml`. Similarly, you must modify +and apply them as shown below, or apply your own. + +??? abstract "Example routes file" + + ``` yaml + --8<-- "etc/gateway-api/routes/http-wildcard-listener.yaml" + ``` + +All routes can be found at `/etc/genestack/gateway-api/routes`. + +!!! example "Example modifying all available Gateway routes with `your.domain.tld`" + + ``` shell + mkdir -p /etc/genestack/gateway-api/routes + for route in $(ls -1 /opt/genestack/etc/gateway-api/routes); do + sed 's/your.domain.tld//g' /opt/genestack/etc/gateway-api/routes/$route > /etc/genestack/gateway-api/routes/$route + done + ``` + +``` shell +kubectl apply -f /etc/genestack/gateway-api/routes +``` + +## Patch Gateway with Let's Encrypt Cluster Issuer + +??? abstract "Example patch to enable LetsEncrypt `/etc/genestack/gateway-api/gateway-letsencrypt.yaml`" + + ``` yaml + --8<-- "etc/gateway-api/gateway-letsencrypt.yaml" + ``` + +``` shell +kubectl patch --namespace nginx-gateway \ + --type merge \ + --patch-file /etc/genestack/gateway-api/gateway-letsencrypt.yaml \ + gateway flex-gateway +``` + +## Example Implementation with Prometheus UI (NGINX Gateway Fabric) + +In this example we will look at how Prometheus UI is exposed through the gateway. For other services the gateway kustomization file for the service. + +First, create the shared gateway and then the httproute resource for prometheus. + +??? abstract "Example patch to enable Prometheus `/etc/genestack/gateway-api/gateway-prometheus.yaml`" + + ``` yaml + --8<-- "etc/gateway-api/gateway-prometheus.yaml" + ``` + +!!! example "Example modifying Prometheus' Gateway deployment" + + ``` shell + mkdir -p /etc/genestack/gateway-api + sed 's/your.domain.tld//g' /opt/genestack/etc/gateway-api/gateway-prometheus.yaml > /etc/genestack/gateway-api/gateway-prometheus.yaml + ``` + +``` shell +kubectl apply -f /etc/genestack/gateway-api/gateway-prometheus.yaml +``` + +At this point, flex-gateway has a listener pointed to the port 80 matching *.your.domain.tld hostname. The HTTPRoute resource configures routes +for this gateway. Here, we match all path and simply pass any request from the matching hostname to kube-prometheus-stack-prometheus backend service. + +## Cross Namespace Routing + +Gateway API has support for multi-ns and cross namespace routing. Routes can be deployed into different Namespaces and Routes can attach to Gateways across +Namespace boundaries. This allows user access control to be applied differently across Namespaces for Routes and Gateways, effectively segmenting access and +control to different parts of the cluster-wide routing configuration. + +More information on cross namespace routing can be found [here](https://gateway-api.sigs.k8s.io/guides/multiple-ns/). diff --git a/docs/monitoring-info.md b/docs/monitoring-info.md index b732d667..9b244887 100644 --- a/docs/monitoring-info.md +++ b/docs/monitoring-info.md @@ -110,7 +110,7 @@ Once we've ran the apply command we will have installed ServiceMonitors for Kube You can view more information about OVN monitoring in the [OVN Monitoring Introduction Docs](ovn-monitoring-introduction.md). * ### Nginx Gateway Monitoring: -Genestack makes use of the [Nginx Gateway Fabric](https://github.com/nginxinc/nginx-gateway-fabric/tree/main/charts/nginx-gateway-fabric) for its implementation of [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io/). Genestack deploys this as part of its infrastructure, view the [Nginx Gateway Deployment Doc](infrastructure-gateway-api.md) for more information. +Genestack makes use of the Gateway API for its implementation of [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io/). Genestack deploys this as part of its infrastructure, view the [Gateway Deployment Doc](infrastructure-gateway-api.md) for more information. Nginx Gateway does expose important metrics for us to gather but it does not do so via a service. Instead we must make use another Prometheus CRD the [PodMonitor](https://prometheus-operator.dev/docs/getting-started/design/#podmonitor). The install is similar to the above OVN monitoring as you can see in the [Nginx Gateway Exporter Deployment Doc](prometheus-nginx-gateway.md). The primary difference is the need to target and match on a pod that's exposing the metrics rather than a service. You can view more information about the metrics exposed by the Nginx Gateway by viewing the [Nginx Gateway Fabric Docs](https://docs.nginx.com/nginx-gateway-fabric/how-to/monitoring/prometheus/). diff --git a/docs/rackspace-infrastructure-gateway-api.md b/docs/rackspace-infrastructure-nginx-gateway-api.md similarity index 100% rename from docs/rackspace-infrastructure-gateway-api.md rename to docs/rackspace-infrastructure-nginx-gateway-api.md diff --git a/etc/gateway-api/gateway-envoy-http-routes.yaml b/etc/gateway-api/gateway-envoy-http-routes.yaml deleted file mode 100644 index 93831c64..00000000 --- a/etc/gateway-api/gateway-envoy-http-routes.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: -name: test_application -namespace: test_app -spec: -parentRefs: -- name: flex-gateway - sectionName: http - namespace: envoy-gateway-system -hostnames: -- "test_application.sjc.your.domain.tld" -rules: - - backendRefs: - - name: test_application - port: 8774 diff --git a/mkdocs.yml b/mkdocs.yml index 097c7854..40a998af 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -181,7 +181,10 @@ nav: - infrastructure-overview.md - Namespace: infrastructure-namespace.md - MetalLB: infrastructure-metallb.md - - Gateway API: infrastructure-gateway-api.md + - Gateway API: + - Gateway API Overview: infrastructure-gateway-api.md + - Envoy Gateway: infrastructure-envoy-gateway-api.md + - NGINX Gateway: infrastructure-nginx-gateway-api.md - MariaDB: - infrastructure-mariadb.md - RabbitMQ: @@ -267,9 +270,10 @@ nav: - MariaDB: - Operations: infrastructure-mariadb-ops.md - Gateway API: - - Custom Routes: infrastructure-gateway-api-custom.md - - Rackspace Example Gateway Overview: rackspace-infrastructure-gateway-api.md - - Creating self-signed CA issuer for Gateway API: gateway-api-ca-issuer.md + - NGINX Gateway: + - Custom Routes: infrastructure-nginx-gateway-api-custom.md + - Rackspace Example Gateway Overview: rackspace-infrastructure-nginx-gateway-api.md + - Creating self-signed CA issuer for Gateway API: infrastructure-nginx-gateway-api-ca-issuer.md - Observability: - Observability Overview: observability-info.md - Monitoring Overview: monitoring-info.md