diff --git a/Makefile b/Makefile index d201becb6..d817cc59a 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ EFFECTIVE_VERSION := $(VERSION)-$(shell git rev-parse HEAD) LD_FLAGS := "-w $(shell bash $(GARDENER_HACK_DIR)/get-build-ld-flags.sh k8s.io/component-base $(REPO_ROOT)/VERSION $(EXTENSION_PREFIX))" LEADER_ELECTION := false IGNORE_OPERATION_ANNOTATION := true -PLATFORM := linux/amd64 +PLATFORM := linux/amd64 TEST_RECONCILER := tf WEBHOOK_CONFIG_PORT := 8443 diff --git a/charts/internal/seed-controlplane/charts/ingress-gce/Chart.yaml b/charts/internal/seed-controlplane/charts/ingress-gce/Chart.yaml new file mode 100644 index 000000000..9b958b01e --- /dev/null +++ b/charts/internal/seed-controlplane/charts/ingress-gce/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Helm chart for ingress-gce +name: ingress-gce +version: 0.1.0 diff --git a/charts/internal/seed-controlplane/charts/ingress-gce/templates/deployment.yaml b/charts/internal/seed-controlplane/charts/ingress-gce/templates/deployment.yaml new file mode 100644 index 000000000..9f2a005a6 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/ingress-gce/templates/deployment.yaml @@ -0,0 +1,111 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingress-gce + namespace: {{ .Release.Namespace }} + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + labels: + app: kubernetes + role: glbc +spec: + revisionHistoryLimit: 0 + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: kubernetes + role: glbc + template: + metadata: +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + labels: + app: kubernetes + role: glbc + gardener.cloud/role: controlplane + networking.gardener.cloud/to-dns: allowed + networking.gardener.cloud/to-public-networks: allowed + networking.resources.gardener.cloud/to-kube-apiserver-tcp-443: allowed +{{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} +{{- end }} + spec: + automountServiceAccountToken: false + serviceAccountName: glbc + containers: + - name: glbc + image: {{ index .Values.images "ingress-gce" }} + command: + - /glbc + - --config-file-path=/etc/kubernetes/cloudprovider/cloudprovider.conf + - --healthz-port=8086 + - --logtostderr + - --sync-period=600s + - --default-backend-service=kube-system/default-http-backend + - --running-in-cluster=false + - --run-l4-netlb-controller=true + - --enable-l4ilb-dual-stack=true + - --enable-l4netlb-dual-stack=true + # services of Type:LoadBalancer with Internal annotation `cloud.google.com/l4-rbs: enabled` + # will be processed by this controller + - --run-l4-controller=true + - --kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig + - --leader-elect=true + - --v=2 + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /srv/cloudprovider/serviceaccount.json + livenessProbe: + httpGet: + path: /healthz + port: 8086 + scheme: HTTP + initialDelaySeconds: 30 + # healthz reaches out to GCE + periodSeconds: 30 + timeoutSeconds: 15 + successThreshold: 1 + failureThreshold: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + {{- with .Values.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + volumeMounts: + - name: cloud-provider-config + mountPath: /etc/kubernetes/cloudprovider + - name: cloudprovider + mountPath: /srv/cloudprovider + - name: kubeconfig + mountPath: /var/run/secrets/gardener.cloud/shoot/generic-kubeconfig + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + volumes: + - name: cloud-provider-config + configMap: + name: cloud-provider-config + - name: cloudprovider + secret: + secretName: cloudprovider + - name: kubeconfig + projected: + defaultMode: 420 + sources: + - secret: + items: + - key: kubeconfig + path: kubeconfig + name: {{ .Values.global.genericTokenKubeconfigSecretName }} + optional: false + - secret: + items: + - key: token + path: token + name: shoot-access-cloud-controller-manager + optional: false diff --git a/charts/internal/seed-controlplane/charts/ingress-gce/templates/service-account.yaml b/charts/internal/seed-controlplane/charts/ingress-gce/templates/service-account.yaml new file mode 100644 index 000000000..5e6f4f2e0 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/ingress-gce/templates/service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: glbc + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/internal/seed-controlplane/charts/ingress-gce/templates/vpa.yaml b/charts/internal/seed-controlplane/charts/ingress-gce/templates/vpa.yaml new file mode 100644 index 000000000..4ddcd39c2 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/ingress-gce/templates/vpa.yaml @@ -0,0 +1,21 @@ +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: ingress-gce + namespace: {{ .Release.Namespace }} +spec: + targetRef: + apiVersion: {{ include "deploymentversion" . }} + kind: Deployment + name: ingress-gce + updatePolicy: + updateMode: Auto + resourcePolicy: + containerPolicies: + - containerName: glbc + minAllowed: + memory: {{ .Values.resources.requests.memory }} + maxAllowed: + cpu: {{ .Values.vpa.resourcePolicy.maxAllowed.cpu }} + memory: {{ .Values.vpa.resourcePolicy.maxAllowed.memory }} + controlledValues: RequestsOnly diff --git a/charts/internal/seed-controlplane/charts/ingress-gce/values.yaml b/charts/internal/seed-controlplane/charts/ingress-gce/values.yaml new file mode 100644 index 000000000..02e440b6a --- /dev/null +++ b/charts/internal/seed-controlplane/charts/ingress-gce/values.yaml @@ -0,0 +1,18 @@ +replicas: 1 + +images: + ingress-gce: image-repository:image-tag + +resources: + requests: + cpu: 100m + memory: 75Mi + +vpa: + resourcePolicy: + maxAllowed: + cpu: 1 + memory: 2G + +podAnnotations: {} +podLabels: {} diff --git a/charts/internal/seed-controlplane/requirements.yaml b/charts/internal/seed-controlplane/requirements.yaml index eb63db30f..3de2d7f9b 100644 --- a/charts/internal/seed-controlplane/requirements.yaml +++ b/charts/internal/seed-controlplane/requirements.yaml @@ -7,3 +7,7 @@ dependencies: repository: http://localhost:10191 version: 0.1.0 condition: csi-driver-controller.enabled +- name: ingress-gce + repository: http://localhost:10191 + version: 0.1.0 + condition: ingress-gce.enabled diff --git a/charts/internal/seed-controlplane/values.yaml b/charts/internal/seed-controlplane/values.yaml index 2d6a15a1f..55b2bba7e 100644 --- a/charts/internal/seed-controlplane/values.yaml +++ b/charts/internal/seed-controlplane/values.yaml @@ -5,3 +5,5 @@ cloud-controller-manager: enabled: true csi-driver-controller: enabled: true +ingress-gce: + enabled: true diff --git a/charts/internal/shoot-system-components/charts/default-http-backend/Chart.yaml b/charts/internal/shoot-system-components/charts/default-http-backend/Chart.yaml new file mode 100644 index 000000000..f1e8a40bb --- /dev/null +++ b/charts/internal/shoot-system-components/charts/default-http-backend/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Helm chart for default-http-backend +name: default-http-backend +version: 0.1.0 diff --git a/charts/internal/shoot-system-components/charts/default-http-backend/templates/default-http-backend.yaml b/charts/internal/shoot-system-components/charts/default-http-backend/templates/default-http-backend.yaml new file mode 100644 index 000000000..3b577ddda --- /dev/null +++ b/charts/internal/shoot-system-components/charts/default-http-backend/templates/default-http-backend.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: l7-default-backend + namespace: {{ .Release.Namespace }} + labels: + k8s-app: glbc + kubernetes.io/name: "GLBC" + app: kubernetes + role: ingress-gce +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: glbc + template: + metadata: + labels: + k8s-app: glbc + name: glbc + spec: + containers: + - name: default-http-backend + # Any image is permissible as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: {{ index .Values.images "default-http-backend" }} + livenessProbe: + httpGet: + path: /healthy + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- +apiVersion: v1 +kind: Service +metadata: + # This must match the --default-backend-service argument of the l7 lb + # controller and is required because GCE mandates a default backend. + name: default-http-backend + namespace: {{ .Release.Namespace }} + labels: + k8s-app: glbc + kubernetes.io/name: "GLBCDefaultBackend" +spec: + # The default backend must be of type NodePort. + type: NodePort + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + k8s-app: glbc diff --git a/charts/internal/shoot-system-components/charts/default-http-backend/templates/rbac.yaml b/charts/internal/shoot-system-components/charts/default-http-backend/templates/rbac.yaml new file mode 100644 index 000000000..19ca4b85e --- /dev/null +++ b/charts/internal/shoot-system-components/charts/default-http-backend/templates/rbac.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: glbc + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: system:controller:glbc + namespace: {{ .Release.Namespace }} + labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "update", "create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:controller:glbc + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system:controller:glbc +subjects: +- kind: ServiceAccount + name: glbc + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:controller:glbc +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +- apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "update", "create", "patch"] +- apiGroups: [""] + resources: ["endpoints", "services", "pods", "nodes", "namespaces"] + verbs: ["get", "list", "watch"] +# TODO: switch to patch services/status +# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/pkg/neg/controller.go#L339-L342 +# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/pkg/neg/controller.go#L359-L361 +- apiGroups: [""] + resources: ["services", "pods"] + verbs: ["update", "patch"] +- apiGroups: ["networking.istio.io"] + resources: ["destinationrules"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: ["extensions", "networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions", "networking.k8s.io"] + resources: ["ingresses"] + verbs: ["update"] +- apiGroups: ["extensions", "networking.k8s.io"] + resources: ["ingresses/status"] + verbs: ["update"] +# GLBC ensures that the `cloud.google.com/backendconfigs` CRD exists in a desired state: +# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/cmd/glbc/main.go#L93 +# TODO(rramkumar1): https://github.com/kubernetes/ingress-gce/issues/744 +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch", "update", "create", "patch"] +- apiGroups: ["cloud.google.com"] + resources: ["backendconfigs"] + verbs: ["get", "list", "watch", "update", "create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:controller:glbc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:glbc +subjects: +- kind: ServiceAccount + name: glbc + namespace: {{ .Release.Namespace }} diff --git a/charts/internal/shoot-system-components/charts/default-http-backend/values.yaml b/charts/internal/shoot-system-components/charts/default-http-backend/values.yaml new file mode 100644 index 000000000..ae34a4b57 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/default-http-backend/values.yaml @@ -0,0 +1,7 @@ +replicas: 1 + +images: + default-http-backend: image-repository:image-tag + +podAnnotations: {} +podLabels: {} diff --git a/charts/internal/shoot-system-components/requirements.yaml b/charts/internal/shoot-system-components/requirements.yaml index 4535cb7e9..338645712 100644 --- a/charts/internal/shoot-system-components/requirements.yaml +++ b/charts/internal/shoot-system-components/requirements.yaml @@ -7,3 +7,7 @@ dependencies: repository: http://locwalhost:10191 version: 0.1.0 condition: csi-driver-node.enabled +- name: default-http-backend + repository: http://localhost:10191 + version: 0.1.0 + condition: default-http-backend.enabled diff --git a/charts/internal/shoot-system-components/values.yaml b/charts/internal/shoot-system-components/values.yaml index b31047e2e..e1c336efb 100644 --- a/charts/internal/shoot-system-components/values.yaml +++ b/charts/internal/shoot-system-components/values.yaml @@ -2,3 +2,5 @@ cloud-controller-manager: enabled: true csi-driver-node: enabled: true +default-http-backend: + enabled: true diff --git a/imagevector/images.yaml b/imagevector/images.yaml index 2b91e6cb0..7703ce048 100644 --- a/imagevector/images.yaml +++ b/imagevector/images.yaml @@ -243,3 +243,29 @@ images: confidentiality_requirement: 'low' integrity_requirement: 'high' availability_requirement: 'low' +- name: ingress-gce + sourceRepository: github.com/kubernetes/ingress-gce + repository: ghcr.io/gardener/ingress-gce + tag: "v1.30.5" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'private' + authentication_enforced: false + user_interaction: 'end-user' + confidentiality_requirement: 'low' + integrity_requirement: 'high' + availability_requirement: 'low' +- name: default-http-backend + sourceRepository: github.com/kubernetes/ingress-gce + repository: europe-docker.pkg.dev/gardener-project/releases/gardener/ingress-default-backend + tag: "0.20.0" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'protected' + authentication_enforced: false + user_interaction: 'end-user' + confidentiality_requirement: 'low' + integrity_requirement: 'high' + availability_requirement: 'low' diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index ccc728c72..e5a5691a9 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -14,7 +14,7 @@ import ( extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" "github.com/gardener/gardener/extensions/pkg/controller/controlplane/genericactuator" extensionssecretsmanager "github.com/gardener/gardener/extensions/pkg/util/secret/manager" - "github.com/gardener/gardener/pkg/apis/core/v1beta1" + gardencorcorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" @@ -151,6 +151,17 @@ var ( {Type: &corev1.Service{}, Name: gcp.CSISnapshotValidationName}, }, }, + { + Name: gcp.IngressGCEName, + Images: []string{ + gcp.IngressGCEImageName, + }, + Objects: []*chart.Object{ + {Type: &appsv1.Deployment{}, Name: gcp.IngressGCEName}, + {Type: &autoscalingv1.VerticalPodAutoscaler{}, Name: gcp.IngressGCEName}, + {Type: &corev1.ServiceAccount{}, Name: "glbc"}, + }, + }, }, } @@ -214,6 +225,21 @@ var ( {Type: &rbacv1.ClusterRoleBinding{}, Name: gcp.UsernamePrefix + gcp.CSISnapshotValidationName}, }, }, + { + Name: "default-http-backend", + Images: []string{ + gcp.DefaultHTTPBackendImageName, + }, + Objects: []*chart.Object{ + {Type: &appsv1.Deployment{}, Name: "l7-default-backend"}, + {Type: &corev1.Service{}, Name: "default-http-backend"}, + {Type: &corev1.ServiceAccount{}, Name: "glbc"}, + {Type: &rbacv1.Role{}, Name: "system:controller:glbc"}, + {Type: &rbacv1.RoleBinding{}, Name: "system:controller:glbc"}, + {Type: &rbacv1.ClusterRole{}, Name: "system:controller:glbc"}, + {Type: &rbacv1.ClusterRoleBinding{}, Name: "system:controller:glbc"}, + }, + }, }, } @@ -333,10 +359,7 @@ func (vp *valuesProvider) GetControlPlaneShootChartValues( cluster *extensionscontroller.Cluster, secretsReader secretsmanager.Reader, _ map[string]string, -) ( - map[string]interface{}, - error, -) { +) (map[string]interface{}, error) { return getControlPlaneShootChartValues(cluster, cp, secretsReader) } @@ -390,9 +413,21 @@ func (vp *valuesProvider) getControlPlaneChartValues( }, gcp.CloudControllerManagerName: ccm, gcp.CSIControllerName: csi, + gcp.IngressGCEName: map[string]interface{}{ + "enabled": isDualstackEnabled(cluster.Shoot.Spec.Networking), + "replicas": extensionscontroller.GetControlPlaneReplicas(cluster, scaledDown, 1), + }, }, nil } +func isDualstackEnabled(networking *gardencorcorev1beta1.Networking) bool { + if networking != nil { + return !gardencorcorev1beta1.IsIPv4SingleStack(networking.IPFamilies) + } + + return false +} + // getCCMChartValues collects and returns the CCM chart values. func (vp *valuesProvider) getCCMChartValues( cpConfig *apisgcp.ControlPlaneConfig, @@ -500,6 +535,9 @@ func getControlPlaneShootChartValues( "caBundle": string(caSecret.Data[secretutils.DataKeyCertificateBundle]), }, }, + gcp.DefaultHTTPBackendImageName: map[string]interface{}{ + "enabled": isDualstackEnabled(cluster.Shoot.Spec.Networking), + }, }, nil } @@ -542,7 +580,7 @@ func getNetworkNames( } subNetworkName := "" - subnet, _ := apihelper.FindSubnetForPurpose(infraStatus.Networks.Subnets, apisgcp.PurposeInternal) + subnet, _ := apihelper.FindSubnetForPurpose(infraStatus.Networks.Subnets, apisgcp.PurposeNodes) if subnet != nil { subNetworkName = subnet.Name } @@ -550,7 +588,7 @@ func getNetworkNames( return networkName, subNetworkName } -func (vp *valuesProvider) isOverlayEnabled(network *v1beta1.Networking) (bool, error) { +func (vp *valuesProvider) isOverlayEnabled(network *gardencorcorev1beta1.Networking) (bool, error) { if network == nil || network.ProviderConfig == nil { return true, nil } diff --git a/pkg/gcp/types.go b/pkg/gcp/types.go index e2ce29457..f5486d3c7 100644 --- a/pkg/gcp/types.go +++ b/pkg/gcp/types.go @@ -14,6 +14,10 @@ const ( // CloudControllerManagerImageName is the name of the cloud-controller-manager image. CloudControllerManagerImageName = "cloud-controller-manager" + // IngressGCEImageName is the name of the ingress-gce image. + IngressGCEImageName = "ingress-gce" + // DefaultHTTPBackendImageName is the name of the csi-driver image. + DefaultHTTPBackendImageName = "default-http-backend" // CSIDriverImageName is the name of the csi-driver image. CSIDriverImageName = "csi-driver" // CSIProvisionerImageName is the name of the csi-provisioner image. @@ -45,6 +49,8 @@ const ( // CloudControllerManagerName is a constant for the name of the CloudController deployed by the worker controller. CloudControllerManagerName = "cloud-controller-manager" + // IngressGCEName is a constant for the name of the ingress-gce deployment in the seed. + IngressGCEName = "ingress-gce" // CSIControllerName is a constant for the name of the CSI controller deployment in the seed. CSIControllerName = "csi-driver-controller" // CSIControllerConfigName is a constant for the name of the CSI controller config in the seed. @@ -83,7 +89,5 @@ const ( SeedAnnotationUseFlowValueNew = "new" ) -var ( - // UsernamePrefix is a constant for the username prefix of components deployed by GCP. - UsernamePrefix = extensionsv1alpha1.SchemeGroupVersion.Group + ":" + Name + ":" -) +// UsernamePrefix is a constant for the username prefix of components deployed by GCP. +var UsernamePrefix = extensionsv1alpha1.SchemeGroupVersion.Group + ":" + Name + ":"