From 27a5b4a96f6f4890461cc988df78e21841c2176f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Mon, 13 Jan 2025 16:03:10 +0100 Subject: [PATCH 01/15] Extending Runtime Custom Resource with Additional Workers --- api/v1/runtime_types.go | 1 + ...cturemanager.kyma-project.io_runtimes.yaml | 672 +++++++++++++++++- pkg/gardener/shoot/extender/provider.go | 6 + 3 files changed, 678 insertions(+), 1 deletion(-) diff --git a/api/v1/runtime_types.go b/api/v1/runtime_types.go index c0847732..743210b9 100644 --- a/api/v1/runtime_types.go +++ b/api/v1/runtime_types.go @@ -174,6 +174,7 @@ type Provider struct { //+kubebuilder:validation:Enum=aws;azure;gcp;openstack Type string `json:"type"` Workers []gardener.Worker `json:"workers"` + AdditionalWorkers *[]gardener.Worker `json:"additionalWorkers,omitempty"` ControlPlaneConfig *runtime.RawExtension `json:"controlPlaneConfig,omitempty"` InfrastructureConfig *runtime.RawExtension `json:"infrastructureConfig,omitempty"` } diff --git a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml index 8fa12e45..e3960f40 100644 --- a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml +++ b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.16.4 name: runtimes.infrastructuremanager.kyma-project.io spec: group: infrastructuremanager.kyma-project.io @@ -311,6 +311,676 @@ spec: type: string provider: properties: + additionalWorkers: + items: + description: Worker is the base definition of a worker group. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a map of key/value pairs + for annotations for all the `Node` objects in this + worker pool. + type: object + caBundle: + description: CABundle is a certificate bundle which + will be installed onto every machine of this worker + pool. + type: string + clusterAutoscaler: + description: ClusterAutoscaler contains the cluster + autoscaler configurations for the worker pool. + properties: + maxNodeProvisionTime: + description: MaxNodeProvisionTime defines how long + CA waits for node to be provisioned. + type: string + scaleDownGpuUtilizationThreshold: + description: ScaleDownGpuUtilizationThreshold defines + the threshold in fraction (0.0 - 1.0) of gpu resources + under which a node is being removed. + type: number + scaleDownUnneededTime: + description: ScaleDownUnneededTime defines how long + a node should be unneeded before it is eligible + for scale down. + type: string + scaleDownUnreadyTime: + description: ScaleDownUnreadyTime defines how long + an unready node should be unneeded before it is + eligible for scale down. + type: string + scaleDownUtilizationThreshold: + description: ScaleDownUtilizationThreshold defines + the threshold in fraction (0.0 - 1.0) under which + a node is being removed. + type: number + type: object + cri: + description: |- + CRI contains configurations of CRI support of every machine in the worker pool. + Defaults to a CRI with name `containerd`. + properties: + containerRuntimes: + description: ContainerRuntimes is the list of the + required container runtimes supported for a worker + pool. + items: + description: ContainerRuntime contains information + about worker's available container runtime + properties: + providerConfig: + description: ProviderConfig is the configuration + passed to container runtime resource. + type: object + x-kubernetes-preserve-unknown-fields: true + type: + description: Type is the type of the Container + Runtime. + type: string + required: + - type + type: object + type: array + name: + description: The name of the CRI library. Supported + values are `containerd`. + type: string + required: + - name + type: object + dataVolumes: + description: DataVolumes contains a list of additional + worker volumes. + items: + description: DataVolume contains information about + a data volume. + properties: + encrypted: + description: Encrypted determines if the volume + should be encrypted. + type: boolean + name: + description: Name of the volume to make it referencable. + type: string + size: + description: VolumeSize is the size of the volume. + type: string + type: + description: Type is the type of the volume. + type: string + required: + - name + - size + type: object + type: array + kubeletDataVolumeName: + description: KubeletDataVolumeName contains the name + of a dataVolume that should be used for storing kubelet + state. + type: string + kubernetes: + description: Kubernetes contains configuration for Kubernetes + components related to this worker pool. + properties: + kubelet: + description: |- + Kubelet contains configuration settings for all kubelets of this worker pool. + If set, all `spec.kubernetes.kubelet` settings will be overwritten for this worker pool (no merge of settings). + properties: + containerLogMaxFiles: + description: Maximum number of container log + files that can be present for a container. + format: int32 + type: integer + containerLogMaxSize: + anyOf: + - type: integer + - type: string + description: |- + A quantity defines the maximum size of the container log file before it is rotated. For example: "5Mi" or "256Ki". + Default: 100Mi + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cpuCFSQuota: + description: CPUCFSQuota allows you to disable/enable + CPU throttling for Pods. + type: boolean + cpuManagerPolicy: + description: 'CPUManagerPolicy allows to set + alternative CPU management policies (default: + none).' + type: string + evictionHard: + description: |- + EvictionHard describes a set of eviction thresholds (e.g. memory.available<1Gi) that if met would trigger a Pod eviction. + Default: + memory.available: "100Mi/1Gi/5%" + nodefs.available: "5%" + nodefs.inodesFree: "5%" + imagefs.available: "5%" + imagefs.inodesFree: "5%" + properties: + imageFSAvailable: + description: ImageFSAvailable is the threshold + for the free disk space in the imagefs + filesystem (docker images and container + writable layers). + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the threshold + for the available inodes in the imagefs + filesystem. + type: string + memoryAvailable: + description: MemoryAvailable is the threshold + for the free memory on the host server. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the threshold + for the free disk space in the nodefs + filesystem (docker volumes, logs, etc). + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the threshold + for the available inodes in the nodefs + filesystem. + type: string + type: object + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod describes the maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met. + Default: 90 + format: int32 + type: integer + evictionMinimumReclaim: + description: |- + EvictionMinimumReclaim configures the amount of resources below the configured eviction threshold that the kubelet attempts to reclaim whenever the kubelet observes resource pressure. + Default: 0 for each resource + properties: + imageFSAvailable: + anyOf: + - type: integer + - type: string + description: ImageFSAvailable is the threshold + for the disk space reclaim in the imagefs + filesystem (docker images and container + writable layers). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + imageFSInodesFree: + anyOf: + - type: integer + - type: string + description: ImageFSInodesFree is the threshold + for the inodes reclaim in the imagefs + filesystem. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryAvailable: + anyOf: + - type: integer + - type: string + description: MemoryAvailable is the threshold + for the memory reclaim on the host server. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + nodeFSAvailable: + anyOf: + - type: integer + - type: string + description: NodeFSAvailable is the threshold + for the disk space reclaim in the nodefs + filesystem (docker volumes, logs, etc). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + nodeFSInodesFree: + anyOf: + - type: integer + - type: string + description: NodeFSInodesFree is the threshold + for the inodes reclaim in the nodefs filesystem. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + evictionPressureTransitionPeriod: + description: |- + EvictionPressureTransitionPeriod is the duration for which the kubelet has to wait before transitioning out of an eviction pressure condition. + Default: 4m0s + type: string + evictionSoft: + description: |- + EvictionSoft describes a set of eviction thresholds (e.g. memory.available<1.5Gi) that if met over a corresponding grace period would trigger a Pod eviction. + Default: + memory.available: "200Mi/1.5Gi/10%" + nodefs.available: "10%" + nodefs.inodesFree: "10%" + imagefs.available: "10%" + imagefs.inodesFree: "10%" + properties: + imageFSAvailable: + description: ImageFSAvailable is the threshold + for the free disk space in the imagefs + filesystem (docker images and container + writable layers). + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the threshold + for the available inodes in the imagefs + filesystem. + type: string + memoryAvailable: + description: MemoryAvailable is the threshold + for the free memory on the host server. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the threshold + for the free disk space in the nodefs + filesystem (docker volumes, logs, etc). + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the threshold + for the available inodes in the nodefs + filesystem. + type: string + type: object + evictionSoftGracePeriod: + description: |- + EvictionSoftGracePeriod describes a set of eviction grace periods (e.g. memory.available=1m30s) that correspond to how long a soft eviction threshold must hold before triggering a Pod eviction. + Default: + memory.available: 1m30s + nodefs.available: 1m30s + nodefs.inodesFree: 1m30s + imagefs.available: 1m30s + imagefs.inodesFree: 1m30s + properties: + imageFSAvailable: + description: ImageFSAvailable is the grace + period for the ImageFSAvailable eviction + threshold. + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the grace + period for the ImageFSInodesFree eviction + threshold. + type: string + memoryAvailable: + description: MemoryAvailable is the grace + period for the MemoryAvailable eviction + threshold. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the grace + period for the NodeFSAvailable eviction + threshold. + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the grace + period for the NodeFSInodesFree eviction + threshold. + type: string + type: object + failSwapOn: + description: FailSwapOn makes the Kubelet fail + to start if swap is enabled on the node. (default + true). + type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information + about enabled feature gates. + type: object + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent describes the percent of the disk usage which triggers image garbage collection. + Default: 50 + format: int32 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent describes the percent of the disk to which garbage collection attempts to free. + Default: 40 + format: int32 + type: integer + kubeReserved: + description: |- + KubeReserved is the configuration for resources reserved for kubernetes node components (mainly kubelet and container runtime). + When updating these values, be aware that cgroup resizes may not succeed on active worker nodes. Look for the NodeAllocatableEnforced event to determine if the configuration was applied. + Default: cpu=80m,memory=1Gi,pid=20k + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU is the reserved cpu. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + ephemeralStorage: + anyOf: + - type: integer + - type: string + description: EphemeralStorage is the reserved + ephemeral-storage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory is the reserved memory. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pid: + anyOf: + - type: integer + - type: string + description: PID is the reserved process-ids. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + maxPods: + description: |- + MaxPods is the maximum number of Pods that are allowed by the Kubelet. + Default: 110 + format: int32 + type: integer + memorySwap: + description: MemorySwap configures swap memory + available to container workloads. + properties: + swapBehavior: + description: |- + SwapBehavior configures swap memory available to container workloads. May be one of {"LimitedSwap", "UnlimitedSwap"} + defaults to: LimitedSwap + type: string + type: object + podPidsLimit: + description: PodPIDsLimit is the maximum number + of process IDs per pod allowed by the kubelet. + format: int64 + type: integer + protectKernelDefaults: + description: |- + ProtectKernelDefaults ensures that the kernel tunables are equal to the kubelet defaults. + Defaults to true for Kubernetes v1.26 or later. + type: boolean + registryBurst: + description: |- + RegistryBurst is the maximum size of bursty pulls, temporarily allows pulls to burst to this number, + while still not exceeding registryPullQPS. The value must not be a negative number. + Only used if registryPullQPS is greater than 0. + Default: 10 + format: int32 + type: integer + registryPullQPS: + description: |- + RegistryPullQPS is the limit of registry pulls per second. The value must not be a negative number. + Setting it to 0 means no limit. + Default: 5 + format: int32 + type: integer + seccompDefault: + description: |- + SeccompDefault enables the use of `RuntimeDefault` as the default seccomp profile for all workloads. + This requires the corresponding SeccompDefault feature gate to be enabled as well. + This field is only available for Kubernetes v1.25 or later. + type: boolean + serializeImagePulls: + description: |- + SerializeImagePulls describes whether the images are pulled one at a time. + Default: true + type: boolean + streamingConnectionIdleTimeout: + description: |- + StreamingConnectionIdleTimeout is the maximum time a streaming connection can be idle before the connection is automatically closed. + This field cannot be set lower than "30s" or greater than "4h". + Default: + "4h" for Kubernetes < v1.26. + "5m" for Kubernetes >= v1.26. + type: string + systemReserved: + description: |- + SystemReserved is the configuration for resources reserved for system processes not managed by kubernetes (e.g. journald). + When updating these values, be aware that cgroup resizes may not succeed on active worker nodes. Look for the NodeAllocatableEnforced event to determine if the configuration was applied. + + Deprecated: Separately configuring resource reservations for system processes is deprecated in Gardener and will be forbidden starting from Kubernetes 1.31. + Please merge existing resource reservations into the kubeReserved field. + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU is the reserved cpu. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + ephemeralStorage: + anyOf: + - type: integer + - type: string + description: EphemeralStorage is the reserved + ephemeral-storage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory is the reserved memory. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pid: + anyOf: + - type: integer + - type: string + description: PID is the reserved process-ids. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + version: + description: |- + Version is the semantic Kubernetes version to use for the Kubelet in this Worker Group. + If not specified the kubelet version is derived from the global shoot cluster kubernetes version. + version must be equal or lower than the version of the shoot kubernetes version. + Only one minor version difference to other worker groups and global kubernetes version is allowed. + type: string + type: object + labels: + additionalProperties: + type: string + description: Labels is a map of key/value pairs for + labels for all the `Node` objects in this worker pool. + type: object + machine: + description: Machine contains information about the + machine type and image. + properties: + architecture: + description: Architecture is CPU architecture of + machines in this worker pool. + type: string + image: + description: |- + Image holds information about the machine image to use for all nodes of this pool. It will default to the + latest version of the first image stated in the referenced CloudProfile if no value has been provided. + properties: + name: + description: Name is the name of the image. + type: string + providerConfig: + description: ProviderConfig is the shoot's individual + configuration passed to an extension resource. + type: object + x-kubernetes-preserve-unknown-fields: true + version: + description: |- + Version is the version of the shoot's image. + If version is not provided, it will be defaulted to the latest version from the CloudProfile. + type: string + required: + - name + type: object + type: + description: Type is the machine type of the worker + group. + type: string + required: + - type + type: object + machineControllerManager: + description: MachineControllerManagerSettings contains + configurations for different worker-pools. Eg. MachineDrainTimeout, + MachineHealthTimeout. + properties: + machineCreationTimeout: + description: MachineCreationTimeout is the period + after which creation of the machine is declared + failed. + type: string + machineDrainTimeout: + description: MachineDrainTimeout is the period after + which machine is forcefully deleted. + type: string + machineHealthTimeout: + description: MachineHealthTimeout is the period + after which machine is declared failed. + type: string + maxEvictRetries: + description: MaxEvictRetries are the number of eviction + retries on a pod after which drain is declared + failed, and forceful deletion is triggered. + format: int32 + type: integer + nodeConditions: + description: NodeConditions are the set of conditions + if set to true for the period of MachineHealthTimeout, + machine will be declared failed. + items: + type: string + type: array + type: object + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + MaxSurge is maximum number of machines that are created during an update. + This value is divided by the number of configured zones for a fair distribution. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of machines that can be unavailable during an update. + This value is divided by the number of configured zones for a fair distribution. + x-kubernetes-int-or-string: true + maximum: + description: |- + Maximum is the maximum number of machines to create. + This value is divided by the number of configured zones for a fair distribution. + format: int32 + type: integer + minimum: + description: |- + Minimum is the minimum number of machines to create. + This value is divided by the number of configured zones for a fair distribution. + format: int32 + type: integer + name: + description: Name is the name of the worker group. + type: string + providerConfig: + description: ProviderConfig is the provider-specific + configuration for this worker pool. + type: object + x-kubernetes-preserve-unknown-fields: true + sysctls: + additionalProperties: + type: string + description: Sysctls is a map of kernel settings to + apply on all machines in this worker pool. + type: object + systemComponents: + description: SystemComponents contains configuration + for system components related to this worker pool + properties: + allow: + description: Allow determines whether the pool should + be allowed to host system components or not (defaults + to true) + type: boolean + required: + - allow + type: object + taints: + description: Taints is a list of taints for all the + `Node` objects in this worker pool. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to + the taint key. + type: string + required: + - effect + - key + type: object + type: array + volume: + description: Volume contains information about the volume + type and size. + properties: + encrypted: + description: Encrypted determines if the volume + should be encrypted. + type: boolean + name: + description: Name of the volume to make it referencable. + type: string + size: + description: VolumeSize is the size of the volume. + type: string + type: + description: Type is the type of the volume. + type: string + required: + - size + type: object + zones: + description: |- + Zones is a list of availability zones that are used to evenly distribute this worker pool. Optional + as not every provider may support availability zones. + items: + type: string + type: array + required: + - machine + - maximum + - minimum + - name + type: object + type: array controlPlaneConfig: type: object x-kubernetes-preserve-unknown-fields: true diff --git a/pkg/gardener/shoot/extender/provider.go b/pkg/gardener/shoot/extender/provider.go index 27a5d6dc..d6f37a78 100644 --- a/pkg/gardener/shoot/extender/provider.go +++ b/pkg/gardener/shoot/extender/provider.go @@ -20,6 +20,12 @@ func NewProviderExtenderForCreateOperation(enableIMDSv2 bool, defMachineImgName, provider.Type = rt.Spec.Shoot.Provider.Type provider.Workers = rt.Spec.Shoot.Provider.Workers + if rt.Spec.Shoot.Provider.AdditionalWorkers != nil { + for _, v := range *rt.Spec.Shoot.Provider.AdditionalWorkers { + provider.Workers = append(provider.Workers, v) + } + } + var err error var controlPlaneConf, infraConfig *runtime.RawExtension zones := getZones(rt.Spec.Shoot.Provider.Workers) From c714647d2dc8525bcc8790637b489dd2ab9830cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Mon, 13 Jan 2025 16:08:44 +0100 Subject: [PATCH 02/15] simplification of the slice merge loop --- pkg/gardener/shoot/extender/provider.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider.go b/pkg/gardener/shoot/extender/provider.go index d6f37a78..e20f8eb6 100644 --- a/pkg/gardener/shoot/extender/provider.go +++ b/pkg/gardener/shoot/extender/provider.go @@ -21,9 +21,7 @@ func NewProviderExtenderForCreateOperation(enableIMDSv2 bool, defMachineImgName, provider.Workers = rt.Spec.Shoot.Provider.Workers if rt.Spec.Shoot.Provider.AdditionalWorkers != nil { - for _, v := range *rt.Spec.Shoot.Provider.AdditionalWorkers { - provider.Workers = append(provider.Workers, v) - } + provider.Workers = append(provider.Workers, *rt.Spec.Shoot.Provider.AdditionalWorkers...) } var err error From abf1e9fc7b6b8f3b0ac790d6dc14525065674c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Tue, 14 Jan 2025 11:29:30 +0100 Subject: [PATCH 03/15] Regenerating DeepCopy method --- api/v1/zz_generated.deepcopy.go | 11 +++++++++++ ...nfrastructuremanager.kyma-project.io_runtimes.yaml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index b0f6969b..7f4247fa 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -285,6 +285,17 @@ func (in *Provider) DeepCopyInto(out *Provider) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.AdditionalWorkers != nil { + in, out := &in.AdditionalWorkers, &out.AdditionalWorkers + *out = new([]v1beta1.Worker) + if **in != nil { + in, out := *in, *out + *out = make([]v1beta1.Worker, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } if in.ControlPlaneConfig != nil { in, out := &in.ControlPlaneConfig, &out.ControlPlaneConfig *out = new(runtime.RawExtension) diff --git a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml index e3960f40..eda7a80a 100644 --- a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml +++ b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.16.5 name: runtimes.infrastructuremanager.kyma-project.io spec: group: infrastructuremanager.kyma-project.io From b65501b9b7743cd8a8520ccfd170f08192736186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Fri, 17 Jan 2025 13:27:42 +0100 Subject: [PATCH 04/15] Processing multiple workers during shoot create and patch --- ...ager.kyma-project.io_gardenerclusters.yaml | 170 --- ...cturemanager.kyma-project.io_runtimes.yaml | 997 ------------------ .../runtime/fsm/runtime_fsm_patch_shoot.go | 50 +- pkg/gardener/shoot/converter.go | 24 +- pkg/gardener/shoot/extender/kubernetes.go | 6 +- pkg/gardener/shoot/extender/provider.go | 181 +++- pkg/gardener/shoot/extender/provider_test.go | 85 +- pkg/gardener/shoot/hyperscaler/aws/config.go | 9 + .../shoot/hyperscaler/azure/config.go | 9 + 9 files changed, 278 insertions(+), 1253 deletions(-) delete mode 100644 config/infrastructuremanager.kyma-project.io_gardenerclusters.yaml delete mode 100644 config/infrastructuremanager.kyma-project.io_runtimes.yaml diff --git a/config/infrastructuremanager.kyma-project.io_gardenerclusters.yaml b/config/infrastructuremanager.kyma-project.io_gardenerclusters.yaml deleted file mode 100644 index ccf23d54..00000000 --- a/config/infrastructuremanager.kyma-project.io_gardenerclusters.yaml +++ /dev/null @@ -1,170 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.12.0 - name: gardenerclusters.infrastructuremanager.kyma-project.io -spec: - group: infrastructuremanager.kyma-project.io - names: - kind: GardenerCluster - listKind: GardenerClusterList - plural: gardenerclusters - singular: gardenercluster - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.state - name: STATE - type: string - - jsonPath: .metadata.labels.kyma-project\.io/runtime-id - name: RUNTIME-ID - type: string - - jsonPath: .metadata.labels.kyma-project\.io/shoot-name - name: SHOOT-NAME - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: GardenerCluster is the Schema for the clusters API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GardenerClusterSpec defines the desired state of GardenerCluster - properties: - kubeconfig: - description: Kubeconfig defines the desired kubeconfig location - properties: - secret: - description: SecretKeyRef defines the location, and structure - of the secret containing kubeconfig - properties: - key: - type: string - name: - type: string - namespace: - type: string - required: - - key - - name - - namespace - type: object - required: - - secret - type: object - shoot: - description: Shoot defines the name of the Shoot resource - properties: - name: - type: string - required: - - name - type: object - required: - - kubeconfig - - shoot - type: object - status: - description: GardenerClusterStatus defines the observed state of GardenerCluster - properties: - conditions: - description: List of status conditions to indicate the status of a - ServiceInstance. - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - state: - description: State signifies current state of Gardener Cluster. Value - can be one of ("Ready", "Processing", "Error", "Deleting"). - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/infrastructuremanager.kyma-project.io_runtimes.yaml b/config/infrastructuremanager.kyma-project.io_runtimes.yaml deleted file mode 100644 index 6d0b5940..00000000 --- a/config/infrastructuremanager.kyma-project.io_runtimes.yaml +++ /dev/null @@ -1,997 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.12.0 - name: runtimes.infrastructuremanager.kyma-project.io -spec: - group: infrastructuremanager.kyma-project.io - names: - kind: Runtime - listKind: RuntimeList - plural: runtimes - singular: runtime - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.state - name: STATE - type: string - - jsonPath: .metadata.labels.kyma-project\.io/shoot-name - name: SHOOT-NAME - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: Runtime is the Schema for the runtimes API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: RuntimeSpec defines the desired state of Runtime - properties: - kubernetes: - properties: - kubeAPIServer: - properties: - additionalOidcConfig: - items: - description: 'OIDCConfig contains configuration settings - for the OIDC provider. Note: Descriptions were taken from - the Kubernetes documentation.' - properties: - caBundle: - description: If set, the OpenID server's certificate - will be verified by one of the authorities in the - oidc-ca-file, otherwise the host's root CA set will - be used. - type: string - clientAuthentication: - description: ClientAuthentication can optionally contain - client configuration used for kubeconfig generation. - properties: - extraConfig: - additionalProperties: - type: string - description: Extra configuration added to kubeconfig's - auth-provider. Must not be any of idp-issuer-url, - client-id, client-secret, idp-certificate-authority, - idp-certificate-authority-data, id-token or refresh-token - type: object - secret: - description: The client Secret for the OpenID Connect - client. - type: string - type: object - clientID: - description: The client ID for the OpenID Connect client, - must be set if oidc-issuer-url is set. - type: string - groupsClaim: - description: If provided, the name of a custom OpenID - Connect claim for specifying user groups. The claim - value is expected to be a string or array of strings. - This flag is experimental, please see the authentication - documentation for further details. - type: string - groupsPrefix: - description: If provided, all groups will be prefixed - with this value to prevent conflicts with other authentication - strategies. - type: string - issuerURL: - description: The URL of the OpenID issuer, only HTTPS - scheme will be accepted. If set, it will be used to - verify the OIDC JSON Web Token (JWT). - type: string - requiredClaims: - additionalProperties: - type: string - description: key=value pairs that describes a required - claim in the ID Token. If set, the claim is verified - to be present in the ID Token with a matching value. - type: object - signingAlgs: - description: List of allowed JOSE asymmetric signing - algorithms. JWTs with a 'alg' header value not in - this list will be rejected. Values are defined by - RFC 7518 https://tools.ietf.org/html/rfc7518#section-3.1 - items: - type: string - type: array - usernameClaim: - description: The OpenID claim to use as the user name. - Note that claims other than the default ('sub') is - not guaranteed to be unique and immutable. This flag - is experimental, please see the authentication documentation - for further details. (default "sub") - type: string - usernamePrefix: - description: If provided, all usernames will be prefixed - with this value. If not provided, username claims - other than 'email' are prefixed by the issuer URL - to avoid clashes. To skip any prefixing, provide the - value '-'. - type: string - type: object - type: array - oidcConfig: - description: 'OIDCConfig contains configuration settings for - the OIDC provider. Note: Descriptions were taken from the - Kubernetes documentation.' - properties: - caBundle: - description: If set, the OpenID server's certificate will - be verified by one of the authorities in the oidc-ca-file, - otherwise the host's root CA set will be used. - type: string - clientAuthentication: - description: ClientAuthentication can optionally contain - client configuration used for kubeconfig generation. - properties: - extraConfig: - additionalProperties: - type: string - description: Extra configuration added to kubeconfig's - auth-provider. Must not be any of idp-issuer-url, - client-id, client-secret, idp-certificate-authority, - idp-certificate-authority-data, id-token or refresh-token - type: object - secret: - description: The client Secret for the OpenID Connect - client. - type: string - type: object - clientID: - description: The client ID for the OpenID Connect client, - must be set if oidc-issuer-url is set. - type: string - groupsClaim: - description: If provided, the name of a custom OpenID - Connect claim for specifying user groups. The claim - value is expected to be a string or array of strings. - This flag is experimental, please see the authentication - documentation for further details. - type: string - groupsPrefix: - description: If provided, all groups will be prefixed - with this value to prevent conflicts with other authentication - strategies. - type: string - issuerURL: - description: The URL of the OpenID issuer, only HTTPS - scheme will be accepted. If set, it will be used to - verify the OIDC JSON Web Token (JWT). - type: string - requiredClaims: - additionalProperties: - type: string - description: key=value pairs that describes a required - claim in the ID Token. If set, the claim is verified - to be present in the ID Token with a matching value. - type: object - signingAlgs: - description: List of allowed JOSE asymmetric signing algorithms. - JWTs with a 'alg' header value not in this list will - be rejected. Values are defined by RFC 7518 https://tools.ietf.org/html/rfc7518#section-3.1 - items: - type: string - type: array - usernameClaim: - description: The OpenID claim to use as the user name. - Note that claims other than the default ('sub') is not - guaranteed to be unique and immutable. This flag is - experimental, please see the authentication documentation - for further details. (default "sub") - type: string - usernamePrefix: - description: If provided, all usernames will be prefixed - with this value. If not provided, username claims other - than 'email' are prefixed by the issuer URL to avoid - clashes. To skip any prefixing, provide the value '-'. - type: string - type: object - required: - - additionalOidcConfig - - oidcConfig - type: object - version: - type: string - required: - - version - type: object - name: - type: string - networking: - properties: - administrators: - items: - type: string - type: array - filiering: - properties: - egress: - properties: - enabled: - type: boolean - required: - - enabled - type: object - ingress: - properties: - enabled: - type: boolean - required: - - enabled - type: object - required: - - egress - - ingress - type: object - required: - - administrators - - filiering - type: object - provider: - properties: - region: - type: string - secretBindingName: - type: string - type: - type: string - required: - - region - - secretBindingName - - type - type: object - purpose: - type: string - workers: - items: - description: Worker is the base definition of a worker group. - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a map of key/value pairs for annotations - for all the `Node` objects in this worker pool. - type: object - caBundle: - description: CABundle is a certificate bundle which will be - installed onto every machine of this worker pool. - type: string - clusterAutoscaler: - description: ClusterAutoscaler contains the cluster autoscaler - configurations for the worker pool. - properties: - maxNodeProvisionTime: - description: MaxNodeProvisionTime defines how long CA waits - for node to be provisioned. - type: string - scaleDownGpuUtilizationThreshold: - description: ScaleDownGpuUtilizationThreshold defines the - threshold in fraction (0.0 - 1.0) of gpu resources under - which a node is being removed. - scaleDownUnneededTime: - description: ScaleDownUnneededTime defines how long a node - should be unneeded before it is eligible for scale down. - type: string - scaleDownUnreadyTime: - description: ScaleDownUnreadyTime defines how long an unready - node should be unneeded before it is eligible for scale - down. - type: string - scaleDownUtilizationThreshold: - description: ScaleDownUtilizationThreshold defines the threshold - in fraction (0.0 - 1.0) under which a node is being removed. - type: object - cri: - description: CRI contains configurations of CRI support of every - machine in the worker pool. Defaults to a CRI with name `containerd`. - properties: - containerRuntimes: - description: ContainerRuntimes is the list of the required - container runtimes supported for a worker pool. - items: - description: ContainerRuntime contains information about - worker's available container runtime - properties: - providerConfig: - description: ProviderConfig is the configuration passed - to container runtime resource. - type: object - x-kubernetes-preserve-unknown-fields: true - type: - description: Type is the type of the Container Runtime. - type: string - required: - - type - type: object - type: array - name: - description: The name of the CRI library. Supported values - are `containerd`. - type: string - required: - - name - type: object - dataVolumes: - description: DataVolumes contains a list of additional worker - volumes. - items: - description: DataVolume contains information about a data - volume. - properties: - encrypted: - description: Encrypted determines if the volume should - be encrypted. - type: boolean - name: - description: Name of the volume to make it referencable. - type: string - size: - description: VolumeSize is the size of the volume. - type: string - type: - description: Type is the type of the volume. - type: string - required: - - name - - size - type: object - type: array - kubeletDataVolumeName: - description: KubeletDataVolumeName contains the name of a dataVolume - that should be used for storing kubelet state. - type: string - kubernetes: - description: Kubernetes contains configuration for Kubernetes - components related to this worker pool. - properties: - kubelet: - description: Kubelet contains configuration settings for - all kubelets of this worker pool. If set, all `spec.kubernetes.kubelet` - settings will be overwritten for this worker pool (no - merge of settings). - properties: - containerLogMaxFiles: - description: Maximum number of container log files that - can be present for a container. - format: int32 - type: integer - containerLogMaxSize: - anyOf: - - type: integer - - type: string - description: 'A quantity defines the maximum size of - the container log file before it is rotated. For example: - "5Mi" or "256Ki". Default: 100Mi' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - cpuCFSQuota: - description: CPUCFSQuota allows you to disable/enable - CPU throttling for Pods. - type: boolean - cpuManagerPolicy: - description: 'CPUManagerPolicy allows to set alternative - CPU management policies (default: none).' - type: string - evictionHard: - description: 'EvictionHard describes a set of eviction - thresholds (e.g. memory.available<1Gi) that if met - would trigger a Pod eviction. Default: memory.available: "100Mi/1Gi/5%" - nodefs.available: "5%" nodefs.inodesFree: "5%" - imagefs.available: "5%" imagefs.inodesFree: "5%"' - properties: - imageFSAvailable: - description: ImageFSAvailable is the threshold for - the free disk space in the imagefs filesystem - (docker images and container writable layers). - type: string - imageFSInodesFree: - description: ImageFSInodesFree is the threshold - for the available inodes in the imagefs filesystem. - type: string - memoryAvailable: - description: MemoryAvailable is the threshold for - the free memory on the host server. - type: string - nodeFSAvailable: - description: NodeFSAvailable is the threshold for - the free disk space in the nodefs filesystem (docker - volumes, logs, etc). - type: string - nodeFSInodesFree: - description: NodeFSInodesFree is the threshold for - the available inodes in the nodefs filesystem. - type: string - type: object - evictionMaxPodGracePeriod: - description: 'EvictionMaxPodGracePeriod describes the - maximum allowed grace period (in seconds) to use when - terminating pods in response to a soft eviction threshold - being met. Default: 90' - format: int32 - type: integer - evictionMinimumReclaim: - description: 'EvictionMinimumReclaim configures the - amount of resources below the configured eviction - threshold that the kubelet attempts to reclaim whenever - the kubelet observes resource pressure. Default: 0 - for each resource' - properties: - imageFSAvailable: - anyOf: - - type: integer - - type: string - description: ImageFSAvailable is the threshold for - the disk space reclaim in the imagefs filesystem - (docker images and container writable layers). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - imageFSInodesFree: - anyOf: - - type: integer - - type: string - description: ImageFSInodesFree is the threshold - for the inodes reclaim in the imagefs filesystem. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - memoryAvailable: - anyOf: - - type: integer - - type: string - description: MemoryAvailable is the threshold for - the memory reclaim on the host server. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - nodeFSAvailable: - anyOf: - - type: integer - - type: string - description: NodeFSAvailable is the threshold for - the disk space reclaim in the nodefs filesystem - (docker volumes, logs, etc). - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - nodeFSInodesFree: - anyOf: - - type: integer - - type: string - description: NodeFSInodesFree is the threshold for - the inodes reclaim in the nodefs filesystem. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - evictionPressureTransitionPeriod: - description: 'EvictionPressureTransitionPeriod is the - duration for which the kubelet has to wait before - transitioning out of an eviction pressure condition. - Default: 4m0s' - type: string - evictionSoft: - description: 'EvictionSoft describes a set of eviction - thresholds (e.g. memory.available<1.5Gi) that if met - over a corresponding grace period would trigger a - Pod eviction. Default: memory.available: "200Mi/1.5Gi/10%" - nodefs.available: "10%" nodefs.inodesFree: "10%" - imagefs.available: "10%" imagefs.inodesFree: "10%"' - properties: - imageFSAvailable: - description: ImageFSAvailable is the threshold for - the free disk space in the imagefs filesystem - (docker images and container writable layers). - type: string - imageFSInodesFree: - description: ImageFSInodesFree is the threshold - for the available inodes in the imagefs filesystem. - type: string - memoryAvailable: - description: MemoryAvailable is the threshold for - the free memory on the host server. - type: string - nodeFSAvailable: - description: NodeFSAvailable is the threshold for - the free disk space in the nodefs filesystem (docker - volumes, logs, etc). - type: string - nodeFSInodesFree: - description: NodeFSInodesFree is the threshold for - the available inodes in the nodefs filesystem. - type: string - type: object - evictionSoftGracePeriod: - description: 'EvictionSoftGracePeriod describes a set - of eviction grace periods (e.g. memory.available=1m30s) - that correspond to how long a soft eviction threshold - must hold before triggering a Pod eviction. Default: - memory.available: 1m30s nodefs.available: 1m30s - nodefs.inodesFree: 1m30s imagefs.available: 1m30s - imagefs.inodesFree: 1m30s' - properties: - imageFSAvailable: - description: ImageFSAvailable is the grace period - for the ImageFSAvailable eviction threshold. - type: string - imageFSInodesFree: - description: ImageFSInodesFree is the grace period - for the ImageFSInodesFree eviction threshold. - type: string - memoryAvailable: - description: MemoryAvailable is the grace period - for the MemoryAvailable eviction threshold. - type: string - nodeFSAvailable: - description: NodeFSAvailable is the grace period - for the NodeFSAvailable eviction threshold. - type: string - nodeFSInodesFree: - description: NodeFSInodesFree is the grace period - for the NodeFSInodesFree eviction threshold. - type: string - type: object - failSwapOn: - description: FailSwapOn makes the Kubelet fail to start - if swap is enabled on the node. (default true). - type: boolean - featureGates: - additionalProperties: - type: boolean - description: FeatureGates contains information about - enabled feature gates. - type: object - imageGCHighThresholdPercent: - description: 'ImageGCHighThresholdPercent describes - the percent of the disk usage which triggers image - garbage collection. Default: 50' - format: int32 - type: integer - imageGCLowThresholdPercent: - description: 'ImageGCLowThresholdPercent describes the - percent of the disk to which garbage collection attempts - to free. Default: 40' - format: int32 - type: integer - kubeReserved: - description: 'KubeReserved is the configuration for - resources reserved for kubernetes node components - (mainly kubelet and container runtime). When updating - these values, be aware that cgroup resizes may not - succeed on active worker nodes. Look for the NodeAllocatableEnforced - event to determine if the configuration was applied. - Default: cpu=80m,memory=1Gi,pid=20k' - properties: - cpu: - anyOf: - - type: integer - - type: string - description: CPU is the reserved cpu. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - ephemeralStorage: - anyOf: - - type: integer - - type: string - description: EphemeralStorage is the reserved ephemeral-storage. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - memory: - anyOf: - - type: integer - - type: string - description: Memory is the reserved memory. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - pid: - anyOf: - - type: integer - - type: string - description: PID is the reserved process-ids. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - maxPods: - description: 'MaxPods is the maximum number of Pods - that are allowed by the Kubelet. Default: 110' - format: int32 - type: integer - memorySwap: - description: MemorySwap configures swap memory available - to container workloads. - properties: - swapBehavior: - description: 'SwapBehavior configures swap memory - available to container workloads. May be one of - {"LimitedSwap", "UnlimitedSwap"} defaults to: - LimitedSwap' - type: string - type: object - podPidsLimit: - description: PodPIDsLimit is the maximum number of process - IDs per pod allowed by the kubelet. - format: int64 - type: integer - protectKernelDefaults: - description: ProtectKernelDefaults ensures that the - kernel tunables are equal to the kubelet defaults. - Defaults to true for Kubernetes v1.26 or later. - type: boolean - registryBurst: - description: 'RegistryBurst is the maximum size of bursty - pulls, temporarily allows pulls to burst to this number, - while still not exceeding registryPullQPS. The value - must not be a negative number. Only used if registryPullQPS - is greater than 0. Default: 10' - format: int32 - type: integer - registryPullQPS: - description: 'RegistryPullQPS is the limit of registry - pulls per second. The value must not be a negative - number. Setting it to 0 means no limit. Default: 5' - format: int32 - type: integer - seccompDefault: - description: SeccompDefault enables the use of `RuntimeDefault` - as the default seccomp profile for all workloads. - This requires the corresponding SeccompDefault feature - gate to be enabled as well. This field is only available - for Kubernetes v1.25 or later. - type: boolean - serializeImagePulls: - description: 'SerializeImagePulls describes whether - the images are pulled one at a time. Default: true' - type: boolean - streamingConnectionIdleTimeout: - description: 'StreamingConnectionIdleTimeout is the - maximum time a streaming connection can be idle before - the connection is automatically closed. This field - cannot be set lower than "30s" or greater than "4h". - Default: "4h" for Kubernetes < v1.26. "5m" for Kubernetes - >= v1.26.' - type: string - systemReserved: - description: SystemReserved is the configuration for - resources reserved for system processes not managed - by kubernetes (e.g. journald). When updating these - values, be aware that cgroup resizes may not succeed - on active worker nodes. Look for the NodeAllocatableEnforced - event to determine if the configuration was applied. - properties: - cpu: - anyOf: - - type: integer - - type: string - description: CPU is the reserved cpu. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - ephemeralStorage: - anyOf: - - type: integer - - type: string - description: EphemeralStorage is the reserved ephemeral-storage. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - memory: - anyOf: - - type: integer - - type: string - description: Memory is the reserved memory. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - pid: - anyOf: - - type: integer - - type: string - description: PID is the reserved process-ids. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - version: - description: Version is the semantic Kubernetes version - to use for the Kubelet in this Worker Group. If not specified - the kubelet version is derived from the global shoot cluster - kubernetes version. version must be equal or lower than - the version of the shoot kubernetes version. Only one - minor version difference to other worker groups and global - kubernetes version is allowed. - type: string - type: object - labels: - additionalProperties: - type: string - description: Labels is a map of key/value pairs for labels for - all the `Node` objects in this worker pool. - type: object - machine: - description: Machine contains information about the machine - type and image. - properties: - architecture: - description: Architecture is CPU architecture of machines - in this worker pool. - type: string - image: - description: Image holds information about the machine image - to use for all nodes of this pool. It will default to - the latest version of the first image stated in the referenced - CloudProfile if no value has been provided. - properties: - name: - description: Name is the name of the image. - type: string - providerConfig: - description: ProviderConfig is the shoot's individual - configuration passed to an extension resource. - type: object - x-kubernetes-preserve-unknown-fields: true - version: - description: Version is the version of the shoot's image. - If version is not provided, it will be defaulted to - the latest version from the CloudProfile. - type: string - required: - - name - type: object - type: - description: Type is the machine type of the worker group. - type: string - required: - - type - type: object - machineControllerManager: - description: MachineControllerManagerSettings contains configurations - for different worker-pools. Eg. MachineDrainTimeout, MachineHealthTimeout. - properties: - machineCreationTimeout: - description: MachineCreationTimeout is the period after - which creation of the machine is declared failed. - type: string - machineDrainTimeout: - description: MachineDrainTimeout is the period after which - machine is forcefully deleted. - type: string - machineHealthTimeout: - description: MachineHealthTimeout is the period after which - machine is declared failed. - type: string - maxEvictRetries: - description: MaxEvictRetries are the number of eviction - retries on a pod after which drain is declared failed, - and forceful deletion is triggered. - format: int32 - type: integer - nodeConditions: - description: NodeConditions are the set of conditions if - set to true for the period of MachineHealthTimeout, machine - will be declared failed. - items: - type: string - type: array - type: object - maxSurge: - anyOf: - - type: integer - - type: string - description: MaxSurge is maximum number of machines that are - created during an update. This value is divided by the number - of configured zones for a fair distribution. - x-kubernetes-int-or-string: true - maxUnavailable: - anyOf: - - type: integer - - type: string - description: MaxUnavailable is the maximum number of machines - that can be unavailable during an update. This value is divided - by the number of configured zones for a fair distribution. - x-kubernetes-int-or-string: true - maximum: - description: Maximum is the maximum number of machines to create. - This value is divided by the number of configured zones for - a fair distribution. - format: int32 - type: integer - minimum: - description: Minimum is the minimum number of machines to create. - This value is divided by the number of configured zones for - a fair distribution. - format: int32 - type: integer - name: - description: Name is the name of the worker group. - type: string - providerConfig: - description: ProviderConfig is the provider-specific configuration - for this worker pool. - type: object - x-kubernetes-preserve-unknown-fields: true - sysctls: - additionalProperties: - type: string - description: Sysctls is a map of kernel settings to apply on - all machines in this worker pool. - type: object - systemComponents: - description: SystemComponents contains configuration for system - components related to this worker pool - properties: - allow: - description: Allow determines whether the pool should be - allowed to host system components or not (defaults to - true) - type: boolean - required: - - allow - type: object - taints: - description: Taints is a list of taints for all the `Node` objects - in this worker pool. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. - properties: - effect: - description: Required. The effect of the taint on pods - that do not tolerate the taint. Valid effects are NoSchedule, - PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the - taint was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - type: array - volume: - description: Volume contains information about the volume type - and size. - properties: - encrypted: - description: Encrypted determines if the volume should be - encrypted. - type: boolean - name: - description: Name of the volume to make it referencable. - type: string - size: - description: VolumeSize is the size of the volume. - type: string - type: - description: Type is the type of the volume. - type: string - required: - - size - type: object - zones: - description: Zones is a list of availability zones that are - used to evenly distribute this worker pool. Optional as not - every provider may support availability zones. - items: - type: string - type: array - required: - - machine - - maximum - - minimum - - name - type: object - type: array - required: - - kubernetes - - name - - networking - - provider - - purpose - type: object - status: - description: RuntimeStatus defines the observed state of Runtime - properties: - conditions: - description: List of status conditions to indicate the status of a - ServiceInstance. - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - state: - description: State signifies current state of Gardener Cluster. Value - can be one of ("Ready", "Processing", "Error", "Deleting"). - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go index 4b2795d8..7c95bc3b 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go @@ -3,8 +3,6 @@ package fsm import ( "context" "fmt" - "slices" - gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" gardener_shoot "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot" @@ -36,17 +34,15 @@ func sFnPatchExistingShoot(ctx context.Context, m *fsm, s *systemState) (stateFn msgFailedToConfigureAuditlogs) } - imgName, imgVersion := getImageNameAndVersion(s.shoot.Spec.Provider.Workers) - updatedShoot, err := convertPatch(&s.instance, gardener_shoot.PatchOpts{ - ConverterConfig: m.ConverterConfig, - AuditLogData: data, - Zones: getZones(s.shoot.Spec.Provider.Workers), - ShootK8SVersion: s.shoot.Spec.Kubernetes.Version, - ShootImageName: imgName, - ShootImageVersion: imgVersion, - Extensions: s.shoot.Spec.Extensions, - Resources: s.shoot.Spec.Resources, + ConverterConfig: m.ConverterConfig, + AuditLogData: data, + Workers: s.shoot.Spec.Provider.Workers, + ShootK8SVersion: s.shoot.Spec.Kubernetes.Version, + Extensions: s.shoot.Spec.Extensions, + Resources: s.shoot.Spec.Resources, + InfrastructureConfig: s.shoot.Spec.Provider.InfrastructureConfig, + ControlPlaneConfig: s.shoot.Spec.Provider.ControlPlaneConfig, }) if err != nil { @@ -90,36 +86,6 @@ func sFnPatchExistingShoot(ctx context.Context, m *fsm, s *systemState) (stateFn return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) } -func getZones(workers []gardener.Worker) []string { - var zones []string - - for _, worker := range workers { - for _, zone := range worker.Zones { - if !slices.Contains(zones, zone) { - zones = append(zones, zone) - } - } - } - - return zones -} - -func getImageNameAndVersion(workers []gardener.Worker) (string, string) { - var imageName, imageVersion string - - for _, worker := range workers { - if worker.Machine.Image != nil { - imageName = worker.Machine.Image.Name - if worker.Machine.Image.Version != nil { - imageVersion = *worker.Machine.Image.Version - } - break - } - } - - return imageName, imageVersion -} - func convertPatch(instance *imv1.Runtime, opts gardener_shoot.PatchOpts) (gardener.Shoot, error) { if err := instance.ValidateRequiredLabels(); err != nil { return gardener.Shoot{}, err diff --git a/pkg/gardener/shoot/converter.go b/pkg/gardener/shoot/converter.go index c610752f..6eb58211 100644 --- a/pkg/gardener/shoot/converter.go +++ b/pkg/gardener/shoot/converter.go @@ -2,6 +2,7 @@ package shoot import ( "fmt" + "k8s.io/apimachinery/pkg/runtime" gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -43,15 +44,20 @@ type CreateOpts struct { auditlogs.AuditLogData } +type WorkerZones struct { + Zones []string + WorkerName string +} + type PatchOpts struct { config.ConverterConfig auditlogs.AuditLogData - Zones []string - ShootK8SVersion string - ShootImageName string - ShootImageVersion string - Extensions []gardener.Extension - Resources []gardener.NamedResourceReference + ShootK8SVersion string + Workers []gardener.Worker + Extensions []gardener.Extension + Resources []gardener.NamedResourceReference + InfrastructureConfig *runtime.RawExtension + ControlPlaneConfig *runtime.RawExtension } func NewConverterCreate(opts CreateOpts) Converter { @@ -90,9 +96,9 @@ func NewConverterPatch(opts PatchOpts) Converter { opts.Provider.AWS.EnableIMDSv2, opts.MachineImage.DefaultName, opts.MachineImage.DefaultVersion, - opts.ShootImageName, - opts.ShootImageVersion, - opts.Zones)) + opts.Workers, + opts.InfrastructureConfig, + opts.ControlPlaneConfig)) extendersForPatch = append(extendersForPatch, extensions.NewExtensionsExtenderForPatch(opts.AuditLogData, opts.Extensions), diff --git a/pkg/gardener/shoot/extender/kubernetes.go b/pkg/gardener/shoot/extender/kubernetes.go index 7e102812..266f9ea3 100644 --- a/pkg/gardener/shoot/extender/kubernetes.go +++ b/pkg/gardener/shoot/extender/kubernetes.go @@ -35,13 +35,13 @@ func NewKubernetesExtender(defaultKubernetesVersion, currentKubernetesVersion st } } -func compareVersions(version1, version2 string) (int, error) { - v1, err := semver.NewVersion(version1) +func compareVersions(prevVersion, currVersion string) (int, error) { + v1, err := semver.NewVersion(prevVersion) if err != nil { return 0, err } - v2, err := semver.NewVersion(version2) + v2, err := semver.NewVersion(currVersion) if err != nil { return 0, err } diff --git a/pkg/gardener/shoot/extender/provider.go b/pkg/gardener/shoot/extender/provider.go index e20f8eb6..5394c15e 100644 --- a/pkg/gardener/shoot/extender/provider.go +++ b/pkg/gardener/shoot/extender/provider.go @@ -1,6 +1,7 @@ package extender import ( + "fmt" "slices" gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -26,7 +27,12 @@ func NewProviderExtenderForCreateOperation(enableIMDSv2 bool, defMachineImgName, var err error var controlPlaneConf, infraConfig *runtime.RawExtension - zones := getZones(rt.Spec.Shoot.Provider.Workers) + zones, err := getNetworkingZonesFromWorkers(provider.Workers) + + if err != nil { + return err + } + infraConfig, controlPlaneConf, err = getConfig(rt.Spec.Shoot, zones) if err != nil { return err @@ -43,7 +49,7 @@ func NewProviderExtenderForCreateOperation(enableIMDSv2 bool, defMachineImgName, provider.ControlPlaneConfig = controlPlaneConf provider.InfrastructureConfig = infraConfig - setMachineImage(provider, defMachineImgName, defMachineImgVer, "", "") + setMachineImage(provider, defMachineImgName, defMachineImgVer) err = setWorkerConfig(provider, provider.Type, enableIMDSv2) setWorkerSettings(provider) @@ -51,20 +57,52 @@ func NewProviderExtenderForCreateOperation(enableIMDSv2 bool, defMachineImgName, } } -func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, defMachineImgVer, currMachineImgName, currMachineImgVer string, zones []string) func(rt imv1.Runtime, shoot *gardener.Shoot) error { +// Zones for patching workes are taken from existing shoot workers +// +// Zones for patching infrastructureConfig are processed as follows: +// For Azure and AWS zones are stored in InfrastructureConfig and can be possibly updated if new workers with new zones are added +// For GCP single zone (one from defined in workers) is stored in ControlPlaneConfig and its value is immutable. +// For Openstack no zone information is stored neither in InfrastructureConfig nor in ControlPlaneConfig +// So only for Azure and AWS zone setup can be changed in InfrastructureConfig scope. +// For other providers we use existing data for patching the shoot +func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, defMachineImgVer string, shootWorkers []gardener.Worker, existingInfraConfig *runtime.RawExtension, existingControlPlaneConfig *runtime.RawExtension) func(rt imv1.Runtime, shoot *gardener.Shoot) error { return func(rt imv1.Runtime, shoot *gardener.Shoot) error { - var err error provider := &shoot.Spec.Provider provider.Type = rt.Spec.Shoot.Provider.Type provider.Workers = rt.Spec.Shoot.Provider.Workers - var controlPlaneConf, infraConfig *runtime.RawExtension + if rt.Spec.Shoot.Provider.AdditionalWorkers != nil { + provider.Workers = append(provider.Workers, *rt.Spec.Shoot.Provider.AdditionalWorkers...) + } - infraConfig, controlPlaneConf, err = getConfig(rt.Spec.Shoot, zones) + var controlPlaneConf, infraConfig *runtime.RawExtension + workerZones, err := getNetworkingZonesFromWorkers(provider.Workers) if err != nil { return err } + if rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAzure || rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAWS { + infraConfigZones, err := getZonesFromInfrastructureConfig(rt.Spec.Shoot.Provider.Type, existingInfraConfig) + if err != nil { + return err + } + // extend infrastructure zones collection if new workers are added with new zones + for _, zone := range workerZones { + if !slices.Contains(infraConfigZones, zone) { + infraConfigZones = append(infraConfigZones, zone) + } + } + + infraConfig, controlPlaneConf, err = getConfig(rt.Spec.Shoot, infraConfigZones) + if err != nil { + return err + } + + } else { + infraConfig = existingInfraConfig.DeepCopy() + controlPlaneConf = existingControlPlaneConfig.DeepCopy() + } + if rt.Spec.Shoot.Provider.ControlPlaneConfig != nil { controlPlaneConf = rt.Spec.Shoot.Provider.ControlPlaneConfig } @@ -76,16 +114,54 @@ func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, def provider.ControlPlaneConfig = controlPlaneConf provider.InfrastructureConfig = infraConfig - setMachineImage(provider, defMachineImgName, defMachineImgVer, currMachineImgName, currMachineImgVer) + setMachineImage(provider, defMachineImgName, defMachineImgVer) + err = setWorkerConfig(provider, provider.Type, enableIMDSv2) setWorkerSettings(provider) - alignWithExistingShoot(provider, zones) + alignWorkersWithShoot(provider, shootWorkers) return err } } +// parse infrastructure config to get current networking zones +func getZonesFromInfrastructureConfig(providerType string, infraConfig *runtime.RawExtension) ([]string, error) { + if infraConfig == nil { + return nil, errors.New("infrastructureConfig is nil") + } + + var zones []string + if providerType == hyperscaler.TypeAWS { + infraConfig, err := aws.DecodeInfrastructureConfig(infraConfig.Raw) + if err != nil { + return nil, err + } + for _, zone := range infraConfig.Networks.Zones { + if !slices.Contains(zones, zone.Name) { + zones = append(zones, zone.Name) + } + } + return zones, nil + } + + if providerType == hyperscaler.TypeAzure { + infraConfig, err := azure.DecodeInfrastructureConfig(infraConfig.Raw) + if err != nil { + return nil, err + } + for _, zone := range infraConfig.Networks.Zones { + name := fmt.Sprint(zone.Name) + if !slices.Contains(zones, name) { + zones = append(zones, name) + } + } + return zones, nil + } + + return []string{}, errors.New("Cannot read zones from infrastructureConfig - provider not supported") +} + type InfrastructureProviderFunc func(workersCidr string, zones []string) ([]byte, error) type ControlPlaneProviderFunc func(zones []string) ([]byte, error) @@ -136,18 +212,39 @@ func getAWSWorkerConfig() (*runtime.RawExtension, error) { return &runtime.RawExtension{Raw: workerConfigBytes}, nil } -func getZones(workers []gardener.Worker) []string { +// Get set of zones from first worker. +// All other workers should have the same zones provided in the same order. +// Otherwise fail +func getNetworkingZonesFromWorkers(workers []gardener.Worker) ([]string, error) { var zones []string + if len(workers) == 0 { + return nil, errors.New("no workers provided") + } + + for _, zone := range workers[0].Zones { + if !slices.Contains(zones, zone) { + zones = append(zones, zone) + } else { + return nil, fmt.Errorf("duplicate zone name detected for worker %s", workers[0].Name) + } + } + + if len(zones) == 0 { + return nil, fmt.Errorf("no networking zones provided for worker %s", workers[0].Name) + } + + if len(workers) == 1 { + return zones, nil + } + for _, worker := range workers { - for _, zone := range worker.Zones { - if !slices.Contains(zones, zone) { - zones = append(zones, zone) - } + if !slices.Equal(worker.Zones, zones) { + return nil, errors.New("workers have specified different zones set, or zones are in different order") } } - return zones + return zones, nil } func setWorkerConfig(provider *gardener.Provider, providerType string, enableIMDSv2 bool) error { @@ -177,8 +274,7 @@ func setWorkerSettings(provider *gardener.Provider) { // It sets the machine image name and version to the values specified in the Runtime worker configuration. // If any value is not specified in the Runtime, it sets it as `machineImage.defaultVersion` or `machineImage.defaultName`, set in `converter_config.json`. -// If the current image version with the same name on Shoot is greater than the version determined above, it sets the version to the current machine image version. -func setMachineImage(provider *gardener.Provider, defMachineImgName, defMachineImgVer, currMachineImgName, currMachineImgVer string) { +func setMachineImage(provider *gardener.Provider, defMachineImgName, defMachineImgVer string) { for i := 0; i < len(provider.Workers); i++ { worker := &provider.Workers[i] @@ -188,31 +284,54 @@ func setMachineImage(provider *gardener.Provider, defMachineImgName, defMachineI Version: &defMachineImgVer, } } - machineImageVersion := worker.Machine.Image.Version - if machineImageVersion == nil || *machineImageVersion == "" { - machineImageVersion = &defMachineImgVer + if worker.Machine.Image.Version == nil || *worker.Machine.Image.Version == "" { + worker.Machine.Image.Version = &defMachineImgVer } if worker.Machine.Image.Name == "" { worker.Machine.Image.Name = defMachineImgName } + } +} - // use current image version from shoot when it is greater than determined above - autoupdate case - if currMachineImgName == worker.Machine.Image.Name && currMachineImgVer != "" && currMachineImgVer != *machineImageVersion { - result, err := compareVersions(*machineImageVersion, currMachineImgVer) - if err == nil && result < 0 { - machineImageVersion = &currMachineImgVer +// We can't predict what will be the order of zones stored by Gardener. +// Without this patch, gardener's admission webhook might reject the request if the zones order does not match. +// If the current image version with the same name on Shoot is greater than the version, it sets the version to the current machine image version. +func alignWorkersWithShoot(provider *gardener.Provider, existingWorkers []gardener.Worker) { + for i, _ := range provider.Workers { + for _, existing := range existingWorkers { + alignedWorker := &provider.Workers[i] + + if alignedWorker.Name == existing.Name { + alignedWorker.Zones = existing.Zones + alignWorkerMachineImageVersion(alignedWorker.Machine.Image, existing.Machine.Image) + break } } - - worker.Machine.Image.Version = machineImageVersion } } -// We can't predict what will be the order of zones stored by Gardener. -// Without this patch, gardener's admission webhook might reject the request if the zones order does not match. -func alignWithExistingShoot(provider *gardener.Provider, zones []string) { - for i := range provider.Workers { - provider.Workers[i].Zones = zones +func alignWorkerMachineImageVersion(workerImage *gardener.ShootMachineImage, shootWorkerImage *gardener.ShootMachineImage) { + if shootWorkerImage == nil || workerImage == nil { + return + } + + if workerImage.Name != shootWorkerImage.Name { + return + } + + workerImgVer := workerImage.Version // both are pointers for *string + shootImgVer := shootWorkerImage.Version + + if shootImgVer == nil { + return + } + + if *shootImgVer != *workerImgVer { + result, err := compareVersions(*workerImgVer, *shootImgVer) + if err == nil && result < 0 { + currMachineImgVer := *shootImgVer + workerImgVer = &currMachineImgVer + } } } diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 48ac307a..9a24b7ed 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -212,7 +212,7 @@ func TestProviderExtender(t *testing.T) { var err error if tc.TestForPatch { - extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentMachineImageName, tc.CurrentMachineImageVersion, tc.CurrentZonesConfig) + extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) err = extender(tc.Runtime, &shoot) } else { extender := NewProviderExtenderForCreateOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) @@ -295,6 +295,7 @@ func fixAWSProviderWithMultipleWorkers() imv1.Provider { Maximum: 3, Zones: []string{ "eu-central-1a", + "eu-central-1b", "eu-central-1c", }, }, @@ -309,6 +310,7 @@ func fixAWSProviderWithMultipleWorkers() imv1.Provider { Zones: []string{ "eu-central-1a", "eu-central-1b", + "eu-central-1c", }, }, { @@ -319,6 +321,7 @@ func fixAWSProviderWithMultipleWorkers() imv1.Provider { Minimum: 1, Maximum: 3, Zones: []string{ + "eu-central-1a", "eu-central-1b", "eu-central-1c", }, @@ -327,6 +330,86 @@ func fixAWSProviderWithMultipleWorkers() imv1.Provider { } } +func TestGetAllWorkersZones(t *testing.T) { + tests := []struct { + name string + workers []gardener.Worker + expected []string + wantErr bool + }{ + { + name: "Single worker with zones", + workers: []gardener.Worker{ + { + Name: "worker1", + Zones: []string{"zone1", "zone2"}, + }, + }, + expected: []string{"zone1", "zone2"}, + wantErr: false, + }, + { + name: "Multiple workers with same zones", + workers: []gardener.Worker{ + { + Name: "worker1", + Zones: []string{"zone1", "zone2"}, + }, + { + Name: "worker2", + Zones: []string{"zone1", "zone2"}, + }, + }, + expected: []string{"zone1", "zone2"}, + wantErr: false, + }, + { + name: "Multiple workers with different zones", + workers: []gardener.Worker{ + { + Name: "worker1", + Zones: []string{"zone1", "zone2"}, + }, + { + Name: "worker2", + Zones: []string{"zone1", "zone3"}, + }, + }, + expected: nil, + wantErr: true, + }, + { + name: "No workers provided", + workers: []gardener.Worker{}, + expected: nil, + wantErr: true, + }, + { + name: "Duplicate zones in a single worker", + workers: []gardener.Worker{ + { + Name: "worker1", + Zones: []string{"zone1", "zone1"}, + }, + }, + expected: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + zones, err := getNetworkingZonesFromWorkers(tt.workers) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expected, zones) + } + }) + } +} + func assertProvider(t *testing.T, runtimeShoot imv1.RuntimeShoot, shoot gardener.Shoot, expectWorkerConfig bool, expectedMachineImageName, expectedMachineImageVersion string) { assert.Equal(t, runtimeShoot.Provider.Type, shoot.Spec.Provider.Type) assert.Equal(t, runtimeShoot.Provider.Workers, shoot.Spec.Provider.Workers) diff --git a/pkg/gardener/shoot/hyperscaler/aws/config.go b/pkg/gardener/shoot/hyperscaler/aws/config.go index c3a96746..07f360f9 100644 --- a/pkg/gardener/shoot/hyperscaler/aws/config.go +++ b/pkg/gardener/shoot/hyperscaler/aws/config.go @@ -27,6 +27,15 @@ func GetWorkerConfig() ([]byte, error) { return json.Marshal(NewWorkerConfig()) } +func DecodeInfrastructureConfig(data []byte) (*v1alpha1.InfrastructureConfig, error) { + infrastructureConfig := &v1alpha1.InfrastructureConfig{} + err := json.Unmarshal(data, infrastructureConfig) + if err != nil { + return nil, err + } + return infrastructureConfig, nil +} + func NewInfrastructureConfig(workersCidr string, zones []string) v1alpha1.InfrastructureConfig { return v1alpha1.InfrastructureConfig{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/gardener/shoot/hyperscaler/azure/config.go b/pkg/gardener/shoot/hyperscaler/azure/config.go index 1a8bad92..6755f95b 100644 --- a/pkg/gardener/shoot/hyperscaler/azure/config.go +++ b/pkg/gardener/shoot/hyperscaler/azure/config.go @@ -27,6 +27,15 @@ func NewControlPlaneConfig() *ControlPlaneConfig { } } +func DecodeInfrastructureConfig(data []byte) (*InfrastructureConfig, error) { + infrastructureConfig := &InfrastructureConfig{} + err := json.Unmarshal(data, infrastructureConfig) + if err != nil { + return nil, err + } + return infrastructureConfig, nil +} + func NewInfrastructureConfig(workerCIDR string, zones []string) InfrastructureConfig { // All Azure shoots are zoned. // No zones - the shoot configuration is invalid. From e880e86426cf83cd0078c0dc1d7da263ca1a5062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Fri, 17 Jan 2025 19:19:45 +0100 Subject: [PATCH 05/15] Optimisation and temporary disable some tests to make it compile --- pkg/gardener/shoot/converter_test.go | 4 +- pkg/gardener/shoot/extender/provider.go | 140 +++++++++---------- pkg/gardener/shoot/extender/provider_test.go | 4 +- 3 files changed, 72 insertions(+), 76 deletions(-) diff --git a/pkg/gardener/shoot/converter_test.go b/pkg/gardener/shoot/converter_test.go index e97014eb..3e07817d 100644 --- a/pkg/gardener/shoot/converter_test.go +++ b/pkg/gardener/shoot/converter_test.go @@ -102,7 +102,7 @@ func TestConverter(t *testing.T) { require.Equalf(t, 5, extensionLen, "unexpected number of extensions: %d, expected: 5", extensionLen) }) - t.Run("Create shoot from Runtime for existing shoot and keep versions", func(t *testing.T) { + /*t.Run("Create shoot from Runtime for existing shoot and keep versions", func(t *testing.T) { // given runtime := fixRuntime() converterConfig := fixConverterConfig() @@ -185,7 +185,7 @@ func TestConverter(t *testing.T) { extensionLen := len(shoot.Spec.Extensions) require.Equalf(t, extensionLen, 5, "unexpected number of extensions: %d, expected: 5", extensionLen) - }) + }) */ } func assertShootFields(t *testing.T, runtime imv1.Runtime, shoot gardener.Shoot) { diff --git a/pkg/gardener/shoot/extender/provider.go b/pkg/gardener/shoot/extender/provider.go index 5394c15e..b21a2d54 100644 --- a/pkg/gardener/shoot/extender/provider.go +++ b/pkg/gardener/shoot/extender/provider.go @@ -75,40 +75,14 @@ func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, def provider.Workers = append(provider.Workers, *rt.Spec.Shoot.Provider.AdditionalWorkers...) } - var controlPlaneConf, infraConfig *runtime.RawExtension workerZones, err := getNetworkingZonesFromWorkers(provider.Workers) if err != nil { return err } - if rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAzure || rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAWS { - infraConfigZones, err := getZonesFromInfrastructureConfig(rt.Spec.Shoot.Provider.Type, existingInfraConfig) - if err != nil { - return err - } - // extend infrastructure zones collection if new workers are added with new zones - for _, zone := range workerZones { - if !slices.Contains(infraConfigZones, zone) { - infraConfigZones = append(infraConfigZones, zone) - } - } - - infraConfig, controlPlaneConf, err = getConfig(rt.Spec.Shoot, infraConfigZones) - if err != nil { - return err - } - - } else { - infraConfig = existingInfraConfig.DeepCopy() - controlPlaneConf = existingControlPlaneConfig.DeepCopy() - } - - if rt.Spec.Shoot.Provider.ControlPlaneConfig != nil { - controlPlaneConf = rt.Spec.Shoot.Provider.ControlPlaneConfig - } - - if rt.Spec.Shoot.Provider.InfrastructureConfig != nil { - infraConfig = rt.Spec.Shoot.Provider.InfrastructureConfig + controlPlaneConf, infraConfig, err := getProviderConfigsForPatch(rt, workerZones, existingInfraConfig, existingControlPlaneConfig) + if err != nil { + return err } provider.ControlPlaneConfig = controlPlaneConf @@ -116,50 +90,81 @@ func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, def setMachineImage(provider, defMachineImgName, defMachineImgVer) - err = setWorkerConfig(provider, provider.Type, enableIMDSv2) - setWorkerSettings(provider) + if err := setWorkerConfig(provider, provider.Type, enableIMDSv2); err != nil { + return err + } + setWorkerSettings(provider) alignWorkersWithShoot(provider, shootWorkers) - return err + return nil } } -// parse infrastructure config to get current networking zones +func getProviderConfigsForPatch(rt imv1.Runtime, workerZones []string, existingInfraConfig, existingControlPlaneConfig *runtime.RawExtension) (*runtime.RawExtension, *runtime.RawExtension, error) { + var controlPlaneConf, infraConfig *runtime.RawExtension + + if rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAzure || rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAWS { + infraConfigZones, err := getZonesFromInfrastructureConfig(rt.Spec.Shoot.Provider.Type, existingInfraConfig) + if err != nil { + return nil, nil, err + } + // extend infrastructure zones collection if new workers are added with new zones + for _, zone := range workerZones { + if !slices.Contains(infraConfigZones, zone) { + infraConfigZones = append(infraConfigZones, zone) + } + } + + infraConfig, controlPlaneConf, err = getConfig(rt.Spec.Shoot, infraConfigZones) + if err != nil { + return nil, nil, err + } + } else { + infraConfig = existingInfraConfig + controlPlaneConf = existingControlPlaneConfig + } + + if rt.Spec.Shoot.Provider.ControlPlaneConfig != nil { + controlPlaneConf = rt.Spec.Shoot.Provider.ControlPlaneConfig + } + + if rt.Spec.Shoot.Provider.InfrastructureConfig != nil { + infraConfig = rt.Spec.Shoot.Provider.InfrastructureConfig + } + + return controlPlaneConf, infraConfig, nil +} + +// parse infrastructure config to get current set of networking zones func getZonesFromInfrastructureConfig(providerType string, infraConfig *runtime.RawExtension) ([]string, error) { if infraConfig == nil { return nil, errors.New("infrastructureConfig is nil") } var zones []string - if providerType == hyperscaler.TypeAWS { + + switch providerType { + case hyperscaler.TypeAWS: infraConfig, err := aws.DecodeInfrastructureConfig(infraConfig.Raw) if err != nil { return nil, err } for _, zone := range infraConfig.Networks.Zones { - if !slices.Contains(zones, zone.Name) { - zones = append(zones, zone.Name) - } + zones = append(zones, zone.Name) } - return zones, nil - } - - if providerType == hyperscaler.TypeAzure { + case hyperscaler.TypeAzure: infraConfig, err := azure.DecodeInfrastructureConfig(infraConfig.Raw) if err != nil { return nil, err } for _, zone := range infraConfig.Networks.Zones { - name := fmt.Sprint(zone.Name) - if !slices.Contains(zones, name) { - zones = append(zones, name) - } + zones = append(zones, fmt.Sprint(zone.Name)) } - return zones, nil + default: + return nil, errors.New("read zones from infrastructureConfig - provider not supported") } - - return []string{}, errors.New("Cannot read zones from infrastructureConfig - provider not supported") + return zones, nil } type InfrastructureProviderFunc func(workersCidr string, zones []string) ([]byte, error) @@ -298,40 +303,31 @@ func setMachineImage(provider *gardener.Provider, defMachineImgName, defMachineI // Without this patch, gardener's admission webhook might reject the request if the zones order does not match. // If the current image version with the same name on Shoot is greater than the version, it sets the version to the current machine image version. func alignWorkersWithShoot(provider *gardener.Provider, existingWorkers []gardener.Worker) { - for i, _ := range provider.Workers { - for _, existing := range existingWorkers { - alignedWorker := &provider.Workers[i] - - if alignedWorker.Name == existing.Name { - alignedWorker.Zones = existing.Zones - alignWorkerMachineImageVersion(alignedWorker.Machine.Image, existing.Machine.Image) - break - } + existingWorkersMap := make(map[string]gardener.Worker) + for _, existing := range existingWorkers { + existingWorkersMap[existing.Name] = existing + } + + for i := range provider.Workers { + alignedWorker := &provider.Workers[i] + + if existing, found := existingWorkersMap[alignedWorker.Name]; found { + alignedWorker.Zones = existing.Zones + alignWorkerMachineImageVersion(alignedWorker.Machine.Image, existing.Machine.Image) } } } func alignWorkerMachineImageVersion(workerImage *gardener.ShootMachineImage, shootWorkerImage *gardener.ShootMachineImage) { - if shootWorkerImage == nil || workerImage == nil { + if shootWorkerImage == nil || workerImage == nil || workerImage.Name != shootWorkerImage.Name { return } - if workerImage.Name != shootWorkerImage.Name { + if shootWorkerImage.Version == nil || *shootWorkerImage.Version == *workerImage.Version { return } - workerImgVer := workerImage.Version // both are pointers for *string - shootImgVer := shootWorkerImage.Version - - if shootImgVer == nil { - return - } - - if *shootImgVer != *workerImgVer { - result, err := compareVersions(*workerImgVer, *shootImgVer) - if err == nil && result < 0 { - currMachineImgVer := *shootImgVer - workerImgVer = &currMachineImgVer - } + if result, err := compareVersions(*workerImage.Version, *shootWorkerImage.Version); err == nil && result < 0 { + workerImage.Version = shootWorkerImage.Version } } diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 9a24b7ed..01cb7b06 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -13,7 +13,7 @@ import ( ) func TestProviderExtender(t *testing.T) { - for tname, tc := range map[string]struct { + /*for tname, tc := range map[string]struct { Runtime imv1.Runtime EnableIMDSv2 bool DefaultMachineImageVersion string @@ -225,7 +225,7 @@ func TestProviderExtender(t *testing.T) { assertProvider(t, tc.Runtime.Spec.Shoot, shoot, tc.EnableIMDSv2, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) assertProviderSpecificConfig(t, shoot, tc.ExpectedZonesCount) }) - } + }*/ t.Run("Return error for unknown provider", func(t *testing.T) { // given From fb44bc9bfdc631d4b680109741a5b67f553abd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sat, 18 Jan 2025 09:48:43 +0100 Subject: [PATCH 06/15] Unit tests update for provider extender - part 1 --- pkg/gardener/shoot/extender/provider_test.go | 108 +++++++++++++++++-- 1 file changed, 101 insertions(+), 7 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 01cb7b06..db7ea04a 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -12,7 +12,105 @@ import ( "github.com/stretchr/testify/require" ) -func TestProviderExtender(t *testing.T) { +func TestProviderExtenderForCreateAWS(t *testing.T) { + + // tests of ProviderExtenderForCreateOperation for AWS provider + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + EnableIMDSv2 bool + DefaultMachineImageVersion string + ExpectedMachineImageVersion string + DefaultMachineImageName string + ExpectedMachineImageName string + CurrentZonesConfig []string + ExpectedZonesCount int + }{ + "Create provider specific config for AWS without worker config and one zone": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixAWSProvider("gardenlinux", "1312.2.0", []string{"eu-central-1a"}), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageVersion: "1312.3.0", + ExpectedMachineImageVersion: "1312.2.0", + ExpectedMachineImageName: "gardenlinux", + ExpectedZonesCount: 1, + }, + "Create provider specific config for AWS without worker config and two zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixAWSProvider("gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b"}), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageVersion: "1312.3.0", + ExpectedMachineImageVersion: "1312.2.0", + ExpectedMachineImageName: "gardenlinux", + ExpectedZonesCount: 2, + }, + "Create provider specific config for AWS without worker config and three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixAWSProvider("gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageVersion: "1312.3.0", + ExpectedMachineImageVersion: "1312.2.0", + ExpectedMachineImageName: "gardenlinux", + ExpectedZonesCount: 3, + }, + "Create provider specific config for AWS with worker config and three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixAWSProvider("", "", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + }, + }, + }, + EnableIMDSv2: true, + DefaultMachineImageVersion: "1312.3.0", + ExpectedMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + }, + "Create provider specific config for AWS with multiple workers - create option": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixAWSProviderWithMultipleWorkers(), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageVersion: "1312.3.0", + ExpectedMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + + extender := NewProviderExtenderForCreateOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProvider(t, tc.Runtime.Spec.Shoot, shoot, tc.EnableIMDSv2, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) + assertProviderSpecificConfig(t, shoot, tc.ExpectedZonesCount) + }) + } + /*for tname, tc := range map[string]struct { Runtime imv1.Runtime EnableIMDSv2 bool @@ -249,7 +347,7 @@ func TestProviderExtender(t *testing.T) { }) } -func fixAWSProvider(machineImageName, machineImageVersion string) imv1.Provider { +func fixAWSProvider(machineImageName, machineImageVersion string, zones []string) imv1.Provider { return imv1.Provider{ Type: hyperscaler.TypeAWS, Workers: []gardener.Worker{ @@ -261,11 +359,7 @@ func fixAWSProvider(machineImageName, machineImageVersion string) imv1.Provider }, Minimum: 1, Maximum: 3, - Zones: []string{ - "eu-central-1a", - "eu-central-1b", - "eu-central-1c", - }, + Zones: zones, }, }, } From 2936d3f8673b4b3b88195e66b5a8fd1d5cc1deee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sat, 18 Jan 2025 19:35:33 +0100 Subject: [PATCH 07/15] Unit tests update for provider extender - part 2 --- pkg/gardener/shoot/extender/provider_test.go | 187 +++++++++++++++---- 1 file changed, 151 insertions(+), 36 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index db7ea04a..6c112666 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -2,6 +2,8 @@ package extender import ( "encoding/json" + "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" + "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/azure" "testing" "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1" @@ -13,7 +15,6 @@ import ( ) func TestProviderExtenderForCreateAWS(t *testing.T) { - // tests of ProviderExtenderForCreateOperation for AWS provider for tname, tc := range map[string]struct { Runtime imv1.Runtime @@ -29,7 +30,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0", []string{"eu-central-1a"}), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a"}), }, }, }, @@ -43,7 +44,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b"}), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b"}), }, }, }, @@ -57,7 +58,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, @@ -71,7 +72,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("", "", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + Provider: fixProvider(hyperscaler.TypeAWS, "", "", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, @@ -84,7 +85,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProviderWithMultipleWorkers(), + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, @@ -107,7 +108,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { require.NoError(t, err) assertProvider(t, tc.Runtime.Spec.Shoot, shoot, tc.EnableIMDSv2, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) - assertProviderSpecificConfig(t, shoot, tc.ExpectedZonesCount) + assertProviderSpecificConfigAWS(t, shoot, tc.ExpectedZonesCount) }) } @@ -128,7 +129,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider("gardenlinux", "1312.2.0"), }, }, }, @@ -143,7 +144,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("", ""), + Provider: fixProvider("", ""), }, }, }, @@ -171,7 +172,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider("gardenlinux", "1312.2.0"), }, }, }, @@ -190,7 +191,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider("gardenlinux", "1312.2.0"), }, }, }, @@ -209,7 +210,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("", ""), + Provider: fixProvider("", ""), }, }, }, @@ -228,7 +229,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider("gardenlinux", "1312.2.0"), }, }, }, @@ -247,7 +248,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("", ""), + Provider: fixProvider("", ""), }, }, }, @@ -266,7 +267,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider("gardenlinux", "1312.2.0"), }, }, }, @@ -285,7 +286,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider("gardenlinux", "1312.2.0"), }, }, }, @@ -347,9 +348,117 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { }) } -func fixAWSProvider(machineImageName, machineImageVersion string, zones []string) imv1.Provider { +func TestProviderExtenderForCreateAzure(t *testing.T) { + // tests of ProviderExtenderForCreateOperation for Azure provider + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + ExpectedMachineImageVersion string + DefaultMachineImageName string + ExpectedMachineImageName string + CurrentZonesConfig []string + ExpectedZonesCount int + }{ + "Create provider specific config for Azure without worker config and one zone": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeAzure, "ubuntu", "18.04", []string{"1"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04", + ExpectedMachineImageName: "ubuntu", + ExpectedZonesCount: 1, + }, + "Create provider specific config for Azure without worker config and two zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeAzure, "ubuntu", "18.04", []string{"1", "2"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04", + ExpectedMachineImageName: "ubuntu", + ExpectedZonesCount: 2, + }, + "Create provider specific config for Azure without worker config and three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeAzure, "ubuntu", "18.04", []string{"1", "2", "3"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04", + ExpectedMachineImageName: "ubuntu", + ExpectedZonesCount: 3, + }, + "Create provider specific config for Azure with worker config and three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeAzure, "", "", []string{"1", "2", "3"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04-LTS", + ExpectedZonesCount: 3, + }, + "Create provider specific config for Azure with multiple workers - create option": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAzure, []string{"1", "2", "3"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04-LTS", + ExpectedZonesCount: 3, + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + + extender := NewProviderExtenderForCreateOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProvider(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) + assertProviderSpecificConfigAzure(t, shoot, tc.ExpectedZonesCount) + }) + } +} + +func fixProvider(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { return imv1.Provider{ - Type: hyperscaler.TypeAWS, + Type: providerType, Workers: []gardener.Worker{ { Name: "worker", @@ -376,9 +485,9 @@ func fixMachineImage(machineImageName, machineImageVersion string) *gardener.Sho return &gardener.ShootMachineImage{} } -func fixAWSProviderWithMultipleWorkers() imv1.Provider { +func fixProviderWithMultipleWorkers(providerType string, zones []string) imv1.Provider { return imv1.Provider{ - Type: hyperscaler.TypeAWS, + Type: providerType, Workers: []gardener.Worker{ { Name: "worker", @@ -387,11 +496,7 @@ func fixAWSProviderWithMultipleWorkers() imv1.Provider { }, Minimum: 1, Maximum: 3, - Zones: []string{ - "eu-central-1a", - "eu-central-1b", - "eu-central-1c", - }, + Zones: zones, }, { Name: "worker", @@ -401,11 +506,7 @@ func fixAWSProviderWithMultipleWorkers() imv1.Provider { }, Minimum: 1, Maximum: 3, - Zones: []string{ - "eu-central-1a", - "eu-central-1b", - "eu-central-1c", - }, + Zones: zones, }, { Name: "worker", @@ -414,11 +515,7 @@ func fixAWSProviderWithMultipleWorkers() imv1.Provider { }, Minimum: 1, Maximum: 3, - Zones: []string{ - "eu-central-1a", - "eu-central-1b", - "eu-central-1c", - }, + Zones: zones, }, }, } @@ -525,7 +622,7 @@ func assertProvider(t *testing.T, runtimeShoot imv1.RuntimeShoot, shoot gardener } } -func assertProviderSpecificConfig(t *testing.T, shoot gardener.Shoot, expectedZonesCount int) { +func assertProviderSpecificConfigAWS(t *testing.T, shoot gardener.Shoot, expectedZonesCount int) { var infrastructureConfig v1alpha1.InfrastructureConfig err := json.Unmarshal(shoot.Spec.Provider.InfrastructureConfig.Raw, &infrastructureConfig) @@ -533,3 +630,21 @@ func assertProviderSpecificConfig(t *testing.T, shoot gardener.Shoot, expectedZo assert.Equal(t, expectedZonesCount, len(infrastructureConfig.Networks.Zones)) } + +func assertProviderSpecificConfigAzure(t *testing.T, shoot gardener.Shoot, expectedZonesCount int) { + var infrastructureConfig azure.InfrastructureConfig + + err := json.Unmarshal(shoot.Spec.Provider.InfrastructureConfig.Raw, &infrastructureConfig) + require.NoError(t, err) + + assert.Equal(t, expectedZonesCount, len(infrastructureConfig.Networks.Zones)) +} + +func assertProviderSpecificConfigGCP(t *testing.T, shoot gardener.Shoot, expectedZonesCount int) { + var infrastructureConfig gcp.InfrastructureConfig + + err := json.Unmarshal(shoot.Spec.Provider.InfrastructureConfig.Raw, &infrastructureConfig) + require.NoError(t, err) + + // validate the networking cidr here +} From d3606defdfd99b68cac6209368822f59761db1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sun, 19 Jan 2025 10:12:51 +0100 Subject: [PATCH 08/15] Unit tests update for provider extender - part 2 --- pkg/gardener/shoot/converter_test.go | 69 +++-- pkg/gardener/shoot/extender/provider_test.go | 269 ++++++++++--------- 2 files changed, 192 insertions(+), 146 deletions(-) diff --git a/pkg/gardener/shoot/converter_test.go b/pkg/gardener/shoot/converter_test.go index 3e07817d..10ca51c5 100644 --- a/pkg/gardener/shoot/converter_test.go +++ b/pkg/gardener/shoot/converter_test.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/extensions" + "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/aws" "io" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" @@ -102,7 +103,7 @@ func TestConverter(t *testing.T) { require.Equalf(t, 5, extensionLen, "unexpected number of extensions: %d, expected: 5", extensionLen) }) - /*t.Run("Create shoot from Runtime for existing shoot and keep versions", func(t *testing.T) { + t.Run("Create shoot from Runtime for existing shoot and keep versions if Shoot has bigger versions then Runtime", func(t *testing.T) { // given runtime := fixRuntime() converterConfig := fixConverterConfig() @@ -114,13 +115,13 @@ func TestConverter(t *testing.T) { } converter := NewConverterPatch(PatchOpts{ - ConverterConfig: converterConfig, - Zones: fixReversedZones(), - ShootK8SVersion: "1.30", - ShootImageName: "gardenlinux", - ShootImageVersion: "1592.2.0", - Extensions: fixAllExtensionsOnTheShoot(), - AuditLogData: auditLogData, + ConverterConfig: converterConfig, + Workers: fixWorkersWithReversedZones("gardenlinux", "1592.2.0"), + ShootK8SVersion: "1.30", + Extensions: fixAllExtensionsOnTheShoot(), + AuditLogData: auditLogData, + InfrastructureConfig: fixAWSInfrastructureConfig("10.250.0.0/16", []string{"eu-central-1c", "eu-central-1b", "eu-central-1a"}), + ControlPlaneConfig: fixAWSControlPlaneConfig(), }) // when @@ -145,7 +146,7 @@ func TestConverter(t *testing.T) { require.Equalf(t, extensionLen, 5, "unexpected number of extensions: %d, expected: 5", extensionLen) }) - t.Run("Create shoot from Runtime for existing shoot and update versions", func(t *testing.T) { + t.Run("Create shoot from Runtime for existing shoot and update versions if Shoot has lesser versions then Runtime", func(t *testing.T) { // given runtime := fixRuntime() converterConfig := fixConverterConfig() @@ -156,13 +157,13 @@ func TestConverter(t *testing.T) { } converter := NewConverterPatch(PatchOpts{ - ConverterConfig: converterConfig, - Zones: fixReversedZones(), - ShootK8SVersion: "1.27", - ShootImageName: "gardenlinux", - ShootImageVersion: "1591.0.0", - Extensions: fixAllExtensionsOnTheShoot(), - AuditLogData: auditLogData, + ConverterConfig: converterConfig, + Workers: fixWorkersWithReversedZones("gardenlinux", "1591.0.0"), + ShootK8SVersion: "1.27", + Extensions: fixAllExtensionsOnTheShoot(), + AuditLogData: auditLogData, + InfrastructureConfig: fixAWSInfrastructureConfig("10.250.0.0/16", []string{"eu-central-1c", "eu-central-1b", "eu-central-1a"}), + ControlPlaneConfig: fixAWSControlPlaneConfig(), }) // when @@ -185,7 +186,7 @@ func TestConverter(t *testing.T) { extensionLen := len(shoot.Spec.Extensions) require.Equalf(t, extensionLen, 5, "unexpected number of extensions: %d, expected: 5", extensionLen) - }) */ + }) } func assertShootFields(t *testing.T, runtime imv1.Runtime, shoot gardener.Shoot) { @@ -200,11 +201,25 @@ func assertShootFields(t *testing.T, runtime imv1.Runtime, shoot gardener.Shoot) assert.Equal(t, "core.gardener.cloud/v1beta1", shoot.TypeMeta.APIVersion) } -func fixReversedZones() []string { - return []string{ - "eu-central-1c", - "eu-central-1b", - "eu-central-1a", +func fixWorkersWithReversedZones(machineImageName, machineImageVersion string) []gardener.Worker { + return []gardener.Worker{ + { + Name: "worker", + Machine: gardener.Machine{ + Type: "m6i.large", + Image: &gardener.ShootMachineImage{ + Name: machineImageName, + Version: &machineImageVersion, + }, + }, + Minimum: 1, + Maximum: 3, + Zones: []string{ + "eu-central-1c", + "eu-central-1b", + "eu-central-1a", + }, + }, } } @@ -263,6 +278,16 @@ func fixAllExtensionsOnTheShoot() []gardener.Extension { } } +func fixAWSInfrastructureConfig(workersCIDR string, zones []string) *runtime.RawExtension { + infraConfig, _ := aws.GetInfrastructureConfig(workersCIDR, zones) + return &runtime.RawExtension{Raw: infraConfig} +} + +func fixAWSControlPlaneConfig() *runtime.RawExtension { + controlPlaneConfig, _ := aws.GetControlPlaneConfig([]string{}) + return &runtime.RawExtension{Raw: controlPlaneConfig} +} + func fixRuntime() imv1.Runtime { kubernetesVersion := "1.28" clientID := "client-id" diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 6c112666..a92d29c1 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -3,7 +3,10 @@ package extender import ( "encoding/json" "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" + aws "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/aws" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/azure" + "k8s.io/apimachinery/pkg/runtime" + "testing" "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1" @@ -112,240 +115,188 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { }) } - /*for tname, tc := range map[string]struct { + t.Run("Return error for unknown provider", func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + runtime := imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: imv1.Provider{ + Type: "unknown", + }, + }, + }, + } + + // when + extender := NewProviderExtenderForCreateOperation(false, "", "") + err := extender(runtime, &shoot) + + // then + require.Error(t, err) + }) +} + +func TestProviderExtenderForPatchAWS(t *testing.T) { + // tests of NewProviderExtenderPatchOperation for AWS only for provider image version patching + for tname, tc := range map[string]struct { Runtime imv1.Runtime EnableIMDSv2 bool DefaultMachineImageVersion string - CurrentMachineImageVersion string ExpectedMachineImageVersion string DefaultMachineImageName string - CurrentMachineImageName string ExpectedMachineImageName string - CurrentZonesConfig []string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkersCount int ExpectedZonesCount int - TestForPatch bool }{ - "Create provider specific config for AWS without worker config - create option": { - Runtime: imv1.Runtime{ - Spec: imv1.RuntimeSpec{ - Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("gardenlinux", "1312.2.0"), - }, - }, - }, - EnableIMDSv2: false, - DefaultMachineImageVersion: "1312.3.0", - ExpectedMachineImageVersion: "1312.2.0", - ExpectedMachineImageName: "gardenlinux", - ExpectedZonesCount: 3, - TestForPatch: false, - }, - "Create provider specific config for AWS with worker config - create option": { - Runtime: imv1.Runtime{ - Spec: imv1.RuntimeSpec{ - Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("", ""), - }, - }, - }, - EnableIMDSv2: true, - DefaultMachineImageVersion: "1312.3.0", - ExpectedMachineImageVersion: "1312.3.0", - ExpectedZonesCount: 3, - TestForPatch: false, - }, - "Create provider specific config for AWS with multiple workers - create option": { - Runtime: imv1.Runtime{ - Spec: imv1.RuntimeSpec{ - Shoot: imv1.RuntimeShoot{ - Provider: fixAWSProviderWithMultipleWorkers(), - }, - }, - }, - EnableIMDSv2: false, - DefaultMachineImageVersion: "1312.3.0", - ExpectedMachineImageVersion: "1312.3.0", - ExpectedZonesCount: 3, - TestForPatch: false, - }, - "Patch option same image name - use bigger current shoot machine image version as image version": { + "Same image name - use bigger current shoot machine image version as image version": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, }, }, }, EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentMachineImageName: "gardenlinux", - CurrentMachineImageVersion: "1312.4.0", ExpectedMachineImageVersion: "1312.4.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, - TestForPatch: true, + CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - "Patch option same image name - override current shoot machine image version with new bigger version from RuntimeCR": { + "Same image name - override current shoot machine image version with new bigger version from RuntimeCR": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentMachineImageName: "gardenlinux", - CurrentMachineImageVersion: "1312.1.0", + CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.2.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, - TestForPatch: true, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - "Patch option same image name - override current shoot machine image version with default version": { + "Same image name - no version is provided override current shoot machine image version with default version": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("", ""), + Provider: fixProvider(hyperscaler.TypeAWS, "", "", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentMachineImageName: "gardenlinux", - CurrentMachineImageVersion: "1312.1.0", + CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, - TestForPatch: true, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - "Patch option different image name - override current shoot machine image and version with new data from RuntimeCR": { + "Different image name - override current shoot machine image and version with new data from RuntimeCR": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentMachineImageName: "ubuntu", - CurrentMachineImageVersion: "1312.4.0", - ExpectedMachineImageVersion: "1312.2.0", + CurrentShootWorkers: fixWorkers("m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, - TestForPatch: true, + ExpectedMachineImageVersion: "1312.2.0", + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - "Patch option different image name - override current shoot machine image and version with default data": { + "Different image name - no data is provided override current shoot machine image and version with default data": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("", ""), + Provider: fixProvider(hyperscaler.TypeAWS, "", "", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentMachineImageName: "ubuntu", - CurrentMachineImageVersion: "1312.4.0", + CurrentShootWorkers: fixWorkers("m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, - TestForPatch: true, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - "Patch option wrong current image name - use data from RuntimeCR": { + "Wrong current image name - use data from RuntimeCR": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentMachineImageName: "", - CurrentMachineImageVersion: "1312.4.0", + CurrentShootWorkers: fixWorkers("m6i.large", "", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.2.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, - TestForPatch: true, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - "Patch option wrong current image version - use data from RuntimeCR": { + "Wrong current image version - use data from RuntimeCR": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider("gardenlinux", "1312.2.0"), + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), }, }, }, EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentMachineImageName: "gardenlinux", - CurrentMachineImageVersion: "", + CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.2.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, - TestForPatch: true, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - // "Patch option different image name - override image name and version with current image name and version": {}, } { t.Run(tname, func(t *testing.T) { // given shoot := fixEmptyGardenerShoot("cluster", "kcp-system") // when - - var err error - if tc.TestForPatch { - extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) - err = extender(tc.Runtime, &shoot) - } else { - extender := NewProviderExtenderForCreateOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) - err = extender(tc.Runtime, &shoot) - } + extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingControlPlaneConfig, tc.ExistingInfraConfig) + err := extender(tc.Runtime, &shoot) // then require.NoError(t, err) assertProvider(t, tc.Runtime.Spec.Shoot, shoot, tc.EnableIMDSv2, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) - assertProviderSpecificConfig(t, shoot, tc.ExpectedZonesCount) + assertProviderSpecificConfigAWS(t, shoot, tc.ExpectedZonesCount) }) - }*/ - - t.Run("Return error for unknown provider", func(t *testing.T) { - // given - shoot := fixEmptyGardenerShoot("cluster", "kcp-system") - runtime := imv1.Runtime{ - Spec: imv1.RuntimeSpec{ - Shoot: imv1.RuntimeShoot{ - Provider: imv1.Provider{ - Type: "unknown", - }, - }, - }, - } - - // when - extender := NewProviderExtenderForCreateOperation(false, "", "") - err := extender(runtime, &shoot) - - // then - require.Error(t, err) - }) + } } func TestProviderExtenderForCreateAzure(t *testing.T) { @@ -456,6 +407,14 @@ func TestProviderExtenderForCreateAzure(t *testing.T) { } } +/*func TestProviderExtenderForCreateGCP(t *testing.T) { + +} + +func TestProviderExtenderForCreateOpenStack(t *testing.T) { + +}*/ + func fixProvider(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { return imv1.Provider{ Type: providerType, @@ -474,6 +433,68 @@ func fixProvider(providerType string, machineImageName, machineImageVersion stri } } +func fixWorkers(machineType, machineImageName, machineImageVersion string, min, max int32, zones []string) []gardener.Worker { + return []gardener.Worker{ + { + Name: "worker", + Machine: gardener.Machine{ + Type: machineType, + Image: &gardener.ShootMachineImage{ + Name: machineImageName, + Version: &machineImageVersion, + }, + }, + Minimum: min, + Maximum: max, + Zones: zones, + }, + } +} + +func fixProviderWithInfrastructureConfig(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { + var infraConfigBytes []byte + switch providerType { + case hyperscaler.TypeAWS: + infraConfigBytes, _ = aws.GetInfrastructureConfig("10.250.0.0/22", zones) + case hyperscaler.TypeAzure: + infraConfigBytes, _ = azure.GetInfrastructureConfig("10.250.0.0/22", zones) + default: + panic("unknown provider type") + } + + return imv1.Provider{ + Type: providerType, + Workers: []gardener.Worker{ + { + Name: "worker", + Machine: gardener.Machine{ + Type: "m6i.large", + Image: fixMachineImage(machineImageName, machineImageVersion), + }, + Minimum: 1, + Maximum: 3, + Zones: zones, + }, + }, + InfrastructureConfig: &runtime.RawExtension{Raw: infraConfigBytes}, + } +} + +func fixAWSInfrastructureConfig(workersCIDR string, zones []string) *runtime.RawExtension { + infraConfig, _ := aws.GetInfrastructureConfig(workersCIDR, zones) + return &runtime.RawExtension{Raw: infraConfig} +} + +func fixAWSControlPlaneConfig() *runtime.RawExtension { + controlPlaneConfig, _ := aws.GetControlPlaneConfig([]string{}) + return &runtime.RawExtension{Raw: controlPlaneConfig} +} + +func fixAzureInfrastructureConfig(workersCIDR string, zones []string) *runtime.RawExtension { + infraConfig, _ := azure.GetInfrastructureConfig(workersCIDR, zones) + return &runtime.RawExtension{Raw: infraConfig} +} + func fixMachineImage(machineImageName, machineImageVersion string) *gardener.ShootMachineImage { if machineImageVersion != "" { return &gardener.ShootMachineImage{ From d54fc340c16b295adab229f63c1a2312a77ab05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sun, 19 Jan 2025 13:06:04 +0100 Subject: [PATCH 09/15] Unit tests update for provider extender - part 4 --- pkg/gardener/shoot/extender/provider_test.go | 313 +++++++++++++++---- 1 file changed, 259 insertions(+), 54 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index a92d29c1..07b4f730 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -18,7 +18,7 @@ import ( ) func TestProviderExtenderForCreateAWS(t *testing.T) { - // tests of ProviderExtenderForCreateOperation for AWS provider + // tests of ProviderExtenderForCreateOperation for AWS provider and single worker for tname, tc := range map[string]struct { Runtime imv1.Runtime EnableIMDSv2 bool @@ -84,7 +84,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, }, - "Create provider specific config for AWS with multiple workers - create option": { + /*"Create provider specific config for AWS with multiple workers - create option": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ @@ -96,7 +96,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { DefaultMachineImageVersion: "1312.3.0", ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, - }, + },*/ } { t.Run(tname, func(t *testing.T) { // given @@ -138,7 +138,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { } func TestProviderExtenderForPatchAWS(t *testing.T) { - // tests of NewProviderExtenderPatchOperation for AWS only for provider image version patching + // tests of NewProviderExtenderPatch for provider image version patching AWS only operation is provider-agnostic for tname, tc := range map[string]struct { Runtime imv1.Runtime EnableIMDSv2 bool @@ -169,7 +169,7 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { ExpectedMachineImageVersion: "1312.4.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", - CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, @@ -184,7 +184,7 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.2.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", @@ -202,7 +202,7 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", @@ -220,7 +220,7 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", ExpectedMachineImageVersion: "1312.2.0", @@ -238,7 +238,7 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", @@ -256,7 +256,7 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("m6i.large", "", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.2.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", @@ -274,13 +274,14 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { EnableIMDSv2: false, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("m6i.large", "gardenlinux", "", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "gardenlinux", "", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExpectedMachineImageVersion: "1312.2.0", ExpectedZonesCount: 3, ExpectedMachineImageName: "gardenlinux", ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, + // TODO: Add tests for override of infrastructure and control plane config with values from RuntimeCR } { t.Run(tname, func(t *testing.T) { // given @@ -299,6 +300,156 @@ func TestProviderExtenderForPatchAWS(t *testing.T) { } } +type workerConfig struct { + Name string + MachineType string + MachineImageName string + MachineImageVersion string + hypescalerMin int32 + hyperscalerMax int32 + Zones []string +} + +func TestProviderExtenderForPatchAWSWorkersUpdate(t *testing.T) { + // tests of NewProviderExtenderPatch for workers update operation + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + EnableIMDSv2 bool + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + ExpectedZonesCount int + }{ + "Add additional worker": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixWorkers("main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, + "Remove additional worker from existing set of workers": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}})), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, + "Update machine image name and version in multiple workers separately": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + })), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExpectedZonesCount: 3, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, + /*"Add new worker to existing set of workers and extend networking zones set in infrastructureConfig": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeAWS, "", "", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExpectedMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + ExpectedMachineImageName: "gardenlinux", + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, + "Remove worker from existing set of workers networking zones set in infrastructureConfig should not change": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExpectedZonesCount: 3, + ExpectedMachineImageName: "gardenlinux", + ExpectedMachineImageVersion: "1312.2.0", + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, */ + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingControlPlaneConfig, tc.ExistingInfraConfig) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, tc.EnableIMDSv2, tc.ExpectedShootWorkers) + assertProviderSpecificConfigAWS(t, shoot, tc.ExpectedZonesCount) + }) + } +} + func TestProviderExtenderForCreateAzure(t *testing.T) { // tests of ProviderExtenderForCreateOperation for Azure provider for tname, tc := range map[string]struct { @@ -373,7 +524,7 @@ func TestProviderExtenderForCreateAzure(t *testing.T) { ExpectedMachineImageVersion: "18.04-LTS", ExpectedZonesCount: 3, }, - "Create provider specific config for Azure with multiple workers - create option": { + /*"Create provider specific config for Azure with multiple workers - create option": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ @@ -387,7 +538,7 @@ func TestProviderExtenderForCreateAzure(t *testing.T) { DefaultMachineImageVersion: "18.04-LTS", ExpectedMachineImageVersion: "18.04-LTS", ExpectedZonesCount: 3, - }, + }, */ } { t.Run(tname, func(t *testing.T) { // given @@ -408,12 +559,17 @@ func TestProviderExtenderForCreateAzure(t *testing.T) { } /*func TestProviderExtenderForCreateGCP(t *testing.T) { - + TBD } func TestProviderExtenderForCreateOpenStack(t *testing.T) { + TBD +} -}*/ +TestProviderExtenderForPatchAWSWorkers(t *testing.T) { + TBD +} +*/ func fixProvider(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { return imv1.Provider{ @@ -433,10 +589,10 @@ func fixProvider(providerType string, machineImageName, machineImageVersion stri } } -func fixWorkers(machineType, machineImageName, machineImageVersion string, min, max int32, zones []string) []gardener.Worker { +func fixWorkers(name, machineType, machineImageName, machineImageVersion string, min, max int32, zones []string) []gardener.Worker { return []gardener.Worker{ { - Name: "worker", + Name: name, Machine: gardener.Machine{ Type: machineType, Image: &gardener.ShootMachineImage{ @@ -451,6 +607,44 @@ func fixWorkers(machineType, machineImageName, machineImageVersion string, min, } } +func fixMultipleWorkers(workers []workerConfig) []gardener.Worker { + var result []gardener.Worker + for _, w := range workers { + result = append(result, gardener.Worker{ + Name: w.Name, + Machine: gardener.Machine{ + Type: w.MachineType, + Image: fixMachineImage(w.MachineImageName, w.MachineImageVersion), + }, + Minimum: w.hypescalerMin, + Maximum: w.hyperscalerMax, + Zones: w.Zones, + }) + } + return result +} + +func fixProviderWithMultipleWorkers(providerType string, workers []gardener.Worker) imv1.Provider { + if len(workers) < 1 { + return imv1.Provider{ + Type: providerType, + } + } + + var addPtr *[]gardener.Worker + + if len(workers) > 1 { + additional := workers[1:] + addPtr = &additional + } + + return imv1.Provider{ + Type: providerType, + Workers: workers[:1], + AdditionalWorkers: addPtr, + } +} + func fixProviderWithInfrastructureConfig(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { var infraConfigBytes []byte switch providerType { @@ -506,42 +700,6 @@ func fixMachineImage(machineImageName, machineImageVersion string) *gardener.Sho return &gardener.ShootMachineImage{} } -func fixProviderWithMultipleWorkers(providerType string, zones []string) imv1.Provider { - return imv1.Provider{ - Type: providerType, - Workers: []gardener.Worker{ - { - Name: "worker", - Machine: gardener.Machine{ - Type: "m6i.large", - }, - Minimum: 1, - Maximum: 3, - Zones: zones, - }, - { - Name: "worker", - Machine: gardener.Machine{ - Type: "m6i.large", - Image: &gardener.ShootMachineImage{}, - }, - Minimum: 1, - Maximum: 3, - Zones: zones, - }, - { - Name: "worker", - Machine: gardener.Machine{ - Type: "m6i.large", - }, - Minimum: 1, - Maximum: 3, - Zones: zones, - }, - }, - } -} - func TestGetAllWorkersZones(t *testing.T) { tests := []struct { name string @@ -630,7 +788,7 @@ func assertProvider(t *testing.T, runtimeShoot imv1.RuntimeShoot, shoot gardener assert.NotEmpty(t, shoot.Spec.Provider.InfrastructureConfig.Raw) assert.NotEmpty(t, shoot.Spec.Provider.ControlPlaneConfig) assert.NotEmpty(t, shoot.Spec.Provider.ControlPlaneConfig.Raw) - assert.NotEmpty(t, shoot.Spec.Provider.ControlPlaneConfig.Raw) + for _, worker := range shoot.Spec.Provider.Workers { if expectWorkerConfig { assert.NotEmpty(t, worker.ProviderConfig) @@ -643,6 +801,53 @@ func assertProvider(t *testing.T, runtimeShoot imv1.RuntimeShoot, shoot gardener } } +func assertProviderMultipleWorkers(t *testing.T, runtimeShoot imv1.RuntimeShoot, shoot gardener.Shoot, expectWorkerConfig bool, expectedWorkers []gardener.Worker) { + assert.Equal(t, len(expectedWorkers), len(shoot.Spec.Provider.Workers)) + assert.Equal(t, shoot.Spec.Provider.Type, runtimeShoot.Provider.Type) + assert.Equal(t, false, shoot.Spec.Provider.WorkersSettings.SSHAccess.Enabled) + assert.NotEmpty(t, shoot.Spec.Provider.InfrastructureConfig) + assert.NotEmpty(t, shoot.Spec.Provider.InfrastructureConfig.Raw) + assert.NotEmpty(t, shoot.Spec.Provider.ControlPlaneConfig) + assert.NotEmpty(t, shoot.Spec.Provider.ControlPlaneConfig.Raw) + + for i, worker := range shoot.Spec.Provider.Workers { + if expectWorkerConfig { + assert.NotEmpty(t, worker.ProviderConfig) + assert.NotEmpty(t, worker.ProviderConfig.Raw) + } else { + assert.Empty(t, worker.ProviderConfig) + } + + expected := expectedWorkers[i] + assert.Equal(t, expected.Name, worker.Name) + assert.Equal(t, expected.Zones, worker.Zones) + assert.Equal(t, expected.Minimum, worker.Minimum) + assert.Equal(t, expected.Maximum, worker.Maximum) + assert.Equal(t, expected.Machine.Type, worker.Machine.Type) + + if expected.MaxSurge != nil { + assert.NotNil(t, worker.MaxSurge) + assert.Equal(t, *expected.MaxSurge, *worker.MaxSurge) + } + + if expected.MaxUnavailable != nil { + assert.NotNil(t, worker.MaxUnavailable) + assert.Equal(t, *expected.MaxUnavailable, *worker.MaxUnavailable) + } + + if expected.Machine.Architecture != nil { + assert.NotNil(t, worker.Machine.Architecture) + assert.Equal(t, *expected.Machine.Architecture, *worker.Machine.Architecture) + } + + assert.NotNil(t, worker.Machine.Image) + assert.Equal(t, expected.Machine.Image.Name, worker.Machine.Image.Name) + assert.NotEmpty(t, expected.Machine.Image.Version) + assert.NotEmpty(t, worker.Machine.Image.Version) + assert.Equal(t, *expected.Machine.Image.Version, *worker.Machine.Image.Version) + } +} + func assertProviderSpecificConfigAWS(t *testing.T, shoot gardener.Shoot, expectedZonesCount int) { var infrastructureConfig v1alpha1.InfrastructureConfig From b2b70089c00d38b2626db5f81ee4e8941fee8e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sun, 19 Jan 2025 18:53:16 +0100 Subject: [PATCH 10/15] Unit tests update for provider extender - part 5 --- pkg/gardener/shoot/extender/provider.go | 28 +-- pkg/gardener/shoot/extender/provider_test.go | 233 ++++++++++++++----- 2 files changed, 186 insertions(+), 75 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider.go b/pkg/gardener/shoot/extender/provider.go index b21a2d54..a501fac4 100644 --- a/pkg/gardener/shoot/extender/provider.go +++ b/pkg/gardener/shoot/extender/provider.go @@ -227,27 +227,27 @@ func getNetworkingZonesFromWorkers(workers []gardener.Worker) ([]string, error) return nil, errors.New("no workers provided") } - for _, zone := range workers[0].Zones { - if !slices.Contains(zones, zone) { - zones = append(zones, zone) - } else { - return nil, fmt.Errorf("duplicate zone name detected for worker %s", workers[0].Name) + for _, worker := range workers { + for _, zone := range worker.Zones { + if !slices.Contains(zones, zone) { + zones = append(zones, zone) + } } } if len(zones) == 0 { - return nil, fmt.Errorf("no networking zones provided for worker %s", workers[0].Name) + return nil, fmt.Errorf("no networking zones detected for Runtime provider workers") } - if len(workers) == 1 { - return zones, nil - } + //if len(workers) == 1 { + // return zones, nil + //} - for _, worker := range workers { - if !slices.Equal(worker.Zones, zones) { - return nil, errors.New("workers have specified different zones set, or zones are in different order") - } - } + //for _, worker := range workers { + // if !slices.Equal(worker.Zones, zones) { + // return nil, errors.New("workers have specified different zones set, or zones are in different order") + // } + //} return zones, nil } diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 07b4f730..76cfe91d 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -71,7 +71,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { ExpectedMachineImageName: "gardenlinux", ExpectedZonesCount: 3, }, - "Create provider specific config for AWS with worker config and three zones": { + "Create provider config for AWS with worker config and three zones": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ @@ -84,19 +84,6 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, }, - /*"Create provider specific config for AWS with multiple workers - create option": { - Runtime: imv1.Runtime{ - Spec: imv1.RuntimeSpec{ - Shoot: imv1.RuntimeShoot{ - Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), - }, - }, - }, - EnableIMDSv2: false, - DefaultMachineImageVersion: "1312.3.0", - ExpectedMachineImageVersion: "1312.3.0", - ExpectedZonesCount: 3, - },*/ } { t.Run(tname, func(t *testing.T) { // given @@ -137,7 +124,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { }) } -func TestProviderExtenderForPatchAWS(t *testing.T) { +func TestProviderExtenderForPatchSingleWorkerAWS(t *testing.T) { // tests of NewProviderExtenderPatch for provider image version patching AWS only operation is provider-agnostic for tname, tc := range map[string]struct { Runtime imv1.Runtime @@ -310,7 +297,86 @@ type workerConfig struct { Zones []string } -func TestProviderExtenderForPatchAWSWorkersUpdate(t *testing.T) { +func TestProviderExtenderForCreateMultipleWorkersAWS(t *testing.T) { + // tests of NewProviderExtenderForCreateOperation for workers create operation + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + EnableIMDSv2 bool + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + ExpectedZonesCount int + }{ + "Create provider config for multiple workers without AWS worker config": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-central-1b", "eu-central-1c"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-central-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-central-1b", "eu-central-1c"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-central-1c"}}}), + }, + "Create provider config for multiple workers with AWS worker config": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-central-1b", "eu-central-1c"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-central-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: true, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-central-1b", "eu-central-1c"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-central-1c"}}}), + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + extender := NewProviderExtenderForCreateOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, tc.EnableIMDSv2, tc.ExpectedShootWorkers) + assertProviderSpecificConfigAWS(t, shoot, tc.ExpectedZonesCount) + }) + } +} + +func TestProviderExtenderForPatchWorkersUpdateAWS(t *testing.T) { // tests of NewProviderExtenderPatch for workers update operation for tname, tc := range map[string]struct { Runtime imv1.Runtime @@ -380,6 +446,9 @@ func TestProviderExtenderForPatchAWSWorkersUpdate(t *testing.T) { {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, }, }, }, @@ -396,49 +465,64 @@ func TestProviderExtenderForPatchAWSWorkersUpdate(t *testing.T) { ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - /*"Add new worker to existing set of workers and extend networking zones set in infrastructureConfig": { + "Add new worker to existing set of workers and extend networking zones set in infrastructureConfig": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider(hyperscaler.TypeAWS, "", "", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, }, }, }, - EnableIMDSv2: false, - DefaultMachineImageName: "gardenlinux", - DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("worker", "m6i.large", "gardenlinux", "1312.1.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), - ExpectedMachineImageVersion: "1312.3.0", - ExpectedZonesCount: 3, - ExpectedMachineImageName: "gardenlinux", - ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), - ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExpectedZonesCount: 3, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, "Remove worker from existing set of workers networking zones set in infrastructureConfig should not change": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProvider(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, }, }, }, - EnableIMDSv2: false, - DefaultMachineImageName: "gardenlinux", - DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixWorkers("worker", "m6i.large", "ubuntu", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), - ExpectedZonesCount: 3, - ExpectedMachineImageName: "gardenlinux", - ExpectedMachineImageVersion: "1312.2.0", - ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), - ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), - }, */ + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}}), + ExpectedZonesCount: 3, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, } { t.Run(tname, func(t *testing.T) { // given shoot := fixEmptyGardenerShoot("cluster", "kcp-system") // when - extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingControlPlaneConfig, tc.ExistingInfraConfig) + extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingInfraConfig, tc.ExistingControlPlaneConfig) err := extender(tc.Runtime, &shoot) // then @@ -524,35 +608,73 @@ func TestProviderExtenderForCreateAzure(t *testing.T) { ExpectedMachineImageVersion: "18.04-LTS", ExpectedZonesCount: 3, }, - /*"Create provider specific config for Azure with multiple workers - create option": { + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + + extender := NewProviderExtenderForCreateOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProvider(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) + assertProviderSpecificConfigAzure(t, shoot, tc.ExpectedZonesCount) + }) + } +} + +func TestProviderExtenderForCreateMultipleWorkersAzure(t *testing.T) { + // tests of NewProviderExtenderForCreateOperation for workers create operation + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + ExpectedZonesCount int + }{ + "Create multiple workers without worker config": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ - Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAzure, []string{"1", "2", "3"}), + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAzure, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"1"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"2", "3"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"3"}}, + })), Networking: imv1.Networking{ Nodes: "10.250.0.0/22", }, }, }, }, - DefaultMachineImageVersion: "18.04-LTS", - ExpectedMachineImageVersion: "18.04-LTS", - ExpectedZonesCount: 3, - }, */ + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"1"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"2", "3"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"3"}}}), + }, } { t.Run(tname, func(t *testing.T) { // given shoot := fixEmptyGardenerShoot("cluster", "kcp-system") // when - extender := NewProviderExtenderForCreateOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) err := extender(tc.Runtime, &shoot) // then require.NoError(t, err) - assertProvider(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedShootWorkers) assertProviderSpecificConfigAzure(t, shoot, tc.ExpectedZonesCount) }) } @@ -742,11 +864,11 @@ func TestGetAllWorkersZones(t *testing.T) { }, { Name: "worker2", - Zones: []string{"zone1", "zone3"}, + Zones: []string{"zone1"}, }, }, - expected: nil, - wantErr: true, + expected: []string{"zone1", "zone2"}, + wantErr: false, }, { name: "No workers provided", @@ -754,17 +876,6 @@ func TestGetAllWorkersZones(t *testing.T) { expected: nil, wantErr: true, }, - { - name: "Duplicate zones in a single worker", - workers: []gardener.Worker{ - { - Name: "worker1", - Zones: []string{"zone1", "zone1"}, - }, - }, - expected: nil, - wantErr: true, - }, } for _, tt := range tests { From 7f6e02f670787b420ac608f166d026d0760cdc55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Wed, 22 Jan 2025 09:01:48 +0100 Subject: [PATCH 11/15] Fixer in unit tests, and small bugfixes --- pkg/gardener/shoot/extender/provider.go | 99 +++++++++----------- pkg/gardener/shoot/extender/provider_test.go | 28 +----- 2 files changed, 47 insertions(+), 80 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider.go b/pkg/gardener/shoot/extender/provider.go index a501fac4..47019fb2 100644 --- a/pkg/gardener/shoot/extender/provider.go +++ b/pkg/gardener/shoot/extender/provider.go @@ -15,42 +15,45 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// InfrastructureConfig and ControlPlaneConfig are generated unless they are specified in the RuntimeCR func NewProviderExtenderForCreateOperation(enableIMDSv2 bool, defMachineImgName, defMachineImgVer string) func(rt imv1.Runtime, shoot *gardener.Shoot) error { return func(rt imv1.Runtime, shoot *gardener.Shoot) error { provider := &shoot.Spec.Provider provider.Type = rt.Spec.Shoot.Provider.Type provider.Workers = rt.Spec.Shoot.Provider.Workers + if len(rt.Spec.Shoot.Provider.Workers) != 1 { + return errors.New("single main worker is required") + } + if rt.Spec.Shoot.Provider.AdditionalWorkers != nil { provider.Workers = append(provider.Workers, *rt.Spec.Shoot.Provider.AdditionalWorkers...) } - var err error - var controlPlaneConf, infraConfig *runtime.RawExtension - zones, err := getNetworkingZonesFromWorkers(provider.Workers) - + workerZones, err := getNetworkingZonesFromWorkers(provider.Workers) if err != nil { return err } - infraConfig, controlPlaneConf, err = getConfig(rt.Spec.Shoot, zones) + infraConfig, controlPlaneConf, err := getConfig(rt.Spec.Shoot, workerZones) if err != nil { return err } - if rt.Spec.Shoot.Provider.ControlPlaneConfig != nil { - controlPlaneConf = rt.Spec.Shoot.Provider.ControlPlaneConfig - } + controlPlaneConf, infraConfig = overrideConfigIfProvided(rt, infraConfig, controlPlaneConf) - if rt.Spec.Shoot.Provider.InfrastructureConfig != nil { - infraConfig = rt.Spec.Shoot.Provider.InfrastructureConfig + // final validation + if err = checkWorkerZonesMatchProviderConfig(rt.Spec.Shoot.Provider.Type, workerZones, controlPlaneConf, infraConfig); err != nil { + return err } provider.ControlPlaneConfig = controlPlaneConf provider.InfrastructureConfig = infraConfig setMachineImage(provider, defMachineImgName, defMachineImgVer) - err = setWorkerConfig(provider, provider.Type, enableIMDSv2) + if err = setWorkerConfig(provider, provider.Type, enableIMDSv2); err != nil { + return err + } setWorkerSettings(provider) return err @@ -58,30 +61,29 @@ func NewProviderExtenderForCreateOperation(enableIMDSv2 bool, defMachineImgName, } // Zones for patching workes are taken from existing shoot workers -// -// Zones for patching infrastructureConfig are processed as follows: -// For Azure and AWS zones are stored in InfrastructureConfig and can be possibly updated if new workers with new zones are added -// For GCP single zone (one from defined in workers) is stored in ControlPlaneConfig and its value is immutable. -// For Openstack no zone information is stored neither in InfrastructureConfig nor in ControlPlaneConfig -// So only for Azure and AWS zone setup can be changed in InfrastructureConfig scope. -// For other providers we use existing data for patching the shoot +// InfrastructureConfig and ControlPlaneConfig are treated as immutable unless they are specified in the RuntimeCR func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, defMachineImgVer string, shootWorkers []gardener.Worker, existingInfraConfig *runtime.RawExtension, existingControlPlaneConfig *runtime.RawExtension) func(rt imv1.Runtime, shoot *gardener.Shoot) error { return func(rt imv1.Runtime, shoot *gardener.Shoot) error { provider := &shoot.Spec.Provider provider.Type = rt.Spec.Shoot.Provider.Type provider.Workers = rt.Spec.Shoot.Provider.Workers + if len(rt.Spec.Shoot.Provider.Workers) != 1 { + return errors.New("single main worker is required") + } + if rt.Spec.Shoot.Provider.AdditionalWorkers != nil { provider.Workers = append(provider.Workers, *rt.Spec.Shoot.Provider.AdditionalWorkers...) } + controlPlaneConf, infraConfig := overrideConfigIfProvided(rt, existingInfraConfig, existingControlPlaneConfig) + workerZones, err := getNetworkingZonesFromWorkers(provider.Workers) if err != nil { return err } - - controlPlaneConf, infraConfig, err := getProviderConfigsForPatch(rt, workerZones, existingInfraConfig, existingControlPlaneConfig) - if err != nil { + // final validation + if err = checkWorkerZonesMatchProviderConfig(rt.Spec.Shoot.Provider.Type, workerZones, controlPlaneConf, infraConfig); err != nil { return err } @@ -101,39 +103,35 @@ func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, def } } -func getProviderConfigsForPatch(rt imv1.Runtime, workerZones []string, existingInfraConfig, existingControlPlaneConfig *runtime.RawExtension) (*runtime.RawExtension, *runtime.RawExtension, error) { - var controlPlaneConf, infraConfig *runtime.RawExtension - - if rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAzure || rt.Spec.Shoot.Provider.Type == hyperscaler.TypeAWS { - infraConfigZones, err := getZonesFromInfrastructureConfig(rt.Spec.Shoot.Provider.Type, existingInfraConfig) +func checkWorkerZonesMatchProviderConfig(providerType string, workerZones []string, ctrlPlaneConfig *runtime.RawExtension, infraConfig *runtime.RawExtension) error { + if providerType == hyperscaler.TypeAzure || providerType == hyperscaler.TypeAWS { + infraConfigZones, err := getZonesFromInfrastructureConfig(providerType, infraConfig) if err != nil { - return nil, nil, err + return err } - // extend infrastructure zones collection if new workers are added with new zones + for _, zone := range workerZones { if !slices.Contains(infraConfigZones, zone) { - infraConfigZones = append(infraConfigZones, zone) + return fmt.Errorf("one of workers is using zone not specified in the infrastructureConfig: %s", zone) } } - - infraConfig, controlPlaneConf, err = getConfig(rt.Spec.Shoot, infraConfigZones) - if err != nil { - return nil, nil, err - } - } else { - infraConfig = existingInfraConfig - controlPlaneConf = existingControlPlaneConfig } + //if providerType == hyperscaler.TypeGCP { - if rt.Spec.Shoot.Provider.ControlPlaneConfig != nil { - controlPlaneConf = rt.Spec.Shoot.Provider.ControlPlaneConfig - } + return nil +} - if rt.Spec.Shoot.Provider.InfrastructureConfig != nil { - infraConfig = rt.Spec.Shoot.Provider.InfrastructureConfig - } +func overrideConfigIfProvided(rt imv1.Runtime, existingInfraConfig, existingControlPlaneConfig *runtime.RawExtension) (*runtime.RawExtension, *runtime.RawExtension) { + controlPlaneConf := getConfigOrDefault(rt.Spec.Shoot.Provider.ControlPlaneConfig, existingControlPlaneConfig) + infraConfig := getConfigOrDefault(rt.Spec.Shoot.Provider.InfrastructureConfig, existingInfraConfig) + return controlPlaneConf, infraConfig +} - return controlPlaneConf, infraConfig, nil +func getConfigOrDefault(config, defaultConfig *runtime.RawExtension) *runtime.RawExtension { + if config != nil { + return config + } + return defaultConfig } // parse infrastructure config to get current set of networking zones @@ -217,9 +215,6 @@ func getAWSWorkerConfig() (*runtime.RawExtension, error) { return &runtime.RawExtension{Raw: workerConfigBytes}, nil } -// Get set of zones from first worker. -// All other workers should have the same zones provided in the same order. -// Otherwise fail func getNetworkingZonesFromWorkers(workers []gardener.Worker) ([]string, error) { var zones []string @@ -235,14 +230,12 @@ func getNetworkingZonesFromWorkers(workers []gardener.Worker) ([]string, error) } } - if len(zones) == 0 { - return nil, fmt.Errorf("no networking zones detected for Runtime provider workers") - } - + // uncomment if we decide to require to validate to have all same zones in all workers + // //if len(workers) == 1 { // return zones, nil //} - + // //for _, worker := range workers { // if !slices.Equal(worker.Zones, zones) { // return nil, errors.New("workers have specified different zones set, or zones are in different order") @@ -301,7 +294,6 @@ func setMachineImage(provider *gardener.Provider, defMachineImgName, defMachineI // We can't predict what will be the order of zones stored by Gardener. // Without this patch, gardener's admission webhook might reject the request if the zones order does not match. -// If the current image version with the same name on Shoot is greater than the version, it sets the version to the current machine image version. func alignWorkersWithShoot(provider *gardener.Provider, existingWorkers []gardener.Worker) { existingWorkersMap := make(map[string]gardener.Worker) for _, existing := range existingWorkers { @@ -318,6 +310,7 @@ func alignWorkersWithShoot(provider *gardener.Provider, existingWorkers []garden } } +// If the current image version with the same name on Shoot is greater than the version, it sets the version to the current machine image version. func alignWorkerMachineImageVersion(workerImage *gardener.ShootMachineImage, shootWorkerImage *gardener.ShootMachineImage) { if shootWorkerImage == nil || workerImage == nil || workerImage.Name != shootWorkerImage.Name { return diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 76cfe91d..01973161 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -275,7 +275,7 @@ func TestProviderExtenderForPatchSingleWorkerAWS(t *testing.T) { shoot := fixEmptyGardenerShoot("cluster", "kcp-system") // when - extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingControlPlaneConfig, tc.ExistingInfraConfig) + extender := NewProviderExtenderPatchOperation(tc.EnableIMDSv2, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingInfraConfig, tc.ExistingControlPlaneConfig) err := extender(tc.Runtime, &shoot) // then @@ -465,32 +465,6 @@ func TestProviderExtenderForPatchWorkersUpdateAWS(t *testing.T) { ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - "Add new worker to existing set of workers and extend networking zones set in infrastructureConfig": { - Runtime: imv1.Runtime{ - Spec: imv1.RuntimeSpec{ - Shoot: imv1.RuntimeShoot{ - Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ - {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}, - {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, - })), - Networking: imv1.Networking{ - Nodes: "10.250.0.0/22", - }, - }, - }, - }, - EnableIMDSv2: false, - DefaultMachineImageName: "gardenlinux", - DefaultMachineImageVersion: "1312.3.0", - CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ - {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a"}}}), - ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ - {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}, - {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), - ExpectedZonesCount: 3, - ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a"}), - ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), - }, "Remove worker from existing set of workers networking zones set in infrastructureConfig should not change": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ From 6409f92407e20ede416f0d2e371f8736d4e8baf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Thu, 23 Jan 2025 13:32:10 +0100 Subject: [PATCH 12/15] Add unit test to switch shoot from no-HA to HA --- pkg/gardener/shoot/extender/provider_test.go | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 01973161..910bd81c 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -414,6 +414,33 @@ func TestProviderExtenderForPatchWorkersUpdateAWS(t *testing.T) { ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, + /*"Add additional worker - extend existing additional worker from non HA setup to HA setup by adding more zones, infrastructureConfig already has three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + },*/ "Remove additional worker from existing set of workers": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ From 46d76a1a7ee604e04e297363c6f994d8fd9f542c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Thu, 23 Jan 2025 19:13:10 +0100 Subject: [PATCH 13/15] GCP worker validation with controlPlaneConfig, unit tests extensions --- pkg/gardener/shoot/extender/provider.go | 37 +- pkg/gardener/shoot/extender/provider_test.go | 481 ++++++++++++++++++- pkg/gardener/shoot/hyperscaler/gcp/config.go | 9 + 3 files changed, 493 insertions(+), 34 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider.go b/pkg/gardener/shoot/extender/provider.go index 47019fb2..a241c094 100644 --- a/pkg/gardener/shoot/extender/provider.go +++ b/pkg/gardener/shoot/extender/provider.go @@ -105,18 +105,31 @@ func NewProviderExtenderPatchOperation(enableIMDSv2 bool, defMachineImgName, def func checkWorkerZonesMatchProviderConfig(providerType string, workerZones []string, ctrlPlaneConfig *runtime.RawExtension, infraConfig *runtime.RawExtension) error { if providerType == hyperscaler.TypeAzure || providerType == hyperscaler.TypeAWS { - infraConfigZones, err := getZonesFromInfrastructureConfig(providerType, infraConfig) + infraConfigZones, err := getZonesFromProviderConfig(providerType, infraConfig) if err != nil { return err } for _, zone := range workerZones { if !slices.Contains(infraConfigZones, zone) { - return fmt.Errorf("one of workers is using zone not specified in the infrastructureConfig: %s", zone) + return fmt.Errorf("one of workers is using networking zone not specified in the %s infrastructureConfig: %s", providerType, zone) } } } - //if providerType == hyperscaler.TypeGCP { + if providerType == hyperscaler.TypeGCP { + ctrlPlaneZones, err := getZonesFromProviderConfig(providerType, ctrlPlaneConfig) + if err != nil { + return err + } + + if len(ctrlPlaneZones) == 0 { + return fmt.Errorf("cannot validate workers zones against GCP controlPlaneConfig, cannot read current networking zone") + } + + if !slices.Contains(workerZones, ctrlPlaneZones[0]) { + return fmt.Errorf("none of workers is using networking zone specified in the controlPlaneConfig: %s", ctrlPlaneZones[0]) + } + } return nil } @@ -134,9 +147,9 @@ func getConfigOrDefault(config, defaultConfig *runtime.RawExtension) *runtime.Ra return defaultConfig } -// parse infrastructure config to get current set of networking zones -func getZonesFromInfrastructureConfig(providerType string, infraConfig *runtime.RawExtension) ([]string, error) { - if infraConfig == nil { +// parse infrastructureConfig or controlPlaneConfig to get current set of networking zones +func getZonesFromProviderConfig(providerType string, extensionConfig *runtime.RawExtension) ([]string, error) { + if extensionConfig == nil { return nil, errors.New("infrastructureConfig is nil") } @@ -144,7 +157,7 @@ func getZonesFromInfrastructureConfig(providerType string, infraConfig *runtime. switch providerType { case hyperscaler.TypeAWS: - infraConfig, err := aws.DecodeInfrastructureConfig(infraConfig.Raw) + infraConfig, err := aws.DecodeInfrastructureConfig(extensionConfig.Raw) if err != nil { return nil, err } @@ -152,13 +165,21 @@ func getZonesFromInfrastructureConfig(providerType string, infraConfig *runtime. zones = append(zones, zone.Name) } case hyperscaler.TypeAzure: - infraConfig, err := azure.DecodeInfrastructureConfig(infraConfig.Raw) + infraConfig, err := azure.DecodeInfrastructureConfig(extensionConfig.Raw) if err != nil { return nil, err } for _, zone := range infraConfig.Networks.Zones { zones = append(zones, fmt.Sprint(zone.Name)) } + case hyperscaler.TypeGCP: + { + ctrlPlaneConfig, err := gcp.DecodeControlPlaneConfig(extensionConfig.Raw) + if err != nil { + return nil, err + } + zones = append(zones, ctrlPlaneConfig.Zone) + } default: return nil, errors.New("read zones from infrastructureConfig - provider not supported") } diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 910bd81c..ba6e07a1 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -2,14 +2,15 @@ package extender import ( "encoding/json" - "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" aws "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/aws" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/azure" + gcp "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/gcp" "k8s.io/apimachinery/pkg/runtime" "testing" - "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1" + awsext "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1" + gcpext "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp/v1alpha1" gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler" @@ -414,6 +415,7 @@ func TestProviderExtenderForPatchWorkersUpdateAWS(t *testing.T) { ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, + // NOTE: uncomment this unit test after the issue of extending the existing infrastructure config with additional zones is implemented /*"Add additional worker - extend existing additional worker from non HA setup to HA setup by adding more zones, infrastructureConfig already has three zones": { Runtime: imv1.Runtime{ Spec: imv1.RuntimeSpec{ @@ -594,21 +596,6 @@ func TestProviderExtenderForCreateAzure(t *testing.T) { ExpectedMachineImageName: "ubuntu", ExpectedZonesCount: 3, }, - "Create provider specific config for Azure with worker config and three zones": { - Runtime: imv1.Runtime{ - Spec: imv1.RuntimeSpec{ - Shoot: imv1.RuntimeShoot{ - Provider: fixProvider(hyperscaler.TypeAzure, "", "", []string{"1", "2", "3"}), - Networking: imv1.Networking{ - Nodes: "10.250.0.0/22", - }, - }, - }, - }, - DefaultMachineImageVersion: "18.04-LTS", - ExpectedMachineImageVersion: "18.04-LTS", - ExpectedZonesCount: 3, - }, } { t.Run(tname, func(t *testing.T) { // given @@ -681,17 +668,443 @@ func TestProviderExtenderForCreateMultipleWorkersAzure(t *testing.T) { } } -/*func TestProviderExtenderForCreateGCP(t *testing.T) { - TBD +// tests of NewProviderExtenderPatch for workers update operation on Azure provider +func TestProviderExtenderForPatchWorkersUpdateAzure(t *testing.T) { + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + ExpectedZonesCount int + }{ + "Add additional worker": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAzure, fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.small", "gardenlinux", "1312.4.0", 1, 3, []string{"1", "2", "3"}}, + {"next-worker", "azure.large", "gardenlinux", "1312.2.0", 1, 3, []string{"1", "2", "3"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixWorkers("main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"1", "2", "3"}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.small", "gardenlinux", "1312.4.0", 1, 3, []string{"1", "2", "3"}}, + {"next-worker", "azure.large", "gardenlinux", "1312.2.0", 1, 3, []string{"1", "2", "3"}}}), + ExistingInfraConfig: fixAzureInfrastructureConfig("10.250.0.0/22", []string{"1", "2", "3"}), + ExistingControlPlaneConfig: fixAzureControlPlaneConfig(), + }, + // NOTE: uncomment this unit test after the issue of extending the existing infrastructure config with additional zones is implemented + /*"Add additional worker - extend existing additional worker from non HA setup to HA setup by adding more zones, infrastructureConfig already has three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + },*/ + "Remove additional worker from existing set of workers": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAzure, fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.small", "gardenlinux", "1312.4.0", 1, 3, []string{"1", "2", "3"}}})), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.small", "gardenlinux", "1312.4.0", 1, 3, []string{"1", "2", "3"}}, + {"next-worker", "azure.large", "gardenlinux", "1312.2.0", 1, 3, []string{"1", "2", "3"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.small", "gardenlinux", "1312.4.0", 1, 3, []string{"1", "2", "3"}}}), + ExistingInfraConfig: fixAzureInfrastructureConfig("10.250.0.0/22", []string{"1", "2", "3"}), + ExistingControlPlaneConfig: fixAzureControlPlaneConfig(), + }, + "Update machine type and image name and version in multiple workers separately": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAzure, fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.large", "gardenlinux", "1313.4.0", 1, 3, []string{"1", "2", "3"}}, + {"next-worker", "azure.big", "gardenlinux", "1313.2.0", 1, 3, []string{"1", "2", "3"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.small", "gardenlinux", "1312.4.0", 1, 3, []string{"1", "2", "3"}}, + {"next-worker", "azure.small", "gardenlinux", "1312.2.0", 1, 3, []string{"1", "2", "3"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.large", "gardenlinux", "1313.4.0", 1, 3, []string{"1", "2", "3"}}, + {"next-worker", "azure.big", "gardenlinux", "1313.2.0", 1, 3, []string{"1", "2", "3"}}}), + ExpectedZonesCount: 3, + ExistingInfraConfig: fixAzureInfrastructureConfig("10.250.0.0/22", []string{"1", "2", "3"}), + ExistingControlPlaneConfig: fixAzureControlPlaneConfig(), + }, + "Remove worker from existing set of workers networking zones set in infrastructureConfig should not change": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAzure, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"1"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"1"}}, + {"next-worker", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"1", "2", "3"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"1"}}}), + ExpectedZonesCount: 3, + ExistingInfraConfig: fixAzureInfrastructureConfig("10.250.0.0/22", []string{"1", "2", "3"}), + ExistingControlPlaneConfig: fixAzureControlPlaneConfig(), + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + extender := NewProviderExtenderPatchOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingInfraConfig, tc.ExistingControlPlaneConfig) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedShootWorkers) + assertProviderSpecificConfigAzure(t, shoot, tc.ExpectedZonesCount) + }) + } } -func TestProviderExtenderForCreateOpenStack(t *testing.T) { - TBD +func TestProviderExtenderForCreateGCP(t *testing.T) { + // tests of ProviderExtenderForCreateOperation for GCP provider + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + ExpectedMachineImageVersion string + DefaultMachineImageName string + ExpectedMachineImageName string + CurrentZonesConfig []string + ExpectedZonesCount int + }{ + "Create provider specific config for GCP without worker config and one zone": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeGCP, "ubuntu", "18.04", []string{"us-central1-a"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04", + ExpectedMachineImageName: "ubuntu", + }, + "Create provider specific config for GCP without worker config and two zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeGCP, "ubuntu", "18.04", []string{"us-central1-a", "us-central1-b"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04", + ExpectedMachineImageName: "ubuntu", + }, + "Create provider specific config for GCP without worker config and three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProvider(hyperscaler.TypeGCP, "ubuntu", "18.04", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageVersion: "18.04-LTS", + ExpectedMachineImageVersion: "18.04", + ExpectedMachineImageName: "ubuntu", + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + + extender := NewProviderExtenderForCreateOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + assertProvider(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedMachineImageName, tc.ExpectedMachineImageVersion) + + assertProviderSpecificConfigGCP(t, shoot) + }) + } +} + +func TestProviderExtenderForCreateMultipleWorkersGCP(t *testing.T) { + // tests of NewProviderExtenderForCreateOperation for workers create operation + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + ExpectedZonesCount int + }{ + "Create multiple GCP workers without worker config": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeGCP, fixMultipleWorkers([]workerConfig{ + {"main-worker", "gcp.large", "gardenlinux", "1310.4.0", 1, 3, []string{"us-central1-a"}}, + {"additional", "gcp.large", "gardenlinux", "1311.2.0", 2, 4, []string{"us-central1-b", "us-central1-c"}}, + {"another", "gcp.large", "gardenlinux", "1312.2.0", 3, 5, []string{"us-central1-c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "gcp.large", "gardenlinux", "1310.4.0", 1, 3, []string{"us-central1-a"}}, + {"additional", "gcp.large", "gardenlinux", "1311.2.0", 2, 4, []string{"us-central1-b", "us-central1-c"}}, + {"another", "gcp.large", "gardenlinux", "1312.2.0", 3, 5, []string{"us-central1-c"}}}), + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + extender := NewProviderExtenderForCreateOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedShootWorkers) + assertProviderSpecificConfigGCP(t, shoot) + }) + } } -TestProviderExtenderForPatchAWSWorkers(t *testing.T) { +func TestProviderExtenderForPatchWorkersUpdateGCP(t *testing.T) { + // tests of NewProviderExtenderPatch for workers update operation for GCP provider + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + }{ + "Add additional worker": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeGCP, fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + {"next-worker", "n2-standard-2", "gardenlinux", "1312.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixWorkers("main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + {"next-worker", "n2-standard-2", "gardenlinux", "1312.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), + }, + // NOTE: uncomment this unit test after the issue of extending the existing infrastructure config with additional zones is implemented + /*"Add additional worker - extend existing additional worker from non HA setup to HA setup by adding more zones, infrastructureConfig already has three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + },*/ + "Remove additional worker from existing set of workers": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeGCP, fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}})), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + {"next-worker", "n2-standard-2", "gardenlinux", "1312.2.0", 2, 4, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), + }, + "Update machine type and image name and version in multiple workers separately": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeGCP, fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-4", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + {"additional", "n2-standard-4", "gardenlinux", "1313.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + {"additional", "n2-standard-2", "gardenlinux", "1312.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-4", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, + {"additional", "n2-standard-4", "gardenlinux", "1313.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), + }, + "Remove worker from existing set of workers networking zones set in infrastructureConfig should not change": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeGCP, fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a"}}, + {"additional", "n2-standard-2", "gardenlinux", "1312.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a"}}}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + extender := NewProviderExtenderPatchOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingInfraConfig, tc.ExistingControlPlaneConfig) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedShootWorkers) + assertProviderSpecificConfigGCP(t, shoot) + }) + } +} + +/*func TestProviderExtenderForCreateOpenStack(t *testing.T) { TBD } + */ func fixProvider(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { @@ -812,6 +1225,21 @@ func fixAzureInfrastructureConfig(workersCIDR string, zones []string) *runtime.R return &runtime.RawExtension{Raw: infraConfig} } +func fixAzureControlPlaneConfig() *runtime.RawExtension { + infraConfig, _ := azure.GetControlPlaneConfig([]string{}) + return &runtime.RawExtension{Raw: infraConfig} +} + +func fixGCPInfrastructureConfig(workersCIDR string, zones []string) *runtime.RawExtension { + infraConfig, _ := gcp.GetInfrastructureConfig(workersCIDR, zones) + return &runtime.RawExtension{Raw: infraConfig} +} + +func fixGCPControlPlaneConfig(zones []string) *runtime.RawExtension { + controlPlaneConfig, _ := gcp.GetControlPlaneConfig(zones) + return &runtime.RawExtension{Raw: controlPlaneConfig} +} + func fixMachineImage(machineImageName, machineImageVersion string) *gardener.ShootMachineImage { if machineImageVersion != "" { return &gardener.ShootMachineImage{ @@ -961,7 +1389,7 @@ func assertProviderMultipleWorkers(t *testing.T, runtimeShoot imv1.RuntimeShoot, } func assertProviderSpecificConfigAWS(t *testing.T, shoot gardener.Shoot, expectedZonesCount int) { - var infrastructureConfig v1alpha1.InfrastructureConfig + var infrastructureConfig awsext.InfrastructureConfig err := json.Unmarshal(shoot.Spec.Provider.InfrastructureConfig.Raw, &infrastructureConfig) require.NoError(t, err) @@ -978,11 +1406,12 @@ func assertProviderSpecificConfigAzure(t *testing.T, shoot gardener.Shoot, expec assert.Equal(t, expectedZonesCount, len(infrastructureConfig.Networks.Zones)) } -func assertProviderSpecificConfigGCP(t *testing.T, shoot gardener.Shoot, expectedZonesCount int) { - var infrastructureConfig gcp.InfrastructureConfig +func assertProviderSpecificConfigGCP(t *testing.T, shoot gardener.Shoot) { + var ctrlPlaneConfig gcpext.ControlPlaneConfig - err := json.Unmarshal(shoot.Spec.Provider.InfrastructureConfig.Raw, &infrastructureConfig) + err := json.Unmarshal(shoot.Spec.Provider.ControlPlaneConfig.Raw, &ctrlPlaneConfig) require.NoError(t, err) + assert.NotEmpty(t, ctrlPlaneConfig.Zone) // validate the networking cidr here } diff --git a/pkg/gardener/shoot/hyperscaler/gcp/config.go b/pkg/gardener/shoot/hyperscaler/gcp/config.go index 5abf1061..23ed5ed0 100644 --- a/pkg/gardener/shoot/hyperscaler/gcp/config.go +++ b/pkg/gardener/shoot/hyperscaler/gcp/config.go @@ -50,3 +50,12 @@ func NewControlPlaneConfig(zones []string) *v1alpha1.ControlPlaneConfig { Zone: zones[0], } } + +func DecodeControlPlaneConfig(data []byte) (*v1alpha1.ControlPlaneConfig, error) { + controlPlaneConfig := &v1alpha1.ControlPlaneConfig{} + err := json.Unmarshal(data, controlPlaneConfig) + if err != nil { + return nil, err + } + return controlPlaneConfig, nil +} From fdccb60a0ac2c929203aca94a30be9b06e380945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sat, 25 Jan 2025 08:18:20 +0100 Subject: [PATCH 14/15] Unit tests update for provider extender - part 6 --- pkg/gardener/shoot/extender/provider_test.go | 199 +++++++++++++++++-- 1 file changed, 180 insertions(+), 19 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index ba6e07a1..370c8532 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -85,6 +85,22 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { ExpectedMachineImageVersion: "1312.3.0", ExpectedZonesCount: 3, }, + "Create provider config for AWS with worker config provided externally": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ //"10.250.0.0/22" + Provider: fixProviderWithConfig(hyperscaler.TypeAWS, "gardenlinux", "1312.3.0", []string{"eu-central-1a"}, + fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + fixAWSControlPlaneConfig()), + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageVersion: "1312.3.0", + ExpectedMachineImageVersion: "1312.3.0", + ExpectedMachineImageName: "gardenlinux", + ExpectedZonesCount: 3, + }, } { t.Run(tname, func(t *testing.T) { // given @@ -269,7 +285,30 @@ func TestProviderExtenderForPatchSingleWorkerAWS(t *testing.T) { ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, - // TODO: Add tests for override of infrastructure and control plane config with values from RuntimeCR + "Override data in controlPlaneConfig and InfrastructureConfig - with data from RuntimeCR": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithConfig(hyperscaler.TypeAWS, "gardenlinux", "1312.2.0", []string{"eu-central-1a"}, + fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + nil), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.2.0", + CurrentShootWorkers: fixWorkers("worker", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}), + ExpectedMachineImageVersion: "1312.2.0", + ExpectedZonesCount: 3, + ExpectedMachineImageName: "gardenlinux", + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/16", []string{"eu-central-1a"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, + // TODO: Add tests for override of control plane config with values from RuntimeCR } { t.Run(tname, func(t *testing.T) { // given @@ -359,6 +398,30 @@ func TestProviderExtenderForCreateMultipleWorkersAWS(t *testing.T) { {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-central-1b", "eu-central-1c"}}, {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-central-1c"}}}), }, + "Create provider config for multiple workers with AWS worker config provided externally": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkersAndConfig(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-central-1a", "eu-central-1b"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-central-1a"}}, + }), fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), fixAWSControlPlaneConfig()), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: true, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m7i.large", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-central-1a", "eu-central-1b"}}, + {"another", "m8i.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-central-1a"}}}), + }, } { t.Run(tname, func(t *testing.T) { // given @@ -519,6 +582,33 @@ func TestProviderExtenderForPatchWorkersUpdateAWS(t *testing.T) { ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), }, + "Edit infrastructure config with value provided externally with 3 zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkersAndConfig(hyperscaler.TypeAWS, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b"}}, + }), fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), fixAWSControlPlaneConfig()), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-central-1a"}}, + {"additional", "m6i.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b"}}}), + ExpectedZonesCount: 3, + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + }, } { t.Run(tname, func(t *testing.T) { // given @@ -908,7 +998,6 @@ func TestProviderExtenderForCreateMultipleWorkersGCP(t *testing.T) { ExistingInfraConfig *runtime.RawExtension ExistingControlPlaneConfig *runtime.RawExtension ExpectedShootWorkers []gardener.Worker - ExpectedZonesCount int }{ "Create multiple GCP workers without worker config": { Runtime: imv1.Runtime{ @@ -927,7 +1016,6 @@ func TestProviderExtenderForCreateMultipleWorkersGCP(t *testing.T) { }, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", - ExpectedZonesCount: 3, ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ {"main-worker", "gcp.large", "gardenlinux", "1310.4.0", 1, 3, []string{"us-central1-a"}}, {"additional", "gcp.large", "gardenlinux", "1311.2.0", 2, 4, []string{"us-central1-b", "us-central1-c"}}, @@ -1101,11 +1189,80 @@ func TestProviderExtenderForPatchWorkersUpdateGCP(t *testing.T) { } } -/*func TestProviderExtenderForCreateOpenStack(t *testing.T) { - TBD +func TestProviderExtenderForCreateOpenstack(t *testing.T) { + // tests of NewProviderExtenderForCreateOperation for workers create operation + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + ExpectedZonesCount int + }{ + "Create single Openstack worker": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + }), + }, + "Create multiple Openstack workers": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a"}}, + {"additional", "openstack.big", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-de-1b", "eu-de-1c"}}, + {"another", "openstack.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-de-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a"}}, + {"additional", "openstack.big", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-de-1b", "eu-de-1c"}}, + {"another", "openstack.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-de-1c"}}}), + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + extender := NewProviderExtenderForCreateOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedShootWorkers) + assertProviderSpecificConfigOpenstack(t, shoot) + }) + } } -*/ +func TestProviderExtenderForPatchWorkersUpdateOpenstack(t *testing.T) { + // TBD +} func fixProvider(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { return imv1.Provider{ @@ -1181,17 +1338,14 @@ func fixProviderWithMultipleWorkers(providerType string, workers []gardener.Work } } -func fixProviderWithInfrastructureConfig(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { - var infraConfigBytes []byte - switch providerType { - case hyperscaler.TypeAWS: - infraConfigBytes, _ = aws.GetInfrastructureConfig("10.250.0.0/22", zones) - case hyperscaler.TypeAzure: - infraConfigBytes, _ = azure.GetInfrastructureConfig("10.250.0.0/22", zones) - default: - panic("unknown provider type") - } +func fixProviderWithMultipleWorkersAndConfig(providerType string, workers []gardener.Worker, infraConfig, ctrlPlaneConfig *runtime.RawExtension) imv1.Provider { + provider := fixProviderWithMultipleWorkers(providerType, workers) + provider.InfrastructureConfig = infraConfig + provider.ControlPlaneConfig = ctrlPlaneConfig + return provider +} +func fixProviderWithConfig(providerType, machineImageName, machineImageVersion string, workerZones []string, infraConfig, ctrlPlaneConfig *runtime.RawExtension) imv1.Provider { return imv1.Provider{ Type: providerType, Workers: []gardener.Worker{ @@ -1203,10 +1357,11 @@ func fixProviderWithInfrastructureConfig(providerType string, machineImageName, }, Minimum: 1, Maximum: 3, - Zones: zones, + Zones: workerZones, }, }, - InfrastructureConfig: &runtime.RawExtension{Raw: infraConfigBytes}, + InfrastructureConfig: infraConfig, + ControlPlaneConfig: ctrlPlaneConfig, } } @@ -1412,6 +1567,12 @@ func assertProviderSpecificConfigGCP(t *testing.T, shoot gardener.Shoot) { err := json.Unmarshal(shoot.Spec.Provider.ControlPlaneConfig.Raw, &ctrlPlaneConfig) require.NoError(t, err) assert.NotEmpty(t, ctrlPlaneConfig.Zone) +} - // validate the networking cidr here +func assertProviderSpecificConfigOpenstack(t *testing.T, shoot gardener.Shoot) { + //var ctrlPlaneConfig gcpext.ControlPlaneConfig + // + //err := json.Unmarshal(shoot.Spec.Provider.ControlPlaneConfig.Raw, &ctrlPlaneConfig) + //require.NoError(t, err) + //assert.NotEmpty(t, ctrlPlaneConfig.Zone) } From 50e31ab54be407a805c03d2cc66ec72f7154018b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sun, 26 Jan 2025 09:30:33 +0100 Subject: [PATCH 15/15] Unit tests update for provider extender - part 7 --- pkg/gardener/shoot/extender/provider_test.go | 302 ++++++++++++++++++- 1 file changed, 286 insertions(+), 16 deletions(-) diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 370c8532..61db737c 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -5,12 +5,14 @@ import ( aws "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/aws" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/azure" gcp "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/gcp" + ops "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler/openstack" "k8s.io/apimachinery/pkg/runtime" "testing" awsext "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1" gcpext "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp/v1alpha1" + ostext "github.com/gardener/gardener-extension-provider-openstack/pkg/apis/openstack/v1alpha1" gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/hyperscaler" @@ -122,7 +124,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { t.Run("Return error for unknown provider", func(t *testing.T) { // given shoot := fixEmptyGardenerShoot("cluster", "kcp-system") - runtime := imv1.Runtime{ + rt := imv1.Runtime{ Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ Provider: imv1.Provider{ @@ -134,7 +136,7 @@ func TestProviderExtenderForCreateAWS(t *testing.T) { // when extender := NewProviderExtenderForCreateOperation(false, "", "") - err := extender(runtime, &shoot) + err := extender(rt, &shoot) // then require.Error(t, err) @@ -895,6 +897,32 @@ func TestProviderExtenderForPatchWorkersUpdateAzure(t *testing.T) { ExistingInfraConfig: fixAzureInfrastructureConfig("10.250.0.0/22", []string{"1", "2", "3"}), ExistingControlPlaneConfig: fixAzureControlPlaneConfig(), }, + "Modify infrastructure config with value provided externally with 3 zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkersAndConfig(hyperscaler.TypeAzure, fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.large", "gardenlinux", "1313.4.0", 1, 3, []string{"1"}}, + {"additional", "azure.large", "gardenlinux", "1313.2.0", 1, 3, []string{"1", "2"}}, + }), fixAzureInfrastructureConfig("10.250.0.0/22", []string{"1", "2", "3"}), fixAWSControlPlaneConfig()), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.large", "gardenlinux", "1312.4.0", 1, 3, []string{"1"}}, + {"additional", "azure.large", "gardenlinux", "1312.2.0", 1, 3, []string{"1", "2"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "azure.large", "gardenlinux", "1313.4.0", 1, 3, []string{"1"}}, + {"additional", "azure.large", "gardenlinux", "1313.2.0", 1, 3, []string{"1", "2"}}}), + ExpectedZonesCount: 3, + ExistingInfraConfig: fixAzureInfrastructureConfig("10.250.0.0/22", []string{"1", "2", "3"}), + ExistingControlPlaneConfig: fixAzureControlPlaneConfig(), + }, } { t.Run(tname, func(t *testing.T) { // given @@ -1070,7 +1098,7 @@ func TestProviderExtenderForPatchWorkersUpdateGCP(t *testing.T) { ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, {"next-worker", "n2-standard-2", "gardenlinux", "1312.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), - ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22"), ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), }, // NOTE: uncomment this unit test after the issue of extending the existing infrastructure config with additional zones is implemented @@ -1120,7 +1148,7 @@ func TestProviderExtenderForPatchWorkersUpdateGCP(t *testing.T) { {"next-worker", "n2-standard-2", "gardenlinux", "1312.2.0", 2, 4, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), - ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22"), ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), }, "Update machine type and image name and version in multiple workers separately": { @@ -1145,7 +1173,7 @@ func TestProviderExtenderForPatchWorkersUpdateGCP(t *testing.T) { ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ {"main-worker", "n2-standard-4", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}, {"additional", "n2-standard-4", "gardenlinux", "1313.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), - ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22"), ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), }, "Remove worker from existing set of workers networking zones set in infrastructureConfig should not change": { @@ -1168,7 +1196,32 @@ func TestProviderExtenderForPatchWorkersUpdateGCP(t *testing.T) { {"additional", "n2-standard-2", "gardenlinux", "1312.2.0", 1, 3, []string{"us-central1-a", "us-central1-b", "us-central1-c"}}}), ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ {"main-worker", "n2-standard-2", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a"}}}), - ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22", []string{"us-central1-a", "us-central1-b", "us-central1-c"}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22"), + ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), + }, + "Modify infrastructure config with value provided externally with 3 zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkersAndConfig(hyperscaler.TypeGCP, fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a"}}, + {"next-worker", "n2-standard-3", "gardenlinux", "1313.2.0", 1, 3, []string{"us-central1-a", "us-central1-b"}}, + }), fixGCPInfrastructureConfig("10.250.0.0/22"), fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"})), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1312.4.0", 1, 3, []string{"us-central1-a"}}, + {"next-worker", "n2-standard-3", "gardenlinux", "1312.2.0", 1, 3, []string{"us-central1-a", "us-central1-b"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "n2-standard-2", "gardenlinux", "1313.4.0", 1, 3, []string{"us-central1-a"}}, + {"next-worker", "n2-standard-3", "gardenlinux", "1313.2.0", 1, 3, []string{"us-central1-a", "us-central1-b"}}}), + ExistingInfraConfig: fixGCPInfrastructureConfig("10.250.0.0/22"), ExistingControlPlaneConfig: fixGCPControlPlaneConfig([]string{"us-central1-a", "us-central1-b", "us-central1-c"}), }, } { @@ -1199,7 +1252,7 @@ func TestProviderExtenderForCreateOpenstack(t *testing.T) { ExistingInfraConfig *runtime.RawExtension ExistingControlPlaneConfig *runtime.RawExtension ExpectedShootWorkers []gardener.Worker - ExpectedZonesCount int + ExpectedInfraConfigCIDR string }{ "Create single Openstack worker": { Runtime: imv1.Runtime{ @@ -1219,6 +1272,7 @@ func TestProviderExtenderForCreateOpenstack(t *testing.T) { ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ {"main-worker", "openstack.large", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, }), + ExpectedInfraConfigCIDR: "10.250.0.0/22", }, "Create multiple Openstack workers": { Runtime: imv1.Runtime{ @@ -1237,6 +1291,30 @@ func TestProviderExtenderForCreateOpenstack(t *testing.T) { }, DefaultMachineImageName: "gardenlinux", DefaultMachineImageVersion: "1312.3.0", + ExpectedInfraConfigCIDR: "10.250.0.0/22", + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a"}}, + {"additional", "openstack.big", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-de-1b", "eu-de-1c"}}, + {"another", "openstack.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-de-1c"}}}), + }, + "Create multiple Openstack workers with custom CIDR Infrastructure Config": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkersAndConfig(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a"}}, + {"additional", "openstack.big", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-de-1b", "eu-de-1c"}}, + {"another", "openstack.large", "gardenlinux", "1312.2.0", 3, 5, []string{"eu-de-1c"}}, + }), fixOpenstackInfrastructureConfig("10.250.0.0/16"), fixOpenstackControlPlaneConfig()), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedInfraConfigCIDR: "10.250.0.0/16", ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ {"main-worker", "openstack.small", "gardenlinux", "1310.4.0", 1, 3, []string{"eu-de-1a"}}, {"additional", "openstack.big", "gardenlinux", "1311.2.0", 2, 4, []string{"eu-de-1b", "eu-de-1c"}}, @@ -1255,13 +1333,190 @@ func TestProviderExtenderForCreateOpenstack(t *testing.T) { require.NoError(t, err) assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedShootWorkers) - assertProviderSpecificConfigOpenstack(t, shoot) + assertProviderSpecificConfigOpenstack(t, shoot, tc.ExpectedInfraConfigCIDR) }) } } func TestProviderExtenderForPatchWorkersUpdateOpenstack(t *testing.T) { - // TBD + // tests of NewProviderExtenderPatch for workers update operation for Openstack provider + for tname, tc := range map[string]struct { + Runtime imv1.Runtime + DefaultMachineImageVersion string + DefaultMachineImageName string + CurrentShootWorkers []gardener.Worker + ExistingInfraConfig *runtime.RawExtension + ExistingControlPlaneConfig *runtime.RawExtension + ExpectedShootWorkers []gardener.Worker + ExpectedInfraConfigCIDR string + }{ + "Add additional worker": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + {"next-worker", "openstack.big", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixWorkers("main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + {"next-worker", "openstack.big", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}}), + ExistingInfraConfig: fixOpenstackInfrastructureConfig("10.250.0.0/22"), + ExistingControlPlaneConfig: fixOpenstackControlPlaneConfig(), + ExpectedInfraConfigCIDR: "10.250.0.0/22", + }, + // NOTE: uncomment this unit test after the issue of extending the existing infrastructure config with additional zones is implemented + /*"Add additional worker - extend existing additional worker from non HA setup to HA setup by adding more zones, infrastructureConfig already has three zones": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + EnableIMDSv2: false, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + ExpectedZonesCount: 3, + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "m6i.large", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}, + {"additional", "m6i.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}}}), + ExistingInfraConfig: fixAWSInfrastructureConfig("10.250.0.0/22", []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}), + ExistingControlPlaneConfig: fixAWSControlPlaneConfig(), + },*/ + "Remove additional worker from existing set of workers": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}})), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + {"next-worker", "openstack.large", "gardenlinux", "1312.2.0", 2, 4, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}}), + ExistingInfraConfig: fixOpenstackInfrastructureConfig("10.250.0.0/22"), + ExistingControlPlaneConfig: fixOpenstackControlPlaneConfig(), + ExpectedInfraConfigCIDR: "10.250.0.0/22", + }, + "Update machine type and image name and version in multiple workers separately": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + {"next-worker", "openstack.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + {"next-worker", "openstack.large", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}, + {"next-worker", "openstack.large", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}}), + ExistingInfraConfig: fixOpenstackInfrastructureConfig("10.250.0.0/22"), + ExistingControlPlaneConfig: fixOpenstackControlPlaneConfig(), + ExpectedInfraConfigCIDR: "10.250.0.0/22", + }, + "Remove worker from existing set of workers networking zones set in infrastructureConfig should not change": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkers(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-de-1a"}}, + })), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a"}}, + {"additional", "openstack.small", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b", "eu-de-1c"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-de-1a"}}}), + ExistingInfraConfig: fixOpenstackInfrastructureConfig("10.250.0.0/22"), + ExistingControlPlaneConfig: fixOpenstackControlPlaneConfig(), + ExpectedInfraConfigCIDR: "10.250.0.0/22", + }, + "Modify infrastructure config with value provided externally with own CIDR": { + Runtime: imv1.Runtime{ + Spec: imv1.RuntimeSpec{ + Shoot: imv1.RuntimeShoot{ + Provider: fixProviderWithMultipleWorkersAndConfig(hyperscaler.TypeOpenStack, fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-de-1a"}}, + {"next-worker", "openstack.big", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b"}}, + }), fixOpenstackInfrastructureConfig("10.250.0.0/16"), fixOpenstackControlPlaneConfig()), + Networking: imv1.Networking{ + Nodes: "10.250.0.0/22", + }, + }, + }, + }, + DefaultMachineImageName: "gardenlinux", + DefaultMachineImageVersion: "1312.3.0", + CurrentShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1312.4.0", 1, 3, []string{"eu-de-1a"}}, + {"next-worker", "openstack.big", "gardenlinux", "1312.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b"}}}), + ExpectedShootWorkers: fixMultipleWorkers([]workerConfig{ + {"main-worker", "openstack.small", "gardenlinux", "1313.4.0", 1, 3, []string{"eu-de-1a"}}, + {"next-worker", "openstack.big", "gardenlinux", "1313.2.0", 1, 3, []string{"eu-de-1a", "eu-de-1b"}}}), + ExistingInfraConfig: fixOpenstackInfrastructureConfig("10.250.0.0/22"), + ExistingControlPlaneConfig: fixOpenstackControlPlaneConfig(), + ExpectedInfraConfigCIDR: "10.250.0.0/16", + }, + } { + t.Run(tname, func(t *testing.T) { + // given + shoot := fixEmptyGardenerShoot("cluster", "kcp-system") + + // when + extender := NewProviderExtenderPatchOperation(false, tc.DefaultMachineImageName, tc.DefaultMachineImageVersion, tc.CurrentShootWorkers, tc.ExistingInfraConfig, tc.ExistingControlPlaneConfig) + err := extender(tc.Runtime, &shoot) + + // then + require.NoError(t, err) + + assertProviderMultipleWorkers(t, tc.Runtime.Spec.Shoot, shoot, false, tc.ExpectedShootWorkers) + assertProviderSpecificConfigOpenstack(t, shoot, tc.ExpectedInfraConfigCIDR) + }) + } } func fixProvider(providerType string, machineImageName, machineImageVersion string, zones []string) imv1.Provider { @@ -1385,8 +1640,8 @@ func fixAzureControlPlaneConfig() *runtime.RawExtension { return &runtime.RawExtension{Raw: infraConfig} } -func fixGCPInfrastructureConfig(workersCIDR string, zones []string) *runtime.RawExtension { - infraConfig, _ := gcp.GetInfrastructureConfig(workersCIDR, zones) +func fixGCPInfrastructureConfig(workersCIDR string) *runtime.RawExtension { + infraConfig, _ := gcp.GetInfrastructureConfig(workersCIDR, []string{}) return &runtime.RawExtension{Raw: infraConfig} } @@ -1395,6 +1650,16 @@ func fixGCPControlPlaneConfig(zones []string) *runtime.RawExtension { return &runtime.RawExtension{Raw: controlPlaneConfig} } +func fixOpenstackInfrastructureConfig(workersCIDR string) *runtime.RawExtension { + infraConfig, _ := ops.GetInfrastructureConfig(workersCIDR, []string{}) + return &runtime.RawExtension{Raw: infraConfig} +} + +func fixOpenstackControlPlaneConfig() *runtime.RawExtension { + controlPlaneConfig, _ := ops.GetControlPlaneConfig([]string{}) + return &runtime.RawExtension{Raw: controlPlaneConfig} +} + func fixMachineImage(machineImageName, machineImageVersion string) *gardener.ShootMachineImage { if machineImageVersion != "" { return &gardener.ShootMachineImage{ @@ -1569,10 +1834,15 @@ func assertProviderSpecificConfigGCP(t *testing.T, shoot gardener.Shoot) { assert.NotEmpty(t, ctrlPlaneConfig.Zone) } -func assertProviderSpecificConfigOpenstack(t *testing.T, shoot gardener.Shoot) { - //var ctrlPlaneConfig gcpext.ControlPlaneConfig +func assertProviderSpecificConfigOpenstack(t *testing.T, shoot gardener.Shoot, expectedWorkersCIDR string) { + var ctrlPlaneConfig ostext.ControlPlaneConfig + var infraConfig ostext.InfrastructureConfig // - //err := json.Unmarshal(shoot.Spec.Provider.ControlPlaneConfig.Raw, &ctrlPlaneConfig) - //require.NoError(t, err) - //assert.NotEmpty(t, ctrlPlaneConfig.Zone) + err := json.Unmarshal(shoot.Spec.Provider.ControlPlaneConfig.Raw, &ctrlPlaneConfig) + require.NoError(t, err) + assert.Equal(t, ctrlPlaneConfig.LoadBalancerProvider, "f5") + + err = json.Unmarshal(shoot.Spec.Provider.InfrastructureConfig.Raw, &infraConfig) + require.NoError(t, err) + assert.Equal(t, infraConfig.Networks.Workers, expectedWorkersCIDR) }