From 56ae9063e185c7164b381c89af753acb4c6d0316 Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Mon, 23 Dec 2024 07:57:29 +0700 Subject: [PATCH 1/5] feat: support topologySpreadConstraints in replication & sentinel Signed-off-by: Husni Alhamdani --- api/v1beta2/redisreplication_types.go | 43 +++-- api/v1beta2/redissentinel_types.go | 41 ++-- api/v1beta2/zz_generated.deepcopy.go | 14 ++ ...edis.opstreelabs.in_redisreplications.yaml | 180 ++++++++++++++++++ ...s.redis.opstreelabs.in_redissentinels.yaml | 180 ++++++++++++++++++ .../redis-replication.yaml | 32 ++++ .../redis-sentinel.yaml | 32 ++++ pkg/k8sutils/redis-replication.go | 1 + pkg/k8sutils/redis-sentinel.go | 1 + 9 files changed, 483 insertions(+), 41 deletions(-) create mode 100644 example/v1beta2/topology_spread_constraints/redis-replication.yaml create mode 100644 example/v1beta2/topology_spread_constraints/redis-sentinel.yaml diff --git a/api/v1beta2/redisreplication_types.go b/api/v1beta2/redisreplication_types.go index 0f19ce3a1..266c2916e 100644 --- a/api/v1beta2/redisreplication_types.go +++ b/api/v1beta2/redisreplication_types.go @@ -7,27 +7,28 @@ import ( ) type RedisReplicationSpec struct { - Size *int32 `json:"clusterSize"` - KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` - RedisExporter *RedisExporter `json:"redisExporter,omitempty"` - RedisConfig *RedisConfig `json:"redisConfig,omitempty"` - Storage *Storage `json:"storage,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` - SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` - PriorityClassName string `json:"priorityClassName,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` - TLS *TLSConfig `json:"TLS,omitempty"` - PodDisruptionBudget *common.RedisPodDisruptionBudget `json:"pdb,omitempty"` - ACL *ACLConfig `json:"acl,omitempty"` - ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"` - LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"` - InitContainer *InitContainer `json:"initContainer,omitempty"` - Sidecars *[]Sidecar `json:"sidecars,omitempty"` - ServiceAccountName *string `json:"serviceAccountName,omitempty"` - TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"` - EnvVars *[]corev1.EnvVar `json:"env,omitempty"` + Size *int32 `json:"clusterSize"` + KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` + RedisExporter *RedisExporter `json:"redisExporter,omitempty"` + RedisConfig *RedisConfig `json:"redisConfig,omitempty"` + Storage *Storage `json:"storage,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + PriorityClassName string `json:"priorityClassName,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` + TLS *TLSConfig `json:"TLS,omitempty"` + PodDisruptionBudget *common.RedisPodDisruptionBudget `json:"pdb,omitempty"` + ACL *ACLConfig `json:"acl,omitempty"` + ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"` + LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"` + InitContainer *InitContainer `json:"initContainer,omitempty"` + Sidecars *[]Sidecar `json:"sidecars,omitempty"` + ServiceAccountName *string `json:"serviceAccountName,omitempty"` + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"` + EnvVars *[]corev1.EnvVar `json:"env,omitempty"` + TopologySpreadConstrains []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` } func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 { diff --git a/api/v1beta2/redissentinel_types.go b/api/v1beta2/redissentinel_types.go index bca8c8672..9babffdcc 100644 --- a/api/v1beta2/redissentinel_types.go +++ b/api/v1beta2/redissentinel_types.go @@ -9,26 +9,27 @@ import ( type RedisSentinelSpec struct { // +kubebuilder:validation:Minimum=1 // +kubebuilder:default=3 - Size *int32 `json:"clusterSize"` - KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` - RedisExporter *RedisExporter `json:"redisExporter,omitempty"` - RedisSentinelConfig *RedisSentinelConfig `json:"redisSentinelConfig,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` - SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` - PriorityClassName string `json:"priorityClassName,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` - TLS *TLSConfig `json:"TLS,omitempty"` - PodDisruptionBudget *common.RedisPodDisruptionBudget `json:"pdb,omitempty"` - ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"` - LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"` - InitContainer *InitContainer `json:"initContainer,omitempty"` - Sidecars *[]Sidecar `json:"sidecars,omitempty"` - ServiceAccountName *string `json:"serviceAccountName,omitempty"` - TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"` - EnvVars *[]corev1.EnvVar `json:"env,omitempty"` - VolumeMount *common.AdditionalVolume `json:"volumeMount,omitempty"` + Size *int32 `json:"clusterSize"` + KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` + RedisExporter *RedisExporter `json:"redisExporter,omitempty"` + RedisSentinelConfig *RedisSentinelConfig `json:"redisSentinelConfig,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + PriorityClassName string `json:"priorityClassName,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` + TLS *TLSConfig `json:"TLS,omitempty"` + PodDisruptionBudget *common.RedisPodDisruptionBudget `json:"pdb,omitempty"` + ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"` + LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"` + InitContainer *InitContainer `json:"initContainer,omitempty"` + Sidecars *[]Sidecar `json:"sidecars,omitempty"` + ServiceAccountName *string `json:"serviceAccountName,omitempty"` + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"` + EnvVars *[]corev1.EnvVar `json:"env,omitempty"` + VolumeMount *common.AdditionalVolume `json:"volumeMount,omitempty"` + TopologySpreadConstrains []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` } func (cr *RedisSentinelSpec) GetSentinelCounts(t string) int32 { diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 331905267..467318adf 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -650,6 +650,13 @@ func (in *RedisReplicationSpec) DeepCopyInto(out *RedisReplicationSpec) { } } } + if in.TopologySpreadConstrains != nil { + in, out := &in.TopologySpreadConstrains, &out.TopologySpreadConstrains + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisReplicationSpec. @@ -866,6 +873,13 @@ func (in *RedisSentinelSpec) DeepCopyInto(out *RedisSentinelSpec) { *out = new(api.AdditionalVolume) (*in).DeepCopyInto(*out) } + if in.TopologySpreadConstrains != nil { + in, out := &in.TopologySpreadConstrains, &out.TopologySpreadConstrains + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisSentinelSpec. diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml index 3dd86791c..f879ce174 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml @@ -9750,6 +9750,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array required: - clusterSize - kubernetesConfig diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml index 349c6b85b..c6c556d91 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml @@ -5238,6 +5238,186 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMount: description: Additional Volume is provided by user that is mounted on the pods diff --git a/example/v1beta2/topology_spread_constraints/redis-replication.yaml b/example/v1beta2/topology_spread_constraints/redis-replication.yaml new file mode 100644 index 000000000..56669bf92 --- /dev/null +++ b/example/v1beta2/topology_spread_constraints/redis-replication.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisReplication +metadata: + name: redis-replication +spec: + clusterSize: 3 + kubernetesConfig: + image: quay.io/opstree/redis:v7.0.12 + imagePullPolicy: IfNotPresent + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + storage: + volumeClaimTemplate: + spec: + # storageClassName: standard + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + redisExporter: + enabled: false + image: quay.io/opstree/redis-exporter:v1.44.0 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + role: replication + app: redis-replication \ No newline at end of file diff --git a/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml b/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml new file mode 100644 index 000000000..adbf1ef84 --- /dev/null +++ b/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml @@ -0,0 +1,32 @@ +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisSentinel +metadata: + name: redis-sentinel +spec: + clusterSize: 3 + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + pdb: + enabled: false + minAvailable: 1 + redisSentinelConfig: + redisReplicationName: redis-replication + kubernetesConfig: + image: quay.io/opstree/redis-sentinel:v7.0.12 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + role: sentinel + app: redis-sentinel-sentinel \ No newline at end of file diff --git a/pkg/k8sutils/redis-replication.go b/pkg/k8sutils/redis-replication.go index ad6a84128..0b4a2e481 100644 --- a/pkg/k8sutils/redis-replication.go +++ b/pkg/k8sutils/redis-replication.go @@ -100,6 +100,7 @@ func generateRedisReplicationParams(cr *redisv1beta2.RedisReplication) statefulS PriorityClassName: cr.Spec.PriorityClassName, Affinity: cr.Spec.Affinity, Tolerations: cr.Spec.Tolerations, + TopologySpreadConstraints: cr.Spec.TopologySpreadConstrains, TerminationGracePeriodSeconds: cr.Spec.TerminationGracePeriodSeconds, UpdateStrategy: cr.Spec.KubernetesConfig.UpdateStrategy, IgnoreAnnotations: cr.Spec.KubernetesConfig.IgnoreAnnotations, diff --git a/pkg/k8sutils/redis-sentinel.go b/pkg/k8sutils/redis-sentinel.go index 4b1a56ff2..c4a019ca8 100644 --- a/pkg/k8sutils/redis-sentinel.go +++ b/pkg/k8sutils/redis-sentinel.go @@ -100,6 +100,7 @@ func generateRedisSentinelParams(ctx context.Context, cr *redisv1beta2.RedisSent Affinity: affinity, TerminationGracePeriodSeconds: cr.Spec.TerminationGracePeriodSeconds, Tolerations: cr.Spec.Tolerations, + TopologySpreadConstraints: cr.Spec.TopologySpreadConstrains, ServiceAccountName: cr.Spec.ServiceAccountName, UpdateStrategy: cr.Spec.KubernetesConfig.UpdateStrategy, IgnoreAnnotations: cr.Spec.KubernetesConfig.IgnoreAnnotations, From 251a2ce008afd7b58ef0383a8f7bf08f55a26a7b Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Mon, 23 Dec 2024 08:23:01 +0700 Subject: [PATCH 2/5] feat: support topologySpreadConstraints in replication & sentinel Signed-off-by: Husni Alhamdani --- pkg/k8sutils/redis-replication_test.go | 22 ++++++++++++++++++++++ pkg/k8sutils/redis-sentinel_test.go | 14 ++++++++++++++ tests/testdata/redis-replication.yaml | 8 ++++++++ tests/testdata/redis-sentinel.yaml | 8 ++++++++ 4 files changed, 52 insertions(+) diff --git a/pkg/k8sutils/redis-replication_test.go b/pkg/k8sutils/redis-replication_test.go index db86f8b6a..a28aab40a 100644 --- a/pkg/k8sutils/redis-replication_test.go +++ b/pkg/k8sutils/redis-replication_test.go @@ -10,10 +10,19 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/utils/ptr" ) +// topologySpreadConstraints: +// - maxSkew: 1 +// topologyKey: kubernetes.io/hostname +// whenUnsatisfiable: ScheduleAnyway +// labelSelector: +// matchLabels: +// role: replication +// app: redis-replication func Test_generateRedisReplicationParams(t *testing.T) { path := filepath.Join("..", "..", "tests", "testdata", "redis-replication.yaml") expected := statefulSetParameters{ @@ -23,6 +32,19 @@ func Test_generateRedisReplicationParams(t *testing.T) { NodeSelector: map[string]string{ "node-role.kubernetes.io/infra": "worker", }, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "kubernetes.io/hostname", + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "role": "replication", + "app": "redis-replication", + }, + }, + }, + }, PodSecurityContext: &corev1.PodSecurityContext{ RunAsUser: ptr.To(int64(1000)), FSGroup: ptr.To(int64(1000)), diff --git a/pkg/k8sutils/redis-sentinel_test.go b/pkg/k8sutils/redis-sentinel_test.go index 5f2160748..c56dde181 100644 --- a/pkg/k8sutils/redis-sentinel_test.go +++ b/pkg/k8sutils/redis-sentinel_test.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/dynamic/fake" @@ -29,6 +30,19 @@ func Test_generateRedisSentinelParams(t *testing.T) { NodeSelector: map[string]string{ "node-role.kubernetes.io/infra": "worker", }, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "kubernetes.io/hostname", + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "role": "sentinel", + "app": "redis-sentinel-sentinel", + }, + }, + }, + }, PodSecurityContext: &corev1.PodSecurityContext{ RunAsUser: ptr.To(int64(1000)), FSGroup: ptr.To(int64(1000)), diff --git a/tests/testdata/redis-replication.yaml b/tests/testdata/redis-replication.yaml index 0c40a1c20..8654056cd 100644 --- a/tests/testdata/redis-replication.yaml +++ b/tests/testdata/redis-replication.yaml @@ -82,6 +82,14 @@ spec: name: example-config nodeSelector: node-role.kubernetes.io/infra: worker + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + role: replication + app: redis-replication priorityClassName: high-priority affinity: nodeAffinity: diff --git a/tests/testdata/redis-sentinel.yaml b/tests/testdata/redis-sentinel.yaml index 517b626e1..1529466c1 100644 --- a/tests/testdata/redis-sentinel.yaml +++ b/tests/testdata/redis-sentinel.yaml @@ -64,6 +64,14 @@ spec: key: username nodeSelector: node-role.kubernetes.io/infra: worker + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + role: sentinel + app: redis-sentinel-sentinel priorityClassName: high-priority affinity: nodeAffinity: From 7e176df77dbb158289b1ba123cad0f91d3e8c928 Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Mon, 23 Dec 2024 08:23:50 +0700 Subject: [PATCH 3/5] feat: support topologySpreadConstraints in replication & sentinel Signed-off-by: Husni Alhamdani --- pkg/k8sutils/redis-replication_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/k8sutils/redis-replication_test.go b/pkg/k8sutils/redis-replication_test.go index a28aab40a..3d12cf044 100644 --- a/pkg/k8sutils/redis-replication_test.go +++ b/pkg/k8sutils/redis-replication_test.go @@ -15,14 +15,6 @@ import ( "k8s.io/utils/ptr" ) -// topologySpreadConstraints: -// - maxSkew: 1 -// topologyKey: kubernetes.io/hostname -// whenUnsatisfiable: ScheduleAnyway -// labelSelector: -// matchLabels: -// role: replication -// app: redis-replication func Test_generateRedisReplicationParams(t *testing.T) { path := filepath.Join("..", "..", "tests", "testdata", "redis-replication.yaml") expected := statefulSetParameters{ From d68d6e32abfbde725ce6bc918d25c4cd64f6b0d6 Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Tue, 24 Dec 2024 15:23:29 +0700 Subject: [PATCH 4/5] feat: support topologySpreadConstraints in replication & sentinel Signed-off-by: Husni Alhamdani --- .../topology_spread_constraints/redis-replication.yaml | 3 ++- .../v1beta2/topology_spread_constraints/redis-sentinel.yaml | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/example/v1beta2/topology_spread_constraints/redis-replication.yaml b/example/v1beta2/topology_spread_constraints/redis-replication.yaml index 56669bf92..6f838cdf2 100644 --- a/example/v1beta2/topology_spread_constraints/redis-replication.yaml +++ b/example/v1beta2/topology_spread_constraints/redis-replication.yaml @@ -29,4 +29,5 @@ spec: labelSelector: matchLabels: role: replication - app: redis-replication \ No newline at end of file + app: redis-replication + \ No newline at end of file diff --git a/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml b/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml index adbf1ef84..8e57d3b71 100644 --- a/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml +++ b/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml @@ -1,3 +1,4 @@ +--- apiVersion: redis.redis.opstreelabs.in/v1beta2 kind: RedisSentinel metadata: @@ -29,4 +30,5 @@ spec: labelSelector: matchLabels: role: sentinel - app: redis-sentinel-sentinel \ No newline at end of file + app: redis-sentinel-sentinel + \ No newline at end of file From cd7de153d607437d8ec25f39ef39bb1bc345fc23 Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Tue, 24 Dec 2024 15:36:41 +0700 Subject: [PATCH 5/5] feat: support topologySpreadConstraints in replication & sentinel Signed-off-by: Husni Alhamdani --- .../v1beta2/topology_spread_constraints/redis-replication.yaml | 1 - example/v1beta2/topology_spread_constraints/redis-sentinel.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/example/v1beta2/topology_spread_constraints/redis-replication.yaml b/example/v1beta2/topology_spread_constraints/redis-replication.yaml index 6f838cdf2..a9f29e138 100644 --- a/example/v1beta2/topology_spread_constraints/redis-replication.yaml +++ b/example/v1beta2/topology_spread_constraints/redis-replication.yaml @@ -30,4 +30,3 @@ spec: matchLabels: role: replication app: redis-replication - \ No newline at end of file diff --git a/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml b/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml index 8e57d3b71..556822a9b 100644 --- a/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml +++ b/example/v1beta2/topology_spread_constraints/redis-sentinel.yaml @@ -31,4 +31,3 @@ spec: matchLabels: role: sentinel app: redis-sentinel-sentinel - \ No newline at end of file