From a6d252f51c0b14e1f12fec80c2994ee5ba278010 Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Tue, 10 Dec 2024 18:55:01 +0530 Subject: [PATCH 01/21] Updated LICENSE today Signed-off-by: xiaozhuang --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index bbbec2b09..058ac3e7c 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [2020] [Opstree Solutions] + Copyright [2024] [Opstree Solutions] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 2472580af3420b0eb671bc28b3b9d12ad6c8f29b Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 17:05:16 +0800 Subject: [PATCH 02/21] feat: enhance RedisReplication controller and CRD with additional status columns and refactor reconciliation logic - Added new printer columns "Master" and "Age" to the RedisReplication CRD for better visibility of the master node and resource age. - Refactored the reconciliation logic in the RedisReplication controller to improve clarity and maintainability by introducing a reconciler struct for handling different reconciliation tasks. - Updated the e2e tests to validate the HA setup of Redis Replication and Sentinel, ensuring consistency in master IP across different sources. - Removed obsolete test files and replaced them with a new HA setup configuration. This update improves the usability and reliability of the Redis replication feature. Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- api/v1beta2/redisreplication_types.go | 2 + ...edis.opstreelabs.in_redisreplications.yaml | 9 +- .../redisreplication_controller.go | 154 +++++++++++++----- pkg/k8sutils/kube.go | 9 + .../v1beta2/setup/ha/chainsaw-test.yaml | 40 ++--- .../v1beta2/setup/ha/cli-pod.yaml | 15 -- .../setup/ha/{replication.yaml => ha.yaml} | 23 +++ .../v1beta2/setup/ha/sentinel.yaml | 23 --- 8 files changed, 174 insertions(+), 101 deletions(-) create mode 100644 pkg/k8sutils/kube.go delete mode 100644 tests/e2e-chainsaw/v1beta2/setup/ha/cli-pod.yaml rename tests/e2e-chainsaw/v1beta2/setup/ha/{replication.yaml => ha.yaml} (53%) delete mode 100644 tests/e2e-chainsaw/v1beta2/setup/ha/sentinel.yaml diff --git a/api/v1beta2/redisreplication_types.go b/api/v1beta2/redisreplication_types.go index 92915e76c..8b2510cc8 100644 --- a/api/v1beta2/redisreplication_types.go +++ b/api/v1beta2/redisreplication_types.go @@ -41,6 +41,8 @@ type RedisReplicationStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Master",type="string",JSONPath=".status.masterNode" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // Redis is the Schema for the redis API type RedisReplication struct { diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml index 48ba4d320..38a26b931 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml @@ -4397,7 +4397,14 @@ spec: storage: false subresources: status: {} - - name: v1beta2 + - additionalPrinterColumns: + - jsonPath: .status.masterNode + name: Master + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 schema: openAPIV3Schema: description: Redis is the Schema for the redis API diff --git a/pkg/controllers/redisreplication/redisreplication_controller.go b/pkg/controllers/redisreplication/redisreplication_controller.go index 6b71bf56f..3959bd677 100644 --- a/pkg/controllers/redisreplication/redisreplication_controller.go +++ b/pkg/controllers/redisreplication/redisreplication_controller.go @@ -28,57 +28,37 @@ type Reconciler struct { } func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx, "Request.Namespace", req.Namespace, "Request.Name", req.Name) instance := &redisv1beta2.RedisReplication{} - err := r.Client.Get(context.TODO(), req.NamespacedName, instance) + err := r.Client.Get(ctx, req.NamespacedName, instance) if err != nil { - return intctrlutil.RequeueWithErrorChecking(ctx, err, "") - } - if instance.ObjectMeta.GetDeletionTimestamp() != nil { - if err = k8sutils.HandleRedisReplicationFinalizer(ctx, r.Client, r.K8sClient, instance); err != nil { - return intctrlutil.RequeueWithError(ctx, err, "") - } - return intctrlutil.Reconciled() - } - if _, found := instance.ObjectMeta.GetAnnotations()["redisreplication.opstreelabs.in/skip-reconcile"]; found { - return intctrlutil.RequeueAfter(ctx, time.Second*10, "found skip reconcile annotation") - } - if err = k8sutils.AddFinalizer(ctx, instance, k8sutils.RedisReplicationFinalizer, r.Client); err != nil { - return intctrlutil.RequeueWithError(ctx, err, "") - } - err = k8sutils.CreateReplicationRedis(ctx, instance, r.K8sClient) - if err != nil { - return intctrlutil.RequeueWithError(ctx, err, "") - } - err = k8sutils.CreateReplicationService(ctx, instance, r.K8sClient) - if err != nil { - return intctrlutil.RequeueWithError(ctx, err, "") - } - if !r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name) { - return intctrlutil.Reconciled() + return intctrlutil.RequeueWithErrorChecking(ctx, err, "failed to get RedisReplication instance") } - var realMaster string - masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, instance, "master") - if len(masterNodes) > 1 { - logger.Info("Creating redis replication by executing replication creation commands") - slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, instance, "slave") - realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, instance, masterNodes) - if len(slaveNodes) == 0 { - realMaster = masterNodes[0] + var reconcilers []reconciler + if k8sutils.IsDeleted(instance) { + reconcilers = []reconciler{ + {typ: "finalizer", rec: r.reconcileFinalizer}, } - if err = k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, instance, masterNodes, realMaster); err != nil { - return intctrlutil.RequeueAfter(ctx, time.Second*60, "") + } else { + reconcilers = []reconciler{ + {typ: "annotation", rec: r.reconcileAnnotation}, + {typ: "statefulset", rec: r.reconcileStatefulSet}, + {typ: "service", rec: r.reconcileService}, + {typ: "redis", rec: r.reconcileRedis}, + {typ: "status", rec: r.reconcileStatus}, } } - realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, instance, masterNodes) - if err = r.UpdateRedisReplicationMaster(ctx, instance, realMaster); err != nil { - return intctrlutil.RequeueWithError(ctx, err, "") - } - if err = r.UpdateRedisPodRoleLabel(ctx, instance, realMaster); err != nil { - return intctrlutil.RequeueWithError(ctx, err, "") + for _, reconciler := range reconcilers { + result, err := reconciler.rec(ctx, instance) + if err != nil { + return intctrlutil.RequeueWithError(ctx, err, "") + } + if result.Requeue { + return result, nil + } } + return intctrlutil.RequeueAfter(ctx, time.Second*10, "") } @@ -86,6 +66,13 @@ func (r *Reconciler) UpdateRedisReplicationMaster(ctx context.Context, instance if instance.Status.MasterNode == masterNode { return nil } + + if instance.Status.MasterNode != masterNode { + logger := log.FromContext(ctx) + logger.Info("Updating master node", + "previous", instance.Status.MasterNode, + "new", masterNode) + } instance.Status.MasterNode = masterNode if err := r.Client.Status().Update(ctx, instance); err != nil { return err @@ -118,6 +105,89 @@ func (r *Reconciler) UpdateRedisPodRoleLabel(ctx context.Context, cr *redisv1bet return nil } +type reconciler struct { + typ string + rec func(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) +} + +func (r *Reconciler) reconcileFinalizer(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { + if k8sutils.IsDeleted(instance) { + if err := k8sutils.HandleRedisReplicationFinalizer(ctx, r.Client, r.K8sClient, instance); err != nil { + return intctrlutil.RequeueWithError(ctx, err, "") + } + return intctrlutil.Reconciled() + } + if err := k8sutils.AddFinalizer(ctx, instance, k8sutils.RedisReplicationFinalizer, r.Client); err != nil { + return intctrlutil.RequeueWithError(ctx, err, "") + } + return intctrlutil.Reconciled() +} + +func (r *Reconciler) reconcileAnnotation(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { + if _, found := instance.ObjectMeta.GetAnnotations()["redisreplication.opstreelabs.in/skip-reconcile"]; found { + return intctrlutil.RequeueAfter(ctx, time.Second*10, "found skip reconcile annotation") + } + return intctrlutil.Reconciled() +} + +func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { + if err := k8sutils.CreateReplicationRedis(ctx, instance, r.K8sClient); err != nil { + return intctrlutil.RequeueAfter(ctx, time.Second*60, "") + } + return intctrlutil.Reconciled() +} + +func (r *Reconciler) reconcileService(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { + if err := k8sutils.CreateReplicationService(ctx, instance, r.K8sClient); err != nil { + return intctrlutil.RequeueAfter(ctx, time.Second*60, "") + } + return intctrlutil.Reconciled() +} + +func (r *Reconciler) reconcileRedis(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + logger := log.FromContext(ctx) + + if !r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name) { + logger.Info("StatefulSet not ready yet, requeuing", + "namespace", instance.Namespace, + "name", instance.Name) + return intctrlutil.RequeueAfter(ctx, time.Second*60, "") + } + + var realMaster string + masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, instance, "master") + if len(masterNodes) > 1 { + log.FromContext(ctx).Info("Creating redis replication by executing replication creation commands") + slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, instance, "slave") + realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, instance, masterNodes) + if len(slaveNodes) == 0 { + realMaster = masterNodes[0] + } + if err := k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, instance, masterNodes, realMaster); err != nil { + return intctrlutil.RequeueAfter(ctx, time.Second*60, "") + } + } + return intctrlutil.Reconciled() +} + +// reconcileStatus update status and label. +func (r *Reconciler) reconcileStatus(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { + var err error + var realMaster string + + masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, instance, "master") + realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, instance, masterNodes) + if err = r.UpdateRedisReplicationMaster(ctx, instance, realMaster); err != nil { + return intctrlutil.RequeueWithError(ctx, err, "") + } + if err = r.UpdateRedisPodRoleLabel(ctx, instance, realMaster); err != nil { + return intctrlutil.RequeueWithError(ctx, err, "") + } + return intctrlutil.Reconciled() +} + // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/pkg/k8sutils/kube.go b/pkg/k8sutils/kube.go new file mode 100644 index 000000000..f70056fb5 --- /dev/null +++ b/pkg/k8sutils/kube.go @@ -0,0 +1,9 @@ +package k8sutils + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func IsDeleted(obj client.Object) bool { + return obj.GetDeletionTimestamp() != nil +} diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml index 050f59672..74205ce69 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml @@ -1,3 +1,11 @@ +# This case is to test the HA setup of the Redis Replication and Sentinel +# It will create a Redis Replication and Sentinel, then terminate the Redis Replication master pod +# and check if the Sentinel can promote a new master pod. +# +# It check three place the same pod IP: +# 1. Status from RedisReplication +# 2. Label from RedisReplication +# 3. get-master-addr-by-name from Sentinel --- apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -7,25 +15,19 @@ spec: steps: - try: - apply: - file: replication.yaml - - apply: - file: sentinel.yaml - - create: - file: cli-pod.yaml + file: ha.yaml - - name: Sleep for 3 minutes + - name: Test Master IP consistency try: - sleep: - duration: 3m - - - name: Test sentinel monitoring - try: + duration: 180s - script: timeout: 10s content: | - export MASTER_IP_FROM_SENTINEL=$(kubectl exec --namespace ${NAMESPACE} redis-sentinel-sentinel-0 -- redis-cli -p 26379 sentinel get-master-addr-by-name myMaster | head -n 1); + export MASTER_IP_FROM_STATUS=$(kubectl -n ${NAMESPACE} get pod $(kubectl -n ${NAMESPACE} get redisreplication redis-replication -o jsonpath='{.status.masterNode}') -o jsonpath='{.status.podIP}'); + export MASTER_IP_FROM_SENTINEL=$(kubectl -n ${NAMESPACE} exec redis-sentinel-sentinel-0 -- redis-cli -p 26379 sentinel get-master-addr-by-name myMaster | head -n 1); export MASTER_IP_FROM_LABEL=$(kubectl -n ${NAMESPACE} get pod -l app=redis-replication,redis-role=master,redis_setup_type=replication -o jsonpath='{.items[0].status.podIP}'); - if [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_LABEL" ]; then echo "OK"; else echo "FAIL"; fi + if [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_LABEL" ] && [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_STATUS" ]; then echo "OK"; else echo "FAIL"; fi check: (contains($stdout, 'OK')): true @@ -35,20 +37,18 @@ spec: - script: timeout: 10s content: | - kubectl --namespace ${NAMESPACE} delete pod redis-replication-0 - - - name: Sleep for 5 minutes - try: + kubectl -n ${NAMESPACE} delete pod redis-replication-0 - sleep: - duration: 5m + duration: 30s - - name: Test sentinel monitoring + - name: Test Master IP consistency try: - script: timeout: 10s content: | - export MASTER_IP_FROM_SENTINEL=$(kubectl exec --namespace ${NAMESPACE} redis-sentinel-sentinel-0 -- redis-cli -p 26379 sentinel get-master-addr-by-name myMaster | head -n 1); + export MASTER_IP_FROM_STATUS=$(kubectl -n ${NAMESPACE} get pod $(kubectl -n ${NAMESPACE} get redisreplication redis-replication -o jsonpath='{.status.masterNode}') -o jsonpath='{.status.podIP}'); + export MASTER_IP_FROM_SENTINEL=$(kubectl -n ${NAMESPACE} exec redis-sentinel-sentinel-0 -- redis-cli -p 26379 sentinel get-master-addr-by-name myMaster | head -n 1); export MASTER_IP_FROM_LABEL=$(kubectl -n ${NAMESPACE} get pod -l app=redis-replication,redis-role=master,redis_setup_type=replication -o jsonpath='{.items[0].status.podIP}'); - if [ $MASTER_IP_FROM_SENTINEL = $MASTER_IP_FROM_LABEL ]; then echo "OK"; else echo "FAIL"; fi + if [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_LABEL" ] && [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_STATUS" ]; then echo "OK"; else echo "FAIL"; fi check: (contains($stdout, 'OK')): true diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/cli-pod.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/cli-pod.yaml deleted file mode 100644 index e3049d88b..000000000 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/cli-pod.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: redis - labels: - app: redis -spec: - containers: - - name: redis - image: redis:alpine - resources: - limits: - cpu: 200m - memory: 500Mi diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/replication.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/ha.yaml similarity index 53% rename from tests/e2e-chainsaw/v1beta2/setup/ha/replication.yaml rename to tests/e2e-chainsaw/v1beta2/setup/ha/ha.yaml index bf7c7e7b4..1ccd498bc 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/replication.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/ha/ha.yaml @@ -25,3 +25,26 @@ spec: resources: requests: storage: 1Gi +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisSentinel +metadata: + name: redis-sentinel +spec: + clusterSize: 1 + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + redisSentinelConfig: + redisReplicationName: redis-replication + quorum: '1' + kubernetesConfig: + image: quay.io/opstree/redis-sentinel:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/sentinel.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/sentinel.yaml deleted file mode 100644 index 994b5626a..000000000 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/sentinel.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: redis.redis.opstreelabs.in/v1beta2 -kind: RedisSentinel -metadata: - name: redis-sentinel -spec: - clusterSize: 1 - podSecurityContext: - runAsUser: 1000 - fsGroup: 1000 - redisSentinelConfig: - redisReplicationName: redis-replication - quorum: '1' - kubernetesConfig: - image: quay.io/opstree/redis-sentinel:latest - imagePullPolicy: Always - resources: - requests: - cpu: 101m - memory: 128Mi - limits: - cpu: 101m - memory: 128Mi From 3250849ceee9d416f6a6c484a37e6da50f7fbecc Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 17:32:35 +0800 Subject: [PATCH 03/21] fix lint Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml index 74205ce69..8d367dfed 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml @@ -24,7 +24,8 @@ spec: - script: timeout: 10s content: | - export MASTER_IP_FROM_STATUS=$(kubectl -n ${NAMESPACE} get pod $(kubectl -n ${NAMESPACE} get redisreplication redis-replication -o jsonpath='{.status.masterNode}') -o jsonpath='{.status.podIP}'); + export MASTER_POD_FROM_STATUS=$(kubectl -n ${NAMESPACE} get redisreplication redis-replication -o jsonpath='{.status.masterNode}'); + export MASTER_IP_FROM_STATUS=$(kubectl -n ${NAMESPACE} get pod ${MASTER_POD_FROM_STATUS} -o jsonpath='{.status.podIP}'); export MASTER_IP_FROM_SENTINEL=$(kubectl -n ${NAMESPACE} exec redis-sentinel-sentinel-0 -- redis-cli -p 26379 sentinel get-master-addr-by-name myMaster | head -n 1); export MASTER_IP_FROM_LABEL=$(kubectl -n ${NAMESPACE} get pod -l app=redis-replication,redis-role=master,redis_setup_type=replication -o jsonpath='{.items[0].status.podIP}'); if [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_LABEL" ] && [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_STATUS" ]; then echo "OK"; else echo "FAIL"; fi @@ -46,7 +47,8 @@ spec: - script: timeout: 10s content: | - export MASTER_IP_FROM_STATUS=$(kubectl -n ${NAMESPACE} get pod $(kubectl -n ${NAMESPACE} get redisreplication redis-replication -o jsonpath='{.status.masterNode}') -o jsonpath='{.status.podIP}'); + export MASTER_POD_FROM_STATUS=$(kubectl -n ${NAMESPACE} get redisreplication redis-replication -o jsonpath='{.status.masterNode}'); + export MASTER_IP_FROM_STATUS=$(kubectl -n ${NAMESPACE} get pod ${MASTER_POD_FROM_STATUS} -o jsonpath='{.status.podIP}'); export MASTER_IP_FROM_SENTINEL=$(kubectl -n ${NAMESPACE} exec redis-sentinel-sentinel-0 -- redis-cli -p 26379 sentinel get-master-addr-by-name myMaster | head -n 1); export MASTER_IP_FROM_LABEL=$(kubectl -n ${NAMESPACE} get pod -l app=redis-replication,redis-role=master,redis_setup_type=replication -o jsonpath='{.items[0].status.podIP}'); if [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_LABEL" ] && [ "$MASTER_IP_FROM_SENTINEL" = "$MASTER_IP_FROM_STATUS" ]; then echo "OK"; else echo "FAIL"; fi From bcc7b4fb0c0862882ca115a875eaf63100bd6182 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 17:45:04 +0800 Subject: [PATCH 04/21] lint Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml index 8d367dfed..eeb088e31 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml @@ -1,7 +1,6 @@ # This case is to test the HA setup of the Redis Replication and Sentinel # It will create a Redis Replication and Sentinel, then terminate the Redis Replication master pod # and check if the Sentinel can promote a new master pod. -# # It check three place the same pod IP: # 1. Status from RedisReplication # 2. Label from RedisReplication From a90cb62c1d116f92b11443343b5431013d385d8c Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 21:54:15 +0800 Subject: [PATCH 05/21] upgrade go Signed-off-by: xiaozhuang --- .github/workflows/ci.yaml | 2 +- .golangci.yml | 2 +- Dockerfile | 2 +- go.mod | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3e403756e..d6f301ae4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ permissions: contents: read env: - GOLANG_VERSION: 1.22 + GOLANG_VERSION: 1.23.4 APPLICATION_NAME: redis-operator DockerImagName: docker.io/opstree/redis-operator BuildDocs: true diff --git a/.golangci.yml b/.golangci.yml index 1ff28d4d5..8e6b32271 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -47,7 +47,7 @@ linters: run: timeout: 15m - go: "1.22" + go: "1.23.4" tests: true show-stats: true skip-files: diff --git a/Dockerfile b/Dockerfile index ae57deaee..b7554aa6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.22 as builder +FROM golang:1.23-alpine as builder ARG BUILDOS ARG BUILDPLATFORM ARG BUILDARCH diff --git a/go.mod b/go.mod index a10f233c7..3e91767ec 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/OT-CONTAINER-KIT/redis-operator -go 1.22 +go 1.23.4 require ( github.com/avast/retry-go v3.0.0+incompatible From 9d128bf5fcd38b5510fd50ad6004c5f29f927c70 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 21:58:57 +0800 Subject: [PATCH 06/21] add Signed-off-by: xiaozhuang --- pkg/controllers/redisreplication/redisreplication_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controllers/redisreplication/redisreplication_controller.go b/pkg/controllers/redisreplication/redisreplication_controller.go index 3959bd677..cca13842c 100644 --- a/pkg/controllers/redisreplication/redisreplication_controller.go +++ b/pkg/controllers/redisreplication/redisreplication_controller.go @@ -43,6 +43,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } else { reconcilers = []reconciler{ {typ: "annotation", rec: r.reconcileAnnotation}, + {typ: "finalizer", rec: r.reconcileFinalizer}, {typ: "statefulset", rec: r.reconcileStatefulSet}, {typ: "service", rec: r.reconcileService}, {typ: "redis", rec: r.reconcileRedis}, From 078377ebb11683ed832691b8cecafd2947af9566 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 22:25:42 +0800 Subject: [PATCH 07/21] disable Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- .golangci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 8e6b32271..2776e5505 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -39,7 +39,6 @@ linters: - tenv - thelper - tparallel - - typecheck - unconvert - unused - wastedassign From 8f4285d7d3dae79c599e84164c30aa36d2ec0a6f Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 22:31:27 +0800 Subject: [PATCH 08/21] update Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d6f301ae4..34d6eaf52 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,7 +32,7 @@ jobs: - name: Run GolangCI-Lint uses: golangci/golangci-lint-action@v6 with: - version: v1.54.0 + version: v1.62.2 gotest: needs: From d64ba5b824e719f397a54e947e9ce087d7dbe76c Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 22:38:41 +0800 Subject: [PATCH 09/21] disable Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- .golangci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 2776e5505..ce2837880 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,7 +24,6 @@ linters: - gofmt - gofumpt - goprintffuncname - - gosec - gosimple - govet - grouper From 67ad139f929073198a94353a32fbf48c1b09157f Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 23:21:43 +0800 Subject: [PATCH 10/21] update Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml index eeb088e31..a97c96b03 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml @@ -39,7 +39,7 @@ spec: content: | kubectl -n ${NAMESPACE} delete pod redis-replication-0 - sleep: - duration: 30s + duration: 60s - name: Test Master IP consistency try: From f434ac3c8759b0be189e5ce1b2448e009662dad5 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 10 Dec 2024 23:22:03 +0800 Subject: [PATCH 11/21] update Signed-off-by: drivebyer Signed-off-by: xiaozhuang --- tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml index a97c96b03..2e4ee24e5 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/ha/chainsaw-test.yaml @@ -39,7 +39,7 @@ spec: content: | kubectl -n ${NAMESPACE} delete pod redis-replication-0 - sleep: - duration: 60s + duration: 120s - name: Test Master IP consistency try: From c88fe2b3dc55034e775e87aaf6c0c8f793b83d1c Mon Sep 17 00:00:00 2001 From: yangw Date: Wed, 11 Dec 2024 09:59:44 +0800 Subject: [PATCH 12/21] chore: update dependabot configuration to change update schedule from daily to monthly for gomod and github-actions Signed-off-by: yangw Signed-off-by: xiaozhuang --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0476c5b94..fe405aaaa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,8 +3,8 @@ updates: - package-ecosystem: gomod directory: / schedule: - interval: daily + interval: monthly - package-ecosystem: github-actions directory: / schedule: - interval: daily \ No newline at end of file + interval: monthly \ No newline at end of file From 6c9d90efa5868f249d102bd05a56f63afcb03460 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 01:43:09 +0000 Subject: [PATCH 13/21] chore(deps): bump github.com/onsi/ginkgo/v2 from 2.20.1 to 2.22.0 Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.20.1 to 2.22.0. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.20.1...v2.22.0) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: xiaozhuang --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 3e91767ec..a99745e3f 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/banzaicloud/k8s-objectmatcher v1.8.0 github.com/go-logr/logr v1.4.2 github.com/go-redis/redismock/v9 v9.2.0 - github.com/onsi/ginkgo/v2 v2.20.1 + github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.35.1 github.com/pkg/errors v0.9.1 github.com/redis/go-redis/v9 v9.7.0 @@ -40,7 +40,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -67,7 +67,7 @@ require ( golang.org/x/term v0.25.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/tools v0.26.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.35.1 // indirect diff --git a/go.sum b/go.sum index ca3356138..c100e7740 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -153,8 +153,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= -github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= @@ -269,8 +269,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 9a1fadce839643f94e6392151bf7a7edc103f2c8 Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Mon, 16 Dec 2024 13:09:31 +0700 Subject: [PATCH 14/21] feat: support PDB in redisreplication (#1166) **Description** Support PDB in redisreplication **Type of change** * Bug fix (non-breaking change which fixes an issue) * New feature (non-breaking change which adds functionality) * Breaking change (fix or feature that would cause existing functionality to not work as expected) **Checklist** - [ ] Tests have been added/modified and all tests pass. - [ ] Functionality/bugs have been confirmed to be unchanged or fixed. - [x] I have performed a self-review of my own code. - [ ] Documentation has been updated or added where necessary. **Additional Context** Signed-off-by: xiaozhuang --- api/v1beta2/redisreplication_types.go | 42 ++++++------- api/v1beta2/zz_generated.deepcopy.go | 5 ++ ...edis.opstreelabs.in_redisreplications.yaml | 13 ++++ .../redisreplication_controller.go | 8 +++ pkg/k8sutils/poddisruption.go | 59 +++++++++++++++++-- 5 files changed, 102 insertions(+), 25 deletions(-) diff --git a/api/v1beta2/redisreplication_types.go b/api/v1beta2/redisreplication_types.go index 8b2510cc8..0f19ce3a1 100644 --- a/api/v1beta2/redisreplication_types.go +++ b/api/v1beta2/redisreplication_types.go @@ -1,31 +1,33 @@ package v1beta2 import ( + common "github.com/OT-CONTAINER-KIT/redis-operator/api" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type RedisReplicationSpec struct { - Size *int32 `json:"clusterSize"` - KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` - RedisExporter *RedisExporter `json:"redisExporter,omitempty"` - RedisConfig *RedisConfig `json:"redisConfig,omitempty"` - Storage *Storage `json:"storage,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` - SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` - PriorityClassName string `json:"priorityClassName,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` - TLS *TLSConfig `json:"TLS,omitempty"` - ACL *ACLConfig `json:"acl,omitempty"` - ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"` - LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"` - InitContainer *InitContainer `json:"initContainer,omitempty"` - Sidecars *[]Sidecar `json:"sidecars,omitempty"` - ServiceAccountName *string `json:"serviceAccountName,omitempty"` - TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"` - EnvVars *[]corev1.EnvVar `json:"env,omitempty"` + Size *int32 `json:"clusterSize"` + KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` + RedisExporter *RedisExporter `json:"redisExporter,omitempty"` + RedisConfig *RedisConfig `json:"redisConfig,omitempty"` + Storage *Storage `json:"storage,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + PriorityClassName string `json:"priorityClassName,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` + TLS *TLSConfig `json:"TLS,omitempty"` + PodDisruptionBudget *common.RedisPodDisruptionBudget `json:"pdb,omitempty"` + ACL *ACLConfig `json:"acl,omitempty"` + ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"` + LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"` + InitContainer *InitContainer `json:"initContainer,omitempty"` + Sidecars *[]Sidecar `json:"sidecars,omitempty"` + ServiceAccountName *string `json:"serviceAccountName,omitempty"` + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"` + EnvVars *[]corev1.EnvVar `json:"env,omitempty"` } func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 { diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 55493e1d9..331905267 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -593,6 +593,11 @@ func (in *RedisReplicationSpec) DeepCopyInto(out *RedisReplicationSpec) { *out = new(TLSConfig) (*in).DeepCopyInto(*out) } + if in.PodDisruptionBudget != nil { + in, out := &in.PodDisruptionBudget, &out.PodDisruptionBudget + *out = new(api.RedisPodDisruptionBudget) + (*in).DeepCopyInto(*out) + } if in.ACL != nil { in, out := &in.ACL, &out.ACL *out = new(ACLConfig) diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml index 38a26b931..3dd86791c 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml @@ -6240,6 +6240,19 @@ spec: additionalProperties: type: string type: object + pdb: + description: RedisPodDisruptionBudget configure a PodDisruptionBudget + on the resource (leader/follower) + properties: + enabled: + type: boolean + maxUnavailable: + format: int32 + type: integer + minAvailable: + format: int32 + type: integer + type: object podSecurityContext: description: |- PodSecurityContext holds pod-level security attributes and common container settings. diff --git a/pkg/controllers/redisreplication/redisreplication_controller.go b/pkg/controllers/redisreplication/redisreplication_controller.go index cca13842c..0e3e0f433 100644 --- a/pkg/controllers/redisreplication/redisreplication_controller.go +++ b/pkg/controllers/redisreplication/redisreplication_controller.go @@ -46,6 +46,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu {typ: "finalizer", rec: r.reconcileFinalizer}, {typ: "statefulset", rec: r.reconcileStatefulSet}, {typ: "service", rec: r.reconcileService}, + {typ: "poddisruptionbudget", rec: r.reconcilePDB}, {typ: "redis", rec: r.reconcileRedis}, {typ: "status", rec: r.reconcileStatus}, } @@ -131,6 +132,13 @@ func (r *Reconciler) reconcileAnnotation(ctx context.Context, instance *redisv1b return intctrlutil.Reconciled() } +func (r *Reconciler) reconcilePDB(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { + if err := k8sutils.ReconcileReplicationPodDisruptionBudget(ctx, instance, instance.Spec.PodDisruptionBudget, r.K8sClient); err != nil { + return intctrlutil.RequeueAfter(ctx, time.Second*60, "") + } + return intctrlutil.Reconciled() +} + func (r *Reconciler) reconcileStatefulSet(ctx context.Context, instance *redisv1beta2.RedisReplication) (ctrl.Result, error) { if err := k8sutils.CreateReplicationRedis(ctx, instance, r.K8sClient); err != nil { return intctrlutil.RequeueAfter(ctx, time.Second*60, "") diff --git a/pkg/k8sutils/poddisruption.go b/pkg/k8sutils/poddisruption.go index 6dc97a970..69bad44cd 100644 --- a/pkg/k8sutils/poddisruption.go +++ b/pkg/k8sutils/poddisruption.go @@ -26,7 +26,7 @@ func ReconcileRedisPodDisruptionBudget(ctx context.Context, cr *redisv1beta2.Red return CreateOrUpdatePodDisruptionBudget(ctx, pdbDef, cl) } else { // Check if one exists, and delete it. - _, err := GetPodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) + _, err := getPodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) if err == nil { return deletePodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) } else if err != nil && errors.IsNotFound(err) { @@ -48,7 +48,29 @@ func ReconcileSentinelPodDisruptionBudget(ctx context.Context, cr *redisv1beta2. return CreateOrUpdatePodDisruptionBudget(ctx, pdbDef, cl) } else { // Check if one exists, and delete it. - _, err := GetPodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) + _, err := getPodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) + if err == nil { + return deletePodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) + } else if err != nil && errors.IsNotFound(err) { + log.FromContext(ctx).V(1).Info("Reconciliation Successful, no PodDisruptionBudget Found.") + // Its ok if its not found, as we're deleting anyway + return nil + } + return err + } +} + +func ReconcileReplicationPodDisruptionBudget(ctx context.Context, cr *redisv1beta2.RedisReplication, pdbParams *commonapi.RedisPodDisruptionBudget, cl kubernetes.Interface) error { + pdbName := cr.ObjectMeta.Name + "-replication" + if pdbParams != nil && pdbParams.Enabled { + labels := getRedisLabels(cr.ObjectMeta.Name, replication, "replication", cr.GetObjectMeta().GetLabels()) + annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations) + pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, annotations) + pdbDef := generateReplicationPodDisruptionBudgetDef(ctx, cr, "replication", pdbMeta, pdbParams) + return CreateOrUpdatePodDisruptionBudget(ctx, pdbDef, cl) + } else { + // Check if one exists, and delete it. + _, err := getPodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) if err == nil { return deletePodDisruptionBudget(ctx, cr.Namespace, pdbName, cl) } else if err != nil && errors.IsNotFound(err) { @@ -87,6 +109,33 @@ func generatePodDisruptionBudgetDef(ctx context.Context, cr *redisv1beta2.RedisC return pdbTemplate } +// generatePodDisruptionBudgetDef will create a PodDisruptionBudget definition +func generateReplicationPodDisruptionBudgetDef(ctx context.Context, cr *redisv1beta2.RedisReplication, role string, pdbMeta metav1.ObjectMeta, pdbParams *commonapi.RedisPodDisruptionBudget) *policyv1.PodDisruptionBudget { + lblSelector := LabelSelectors(map[string]string{ + "app": fmt.Sprintf("%s-%s", cr.ObjectMeta.Name, role), + "role": role, + }) + pdbTemplate := &policyv1.PodDisruptionBudget{ + TypeMeta: generateMetaInformation("PodDisruptionBudget", "policy/v1"), + ObjectMeta: pdbMeta, + Spec: policyv1.PodDisruptionBudgetSpec{ + Selector: lblSelector, + }, + } + if pdbParams.MinAvailable != nil { + pdbTemplate.Spec.MinAvailable = &intstr.IntOrString{Type: intstr.Int, IntVal: *pdbParams.MinAvailable} + } + if pdbParams.MaxUnavailable != nil { + pdbTemplate.Spec.MaxUnavailable = &intstr.IntOrString{Type: intstr.Int, IntVal: *pdbParams.MaxUnavailable} + } + // If we don't have a value for either, assume quorum: (N/2)+1 + if pdbTemplate.Spec.MaxUnavailable == nil && pdbTemplate.Spec.MinAvailable == nil { + pdbTemplate.Spec.MinAvailable = &intstr.IntOrString{Type: intstr.Int, IntVal: (*cr.Spec.Size / 2) + 1} + } + AddOwnerRefToObject(pdbTemplate, redisReplicationAsOwner(cr)) + return pdbTemplate +} + // generatePodDisruptionBudgetDef will create a PodDisruptionBudget definition func generateSentinelPodDisruptionBudgetDef(ctx context.Context, cr *redisv1beta2.RedisSentinel, role string, pdbMeta metav1.ObjectMeta, pdbParams *commonapi.RedisPodDisruptionBudget) *policyv1.PodDisruptionBudget { lblSelector := LabelSelectors(map[string]string{ @@ -116,7 +165,7 @@ func generateSentinelPodDisruptionBudgetDef(ctx context.Context, cr *redisv1beta // CreateOrUpdateService method will create or update Redis service func CreateOrUpdatePodDisruptionBudget(ctx context.Context, pdbDef *policyv1.PodDisruptionBudget, cl kubernetes.Interface) error { - storedPDB, err := GetPodDisruptionBudget(ctx, pdbDef.Namespace, pdbDef.Name, cl) + storedPDB, err := getPodDisruptionBudget(ctx, pdbDef.Namespace, pdbDef.Name, cl) if err != nil { if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(pdbDef); err != nil { //nolint log.FromContext(ctx).Error(err, "Unable to patch redis PodDisruptionBudget with comparison object") @@ -204,8 +253,8 @@ func deletePodDisruptionBudget(ctx context.Context, namespace string, pdbName st return nil } -// GetPodDisruptionBudget is a method to get PodDisruptionBudgets in Kubernetes -func GetPodDisruptionBudget(ctx context.Context, namespace string, pdb string, cl kubernetes.Interface) (*policyv1.PodDisruptionBudget, error) { +// getPodDisruptionBudget is a method to get PodDisruptionBudgets in Kubernetes +func getPodDisruptionBudget(ctx context.Context, namespace string, pdb string, cl kubernetes.Interface) (*policyv1.PodDisruptionBudget, error) { getOpts := metav1.GetOptions{ TypeMeta: generateMetaInformation("PodDisruptionBudget", "policy/v1"), } From 170773fae3f6a0a6bd085d7db52f606bafd5b54e Mon Sep 17 00:00:00 2001 From: xiaozhuang <61875985+xiaozhuang-a@users.noreply.github.com> Date: Mon, 16 Dec 2024 21:29:16 +0800 Subject: [PATCH 15/21] fix: redis-cluster unexpected downscaling (#1167) (#1173) Signed-off-by: xiaozhuang Co-authored-by: xiaozhuang Signed-off-by: xiaozhuang --- .../rediscluster/rediscluster_controller.go | 10 +++++++++- pkg/k8sutils/cluster-scaling.go | 18 +++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/rediscluster/rediscluster_controller.go b/pkg/controllers/rediscluster/rediscluster_controller.go index 0c79a2a56..76d51f2be 100644 --- a/pkg/controllers/rediscluster/rediscluster_controller.go +++ b/pkg/controllers/rediscluster/rediscluster_controller.go @@ -73,6 +73,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // Check if the cluster is downscaled if leaderCount := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, instance, "leader"); leaderReplicas < leaderCount { + if !(r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-follower")) { + return intctrlutil.Reconciled() + } + logger.Info("Redis cluster is downscaling...", "Current.LeaderReplicas", leaderCount, "Desired.LeaderReplicas", leaderReplicas) for shardIdx := leaderCount - 1; shardIdx >= leaderReplicas; shardIdx-- { logger.Info("Remove the shard", "Shard.Index", shardIdx) @@ -83,7 +87,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // lastLeaderPod is slaving right now Make it the master Pod // We have to bring a manual failover here to make it a leaderPod // clusterFailover should also include the clusterReplicate since we have to map the followers to new leader - k8sutils.ClusterFailover(ctx, r.K8sClient, instance) + logger.Info("Cluster Failover is initiated", "Shard.Index", shardIdx) + if err = k8sutils.ClusterFailover(ctx, r.K8sClient, instance); err != nil { + logger.Error(err, "Failed to initiate cluster failover") + return intctrlutil.RequeueWithError(ctx, err, "") + } } // Step 1 Remove the Follower Node k8sutils.RemoveRedisFollowerNodesFromCluster(ctx, r.K8sClient, instance) diff --git a/pkg/k8sutils/cluster-scaling.go b/pkg/k8sutils/cluster-scaling.go index b2bd5a0da..3e7cd7097 100644 --- a/pkg/k8sutils/cluster-scaling.go +++ b/pkg/k8sutils/cluster-scaling.go @@ -391,7 +391,7 @@ func verifyLeaderPodInfo(ctx context.Context, redisClient *redis.Client, podName return false } -func ClusterFailover(ctx context.Context, client kubernetes.Interface, cr *redisv1beta2.RedisCluster) { +func ClusterFailover(ctx context.Context, client kubernetes.Interface, cr *redisv1beta2.RedisCluster) error { slavePodName := cr.Name + "-leader-" + strconv.Itoa(int(CheckRedisNodeCount(ctx, client, cr, "leader"))-1) // cmd = redis-cli cluster failover -a var cmd []string @@ -400,13 +400,15 @@ func ClusterFailover(ctx context.Context, client kubernetes.Interface, cr *redis Namespace: cr.Namespace, } - cmd = []string{"redis-cli", "cluster", "failover"} + cmd = []string{"redis-cli", "-h"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(pod, cr, "leader")) } else { - cmd = append(cmd, getRedisServerAddress(ctx, client, pod, *cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(ctx, client, pod)) } + cmd = append(cmd, "-p") + cmd = append(cmd, strconv.Itoa(*cr.Spec.Port)) if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) @@ -418,7 +420,13 @@ func ClusterFailover(ctx context.Context, client kubernetes.Interface, cr *redis } cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, slavePodName)...) + cmd = append(cmd, "cluster", "failover") log.FromContext(ctx).V(1).Info("Redis cluster failover command is", "Command", cmd) - executeCommand(ctx, client, cr, cmd, slavePodName) + execOut, err := executeCommand1(ctx, client, cr, cmd, slavePodName) + if err != nil { + log.FromContext(ctx).Error(err, "Could not execute command", "Command", cmd, "Output", execOut) + return err + } + return nil } From 182514efd099efc2363ccf6928fa28b185cc0d04 Mon Sep 17 00:00:00 2001 From: xiaozhuang Date: Tue, 17 Dec 2024 23:40:57 +0800 Subject: [PATCH 16/21] feat: add podAntiAffinity webhook Signed-off-by: xiaozhuang --- PROJECT | 7 ++ config/default/kustomization.yaml | 8 +- config/webhook/manifests.yaml | 30 ++++++ main.go | 4 + pkg/webhook/pod_webhook.go | 146 ++++++++++++++++++++++++++++++ 5 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 config/webhook/manifests.yaml create mode 100644 pkg/webhook/pod_webhook.go diff --git a/PROJECT b/PROJECT index ef0b3b2eb..82500e9e4 100644 --- a/PROJECT +++ b/PROJECT @@ -88,4 +88,11 @@ resources: kind: RedisSentinel path: redis-operator/api/v1beta1 version: v1beta1 +- group: core + kind: Pod + path: k8s.io/api/core/v1 + version: v1 + webhooks: + defaulting: true + webhookVersion: v1 version: "3" diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 44dba7b21..d94a827b1 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -18,9 +18,9 @@ bases: - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -36,12 +36,12 @@ bases: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- manager_webhook_patch.yaml +- manager_webhook_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml +- webhookcainjection_patch.yaml # the following config is for teaching kustomize how to do var substitution #vars: diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 000000000..a5705704d --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: kv-o2btjndvnskg5zakups4/serving-cert +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: kv-o2btjndvnskg5zakups4 + path: /mutate-core-v1-pod + failurePolicy: Fail + name: mpod.kb.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + namespaceSelector: + matchLabels: + pod-admission-webhook-injection: enabled \ No newline at end of file diff --git a/main.go b/main.go index 576d16fbf..f69a7883a 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ package main import ( "flag" "os" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "strings" redisv1beta1 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta1" @@ -29,6 +30,7 @@ import ( "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllers/redissentinel" intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil" "github.com/OT-CONTAINER-KIT/redis-operator/pkg/k8sutils" + coreWebhook "github.com/OT-CONTAINER-KIT/redis-operator/pkg/webhook" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -176,6 +178,8 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "RedisSentinel") os.Exit(1) } + + mgr.GetWebhookServer().Register("/mutate-core-v1-pod", &webhook.Admission{Handler: coreWebhook.NewPodAffiniytMutate(mgr.GetClient(), admission.NewDecoder(scheme))}) } // +kubebuilder:scaffold:builder diff --git a/pkg/webhook/pod_webhook.go b/pkg/webhook/pod_webhook.go new file mode 100644 index 000000000..c94d471a5 --- /dev/null +++ b/pkg/webhook/pod_webhook.go @@ -0,0 +1,146 @@ +/* +Copyright 2020 Opstree Solutions. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "encoding/json" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "net/http" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "strings" +) + +//+kubebuilder:webhook:path=/mutate-core-v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,groups=core,resources=pods,verbs=create,versions=v1,name=mpod.kb.io,admissionReviewVersions=v1 + +// PodAntiAffiniytMutate mutate Pods +type PodAntiAffiniytMutate struct { + Client client.Client + decoder *admission.Decoder + logger logr.Logger +} + +func NewPodAffiniytMutate(c client.Client, d *admission.Decoder) admission.Handler { + return &PodAntiAffiniytMutate{Client: c, decoder: d} +} + +const ( + podAnnotationsRedisClusterApp = "redis.opstreelabs.instance" + podNameLabel = "statefulset.kubernetes.io/pod-name" +) + +func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Request) admission.Response { + logger := v.logger.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name) + + pod := &corev1.Pod{} + err := v.decoder.Decode(req, pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + if v.isRedisClusterApp(pod) == "" { + return admission.Allowed("") + } + + old := pod.DeepCopy() + + v.AddPodAntiAffinity(pod) + if !reflect.DeepEqual(old, pod) { + marshaledPod, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + + logger.Info("mutate pod with anti-affinity") + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) + } + + return admission.Allowed("") +} + +// PodAntiAffiniytMutate implements admission.DecoderInjector. +// A decoder will be automatically injected. + +// InjectDecoder injects the decoder. +func (v *PodAntiAffiniytMutate) InjectDecoder(d *admission.Decoder) error { + v.decoder = d + return nil +} + +func (m *PodAntiAffiniytMutate) InjectLogger(l logr.Logger) error { + m.logger = l + return nil +} + +func (v *PodAntiAffiniytMutate) AddPodAntiAffinity(pod *corev1.Pod) { + // todo: determine whether to add anti affinity + + labels := pod.GetLabels() + if labels == nil || labels[podNameLabel] == "" { + return + } + antiAppLabel := v.reverseAppValue(labels[podNameLabel]) + + if pod.Spec.Affinity == nil { + pod.Spec.Affinity = &corev1.Affinity{} + } + if pod.Spec.Affinity.PodAntiAffinity == nil { + pod.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{} + } + if pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = make([]corev1.PodAffinityTerm, 0) + } + addAntiAffinity := corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{antiAppLabel}, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + } + + pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, addAntiAffinity) +} + +func (v *PodAntiAffiniytMutate) getPodAnnotations(pod *corev1.Pod) map[string]string { + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + return pod.Annotations +} + +func (v *PodAntiAffiniytMutate) isRedisClusterApp(pod *corev1.Pod) string { + annotations := v.getPodAnnotations(pod) + return annotations[podAnnotationsRedisClusterApp] +} + +func (v *PodAntiAffiniytMutate) reverseAppValue(app string) string { + if strings.Contains(app, "follower") { + return strings.Replace(app, "follower", "leader", -1) + } + if strings.Contains(app, "leader") { + return strings.Replace(app, "leader", "follower", -1) + } + return app +} From da3ca1673313c70b84a182fb5a1def2f11489b9d Mon Sep 17 00:00:00 2001 From: xiaozhuang Date: Wed, 18 Dec 2024 01:13:40 +0800 Subject: [PATCH 17/21] feat: add podAntiAffinity webhook Signed-off-by: xiaozhuang --- config/webhook/manifests.yaml | 8 ++++++-- main.go | 4 +++- pkg/webhook/pod_webhook.go | 34 +++++++++++++++++----------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index a5705704d..02a5f3fab 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -13,7 +13,7 @@ webhooks: name: webhook-service namespace: kv-o2btjndvnskg5zakups4 path: /mutate-core-v1-pod - failurePolicy: Fail + failurePolicy: Ignore name: mpod.kb.io rules: - apiGroups: @@ -27,4 +27,8 @@ webhooks: sideEffects: None namespaceSelector: matchLabels: - pod-admission-webhook-injection: enabled \ No newline at end of file + db-pod-webhook-injection: enabled + objectSelector: + matchExpressions: + - key: redis_setup_type + operator: Exists \ No newline at end of file diff --git a/main.go b/main.go index f69a7883a..6984eb598 100644 --- a/main.go +++ b/main.go @@ -179,7 +179,9 @@ func main() { os.Exit(1) } - mgr.GetWebhookServer().Register("/mutate-core-v1-pod", &webhook.Admission{Handler: coreWebhook.NewPodAffiniytMutate(mgr.GetClient(), admission.NewDecoder(scheme))}) + wblog := ctrl.Log.WithName("webhook").WithName("PodAffiniytMutate") + mgr.GetWebhookServer().Register("/mutate-core-v1-pod", &webhook.Admission{ + Handler: coreWebhook.NewPodAffiniytMutate(mgr.GetClient(), admission.NewDecoder(scheme), wblog)}) } // +kubebuilder:scaffold:builder diff --git a/pkg/webhook/pod_webhook.go b/pkg/webhook/pod_webhook.go index c94d471a5..d56845913 100644 --- a/pkg/webhook/pod_webhook.go +++ b/pkg/webhook/pod_webhook.go @@ -38,13 +38,16 @@ type PodAntiAffiniytMutate struct { logger logr.Logger } -func NewPodAffiniytMutate(c client.Client, d *admission.Decoder) admission.Handler { - return &PodAntiAffiniytMutate{Client: c, decoder: d} +func NewPodAffiniytMutate(c client.Client, d *admission.Decoder, log logr.Logger) admission.Handler { + return &PodAntiAffiniytMutate{ + Client: c, + decoder: d, + logger: log} } const ( podAnnotationsRedisClusterApp = "redis.opstreelabs.instance" - podNameLabel = "statefulset.kubernetes.io/pod-name" + podLabelsPodName = "statefulset.kubernetes.io/pod-name" ) func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Request) admission.Response { @@ -90,13 +93,10 @@ func (m *PodAntiAffiniytMutate) InjectLogger(l logr.Logger) error { } func (v *PodAntiAffiniytMutate) AddPodAntiAffinity(pod *corev1.Pod) { - // todo: determine whether to add anti affinity + // todo: determine whether to add anti affinity,need add parameters to control - labels := pod.GetLabels() - if labels == nil || labels[podNameLabel] == "" { - return - } - antiAppLabel := v.reverseAppValue(labels[podNameLabel]) + podName := pod.ObjectMeta.Name + antiLabelValue := v.getAntiAffinityValue(podName) if pod.Spec.Affinity == nil { pod.Spec.Affinity = &corev1.Affinity{} @@ -111,9 +111,9 @@ func (v *PodAntiAffiniytMutate) AddPodAntiAffinity(pod *corev1.Pod) { LabelSelector: &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "app", + Key: podLabelsPodName, Operator: metav1.LabelSelectorOpIn, - Values: []string{antiAppLabel}, + Values: []string{antiLabelValue}, }, }, }, @@ -135,12 +135,12 @@ func (v *PodAntiAffiniytMutate) isRedisClusterApp(pod *corev1.Pod) string { return annotations[podAnnotationsRedisClusterApp] } -func (v *PodAntiAffiniytMutate) reverseAppValue(app string) string { - if strings.Contains(app, "follower") { - return strings.Replace(app, "follower", "leader", -1) +func (v *PodAntiAffiniytMutate) getAntiAffinityValue(podName string) string { + if strings.Contains(podName, "follower") { + return strings.Replace(podName, "follower", "leader", -1) } - if strings.Contains(app, "leader") { - return strings.Replace(app, "leader", "follower", -1) + if strings.Contains(podName, "leader") { + return strings.Replace(podName, "leader", "follower", -1) } - return app + return "" } From cdfc66e60d6e6448e1bcfe6e03ab938401b71ccf Mon Sep 17 00:00:00 2001 From: xiaozhuang Date: Wed, 18 Dec 2024 01:24:03 +0800 Subject: [PATCH 18/21] chore: only mutate pods that belong to redis cluster Signed-off-by: xiaozhuang --- pkg/webhook/pod_webhook.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/webhook/pod_webhook.go b/pkg/webhook/pod_webhook.go index d56845913..357ae7f14 100644 --- a/pkg/webhook/pod_webhook.go +++ b/pkg/webhook/pod_webhook.go @@ -48,6 +48,7 @@ func NewPodAffiniytMutate(c client.Client, d *admission.Decoder, log logr.Logger const ( podAnnotationsRedisClusterApp = "redis.opstreelabs.instance" podLabelsPodName = "statefulset.kubernetes.io/pod-name" + podLabelsRedisType = "redis_setup_type" ) func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Request) admission.Response { @@ -58,7 +59,9 @@ func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Reques if err != nil { return admission.Errored(http.StatusBadRequest, err) } - if v.isRedisClusterApp(pod) == "" { + + // only mutate pods that belong to redis cluster + if !v.isRedisClusterPod(pod) { return admission.Allowed("") } @@ -130,9 +133,18 @@ func (v *PodAntiAffiniytMutate) getPodAnnotations(pod *corev1.Pod) map[string]st return pod.Annotations } -func (v *PodAntiAffiniytMutate) isRedisClusterApp(pod *corev1.Pod) string { +func (v *PodAntiAffiniytMutate) isRedisClusterPod(pod *corev1.Pod) bool { annotations := v.getPodAnnotations(pod) - return annotations[podAnnotationsRedisClusterApp] + if _, ok := annotations[podAnnotationsRedisClusterApp]; !ok { + return false + } + + labels := pod.GetLabels() + if _, ok := labels[podLabelsRedisType]; !ok { + return false + } + + return true } func (v *PodAntiAffiniytMutate) getAntiAffinityValue(podName string) string { From e394a9069ead7c0999b7e40d82b1806b7c6c9c93 Mon Sep 17 00:00:00 2001 From: xiaozhuang Date: Wed, 18 Dec 2024 01:57:27 +0800 Subject: [PATCH 19/21] feat: redis-cluster crd add LeaderFollowerPodAntiAffinity Signed-off-by: xiaozhuang --- api/v1beta2/rediscluster_types.go | 31 ++++++++++--------- api/v1beta2/zz_generated.deepcopy.go | 5 +++ ...is.redis.opstreelabs.in_redisclusters.yaml | 2 ++ pkg/webhook/pod_webhook.go | 31 +++++++++++++++++-- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/api/v1beta2/rediscluster_types.go b/api/v1beta2/rediscluster_types.go index e49141d15..b7faf7564 100644 --- a/api/v1beta2/rediscluster_types.go +++ b/api/v1beta2/rediscluster_types.go @@ -31,21 +31,22 @@ type RedisClusterSpec struct { // +kubebuilder:default:=6379 Port *int `json:"port,omitempty"` // +kubebuilder:default:=v7 - ClusterVersion *string `json:"clusterVersion,omitempty"` - RedisLeader RedisLeader `json:"redisLeader,omitempty"` - RedisFollower RedisFollower `json:"redisFollower,omitempty"` - RedisExporter *RedisExporter `json:"redisExporter,omitempty"` - Storage *ClusterStorage `json:"storage,omitempty"` - PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` - PriorityClassName string `json:"priorityClassName,omitempty"` - Resources *corev1.ResourceRequirements `json:"resources,omitempty"` - TLS *TLSConfig `json:"TLS,omitempty"` - ACL *ACLConfig `json:"acl,omitempty"` - InitContainer *InitContainer `json:"initContainer,omitempty"` - Sidecars *[]Sidecar `json:"sidecars,omitempty"` - ServiceAccountName *string `json:"serviceAccountName,omitempty"` - PersistenceEnabled *bool `json:"persistenceEnabled,omitempty"` - EnvVars *[]corev1.EnvVar `json:"env,omitempty"` + ClusterVersion *string `json:"clusterVersion,omitempty"` + RedisLeader RedisLeader `json:"redisLeader,omitempty"` + RedisFollower RedisFollower `json:"redisFollower,omitempty"` + RedisExporter *RedisExporter `json:"redisExporter,omitempty"` + Storage *ClusterStorage `json:"storage,omitempty"` + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + PriorityClassName string `json:"priorityClassName,omitempty"` + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + TLS *TLSConfig `json:"TLS,omitempty"` + ACL *ACLConfig `json:"acl,omitempty"` + InitContainer *InitContainer `json:"initContainer,omitempty"` + Sidecars *[]Sidecar `json:"sidecars,omitempty"` + ServiceAccountName *string `json:"serviceAccountName,omitempty"` + PersistenceEnabled *bool `json:"persistenceEnabled,omitempty"` + EnvVars *[]corev1.EnvVar `json:"env,omitempty"` + LeaderFollowerPodAntiAffinity *bool `json:"leaderFollowerPodAntiAffinity,omitempty"` } func (cr *RedisClusterSpec) GetReplicaCounts(t string) int32 { diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 331905267..b7ca6d114 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -329,6 +329,11 @@ func (in *RedisClusterSpec) DeepCopyInto(out *RedisClusterSpec) { } } } + if in.LeaderFollowerPodAntiAffinity != nil { + in, out := &in.LeaderFollowerPodAntiAffinity, &out.LeaderFollowerPodAntiAffinity + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisClusterSpec. diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml index 9da67022d..775026200 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml @@ -6598,6 +6598,8 @@ spec: required: - image type: object + leaderFollowerPodAntiAffinity: + type: boolean persistenceEnabled: type: boolean podSecurityContext: diff --git a/pkg/webhook/pod_webhook.go b/pkg/webhook/pod_webhook.go index 357ae7f14..15222cbc1 100644 --- a/pkg/webhook/pod_webhook.go +++ b/pkg/webhook/pod_webhook.go @@ -19,6 +19,7 @@ package webhook import ( "context" "encoding/json" + "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -65,6 +66,17 @@ func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Reques return admission.Allowed("") } + // determine whether to add anti affinity + redisCluster, err := v.getRedisClusterSpec(ctx, pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + if !(redisCluster != nil && redisCluster.Spec.LeaderFollowerPodAntiAffinity != nil && + *redisCluster.Spec.LeaderFollowerPodAntiAffinity == true) { + v.logger.V(1).Info("leader follower pod anti affinity is disabled") + return admission.Allowed("") + } + old := pod.DeepCopy() v.AddPodAntiAffinity(pod) @@ -96,8 +108,6 @@ func (m *PodAntiAffiniytMutate) InjectLogger(l logr.Logger) error { } func (v *PodAntiAffiniytMutate) AddPodAntiAffinity(pod *corev1.Pod) { - // todo: determine whether to add anti affinity,need add parameters to control - podName := pod.ObjectMeta.Name antiLabelValue := v.getAntiAffinityValue(podName) @@ -156,3 +166,20 @@ func (v *PodAntiAffiniytMutate) getAntiAffinityValue(podName string) string { } return "" } + +func (v *PodAntiAffiniytMutate) getRedisClusterSpec(ctx context.Context, pod *corev1.Pod) (*v1beta2.RedisCluster, error) { + namespaceKey := client.ObjectKey{ + Namespace: pod.Namespace, + Name: pod.Annotations[podAnnotationsRedisClusterApp], + } + + redisCluster := &v1beta2.RedisCluster{} + + err := v.Client.Get(ctx, namespaceKey, redisCluster) + if err != nil { + v.logger.Error(err, "failed to get redis cluster") + return nil, err + } + + return redisCluster, nil +} From 3a1994bbf31801e45e39e44abcae27c6e9996a2c Mon Sep 17 00:00:00 2001 From: xiaozhuang Date: Wed, 18 Dec 2024 02:00:17 +0800 Subject: [PATCH 20/21] chore: delete manifests Signed-off-by: xiaozhuang --- config/webhook/manifests.yaml | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 config/webhook/manifests.yaml diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml deleted file mode 100644 index 02a5f3fab..000000000 --- a/config/webhook/manifests.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: kv-o2btjndvnskg5zakups4/serving-cert -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: kv-o2btjndvnskg5zakups4 - path: /mutate-core-v1-pod - failurePolicy: Ignore - name: mpod.kb.io - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None - namespaceSelector: - matchLabels: - db-pod-webhook-injection: enabled - objectSelector: - matchExpressions: - - key: redis_setup_type - operator: Exists \ No newline at end of file From 7e74516f3610993d3e8425b6ac6ca2c5cbbec8ed Mon Sep 17 00:00:00 2001 From: xiaozhuang Date: Wed, 18 Dec 2024 02:24:19 +0800 Subject: [PATCH 21/21] chore: lint Signed-off-by: xiaozhuang --- pkg/webhook/pod_webhook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/webhook/pod_webhook.go b/pkg/webhook/pod_webhook.go index 15222cbc1..c22c4f31e 100644 --- a/pkg/webhook/pod_webhook.go +++ b/pkg/webhook/pod_webhook.go @@ -72,7 +72,7 @@ func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Reques return admission.Errored(http.StatusInternalServerError, err) } if !(redisCluster != nil && redisCluster.Spec.LeaderFollowerPodAntiAffinity != nil && - *redisCluster.Spec.LeaderFollowerPodAntiAffinity == true) { + *redisCluster.Spec.LeaderFollowerPodAntiAffinity) { v.logger.V(1).Info("leader follower pod anti affinity is disabled") return admission.Allowed("") }