From ee44c3618518f2d5a9803037139fd93bcaa6fae5 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Thu, 7 Dec 2023 14:19:35 +0800 Subject: [PATCH 01/21] add cr filed, default port to 6379 Signed-off-by: drivebyer --- api/v1beta2/rediscluster_types.go | 3 +++ .../crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml | 6 ++++++ k8sutils/redis-cluster.go | 1 + k8sutils/statefulset.go | 2 ++ 4 files changed, 12 insertions(+) diff --git a/api/v1beta2/rediscluster_types.go b/api/v1beta2/rediscluster_types.go index 913e4f2a1..cf55e594a 100644 --- a/api/v1beta2/rediscluster_types.go +++ b/api/v1beta2/rediscluster_types.go @@ -27,6 +27,9 @@ import ( type RedisClusterSpec struct { Size *int32 `json:"clusterSize"` KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` + HostNetwork bool `json:"hostNetwork,omitempty"` + // +kubebuilder:default:=6379 + Port int32 `json:"port,omitempty"` // +kubebuilder:default:=v7 ClusterVersion *string `json:"clusterVersion,omitempty"` // +kubebuilder:default:={livenessProbe:{initialDelaySeconds: 1, timeoutSeconds: 1, periodSeconds: 10, successThreshold: 1, failureThreshold:3}, readinessProbe:{initialDelaySeconds: 1, timeoutSeconds: 1, periodSeconds: 10, successThreshold: 1, failureThreshold:3}} diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml index 2b9e3c39e..081f77cf6 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml @@ -5297,6 +5297,8 @@ spec: - name type: object type: array + hostNetwork: + type: boolean initContainer: description: InitContainer for each Redis pods properties: @@ -5792,6 +5794,10 @@ spec: type: string type: object type: object + port: + default: 6379 + format: int32 + type: integer priorityClassName: type: string redisExporter: diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go index d3a523cc2..ddb38b13e 100644 --- a/k8sutils/redis-cluster.go +++ b/k8sutils/redis-cluster.go @@ -39,6 +39,7 @@ func generateRedisClusterParams(cr *redisv1beta2.RedisCluster, replicas int32, e ServiceAccountName: cr.Spec.ServiceAccountName, UpdateStrategy: cr.Spec.KubernetesConfig.UpdateStrategy, IgnoreAnnotations: cr.Spec.KubernetesConfig.IgnoreAnnotations, + HostNetwork: cr.Spec.HostNetwork, } if cr.Spec.RedisExporter != nil { res.EnableMetrics = cr.Spec.RedisExporter.Enabled diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 3be4b54f2..daab66fde 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -45,6 +45,7 @@ type statefulSetParameters struct { RecreateStatefulSet bool TerminationGracePeriodSeconds *int64 IgnoreAnnotations []string + HostNetwork bool } // containerParameters will define container input params @@ -242,6 +243,7 @@ func generateStatefulSetsDef(stsMeta metav1.ObjectMeta, params statefulSetParame PriorityClassName: params.PriorityClassName, Affinity: params.Affinity, TerminationGracePeriodSeconds: params.TerminationGracePeriodSeconds, + HostNetwork: params.HostNetwork, }, }, }, From 39da742b8080078fe3170c26f26a7135e829fdf9 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Thu, 7 Dec 2023 15:15:27 +0800 Subject: [PATCH 02/21] replace with custom port Signed-off-by: drivebyer --- k8sutils/cluster-scaling.go | 33 +++++++++++++++++---------------- k8sutils/redis.go | 16 ++++++++-------- k8sutils/redis_test.go | 4 ++++ 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/k8sutils/cluster-scaling.go b/k8sutils/cluster-scaling.go index 395a5c942..7c3b244e0 100644 --- a/k8sutils/cluster-scaling.go +++ b/k8sutils/cluster-scaling.go @@ -2,6 +2,7 @@ package k8sutils import ( "context" + "fmt" "strconv" "strings" @@ -32,9 +33,9 @@ func ReshardRedisCluster(client kubernetes.Interface, logger logr.Logger, cr *re cmd = []string{"redis-cli", "--cluster", "reshard"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+":6379") + cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, transferPOD)+":6379") + cmd = append(cmd, getRedisServerIP(client, logger, transferPOD)+fmt.Sprintf(":%d", cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { @@ -157,9 +158,9 @@ func RebalanceRedisClusterEmptyMasters(client kubernetes.Interface, logger logr. cmd = []string{"redis-cli", "--cluster", "rebalance"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+":6379") + cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, pod)+":6379") + cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", cr.Spec.Port)) } cmd = append(cmd, "--cluster-use-empty-masters") @@ -209,9 +210,9 @@ func RebalanceRedisCluster(client kubernetes.Interface, logger logr.Logger, cr * cmd = []string{"redis-cli", "--cluster", "rebalance"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+":6379") + cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, pod)+":6379") + cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { @@ -246,11 +247,11 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, log cmd = []string{"redis-cli", "--cluster", "add-node"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+":6379") - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+":6379") + cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, newPod)+":6379") - cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+":6379") + cmd = append(cmd, getRedisServerIP(client, logger, newPod)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { @@ -327,9 +328,9 @@ func RemoveRedisFollowerNodesFromCluster(ctx context.Context, client kubernetes. cmd = append(cmd, "--cluster", "del-node") if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+":6379") + cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+":6379") + cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", cr.Spec.Port)) } for _, followerNodeID := range followerNodeIDs { @@ -358,9 +359,9 @@ func RemoveRedisNodeFromCluster(ctx context.Context, client kubernetes.Interface cmd = []string{"redis-cli", "--cluster", "del-node"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+":6379") + cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+":6379") + cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", cr.Spec.Port)) } removePodNodeID := getRedisNodeID(ctx, client, logger, cr, removePod) @@ -419,9 +420,9 @@ func ClusterFailover(ctx context.Context, client kubernetes.Interface, logger lo cmd = []string{"redis-cli", "cluster", "failover"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+":6379") + cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, pod)+":6379") + cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { diff --git a/k8sutils/redis.go b/k8sutils/redis.go index 0ec4433e9..9023799b4 100644 --- a/k8sutils/redis.go +++ b/k8sutils/redis.go @@ -83,9 +83,9 @@ func CreateMultipleLeaderRedisCommand(client kubernetes.Interface, logger logr.L podName := cr.ObjectMeta.Name + "-leader-" + strconv.Itoa(podCount) var address string if cr.Spec.ClusterVersion != nil && *cr.Spec.ClusterVersion == "v7" { - address = getRedisHostname(RedisDetails{PodName: podName, Namespace: cr.Namespace}, cr, "leader") + ":6379" + address = getRedisHostname(RedisDetails{PodName: podName, Namespace: cr.Namespace}, cr, "leader") + fmt.Sprintf(":%d", cr.Spec.Port) } else { - address = getRedisServerIP(client, logger, RedisDetails{PodName: podName, Namespace: cr.Namespace}) + ":6379" + address = getRedisServerIP(client, logger, RedisDetails{PodName: podName, Namespace: cr.Namespace}) + fmt.Sprintf(":%d", cr.Spec.Port) } cmd = append(cmd, address) } @@ -141,11 +141,11 @@ func createRedisReplicationCommand(client kubernetes.Interface, logger logr.Logg var followerAddress, leaderAddress string if cr.Spec.ClusterVersion != nil && *cr.Spec.ClusterVersion == "v7" { - followerAddress = getRedisHostname(followerPod, cr, "follower") + ":6379" - leaderAddress = getRedisHostname(leaderPod, cr, "leader") + ":6379" + followerAddress = getRedisHostname(followerPod, cr, "follower") + fmt.Sprintf(":%d", cr.Spec.Port) + leaderAddress = getRedisHostname(leaderPod, cr, "leader") + fmt.Sprintf(":%d", cr.Spec.Port) } else { - followerAddress = getRedisServerIP(client, logger, followerPod) + ":6379" - leaderAddress = getRedisServerIP(client, logger, leaderPod) + ":6379" + followerAddress = getRedisServerIP(client, logger, followerPod) + fmt.Sprintf(":%d", cr.Spec.Port) + leaderAddress = getRedisServerIP(client, logger, leaderPod) + fmt.Sprintf(":%d", cr.Spec.Port) } cmd = append(cmd, followerAddress, leaderAddress, "--cluster-slave") @@ -340,14 +340,14 @@ func configureRedisClient(client kubernetes.Interface, logger logr.Logger, cr *r logger.Error(err, "Error in getting redis password") } redisClient = redis.NewClient(&redis.Options{ - Addr: getRedisServerIP(client, logger, redisInfo) + ":6379", + Addr: getRedisServerIP(client, logger, redisInfo) + fmt.Sprintf(":%d", cr.Spec.Port), Password: pass, DB: 0, TLSConfig: getRedisTLSConfig(client, logger, cr, redisInfo), }) } else { redisClient = redis.NewClient(&redis.Options{ - Addr: getRedisServerIP(client, logger, redisInfo) + ":6379", + Addr: getRedisServerIP(client, logger, redisInfo) + fmt.Sprintf(":%d", cr.Spec.Port), Password: "", DB: 0, TLSConfig: getRedisTLSConfig(client, logger, cr, redisInfo), diff --git a/k8sutils/redis_test.go b/k8sutils/redis_test.go index 26628290d..76c0ec2c2 100644 --- a/k8sutils/redis_test.go +++ b/k8sutils/redis_test.go @@ -206,6 +206,7 @@ func TestCreateMultipleLeaderRedisCommand(t *testing.T) { Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), ClusterVersion: pointer.String("v7"), + Port: 6379, }, }, expectedCommands: []string{ @@ -225,6 +226,7 @@ func TestCreateMultipleLeaderRedisCommand(t *testing.T) { }, Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), + Port: 6379, }, }, expectedCommands: []string{ @@ -295,6 +297,7 @@ func TestCreateRedisReplicationCommand(t *testing.T) { Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), ClusterVersion: pointer.String("v7"), + Port: 6379, }, }, leaderPod: RedisDetails{ @@ -321,6 +324,7 @@ func TestCreateRedisReplicationCommand(t *testing.T) { }, Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), + Port: 6379, }, }, leaderPod: RedisDetails{ From 9646647fa2db5398f7cb6f555cdfa453a1d66233 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 17:16:19 +0800 Subject: [PATCH 03/21] service custom port Signed-off-by: drivebyer --- controllers/rediscluster_controller.go | 1 + k8sutils/redis-cluster.go | 7 ++++--- k8sutils/redis-replication.go | 6 +++--- k8sutils/redis-sentinel.go | 6 +++--- k8sutils/redis-standalone.go | 6 +++--- k8sutils/services.go | 13 +++++-------- k8sutils/statefulset.go | 6 ++++-- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go index 25549e8b2..0a501170d 100644 --- a/controllers/rediscluster_controller.go +++ b/controllers/rediscluster_controller.go @@ -55,6 +55,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } return ctrl.Result{}, err } + instance.SetDefault() if _, found := instance.ObjectMeta.GetAnnotations()["rediscluster.opstreelabs.in/skip-reconcile"]; found { reqLogger.Info("Found annotations rediscluster.opstreelabs.in/skip-reconcile, so skipping reconcile") diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go index ddb38b13e..b0495fa74 100644 --- a/k8sutils/redis-cluster.go +++ b/k8sutils/redis-cluster.go @@ -101,6 +101,7 @@ func generateRedisClusterContainerParams(cr *redisv1beta2.RedisCluster, security ImagePullPolicy: cr.Spec.KubernetesConfig.ImagePullPolicy, Resources: cr.Spec.KubernetesConfig.Resources, SecurityContext: securityContext, + Port: cr.Spec.Port, } if cr.Spec.EnvVars != nil { containerProp.EnvVars = cr.Spec.EnvVars @@ -247,12 +248,12 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations) headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), false, true, "ClusterIP") + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), false, true, "ClusterIP", cr.Spec.Port) if err != nil { logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole) return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics, false, "ClusterIP") + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics, false, "ClusterIP", cr.Spec.Port) if err != nil { logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole) return err @@ -261,7 +262,7 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), false, false, additionalServiceType) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), false, false, additionalServiceType, cr.Spec.Port) if err != nil { logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole) return err diff --git a/k8sutils/redis-replication.go b/k8sutils/redis-replication.go index df48c5d18..5e5223a3c 100644 --- a/k8sutils/redis-replication.go +++ b/k8sutils/redis-replication.go @@ -19,12 +19,12 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error { objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations) headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), false, true, "ClusterIP") + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), false, true, "ClusterIP", redisPort) if err != nil { logger.Error(err, "Cannot create replication headless service for Redis") return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), enableMetrics, false, "ClusterIP") + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), enableMetrics, false, "ClusterIP", redisPort) if err != nil { logger.Error(err, "Cannot create replication service for Redis") return err @@ -33,7 +33,7 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error { if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), false, false, additionalServiceType) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), false, false, additionalServiceType, redisPort) if err != nil { logger.Error(err, "Cannot create additional service for Redis Replication") return err diff --git a/k8sutils/redis-sentinel.go b/k8sutils/redis-sentinel.go index 9977fa644..6eef41716 100644 --- a/k8sutils/redis-sentinel.go +++ b/k8sutils/redis-sentinel.go @@ -215,12 +215,12 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2. headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), false, true, "ClusterIP") + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), false, true, "ClusterIP", sentinelPort) if err != nil { logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole) return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), enableMetrics, false, "ClusterIP") + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), enableMetrics, false, "ClusterIP", sentinelPort) if err != nil { logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole) return err @@ -230,7 +230,7 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2. if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), false, false, additionalServiceType) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), false, false, additionalServiceType, sentinelPort) if err != nil { logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole) return err diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go index bd535e227..33d2e4c03 100644 --- a/k8sutils/redis-standalone.go +++ b/k8sutils/redis-standalone.go @@ -23,12 +23,12 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error { objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations) headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), false, true, "ClusterIP") + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), false, true, "ClusterIP", redisPort) if err != nil { logger.Error(err, "Cannot create standalone headless service for Redis") return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), enableMetrics, false, "ClusterIP") + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), enableMetrics, false, "ClusterIP", redisPort) if err != nil { logger.Error(err, "Cannot create standalone service for Redis") return err @@ -37,7 +37,7 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error { if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), false, false, additionalServiceType) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), false, false, additionalServiceType, redisPort) if err != nil { logger.Error(err, "Cannot create additional service for Redis") return err diff --git a/k8sutils/services.go b/k8sutils/services.go index 0bcf56944..e984b0f32 100644 --- a/k8sutils/services.go +++ b/k8sutils/services.go @@ -23,15 +23,12 @@ var ( ) // generateServiceDef generates service definition for Redis -func generateServiceDef(serviceMeta metav1.ObjectMeta, enableMetrics bool, ownerDef metav1.OwnerReference, headless bool, serviceType string) *corev1.Service { +func generateServiceDef(serviceMeta metav1.ObjectMeta, enableMetrics bool, ownerDef metav1.OwnerReference, headless bool, serviceType string, port int32) *corev1.Service { var PortName string - var PortNum int32 if serviceMeta.Labels["role"] == "sentinel" { PortName = "sentinel-client" - PortNum = sentinelPort } else { PortName = "redis-client" - PortNum = redisPort } service := &corev1.Service{ TypeMeta: generateMetaInformation("Service", "v1"), @@ -43,8 +40,8 @@ func generateServiceDef(serviceMeta metav1.ObjectMeta, enableMetrics bool, owner Ports: []corev1.ServicePort{ { Name: PortName, - Port: PortNum, - TargetPort: intstr.FromInt(int(PortNum)), + Port: port, + TargetPort: intstr.FromInt(int(port)), Protocol: corev1.ProtocolTCP, }, }, @@ -146,9 +143,9 @@ func serviceLogger(namespace string, name string) logr.Logger { } // CreateOrUpdateService method will create or update Redis service -func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, enableMetrics, headless bool, serviceType string) error { +func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, enableMetrics, headless bool, serviceType string, port int32) error { logger := serviceLogger(namespace, serviceMeta.Name) - serviceDef := generateServiceDef(serviceMeta, enableMetrics, ownerDef, headless, serviceType) + serviceDef := generateServiceDef(serviceMeta, enableMetrics, ownerDef, headless, serviceType, port) storedService, err := getService(namespace, serviceMeta.Name) if err != nil { if errors.IsNotFound(err) { diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index daab66fde..43b4308fb 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -71,6 +71,7 @@ type containerParameters struct { AdditionalVolume []corev1.Volume AdditionalMountPath []corev1.VolumeMount EnvVars *[]corev1.EnvVar + Port int32 } type initContainerParameters struct { @@ -355,6 +356,7 @@ func generateContainerDef(name string, containerParams containerParameters, clus containerParams.TLSConfig, containerParams.ACLConfig, containerParams.EnvVars, + containerParams.Port, ), ReadinessProbe: getProbeInfo(containerParams.ReadinessProbe), LivenessProbe: getProbeInfo(containerParams.LivenessProbe), @@ -576,12 +578,12 @@ func getProbeInfo(probe *commonapi.Probe) *corev1.Probe { // getEnvironmentVariables returns all the required Environment Variables func getEnvironmentVariables(role string, enabledPassword *bool, secretName *string, secretKey *string, persistenceEnabled *bool, tlsConfig *redisv1beta2.TLSConfig, - aclConfig *redisv1beta2.ACLConfig, envVar *[]corev1.EnvVar) []corev1.EnvVar { + aclConfig *redisv1beta2.ACLConfig, envVar *[]corev1.EnvVar, port int32) []corev1.EnvVar { envVars := []corev1.EnvVar{ {Name: "SERVER_MODE", Value: role}, {Name: "SETUP_MODE", Value: role}, + {Name: "REDIS_PORT", Value: strconv.Itoa(int(port))}, } - var redisHost string if role == "sentinel" { redisHost = "redis://localhost:" + strconv.Itoa(sentinelPort) From 85dcfcaab5c13ad28ad300b0cb47f754c5a36778 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 17:16:42 +0800 Subject: [PATCH 04/21] make default to 6379 when reconcile old cluster Signed-off-by: drivebyer --- api/v1beta2/rediscluster_default.go | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 api/v1beta2/rediscluster_default.go diff --git a/api/v1beta2/rediscluster_default.go b/api/v1beta2/rediscluster_default.go new file mode 100644 index 000000000..745ddd1bb --- /dev/null +++ b/api/v1beta2/rediscluster_default.go @@ -0,0 +1,8 @@ +package v1beta2 + +// SetDefault sets default values for the RedisCluster object. +func (r *RedisCluster) SetDefault() { + if r.Spec.Port == 0 { + r.Spec.Port = 6379 + } +} From c1a116c57d6406234d18c4f2e4f1247daf45ae59 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 17:17:32 +0800 Subject: [PATCH 05/21] add e2e Signed-off-by: drivebyer --- .github/workflows/e2e.yaml | 1 + tests/_config/kind-config.yaml | 4 + tests/_config/kuttl-test.yaml | 1 + .../hostnetwork/redis-cluster/00-install.yaml | 11 + .../hostnetwork/redis-cluster/cluster.yaml | 49 +++++ .../redis-cluster/ready-cluster.yaml | 7 + .../hostnetwork/redis-cluster/ready-pod.yaml | 95 +++++++++ .../hostnetwork/redis-cluster/ready-pvc.yaml | 181 ++++++++++++++++ .../hostnetwork/redis-cluster/ready-sts.yaml | 25 +++ .../hostnetwork/redis-cluster/ready-svc.yaml | 201 ++++++++++++++++++ 10 files changed, 575 insertions(+) create mode 100644 tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml create mode 100644 tests/e2e/v1beta2/hostnetwork/redis-cluster/cluster.yaml create mode 100644 tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml create mode 100644 tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml create mode 100644 tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml create mode 100644 tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml create mode 100644 tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1e82126eb..c9c271f0a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -19,6 +19,7 @@ jobs: - ./tests/e2e/v1beta2/teardown - ./tests/e2e/v1beta2/ignore-annots - ./tests/e2e/v1beta2/scaling + - ./tests/e2e/v1beta2/hostnetwork steps: - name: Checkout code diff --git a/tests/_config/kind-config.yaml b/tests/_config/kind-config.yaml index da75a970a..5553b6587 100644 --- a/tests/_config/kind-config.yaml +++ b/tests/_config/kind-config.yaml @@ -5,3 +5,7 @@ nodes: - role: control-plane - role: worker - role: worker +- role: worker +- role: worker +- role: worker +- role: worker \ No newline at end of file diff --git a/tests/_config/kuttl-test.yaml b/tests/_config/kuttl-test.yaml index 984bba05e..8e560ef77 100644 --- a/tests/_config/kuttl-test.yaml +++ b/tests/_config/kuttl-test.yaml @@ -9,5 +9,6 @@ testDirs: - tests/e2e/v1beta2/teardown - tests/e2e/v1beta2/ignore-annots - tests/e2e/v1beta2/scaling + - tests/e2e/v1beta2/hostnetwork suppress : - events \ No newline at end of file diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml new file mode 100644 index 000000000..c927a0b30 --- /dev/null +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +apply : + - cluster.yaml + - configmap.yaml +assert : + - ready-cluster.yaml + - ready-sts.yaml + - ready-pvc.yaml + - ready-svc.yaml + - ready-pod.yaml \ No newline at end of file diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/cluster.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/cluster.yaml new file mode 100644 index 000000000..afe493178 --- /dev/null +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/cluster.yaml @@ -0,0 +1,49 @@ +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +spec: + hostNetwork: true + port: 6380 + clusterSize: 3 + clusterVersion: v6 # hostNetwork only works with v6 + persistenceEnabled: true + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + # storageClassName: standard + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + nodeConfVolume: true + nodeConfVolumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml new file mode 100644 index 000000000..49e754e81 --- /dev/null +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml @@ -0,0 +1,7 @@ +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +status: + readyFollowerReplicas: 3 + readyLeaderReplicas: 3 diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml new file mode 100644 index 000000000..9c98f4393 --- /dev/null +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-leader-0 +spec: + containers: + - env: + - name: REDIS_PORT + value: "6380" + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-leader-1 +spec: + containers: + - env: + - name: REDIS_PORT + value: "6380" + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-leader-2 +spec: + containers: + - env: + - name: REDIS_PORT + value: "6380" + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-follower-0 +spec: + containers: + - env: + - name: REDIS_PORT + value: "6380" + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-follower-1 +spec: + containers: + - env: + - name: REDIS_PORT + value: "6380" + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-follower-2 +spec: + containers: + - env: + - name: REDIS_PORT + value: "6380" + hostNetwork: true +status: + phase: Running \ No newline at end of file diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml new file mode 100644 index 000000000..a89bfa22a --- /dev/null +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-leader-0 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-leader-1 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-leader-2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-follower-0 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-follower-1 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-follower-2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound + +--- + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-leader-redis-cluster-v1beta2-leader-0 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-leader-redis-cluster-v1beta2-leader-1 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-leader-redis-cluster-v1beta2-leader-2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-follower-redis-cluster-v1beta2-follower-0 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-follower-redis-cluster-v1beta2-follower-1 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-follower-redis-cluster-v1beta2-follower-2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml new file mode 100644 index 000000000..1053eb784 --- /dev/null +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-cluster-v1beta2-leader + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + replicas: 3 + readyReplicas: 3 + +--- + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-cluster-v1beta2-follower + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + replicas: 3 + readyReplicas: 3 diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml new file mode 100644 index 000000000..7dd11eb37 --- /dev/null +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml @@ -0,0 +1,201 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + name: redis-cluster-v1beta2-leader-additional + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + name: redis-cluster-v1beta2-leader + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + name: redis-cluster-v1beta2-leader-headless + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + clusterIP: None + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + name: redis-cluster-v1beta2-follower + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + name: redis-cluster-v1beta2-follower-additional + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + name: redis-cluster-v1beta2-follower-headless + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + clusterIP: None + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + type: ClusterIP +status: + loadBalancer: {} \ No newline at end of file From 4b8f0d74fa5c0f97124d096587658558d05b9ec9 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 17:50:18 +0800 Subject: [PATCH 06/21] fix ut Signed-off-by: drivebyer --- k8sutils/services_test.go | 8 +++++++- k8sutils/statefulset.go | 9 ++++++++- k8sutils/statefulset_test.go | 10 +++++++++- pkg/util/coalesce.go | 9 +++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 pkg/util/coalesce.go diff --git a/k8sutils/services_test.go b/k8sutils/services_test.go index 035583779..9ae24b845 100644 --- a/k8sutils/services_test.go +++ b/k8sutils/services_test.go @@ -16,6 +16,7 @@ func TestGenerateServiceDef(t *testing.T) { enableMetrics bool headless bool serviceType string + port int32 expected *corev1.Service }{ { @@ -29,6 +30,7 @@ func TestGenerateServiceDef(t *testing.T) { enableMetrics: false, headless: false, serviceType: "ClusterIP", + port: sentinelPort, expected: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -69,6 +71,7 @@ func TestGenerateServiceDef(t *testing.T) { enableMetrics: false, headless: true, serviceType: "ClusterIP", + port: sentinelPort, expected: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -109,6 +112,7 @@ func TestGenerateServiceDef(t *testing.T) { enableMetrics: false, headless: false, serviceType: "ClusterIP", + port: redisPort, expected: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -149,6 +153,7 @@ func TestGenerateServiceDef(t *testing.T) { enableMetrics: false, headless: true, serviceType: "ClusterIP", + port: redisPort, expected: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -189,6 +194,7 @@ func TestGenerateServiceDef(t *testing.T) { enableMetrics: true, headless: false, serviceType: "ClusterIP", + port: redisPort, expected: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -223,7 +229,7 @@ func TestGenerateServiceDef(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual := generateServiceDef(tt.serviceMeta, tt.enableMetrics, metav1.OwnerReference{}, tt.headless, tt.serviceType) + actual := generateServiceDef(tt.serviceMeta, tt.enableMetrics, metav1.OwnerReference{}, tt.headless, tt.serviceType, tt.port) assert.Equal(t, tt.expected, actual) }) } diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 43b4308fb..caa114645 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -3,6 +3,7 @@ package k8sutils import ( "context" "fmt" + "github.com/OT-CONTAINER-KIT/redis-operator/pkg/util" "path" "sort" "strconv" @@ -582,13 +583,19 @@ func getEnvironmentVariables(role string, enabledPassword *bool, secretName *str envVars := []corev1.EnvVar{ {Name: "SERVER_MODE", Value: role}, {Name: "SETUP_MODE", Value: role}, - {Name: "REDIS_PORT", Value: strconv.Itoa(int(port))}, } + var redisHost string if role == "sentinel" { redisHost = "redis://localhost:" + strconv.Itoa(sentinelPort) + envVars = append(envVars, corev1.EnvVar{ + Name: "SENTINEL_PORT", Value: strconv.Itoa(int(util.Coalesce(port, sentinelPort))), + }) } else { redisHost = "redis://localhost:" + strconv.Itoa(redisPort) + envVars = append(envVars, corev1.EnvVar{ + Name: "REDIS_PORT", Value: strconv.Itoa(int(util.Coalesce(port, redisPort))), + }) } if tlsConfig != nil { diff --git a/k8sutils/statefulset_test.go b/k8sutils/statefulset_test.go index 09a39ced9..7474476eb 100644 --- a/k8sutils/statefulset_test.go +++ b/k8sutils/statefulset_test.go @@ -229,6 +229,7 @@ func TestGetEnvironmentVariables(t *testing.T) { tlsConfig *redisv1beta2.TLSConfig aclConfig *redisv1beta2.ACLConfig envVar *[]corev1.EnvVar + port int32 expectedEnvironment []corev1.EnvVar }{ { @@ -275,6 +276,7 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "SERVER_MODE", Value: "sentinel"}, {Name: "SETUP_MODE", Value: "sentinel"}, {Name: "TEST_ENV", Value: "test-value"}, + {Name: "SENTINEL_PORT", Value: "26379"}, }, }, { @@ -287,10 +289,12 @@ func TestGetEnvironmentVariables(t *testing.T) { tlsConfig: nil, aclConfig: nil, envVar: nil, + port: 0, expectedEnvironment: []corev1.EnvVar{ {Name: "REDIS_ADDR", Value: "redis://localhost:6379"}, {Name: "SERVER_MODE", Value: "redis"}, {Name: "SETUP_MODE", Value: "redis"}, + {Name: "REDIS_PORT", Value: "6379"}, }, }, { @@ -307,6 +311,7 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "REDIS_ADDR", Value: "redis://localhost:26379"}, {Name: "SERVER_MODE", Value: "sentinel"}, {Name: "SETUP_MODE", Value: "sentinel"}, + {Name: "SENTINEL_PORT", Value: "26379"}, }, }, { @@ -321,6 +326,7 @@ func TestGetEnvironmentVariables(t *testing.T) { envVar: &[]corev1.EnvVar{ {Name: "TEST_ENV", Value: "test-value"}, }, + port: 6380, expectedEnvironment: []corev1.EnvVar{ {Name: "ACL_MODE", Value: "true"}, {Name: "PERSISTENCE_ENABLED", Value: "true"}, @@ -336,6 +342,7 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "SERVER_MODE", Value: "cluster"}, {Name: "SETUP_MODE", Value: "cluster"}, {Name: "TEST_ENV", Value: "test-value"}, + {Name: "REDIS_PORT", Value: "6380"}, }, }, { @@ -352,6 +359,7 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "REDIS_ADDR", Value: "redis://localhost:6379"}, {Name: "SERVER_MODE", Value: "cluster"}, {Name: "SETUP_MODE", Value: "cluster"}, + {Name: "REDIS_PORT", Value: "6379"}, }, }, } @@ -359,7 +367,7 @@ func TestGetEnvironmentVariables(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { actualEnvironment := getEnvironmentVariables(tt.role, tt.enabledPassword, tt.secretName, - tt.secretKey, tt.persistenceEnabled, tt.tlsConfig, tt.aclConfig, tt.envVar) + tt.secretKey, tt.persistenceEnabled, tt.tlsConfig, tt.aclConfig, tt.envVar, tt.port) assert.ElementsMatch(t, tt.expectedEnvironment, actualEnvironment) }) diff --git a/pkg/util/coalesce.go b/pkg/util/coalesce.go new file mode 100644 index 000000000..f95293825 --- /dev/null +++ b/pkg/util/coalesce.go @@ -0,0 +1,9 @@ +package util + +func Coalesce[T comparable](val, defaultVal T) T { + var t T + if val == t { + return defaultVal + } + return val +} From ce035ec38049599838d783c4c05739700de8cd81 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 17:59:13 +0800 Subject: [PATCH 07/21] fix error: no required module provides package github.com/OT-CONTAINER-KIT/redis-operator/pkg/util Signed-off-by: drivebyer --- k8sutils/statefulset.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index caa114645..858067e40 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -3,7 +3,6 @@ package k8sutils import ( "context" "fmt" - "github.com/OT-CONTAINER-KIT/redis-operator/pkg/util" "path" "sort" "strconv" @@ -588,13 +587,19 @@ func getEnvironmentVariables(role string, enabledPassword *bool, secretName *str var redisHost string if role == "sentinel" { redisHost = "redis://localhost:" + strconv.Itoa(sentinelPort) + if port == 0 { + port = sentinelPort + } envVars = append(envVars, corev1.EnvVar{ - Name: "SENTINEL_PORT", Value: strconv.Itoa(int(util.Coalesce(port, sentinelPort))), + Name: "SENTINEL_PORT", Value: strconv.Itoa(int(port)), }) } else { redisHost = "redis://localhost:" + strconv.Itoa(redisPort) + if port == 0 { + port = redisPort + } envVars = append(envVars, corev1.EnvVar{ - Name: "REDIS_PORT", Value: strconv.Itoa(int(util.Coalesce(port, redisPort))), + Name: "REDIS_PORT", Value: strconv.Itoa(int(port)), }) } From 155194b78112ad05d1a235c38e86cc1c4cd68916 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 18:05:41 +0800 Subject: [PATCH 08/21] remove Signed-off-by: drivebyer --- tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml index c927a0b30..56b451984 100644 --- a/tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/00-install.yaml @@ -2,7 +2,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep apply : - cluster.yaml - - configmap.yaml assert : - ready-cluster.yaml - ready-sts.yaml From c39faae84ba4f310c5f4f0a319935782059c958d Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 22:36:52 +0800 Subject: [PATCH 09/21] make port to pointer Signed-off-by: drivebyer --- api/v1beta2/rediscluster_default.go | 6 ++-- api/v1beta2/rediscluster_types.go | 2 +- api/v1beta2/zz_generated.deepcopy.go | 5 +++ ...is.redis.opstreelabs.in_redisclusters.yaml | 1 - k8sutils/cluster-scaling.go | 32 +++++++++---------- k8sutils/redis-cluster.go | 6 ++-- k8sutils/redis.go | 16 +++++----- k8sutils/redis_test.go | 8 ++--- k8sutils/services.go | 6 ++-- k8sutils/services_test.go | 2 +- k8sutils/statefulset.go | 22 ++++++------- k8sutils/statefulset_test.go | 10 ++---- 12 files changed, 58 insertions(+), 58 deletions(-) diff --git a/api/v1beta2/rediscluster_default.go b/api/v1beta2/rediscluster_default.go index 745ddd1bb..7dbc9ea8f 100644 --- a/api/v1beta2/rediscluster_default.go +++ b/api/v1beta2/rediscluster_default.go @@ -1,8 +1,10 @@ package v1beta2 +import "k8s.io/utils/pointer" + // SetDefault sets default values for the RedisCluster object. func (r *RedisCluster) SetDefault() { - if r.Spec.Port == 0 { - r.Spec.Port = 6379 + if r.Spec.Port == nil { + r.Spec.Port = pointer.Int(6379) } } diff --git a/api/v1beta2/rediscluster_types.go b/api/v1beta2/rediscluster_types.go index cf55e594a..b9e629f7a 100644 --- a/api/v1beta2/rediscluster_types.go +++ b/api/v1beta2/rediscluster_types.go @@ -29,7 +29,7 @@ type RedisClusterSpec struct { KubernetesConfig KubernetesConfig `json:"kubernetesConfig"` HostNetwork bool `json:"hostNetwork,omitempty"` // +kubebuilder:default:=6379 - Port int32 `json:"port,omitempty"` + Port *int `json:"port,omitempty"` // +kubebuilder:default:=v7 ClusterVersion *string `json:"clusterVersion,omitempty"` // +kubebuilder:default:={livenessProbe:{initialDelaySeconds: 1, timeoutSeconds: 1, periodSeconds: 10, successThreshold: 1, failureThreshold:3}, readinessProbe:{initialDelaySeconds: 1, timeoutSeconds: 1, periodSeconds: 10, successThreshold: 1, failureThreshold:3}} diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 5e5fe2954..6fc14b16b 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -262,6 +262,11 @@ func (in *RedisClusterSpec) DeepCopyInto(out *RedisClusterSpec) { **out = **in } in.KubernetesConfig.DeepCopyInto(&out.KubernetesConfig) + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int) + **out = **in + } if in.ClusterVersion != nil { in, out := &in.ClusterVersion, &out.ClusterVersion *out = new(string) diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml index 081f77cf6..cdf11fc59 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml @@ -5796,7 +5796,6 @@ spec: type: object port: default: 6379 - format: int32 type: integer priorityClassName: type: string diff --git a/k8sutils/cluster-scaling.go b/k8sutils/cluster-scaling.go index 7c3b244e0..957b74b28 100644 --- a/k8sutils/cluster-scaling.go +++ b/k8sutils/cluster-scaling.go @@ -33,9 +33,9 @@ func ReshardRedisCluster(client kubernetes.Interface, logger logr.Logger, cr *re cmd = []string{"redis-cli", "--cluster", "reshard"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, transferPOD)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, transferPOD)+fmt.Sprintf(":%d", *cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { @@ -158,9 +158,9 @@ func RebalanceRedisClusterEmptyMasters(client kubernetes.Interface, logger logr. cmd = []string{"redis-cli", "--cluster", "rebalance"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", *cr.Spec.Port)) } cmd = append(cmd, "--cluster-use-empty-masters") @@ -210,9 +210,9 @@ func RebalanceRedisCluster(client kubernetes.Interface, logger logr.Logger, cr * cmd = []string{"redis-cli", "--cluster", "rebalance"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", *cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { @@ -247,11 +247,11 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, log cmd = []string{"redis-cli", "--cluster", "add-node"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, newPod)+fmt.Sprintf(":%d", cr.Spec.Port)) - cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, newPod)+fmt.Sprintf(":%d", *cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", *cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { @@ -328,9 +328,9 @@ func RemoveRedisFollowerNodesFromCluster(ctx context.Context, client kubernetes. cmd = append(cmd, "--cluster", "del-node") if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", *cr.Spec.Port)) } for _, followerNodeID := range followerNodeIDs { @@ -359,9 +359,9 @@ func RemoveRedisNodeFromCluster(ctx context.Context, client kubernetes.Interface cmd = []string{"redis-cli", "--cluster", "del-node"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", *cr.Spec.Port)) } removePodNodeID := getRedisNodeID(ctx, client, logger, cr, removePod) @@ -420,9 +420,9 @@ func ClusterFailover(ctx context.Context, client kubernetes.Interface, logger lo cmd = []string{"redis-cli", "cluster", "failover"} if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) } else { - cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", cr.Spec.Port)) + cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", *cr.Spec.Port)) } if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go index b0495fa74..b6ae773b3 100644 --- a/k8sutils/redis-cluster.go +++ b/k8sutils/redis-cluster.go @@ -248,12 +248,12 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations) headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), false, true, "ClusterIP", cr.Spec.Port) + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), false, true, "ClusterIP", *cr.Spec.Port) if err != nil { logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole) return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics, false, "ClusterIP", cr.Spec.Port) + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics, false, "ClusterIP", *cr.Spec.Port) if err != nil { logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole) return err @@ -262,7 +262,7 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), false, false, additionalServiceType, cr.Spec.Port) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), false, false, additionalServiceType, *cr.Spec.Port) if err != nil { logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole) return err diff --git a/k8sutils/redis.go b/k8sutils/redis.go index 9023799b4..fa1986bd6 100644 --- a/k8sutils/redis.go +++ b/k8sutils/redis.go @@ -83,9 +83,9 @@ func CreateMultipleLeaderRedisCommand(client kubernetes.Interface, logger logr.L podName := cr.ObjectMeta.Name + "-leader-" + strconv.Itoa(podCount) var address string if cr.Spec.ClusterVersion != nil && *cr.Spec.ClusterVersion == "v7" { - address = getRedisHostname(RedisDetails{PodName: podName, Namespace: cr.Namespace}, cr, "leader") + fmt.Sprintf(":%d", cr.Spec.Port) + address = getRedisHostname(RedisDetails{PodName: podName, Namespace: cr.Namespace}, cr, "leader") + fmt.Sprintf(":%d", *cr.Spec.Port) } else { - address = getRedisServerIP(client, logger, RedisDetails{PodName: podName, Namespace: cr.Namespace}) + fmt.Sprintf(":%d", cr.Spec.Port) + address = getRedisServerIP(client, logger, RedisDetails{PodName: podName, Namespace: cr.Namespace}) + fmt.Sprintf(":%d", *cr.Spec.Port) } cmd = append(cmd, address) } @@ -141,11 +141,11 @@ func createRedisReplicationCommand(client kubernetes.Interface, logger logr.Logg var followerAddress, leaderAddress string if cr.Spec.ClusterVersion != nil && *cr.Spec.ClusterVersion == "v7" { - followerAddress = getRedisHostname(followerPod, cr, "follower") + fmt.Sprintf(":%d", cr.Spec.Port) - leaderAddress = getRedisHostname(leaderPod, cr, "leader") + fmt.Sprintf(":%d", cr.Spec.Port) + followerAddress = getRedisHostname(followerPod, cr, "follower") + fmt.Sprintf(":%d", *cr.Spec.Port) + leaderAddress = getRedisHostname(leaderPod, cr, "leader") + fmt.Sprintf(":%d", *cr.Spec.Port) } else { - followerAddress = getRedisServerIP(client, logger, followerPod) + fmt.Sprintf(":%d", cr.Spec.Port) - leaderAddress = getRedisServerIP(client, logger, leaderPod) + fmt.Sprintf(":%d", cr.Spec.Port) + followerAddress = getRedisServerIP(client, logger, followerPod) + fmt.Sprintf(":%d", *cr.Spec.Port) + leaderAddress = getRedisServerIP(client, logger, leaderPod) + fmt.Sprintf(":%d", *cr.Spec.Port) } cmd = append(cmd, followerAddress, leaderAddress, "--cluster-slave") @@ -340,14 +340,14 @@ func configureRedisClient(client kubernetes.Interface, logger logr.Logger, cr *r logger.Error(err, "Error in getting redis password") } redisClient = redis.NewClient(&redis.Options{ - Addr: getRedisServerIP(client, logger, redisInfo) + fmt.Sprintf(":%d", cr.Spec.Port), + Addr: getRedisServerIP(client, logger, redisInfo) + fmt.Sprintf(":%d", *cr.Spec.Port), Password: pass, DB: 0, TLSConfig: getRedisTLSConfig(client, logger, cr, redisInfo), }) } else { redisClient = redis.NewClient(&redis.Options{ - Addr: getRedisServerIP(client, logger, redisInfo) + fmt.Sprintf(":%d", cr.Spec.Port), + Addr: getRedisServerIP(client, logger, redisInfo) + fmt.Sprintf(":%d", *cr.Spec.Port), Password: "", DB: 0, TLSConfig: getRedisTLSConfig(client, logger, cr, redisInfo), diff --git a/k8sutils/redis_test.go b/k8sutils/redis_test.go index 76c0ec2c2..8dacd2032 100644 --- a/k8sutils/redis_test.go +++ b/k8sutils/redis_test.go @@ -206,7 +206,7 @@ func TestCreateMultipleLeaderRedisCommand(t *testing.T) { Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), ClusterVersion: pointer.String("v7"), - Port: 6379, + Port: pointer.Int(6379), }, }, expectedCommands: []string{ @@ -226,7 +226,7 @@ func TestCreateMultipleLeaderRedisCommand(t *testing.T) { }, Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), - Port: 6379, + Port: pointer.Int(6379), }, }, expectedCommands: []string{ @@ -297,7 +297,7 @@ func TestCreateRedisReplicationCommand(t *testing.T) { Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), ClusterVersion: pointer.String("v7"), - Port: 6379, + Port: pointer.Int(6379), }, }, leaderPod: RedisDetails{ @@ -324,7 +324,7 @@ func TestCreateRedisReplicationCommand(t *testing.T) { }, Spec: redisv1beta2.RedisClusterSpec{ Size: pointer.Int32(3), - Port: 6379, + Port: pointer.Int(6379), }, }, leaderPod: RedisDetails{ diff --git a/k8sutils/services.go b/k8sutils/services.go index e984b0f32..c28e6ba29 100644 --- a/k8sutils/services.go +++ b/k8sutils/services.go @@ -23,7 +23,7 @@ var ( ) // generateServiceDef generates service definition for Redis -func generateServiceDef(serviceMeta metav1.ObjectMeta, enableMetrics bool, ownerDef metav1.OwnerReference, headless bool, serviceType string, port int32) *corev1.Service { +func generateServiceDef(serviceMeta metav1.ObjectMeta, enableMetrics bool, ownerDef metav1.OwnerReference, headless bool, serviceType string, port int) *corev1.Service { var PortName string if serviceMeta.Labels["role"] == "sentinel" { PortName = "sentinel-client" @@ -40,7 +40,7 @@ func generateServiceDef(serviceMeta metav1.ObjectMeta, enableMetrics bool, owner Ports: []corev1.ServicePort{ { Name: PortName, - Port: port, + Port: int32(port), TargetPort: intstr.FromInt(int(port)), Protocol: corev1.ProtocolTCP, }, @@ -143,7 +143,7 @@ func serviceLogger(namespace string, name string) logr.Logger { } // CreateOrUpdateService method will create or update Redis service -func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, enableMetrics, headless bool, serviceType string, port int32) error { +func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, enableMetrics, headless bool, serviceType string, port int) error { logger := serviceLogger(namespace, serviceMeta.Name) serviceDef := generateServiceDef(serviceMeta, enableMetrics, ownerDef, headless, serviceType, port) storedService, err := getService(namespace, serviceMeta.Name) diff --git a/k8sutils/services_test.go b/k8sutils/services_test.go index 9ae24b845..cbd012b2f 100644 --- a/k8sutils/services_test.go +++ b/k8sutils/services_test.go @@ -16,7 +16,7 @@ func TestGenerateServiceDef(t *testing.T) { enableMetrics bool headless bool serviceType string - port int32 + port int expected *corev1.Service }{ { diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 858067e40..72cbba5f6 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -71,7 +71,7 @@ type containerParameters struct { AdditionalVolume []corev1.Volume AdditionalMountPath []corev1.VolumeMount EnvVars *[]corev1.EnvVar - Port int32 + Port *int } type initContainerParameters struct { @@ -578,7 +578,7 @@ func getProbeInfo(probe *commonapi.Probe) *corev1.Probe { // getEnvironmentVariables returns all the required Environment Variables func getEnvironmentVariables(role string, enabledPassword *bool, secretName *string, secretKey *string, persistenceEnabled *bool, tlsConfig *redisv1beta2.TLSConfig, - aclConfig *redisv1beta2.ACLConfig, envVar *[]corev1.EnvVar, port int32) []corev1.EnvVar { + aclConfig *redisv1beta2.ACLConfig, envVar *[]corev1.EnvVar, port *int) []corev1.EnvVar { envVars := []corev1.EnvVar{ {Name: "SERVER_MODE", Value: role}, {Name: "SETUP_MODE", Value: role}, @@ -587,20 +587,18 @@ func getEnvironmentVariables(role string, enabledPassword *bool, secretName *str var redisHost string if role == "sentinel" { redisHost = "redis://localhost:" + strconv.Itoa(sentinelPort) - if port == 0 { - port = sentinelPort + if port != nil { + envVars = append(envVars, corev1.EnvVar{ + Name: "SENTINEL_PORT", Value: strconv.Itoa(*port), + }) } - envVars = append(envVars, corev1.EnvVar{ - Name: "SENTINEL_PORT", Value: strconv.Itoa(int(port)), - }) } else { redisHost = "redis://localhost:" + strconv.Itoa(redisPort) - if port == 0 { - port = redisPort + if port != nil { + envVars = append(envVars, corev1.EnvVar{ + Name: "REDIS_PORT", Value: strconv.Itoa(*port), + }) } - envVars = append(envVars, corev1.EnvVar{ - Name: "REDIS_PORT", Value: strconv.Itoa(int(port)), - }) } if tlsConfig != nil { diff --git a/k8sutils/statefulset_test.go b/k8sutils/statefulset_test.go index 7474476eb..66672b475 100644 --- a/k8sutils/statefulset_test.go +++ b/k8sutils/statefulset_test.go @@ -229,7 +229,7 @@ func TestGetEnvironmentVariables(t *testing.T) { tlsConfig *redisv1beta2.TLSConfig aclConfig *redisv1beta2.ACLConfig envVar *[]corev1.EnvVar - port int32 + port *int expectedEnvironment []corev1.EnvVar }{ { @@ -276,7 +276,6 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "SERVER_MODE", Value: "sentinel"}, {Name: "SETUP_MODE", Value: "sentinel"}, {Name: "TEST_ENV", Value: "test-value"}, - {Name: "SENTINEL_PORT", Value: "26379"}, }, }, { @@ -289,12 +288,11 @@ func TestGetEnvironmentVariables(t *testing.T) { tlsConfig: nil, aclConfig: nil, envVar: nil, - port: 0, + port: nil, expectedEnvironment: []corev1.EnvVar{ {Name: "REDIS_ADDR", Value: "redis://localhost:6379"}, {Name: "SERVER_MODE", Value: "redis"}, {Name: "SETUP_MODE", Value: "redis"}, - {Name: "REDIS_PORT", Value: "6379"}, }, }, { @@ -311,7 +309,6 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "REDIS_ADDR", Value: "redis://localhost:26379"}, {Name: "SERVER_MODE", Value: "sentinel"}, {Name: "SETUP_MODE", Value: "sentinel"}, - {Name: "SENTINEL_PORT", Value: "26379"}, }, }, { @@ -326,7 +323,7 @@ func TestGetEnvironmentVariables(t *testing.T) { envVar: &[]corev1.EnvVar{ {Name: "TEST_ENV", Value: "test-value"}, }, - port: 6380, + port: pointer.Int(6380), expectedEnvironment: []corev1.EnvVar{ {Name: "ACL_MODE", Value: "true"}, {Name: "PERSISTENCE_ENABLED", Value: "true"}, @@ -359,7 +356,6 @@ func TestGetEnvironmentVariables(t *testing.T) { {Name: "REDIS_ADDR", Value: "redis://localhost:6379"}, {Name: "SERVER_MODE", Value: "cluster"}, {Name: "SETUP_MODE", Value: "cluster"}, - {Name: "REDIS_PORT", Value: "6379"}, }, }, } From d6579b555b22fd06d6349d6e060c4e2abe83bfa5 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Tue, 12 Dec 2023 22:49:05 +0800 Subject: [PATCH 10/21] fix e2e Signed-off-by: drivebyer --- .../hostnetwork/redis-cluster/ready-pod.yaml | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml index 9c98f4393..b1b30e27e 100644 --- a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml @@ -7,9 +7,11 @@ metadata: name: redis-cluster-v1beta2-leader-0 spec: containers: - - env: - - name: REDIS_PORT - value: "6380" + - name: redis-cluster-v1beta2-leader + env: + - name: REDIS_PORT + value: "6380" + - name: redis-exporter hostNetwork: true status: phase: Running @@ -23,9 +25,11 @@ metadata: name: redis-cluster-v1beta2-leader-1 spec: containers: - - env: - - name: REDIS_PORT - value: "6380" + - name: redis-cluster-v1beta2-leader + env: + - name: REDIS_PORT + value: "6380" + - name: redis-exporter hostNetwork: true status: phase: Running @@ -39,9 +43,11 @@ metadata: name: redis-cluster-v1beta2-leader-2 spec: containers: - - env: - - name: REDIS_PORT - value: "6380" + - name: redis-cluster-v1beta2-leader + env: + - name: REDIS_PORT + value: "6380" + - name: redis-exporter hostNetwork: true status: phase: Running @@ -55,9 +61,11 @@ metadata: name: redis-cluster-v1beta2-follower-0 spec: containers: - - env: - - name: REDIS_PORT - value: "6380" + - name: redis-cluster-v1beta2-follower + env: + - name: REDIS_PORT + value: "6380" + - name: redis-exporter hostNetwork: true status: phase: Running @@ -71,9 +79,11 @@ metadata: name: redis-cluster-v1beta2-follower-1 spec: containers: - - env: - - name: REDIS_PORT - value: "6380" + - name: redis-cluster-v1beta2-follower + env: + - name: REDIS_PORT + value: "6380" + - name: redis-exporter hostNetwork: true status: phase: Running @@ -87,9 +97,11 @@ metadata: name: redis-cluster-v1beta2-follower-2 spec: containers: - - env: - - name: REDIS_PORT - value: "6380" + - name: redis-cluster-v1beta2-follower + env: + - name: REDIS_PORT + value: "6380" + - name: redis-exporter hostNetwork: true status: phase: Running \ No newline at end of file From c24de1438be90a13d50548cbf81872b13b833036 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Wed, 13 Dec 2023 10:11:10 +0800 Subject: [PATCH 11/21] fix e2e Signed-off-by: drivebyer --- .../hostnetwork/redis-cluster/ready-pod.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml index b1b30e27e..a2da1b7c0 100644 --- a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml @@ -47,6 +47,14 @@ spec: env: - name: REDIS_PORT value: "6380" + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" - name: redis-exporter hostNetwork: true status: @@ -65,6 +73,14 @@ spec: env: - name: REDIS_PORT value: "6380" + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" - name: redis-exporter hostNetwork: true status: @@ -83,6 +99,14 @@ spec: env: - name: REDIS_PORT value: "6380" + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" - name: redis-exporter hostNetwork: true status: @@ -101,6 +125,14 @@ spec: env: - name: REDIS_PORT value: "6380" + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" - name: redis-exporter hostNetwork: true status: From cdd4853161f7516cc489c5c746f90518e9a7f175 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Wed, 13 Dec 2023 10:34:00 +0800 Subject: [PATCH 12/21] add unit test Signed-off-by: drivebyer --- k8sutils/redis-sentinel_test.go | 86 +++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/k8sutils/redis-sentinel_test.go b/k8sutils/redis-sentinel_test.go index 9aedb48b4..f30d7a267 100644 --- a/k8sutils/redis-sentinel_test.go +++ b/k8sutils/redis-sentinel_test.go @@ -2,8 +2,10 @@ package k8sutils import ( "context" + "k8s.io/client-go/kubernetes" "os" "path/filepath" + "reflect" "testing" common "github.com/OT-CONTAINER-KIT/redis-operator/api" @@ -246,3 +248,87 @@ func Test_generateRedisSentinelInitContainerParams(t *testing.T) { actual := generateRedisSentinelInitContainerParams(input) assert.EqualValues(t, expected, actual, "Expected %+v, got %+v", expected, actual) } + +func Test_getSentinelEnvVariable(t *testing.T) { + type args struct { + ctx context.Context + client kubernetes.Interface + logger logr.Logger + cr *redisv1beta2.RedisSentinel + } + tests := []struct { + name string + args args + want *[]corev1.EnvVar + }{ + { + name: "When RedisSentinelConfig is nil", + args: args{ + ctx: context.TODO(), + client: nil, + logger: logr.Logger{}, + cr: &redisv1beta2.RedisSentinel{}, + }, + want: &[]corev1.EnvVar{}, + }, + { + name: "When RedisSentinelConfig is not nil", + args: args{ + ctx: context.TODO(), + client: nil, + logger: logr.Logger{}, + cr: &redisv1beta2.RedisSentinel{ + Spec: redisv1beta2.RedisSentinelSpec{ + RedisSentinelConfig: &redisv1beta2.RedisSentinelConfig{ + RedisSentinelConfig: common.RedisSentinelConfig{ + MasterGroupName: "master", + RedisPort: "6379", + Quorum: "2", + DownAfterMilliseconds: "30000", + ParallelSyncs: "1", + FailoverTimeout: "180000", + }, + }, + }, + }, + }, + want: &[]corev1.EnvVar{ + { + Name: "MASTER_GROUP_NAME", + Value: "master", + }, + { + Name: "IP", + Value: "", + }, + { + Name: "PORT", + Value: "6379", + }, + { + Name: "QUORUM", + Value: "2", + }, + { + Name: "DOWN_AFTER_MILLISECONDS", + Value: "30000", + }, + { + Name: "PARALLEL_SYNCS", + Value: "1", + }, + { + Name: "FAILOVER_TIMEOUT", + Value: "180000", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getSentinelEnvVariable(tt.args.ctx, tt.args.client, tt.args.logger, tt.args.cr); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getSentinelEnvVariable() = %v, want %v", got, tt.want) + } + }) + } +} From 3d60b26838a4daa606312fc4a238b610c5395893 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Wed, 13 Dec 2023 10:38:20 +0800 Subject: [PATCH 13/21] fix e2e Signed-off-by: drivebyer --- .../hostnetwork/redis-cluster/ready-pod.yaml | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml index a2da1b7c0..e789ed06f 100644 --- a/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml +++ b/tests/e2e/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml @@ -9,8 +9,16 @@ spec: containers: - name: redis-cluster-v1beta2-leader env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" - name: REDIS_PORT value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" - name: redis-exporter hostNetwork: true status: @@ -27,8 +35,16 @@ spec: containers: - name: redis-cluster-v1beta2-leader env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" - name: REDIS_PORT value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" - name: redis-exporter hostNetwork: true status: @@ -45,12 +61,12 @@ spec: containers: - name: redis-cluster-v1beta2-leader env: - - name: REDIS_PORT - value: "6380" - name: PERSISTENCE_ENABLED value: "true" - name: REDIS_ADDR value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" - name: SERVER_MODE value: "cluster" - name: SETUP_MODE @@ -71,12 +87,12 @@ spec: containers: - name: redis-cluster-v1beta2-follower env: - - name: REDIS_PORT - value: "6380" - name: PERSISTENCE_ENABLED value: "true" - name: REDIS_ADDR value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" - name: SERVER_MODE value: "cluster" - name: SETUP_MODE @@ -97,12 +113,12 @@ spec: containers: - name: redis-cluster-v1beta2-follower env: - - name: REDIS_PORT - value: "6380" - name: PERSISTENCE_ENABLED value: "true" - name: REDIS_ADDR value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" - name: SERVER_MODE value: "cluster" - name: SETUP_MODE @@ -123,12 +139,12 @@ spec: containers: - name: redis-cluster-v1beta2-follower env: - - name: REDIS_PORT - value: "6380" - name: PERSISTENCE_ENABLED value: "true" - name: REDIS_ADDR value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" - name: SERVER_MODE value: "cluster" - name: SETUP_MODE From 635c0b4e1246fc69449006e84dd5aab8a1af6256 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Wed, 13 Dec 2023 22:18:43 +0800 Subject: [PATCH 14/21] add chainsaw e2e Signed-off-by: drivebyer --- .github/workflows/e2e-chainsaw.yml | 1 + .../redis-cluster/chainsaw-test.yaml | 36 ++++ .../hostnetwork/redis-cluster/cluster.yaml | 49 +++++ .../redis-cluster/ready-cluster.yaml | 7 + .../hostnetwork/redis-cluster/ready-pod.yaml | 155 ++++++++++++++ .../hostnetwork/redis-cluster/ready-pvc.yaml | 181 ++++++++++++++++ .../hostnetwork/redis-cluster/ready-sts.yaml | 25 +++ .../hostnetwork/redis-cluster/ready-svc.yaml | 201 ++++++++++++++++++ 8 files changed, 655 insertions(+) create mode 100644 tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/cluster.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml diff --git a/.github/workflows/e2e-chainsaw.yml b/.github/workflows/e2e-chainsaw.yml index b61bde8cf..dd6556b3f 100644 --- a/.github/workflows/e2e-chainsaw.yml +++ b/.github/workflows/e2e-chainsaw.yml @@ -17,6 +17,7 @@ jobs: testpath: - ./tests/e2e-chainsaw/v1beta2/teardown/ - ./tests/e2e-chainsaw/v1beta2/setup/ + - ./tests/e2e-chainsaw/v1beta2/hostnetwork/ steps: - name: Checkout code diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml new file mode 100644 index 000000000..0ab777b4f --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json + +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: redis-cluster-setup +spec: + steps: + - name: Apply and Assert + try: + - apply: + file: cluster.yaml + - assert: + file: ready-cluster.yaml + - assert: + file: ready-sts.yaml + - assert: + file: ready-svc.yaml + - assert: + file: ready-pvc.yaml + - assert: + file: ready-pod.yaml + + - name: Install Redis Cli + try: + - script: + content: | + apt-get update && apt-get install -y redis-tools + + - name: Ping Cluster + try: + - script: + content: | + kubectl get node -o wide | grep 'worker' | awk '{print $6}' | head -n 1 | xargs -I {} redis-cli -h {} -p 6380 ping + check: + ($stdout=='PONG'): true \ No newline at end of file diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/cluster.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/cluster.yaml new file mode 100644 index 000000000..afe493178 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/cluster.yaml @@ -0,0 +1,49 @@ +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +spec: + hostNetwork: true + port: 6380 + clusterSize: 3 + clusterVersion: v6 # hostNetwork only works with v6 + persistenceEnabled: true + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + # storageClassName: standard + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + nodeConfVolume: true + nodeConfVolumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml new file mode 100644 index 000000000..49e754e81 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml @@ -0,0 +1,7 @@ +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +status: + readyFollowerReplicas: 3 + readyLeaderReplicas: 3 diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml new file mode 100644 index 000000000..e789ed06f --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pod.yaml @@ -0,0 +1,155 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-leader-0 +spec: + containers: + - name: redis-cluster-v1beta2-leader + env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" + - name: redis-exporter + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-leader-1 +spec: + containers: + - name: redis-cluster-v1beta2-leader + env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" + - name: redis-exporter + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-leader-2 +spec: + containers: + - name: redis-cluster-v1beta2-leader + env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" + - name: redis-exporter + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-follower-0 +spec: + containers: + - name: redis-cluster-v1beta2-follower + env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" + - name: redis-exporter + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-follower-1 +spec: + containers: + - name: redis-cluster-v1beta2-follower + env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" + - name: redis-exporter + hostNetwork: true +status: + phase: Running +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + name: redis-cluster-v1beta2-follower-2 +spec: + containers: + - name: redis-cluster-v1beta2-follower + env: + - name: PERSISTENCE_ENABLED + value: "true" + - name: REDIS_ADDR + value: "redis://localhost:6379" + - name: REDIS_PORT + value: "6380" + - name: SERVER_MODE + value: "cluster" + - name: SETUP_MODE + value: "cluster" + - name: redis-exporter + hostNetwork: true +status: + phase: Running \ No newline at end of file diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml new file mode 100644 index 000000000..a89bfa22a --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-pvc.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-leader-0 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-leader-1 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-leader-2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-follower-0 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-follower-1 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-conf-redis-cluster-v1beta2-follower-2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound + +--- + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-leader-redis-cluster-v1beta2-leader-0 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-leader-redis-cluster-v1beta2-leader-1 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-leader-redis-cluster-v1beta2-leader-2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-follower-redis-cluster-v1beta2-follower-0 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-follower-redis-cluster-v1beta2-follower-1 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-cluster-v1beta2-follower-redis-cluster-v1beta2-follower-2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + phase: Bound diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml new file mode 100644 index 000000000..1053eb784 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-sts.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-cluster-v1beta2-leader + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader +status: + replicas: 3 + readyReplicas: 3 + +--- + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-cluster-v1beta2-follower + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower +status: + replicas: 3 + readyReplicas: 3 diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml new file mode 100644 index 000000000..7dd11eb37 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-svc.yaml @@ -0,0 +1,201 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + name: redis-cluster-v1beta2-leader-additional + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + name: redis-cluster-v1beta2-leader + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + name: redis-cluster-v1beta2-leader-headless + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + clusterIP: None + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-leader + redis_setup_type: cluster + role: leader + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + name: redis-cluster-v1beta2-follower + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + - name: redis-exporter + port: 9121 + protocol: TCP + targetPort: 9121 + selector: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + name: redis-cluster-v1beta2-follower-additional + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + redis.opstreelabs.in: "true" + redis.opstreelabs.instance: redis-cluster-v1beta2 + labels: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + name: redis-cluster-v1beta2-follower-headless + ownerReferences: + - apiVersion: redis.redis.opstreelabs.in/v1beta2 + controller: true + kind: RedisCluster + name: redis-cluster-v1beta2 +spec: + clusterIP: None + ports: + - name: redis-client + port: 6380 + protocol: TCP + targetPort: 6380 + selector: + app: redis-cluster-v1beta2-follower + redis_setup_type: cluster + role: follower + type: ClusterIP +status: + loadBalancer: {} \ No newline at end of file From 356103a97f66ffa05d438c987bad814eef911054 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Wed, 13 Dec 2023 22:31:18 +0800 Subject: [PATCH 15/21] install redis cli Signed-off-by: drivebyer --- .../v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index 0ab777b4f..c3a66bc11 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -25,7 +25,7 @@ spec: try: - script: content: | - apt-get update && apt-get install -y redis-tools + sudo apt-get install redis-tools -y - name: Ping Cluster try: From 0e9e1bdacb87840e5640b2df2d2d40d83e743c3a Mon Sep 17 00:00:00 2001 From: drivebyer Date: Wed, 13 Dec 2023 22:50:48 +0800 Subject: [PATCH 16/21] setup redis action Signed-off-by: drivebyer --- .github/workflows/e2e-chainsaw.yml | 3 +++ .../hostnetwork/redis-cluster/chainsaw-test.yaml | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-chainsaw.yml b/.github/workflows/e2e-chainsaw.yml index dd6556b3f..c1a0b50e2 100644 --- a/.github/workflows/e2e-chainsaw.yml +++ b/.github/workflows/e2e-chainsaw.yml @@ -40,6 +40,9 @@ jobs: chmod +x kubectl sudo mv kubectl /usr/local/bin/ + - name: Install Redis + uses: shogo82148/actions-setup-redis@v1 + - name: Create k8s Kind Cluster uses: helm/kind-action@v1.5.0 with: diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index c3a66bc11..7df8a5477 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -21,11 +21,11 @@ spec: - assert: file: ready-pod.yaml - - name: Install Redis Cli - try: - - script: - content: | - sudo apt-get install redis-tools -y +# - name: Install Redis Cli +# try: +# - script: +# content: | +# sudo apt-get install redis-tools -y - name: Ping Cluster try: From bacb27180e4602c084ec53f42f8b15996eeac068 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 13 Dec 2023 23:37:35 +0530 Subject: [PATCH 17/21] remove redis cli from CI Signed-off-by: Shubham Gupta --- .github/workflows/e2e-chainsaw.yml | 4 ++-- .../hostnetwork/redis-cluster/chainsaw-test.yaml | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e-chainsaw.yml b/.github/workflows/e2e-chainsaw.yml index c1a0b50e2..dcf25b81a 100644 --- a/.github/workflows/e2e-chainsaw.yml +++ b/.github/workflows/e2e-chainsaw.yml @@ -40,8 +40,8 @@ jobs: chmod +x kubectl sudo mv kubectl /usr/local/bin/ - - name: Install Redis - uses: shogo82148/actions-setup-redis@v1 + # - name: Install Redis + # uses: shogo82148/actions-setup-redis@v1 - name: Create k8s Kind Cluster uses: helm/kind-action@v1.5.0 diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index 7df8a5477..5bc28ab50 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -6,7 +6,7 @@ metadata: name: redis-cluster-setup spec: steps: - - name: Apply and Assert + - name: Setup redis cluster on host network try: - apply: file: cluster.yaml @@ -21,11 +21,13 @@ spec: - assert: file: ready-pod.yaml -# - name: Install Redis Cli -# try: -# - script: -# content: | -# sudo apt-get install redis-tools -y + - name: Install Redis Cli + try: + - script: + content: | + sudo apt install redis-tools -y + check: + ($error != null): false - name: Ping Cluster try: From ee6018fc01f7c54b466d9c0882556d7112b7edbf Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 13 Dec 2023 23:44:08 +0530 Subject: [PATCH 18/21] fix Signed-off-by: Shubham Gupta --- .../v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index 5bc28ab50..59a342720 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: redis-cluster-setup + name: host network spec: steps: - name: Setup redis cluster on host network @@ -26,9 +26,7 @@ spec: - script: content: | sudo apt install redis-tools -y - check: - ($error != null): false - + - name: Ping Cluster try: - script: From c36ca8d1d8f371ea2ebf049a5f41e989f443ef37 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 13 Dec 2023 23:48:19 +0530 Subject: [PATCH 19/21] try on every node Signed-off-by: Shubham Gupta --- .../hostnetwork/redis-cluster/chainsaw-test.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index 59a342720..11ff282e4 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: host network + name: host-network spec: steps: - name: Setup redis cluster on host network @@ -31,6 +31,14 @@ spec: try: - script: content: | - kubectl get node -o wide | grep 'worker' | awk '{print $6}' | head -n 1 | xargs -I {} redis-cli -h {} -p 6380 ping + kubectl get node -o wide | grep 'worker' | awk '{print $6}' | xargs -I {} redis-cli -h {} -p 6380 ping check: - ($stdout=='PONG'): true \ No newline at end of file + ($stdout=='PONG'): true + + # - name: Try saving a key + # try: + # - script: + # content: | + # kubectl get node -o wide | grep 'worker' | awk '{print $6}' | xargs -I {} redis-cli -h {} -p 6380 set foo bar + # check: + # ($stdout=='OK'): true \ No newline at end of file From ac99b8782665f0e6b2cd9983de5fc886a59a1c3d Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 13 Dec 2023 23:56:11 +0530 Subject: [PATCH 20/21] use cluster client Signed-off-by: Shubham Gupta --- .../hostnetwork/redis-cluster/chainsaw-test.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index 11ff282e4..085862ea4 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -31,14 +31,14 @@ spec: try: - script: content: | - kubectl get node -o wide | grep 'worker' | awk '{print $6}' | xargs -I {} redis-cli -h {} -p 6380 ping + kubectl get node -o wide | grep 'worker' | awk '{print $6}' | head -n 1 | xargs -I {} redis-cli -h {} -p 6380 -c ping check: ($stdout=='PONG'): true - # - name: Try saving a key - # try: - # - script: - # content: | - # kubectl get node -o wide | grep 'worker' | awk '{print $6}' | xargs -I {} redis-cli -h {} -p 6380 set foo bar - # check: - # ($stdout=='OK'): true \ No newline at end of file + - name: Try saving a key + try: + - script: + content: | + kubectl get node -o wide | grep 'worker' | awk '{print $6}' | xargs -I {} redis-cli -h {} -p 6380 -c set foo bar + check: + ($stdout=='OK'): true \ No newline at end of file From 2946c6bdb354a15eb24fd90fa43d2a7da31da575 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 14 Dec 2023 00:14:12 +0530 Subject: [PATCH 21/21] fix Signed-off-by: Shubham Gupta --- tests/_config/chainsaw-configuration.yaml | 8 ++++---- .../v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/_config/chainsaw-configuration.yaml b/tests/_config/chainsaw-configuration.yaml index f2637f495..1d52cf41b 100644 --- a/tests/_config/chainsaw-configuration.yaml +++ b/tests/_config/chainsaw-configuration.yaml @@ -8,7 +8,7 @@ spec: parallel: 1 delayBeforeCleanup: 10s timeouts: - apply: 100s - delete: 100s - assert: 900s - error: 900s + apply: 5m + delete: 5m + assert: 15m + error: 15m diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index 085862ea4..e9fc5b562 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -24,6 +24,7 @@ spec: - name: Install Redis Cli try: - script: + timeout: 5m content: | sudo apt install redis-tools -y @@ -31,7 +32,7 @@ spec: try: - script: content: | - kubectl get node -o wide | grep 'worker' | awk '{print $6}' | head -n 1 | xargs -I {} redis-cli -h {} -p 6380 -c ping + kubectl get node -o wide | grep 'worker' | awk '{print $6}' | head -n 1 | xargs -I {} redis-cli -h {} -c -p 6380 ping check: ($stdout=='PONG'): true @@ -39,6 +40,6 @@ spec: try: - script: content: | - kubectl get node -o wide | grep 'worker' | awk '{print $6}' | xargs -I {} redis-cli -h {} -p 6380 -c set foo bar + kubectl get node -o wide | grep 'worker' | awk '{print $6}' | head -n 1 | xargs -I {} redis-cli -h {} -c -p 6380 set foo bar check: ($stdout=='OK'): true \ No newline at end of file