review fixes

go.mod

@@ -21,6 +21,7 @@ require (
 	cloud.google.com/go/storage v1.39.1 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/avast/retry-go v3.0.0+incompatible // indirect	
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect

go.sum

@@ -193,6 +193,8 @@ github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go v1.51.1 h1:AFvTihcDPanvptoKS09a4yYmNtPm3+pXlk6uYHmZiFk=
 github.com/aws/aws-sdk-go v1.51.1/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=

pkg/testcase/secretsencrypt.go

@@ -19,10 +19,10 @@ func TestSecretsEncryption() {
 	product, err := shared.Product()
 	Expect(err).NotTo(HaveOccurred(), "error getting product from config")
 
-	errSecret := shared.CreateSecret("secret1", "default", false)
+	errSecret := shared.CreateSecret("secret1", "default")
 	Expect(errSecret).NotTo(HaveOccurred(), "error creating secret")
 
-	shared.LogLevel("INFO", "TEST: 'CLASSIC' Secrets Encryption method")
+	shared.LogLevel("info", "TEST: 'CLASSIC' Secrets Encryption method")
 
 	index := len(nodes) - 1
 	cpIp := nodes[index].ExternalIP
@@ -31,13 +31,13 @@ func TestSecretsEncryption() {
 	secretsEncryptOps("reencrypt", product, cpIp, nodes)
 
 	if strings.Contains(os.Getenv("TEST_TYPE"), "both") {
-		shared.LogLevel("INFO", "TEST: 'NEW' Secrets Encryption method")
+		shared.LogLevel("info", "TEST: 'NEW' Secrets Encryption method")
 		secretsEncryptOps("rotate-keys", product, cpIp, nodes)
 	}
 }
 
 func secretsEncryptOps(action, product, cpIp string, nodes []shared.Node) {
-	shared.LogLevel("INFO", fmt.Sprintf("TEST: Secrets-Encryption: %s", action))
+	shared.LogLevel("info", fmt.Sprintf("TEST: Secrets-Encryption: %s", action))
 	_, errStatusB4 := shared.SecretEncryptOps("status", cpIp, product)
 	Expect(errStatusB4).NotTo(HaveOccurred(), "error getting secret-encryption status before action")
 
@@ -55,33 +55,36 @@ func secretsEncryptOps(action, product, cpIp string, nodes []shared.Node) {
 		// Order of reboot matters. Etcd first then control plane nodes.
 		// Little lag needed between node restarts to avoid issues.
 		time.Sleep(30 * time.Second)
-		waitEtcdErr := shared.WaitForPodsRunning(5, 4)
+		waitEtcdErr := shared.WaitForPodsRunning(10, 3)
 		if waitEtcdErr != nil {
-			shared.LogLevel("WARN", "pods not up after 20 seconds.")
+			shared.LogLevel("WARN", "pods not up after 30 seconds.")
 		}
 	}
 	switch product {
 	case "k3s":
-		waitPodsErr := shared.WaitForPodsRunning(5, 6)
+		waitPodsErr := shared.WaitForPodsRunning(10, 3)
 		if waitPodsErr != nil {
 			shared.LogLevel("WARN", "pods not up after 30 seconds")
 		}
 	case "rke2":
-		waitPodsErr := shared.WaitForPodsRunning(5, 12)
+		waitPodsErr := shared.WaitForPodsRunning(10, 6)
 		if waitPodsErr != nil {
 			shared.LogLevel("WARN", "pods not up after 60 seconds")
 		}
 	}
 
-	secretEncryptStatus, errGetStatus := waitForHashMatch(cpIp, product, 5, 36) // Max 3 minute wait time for hash to match
+	secretEncryptStatus, errGetStatus := waitForHashMatch(cpIp, product)
 	Expect(errGetStatus).NotTo(HaveOccurred(), "error getting secret-encryption status")
 	verifyStatusStdOut(action, secretEncryptStatus)
 
 	errLog := logEncryptionFileContents(nodes, product)
 	Expect(errLog).NotTo(HaveOccurred())
 }
 
-func waitForHashMatch(cpIp, product string, defaultTime time.Duration, times int) (string, error) {
+func waitForHashMatch(cpIp, product string) (string, error) {
+	// Max 3 minute wait time for hash match
+	defaultTime := time.Duration(10)
+	times := 6 * 3
 	var secretEncryptStatus string
 	var errGetStatus error
 	for i := 1; i <= times; i++ {

shared/cluster.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/avast/retry-go"
 	"github.com/rancher/distros-test-framework/config"
 )
 
@@ -673,48 +674,45 @@ func CreateSecret(secret, namespace string) error {
 	return nil
 }
 
-func checkPodStatus(print bool) bool {
-	podsRunningStatus := false
-	pods, errGetPods := GetPods(print)
+func checkPodStatus() bool {
+	pods, errGetPods := GetPods(false)
 	if errGetPods != nil || len(pods) == 0 {
 		LogLevel("DEBUG", "Error getting pods. Retry.")
-		return podsRunningStatus
+		return false
 	}
 	podReady := 0
 	podNotReady := 0
 	for _, pod := range pods {
 		if pod.Status == "Running" || pod.Status == "Completed" {
-			podReady = podReady + 1
+			podReady++
 		} else {
-			podNotReady = podNotReady + 1
+			podNotReady++
 			LogLevel("DEBUG", "Pod Not Ready. Pod details: Name: %s Status: %s", pod.Name, pod.Status)
 		}
 	}
 	if podReady+podNotReady != len(pods) {
 		LogLevel("DEBUG", "Length of pods %d != Ready pods: %d + Not Ready Pods: %d", len(pods), podReady, podNotReady)
 	}
 	if podNotReady == 0 {
-		podsRunningStatus = true
+		return true
 	}
 
-	return podsRunningStatus
+	return true
 }
 
 // WaitForPodsRunning Waits for pods to reach running state.
-func WaitForPodsRunning(defaultTime time.Duration, times int) error {
-	var podsRunning bool
-	print := false
-	for i := 1; i <= times; i++ {
-		time.Sleep(defaultTime * time.Second)
-		podsRunning = checkPodStatus(print)
-		if podsRunning {
-			LogLevel("DEBUG", "All pods are up. Exiting sleep cycle after: %d seconds", i*int(defaultTime))
-			break
-		}
-	}
-	if !podsRunning {
-		return ReturnLogError("All pods were not up at the end of wait period %d", int(defaultTime)*times)
-	}
-
-	return nil
+func WaitForPodsRunning(defaultTime time.Duration, attempts uint) error {
+	return retry.Do(
+		func() error {
+			if !checkPodStatus() {
+				return ReturnLogError("Not all pods are ready yet")
+			}
+			return nil
+		},
+		retry.Attempts(attempts),
+		retry.Delay(defaultTime),
+		retry.OnRetry(func(n uint, err error) {
+			LogLevel("DEBUG", "Attempt %d: Pods not ready, retrying...", n+1)
+		}),
+	)
 }