Node replacement test (rancher#70)

 add Node replacement test 
---------

Signed-off-by: Francisco <[email protected]>

.github/workflows/run-distros.yaml

@@ -55,4 +55,4 @@ jobs:
         with:
           version: v1.55.2
           args: --timeout 10m
-        continue-on-error: false
+        continue-on-error: false

.golangci.yaml

@@ -24,7 +24,7 @@ linters-settings:
     ignore-generated-header: true
     rules:
       - name: line-length-limit
-        arguments: [110]
+        arguments: [125]
       - name: bare-return
       - name: blank-imports
       - name: confusing-results

Makefile

@@ -72,11 +72,13 @@ test-validate:
 test-upgrade-suc:
 	@go test -timeout=45m -v -tags=upgradesuc -count=1 ./entrypoint/upgradecluster/... -sucUpgradeVersion ${SUC_UPGRADE_VERSION}
 
-
 .PHONY: test-upgrade-manual
 test-upgrade-manual:
 	@go test -timeout=45m -v -tags=upgrademanual -count=1 ./entrypoint/upgradecluster/... -installVersionOrCommit ${INSTALL_VERSION_OR_COMMIT} -channel ${CHANNEL}
 
+.PHONY: test-upgrade-node-replacement
+test-upgrade-node-replacement:
+	@go test -timeout=60m -v -tags=upgradereplacement -count=1 ./entrypoint/upgradecluster/... -installVersionOrCommit ${INSTALL_VERSION_OR_COMMIT}
 
 .PHONY: test-create-mixedos
 test-create-mixedos:

entrypoint/upgradecluster/upgrademanual_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/rancher/distros-test-framework/pkg/assert"
 	"github.com/rancher/distros-test-framework/pkg/customflag"
 	"github.com/rancher/distros-test-framework/pkg/testcase"
-	"github.com/rancher/distros-test-framework/shared"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -77,9 +76,8 @@ var _ = Describe("Test:", func() {
 	}
 
 	It("Upgrade Manual", func() {
-		fmt.Println("Current cluster state before upgrade:")
-		shared.PrintClusterState()
-		_ = testcase.TestUpgradeClusterManually(customflag.ServiceFlag.InstallMode.String())
+		err := testcase.TestUpgradeClusterManually(customflag.ServiceFlag.InstallMode.String())
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("Checks Node Status after upgrade and validate version", func() {

entrypoint/upgradecluster/upgradenodereplacement_test.go

@@ -0,0 +1,91 @@
+//go:build upgradereplacement
+
+package upgradecluster
+
+import (
+	"fmt"
+
+	"github.com/rancher/distros-test-framework/pkg/assert"
+	"github.com/rancher/distros-test-framework/pkg/customflag"
+	"github.com/rancher/distros-test-framework/pkg/testcase"
+
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("Test:", func() {
+
+	It("Start Up with no issues", func() {
+		testcase.TestBuildCluster(GinkgoT())
+	})
+
+	It("Validate Node", func() {
+		testcase.TestNodeStatus(
+			assert.NodeAssertReadyStatus(),
+			nil)
+	})
+
+	It("Validate Pod", func() {
+		testcase.TestPodStatus(
+			assert.PodAssertRestart(),
+			assert.PodAssertReady(),
+			assert.PodAssertStatus())
+	})
+
+	It("Verifies ClusterIP Service pre-upgrade", func() {
+		testcase.TestServiceClusterIp(true, false)
+	})
+
+	if cfg.Product == "k3s" {
+		It("Verifies LoadBalancer Service pre-upgrade", func() {
+			testcase.TestServiceLoadBalancer(true, false)
+		})
+	}
+
+	It("Verifies Ingress pre-upgrade", func() {
+		testcase.TestIngress(true, false)
+	})
+
+	It("Upgrade by Node replacement", func() {
+		testcase.TestUpgradeReplaceNode(customflag.ServiceFlag.InstallMode.String())
+	})
+
+	It("Checks Node Status after upgrade and validate version", func() {
+		testcase.TestNodeStatus(
+			assert.NodeAssertReadyStatus(),
+			assert.NodeAssertVersionTypeUpgrade(customflag.ServiceFlag))
+	})
+
+	It("Checks Pod Status after upgrade", func() {
+		testcase.TestPodStatus(
+			assert.PodAssertRestart(),
+			assert.PodAssertReady(),
+			assert.PodAssertStatus(),
+		)
+	})
+
+	It("Verifies ClusterIP Service after upgrade", func() {
+		testcase.TestServiceClusterIp(false, true)
+	})
+
+	It("Verifies NodePort Service after upgrade applying and deleting workload", func() {
+		testcase.TestServiceNodePort(true, true)
+	})
+
+	It("Verifies Ingress after upgrade", func() {
+		testcase.TestIngress(false, true)
+	})
+
+	if cfg.Product == "k3s" {
+		It("Verifies LoadBalancer Service after upgrade", func() {
+			testcase.TestServiceLoadBalancer(false, true)
+		})
+	}
+})
+
+var _ = AfterEach(func() {
+	if CurrentSpecReport().Failed() {
+		fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText())
+	} else {
+		fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText())
+	}
+})

entrypoint/upgradecluster/upgradesuc_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/rancher/distros-test-framework/pkg/assert"
 	"github.com/rancher/distros-test-framework/pkg/customflag"
 	"github.com/rancher/distros-test-framework/pkg/testcase"
-	"github.com/rancher/distros-test-framework/shared"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -77,9 +76,8 @@ var _ = Describe("SUC Upgrade Tests:", func() {
 	}
 
 	It("\nUpgrade via SUC", func() {
-		fmt.Println("Current cluster state before upgrade:")
-		shared.PrintClusterState()
-		_ = testcase.TestUpgradeClusterSUC(customflag.ServiceFlag.SUCUpgradeVersion.String())
+		err := testcase.TestUpgradeClusterSUC(customflag.ServiceFlag.SUCUpgradeVersion.String())
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("Checks Node status post-upgrade", func() {

pkg/aws/aws.go

@@ -24,9 +24,10 @@ type Client struct {
 type response struct {
 	nodeId     string
 	externalIp string
+	privateIp  string
 }
 
-func AddAwsNode() (*Client, error) {
+func AddNode() (*Client, error) {
 	c := factory.ClusterConfig(GinkgoT())
 
 	sess, err := session.NewSession(&aws.Config{
@@ -41,9 +42,9 @@ func AddAwsNode() (*Client, error) {
 	}, nil
 }
 
-func (c Client) CreateInstances(names ...string) (ids, ips []string, err error) {
+func (c Client) CreateInstances(names ...string) (externalIPs, privateIPs, ids []string, err error) {
 	if len(names) == 0 {
-		return nil, nil, shared.ReturnLogError("must sent a name: %s\n", names)
+		return nil, nil, nil, shared.ReturnLogError("must sent name for the instance")
 	}
 
 	errChan := make(chan error, len(names))
@@ -67,13 +68,13 @@ func (c Client) CreateInstances(names ...string) (ids, ips []string, err error)
 				return
 			}
 
-			externalIp, err := c.fetchIP(nodeID)
+			externalIp, privateIp, err := c.fetchIP(nodeID)
 			if err != nil {
 				errChan <- shared.ReturnLogError("error fetching ip: %w\n", err)
 				return
 			}
 
-			resChan <- response{nodeId: nodeID, externalIp: externalIp}
+			resChan <- response{nodeId: nodeID, externalIp: externalIp, privateIp: privateIp}
 		}(n)
 	}
 	go func() {
@@ -84,17 +85,18 @@ func (c Client) CreateInstances(names ...string) (ids, ips []string, err error)
 
 	for e := range errChan {
 		if e != nil {
-			return nil, nil, shared.ReturnLogError("error from errChan: %w\n", e)
+			return nil, nil, nil, shared.ReturnLogError("error from errChan: %w\n", e)
 		}
 	}
 
-	var nodeIps, nodeIds []string
+	var externalIps, privateIps, nodeIds []string
 	for i := range resChan {
 		nodeIds = append(nodeIds, i.nodeId)
-		nodeIps = append(nodeIps, i.externalIp)
+		externalIps = append(externalIps, i.externalIp)
+		privateIps = append(privateIps, i.privateIp)
 	}
 
-	return nodeIps, nodeIds, nil
+	return externalIps, privateIps, nodeIds, nil
 }
 
 func (c Client) DeleteInstance(ip string) error {
@@ -133,7 +135,7 @@ func (c Client) DeleteInstance(ip string) error {
 				if len(node.Tags) > 0 {
 					instanceName = *node.Tags[0].Value
 				}
-				shared.LogLevel("info", fmt.Sprintf("\nTerminated instance: %s (ID: %s)",
+				shared.LogLevel("info", fmt.Sprintf("Terminated instance: %s (ID: %s)",
 					instanceName, *node.InstanceId))
 			}
 		}
@@ -229,29 +231,29 @@ func (c Client) create(name string) (*ec2.Reservation, error) {
 	return c.ec2.RunInstances(input)
 }
 
-func (c Client) fetchIP(nodeID string) (string, error) {
+func (c Client) fetchIP(nodeID string) (publicIP string, privateIP string, err error) {
 	waitErr := c.WaitForInstanceRunning(nodeID)
 	if waitErr != nil {
-		return "", shared.ReturnLogError("error waiting for instance to be in running state: %w\n", waitErr)
+		return "", "", shared.ReturnLogError("error waiting for instance to be running: %w\n", waitErr)
 	}
 
 	id := &ec2.DescribeInstancesInput{
 		InstanceIds: aws.StringSlice([]string{nodeID}),
 	}
 	result, err := c.ec2.DescribeInstances(id)
 	if err != nil {
-		return "", shared.ReturnLogError("error describing instances: %w\n", err)
+		return "", "", shared.ReturnLogError("error describing instances: %w\n", err)
 	}
 
 	for _, r := range result.Reservations {
 		for _, i := range r.Instances {
-			if i.PublicIpAddress != nil {
-				return *i.PublicIpAddress, nil
+			if i.PublicIpAddress != nil && i.PrivateIpAddress != nil {
+				return *i.PublicIpAddress, *i.PrivateIpAddress, nil
 			}
 		}
 	}
 
-	return "", shared.ReturnLogError("no public ip found for instance: %s\n", nodeID)
+	return "", "", shared.ReturnLogError("no ip found for instance: %s\n", nodeID)
 }
 
 func extractID(reservation *ec2.Reservation) (string, error) {

pkg/testcase/upgradecluster.go → pkg/testcase/upgrademanually.go

@@ -2,66 +2,24 @@ package testcase
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"sync"
 
 	"github.com/rancher/distros-test-framework/factory"
-	"github.com/rancher/distros-test-framework/pkg/assert"
 	"github.com/rancher/distros-test-framework/pkg/customflag"
 	"github.com/rancher/distros-test-framework/shared"
 
 	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
 )
 
-// TestUpgradeClusterSUC upgrades cluster using the system-upgrade-controller.
-func TestUpgradeClusterSUC(version string) error {
-	fmt.Printf("\nUpgrading cluster to: %s\n", version)
-
-	workloadErr := shared.ManageWorkload("apply", "suc.yaml")
-	Expect(workloadErr).NotTo(HaveOccurred(),
-		"system-upgrade-controller manifest did not deploy successfully")
-
-	getPodsSystemUpgrade := "kubectl get pods -n system-upgrade --kubeconfig="
-	err := assert.CheckComponentCmdHost(
-		getPodsSystemUpgrade+shared.KubeConfigFile,
-		"system-upgrade-controller",
-		statusRunning,
-	)
-	Expect(err).NotTo(HaveOccurred(), err)
-
-	product, err := shared.Product()
-	Expect(err).NotTo(HaveOccurred())
-
-	originalFilePath := shared.BasePath() +
-		fmt.Sprintf("/workloads/amd64/%s-upgrade-plan.yaml", product)
-	newFilePath := shared.BasePath() + "/workloads/amd64/plan.yaml"
-
-	content, err := os.ReadFile(originalFilePath)
-	if err != nil {
-		return fmt.Errorf("failed to read file: %s", err)
-	}
-
-	newContent := strings.ReplaceAll(string(content), "$UPGRADEVERSION", version)
-	err = os.WriteFile(newFilePath, []byte(newContent), 0644)
-	if err != nil {
-		return fmt.Errorf("failed to write file: %s", err)
-	}
-
-	workloadErr = shared.ManageWorkload("apply", "plan.yaml")
-	Expect(workloadErr).NotTo(HaveOccurred(), "failed to upgrade cluster.")
-
-	return nil
-}
-
 // TestUpgradeClusterManually upgrades the cluster "manually"
 func TestUpgradeClusterManually(version string) error {
-	fmt.Printf("\nUpgrading cluster to: %s", version)
-
 	if version == "" {
 		return shared.ReturnLogError("please provide a non-empty version or commit to upgrade to")
 	}
+	shared.PrintClusterState()
+	fmt.Printf("\nUpgrading cluster to: %s\n", version)
+
 	cluster := factory.ClusterConfig(GinkgoT())
 
 	if cluster.NumServers == 0 && cluster.NumAgents == 0 {