diff --git a/terratest/examples/failover-playground-ref-gslb.yaml b/terratest/examples/failover-playground-ref-gslb.yaml new file mode 100644 index 0000000000..fa77797c58 --- /dev/null +++ b/terratest/examples/failover-playground-ref-gslb.yaml @@ -0,0 +1,13 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: test-gslb + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" diff --git a/terratest/examples/failover-playground-ref-ingress.yaml b/terratest/examples/failover-playground-ref-ingress.yaml new file mode 100644 index 0000000000..509542e81e --- /dev/null +++ b/terratest/examples/failover-playground-ref-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-gslb + labels: + app: test-gslb +spec: + ingressClassName: nginx + rules: + - host: playground-failover.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/examples/failover_ref_gslb.yaml b/terratest/examples/failover_ref_gslb.yaml new file mode 100644 index 0000000000..fa77797c58 --- /dev/null +++ b/terratest/examples/failover_ref_gslb.yaml @@ -0,0 +1,13 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: test-gslb + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" diff --git a/terratest/examples/failover_ref_ingress.yaml b/terratest/examples/failover_ref_ingress.yaml new file mode 100644 index 0000000000..99b276c2cf --- /dev/null +++ b/terratest/examples/failover_ref_ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-gslb + labels: + app: test-gslb +spec: + ingressClassName: nginx + rules: + - host: terratest-failover.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/examples/roundrobin-weight1-ref-gslb.yaml b/terratest/examples/roundrobin-weight1-ref-gslb.yaml new file mode 100644 index 0000000000..0ddb8d10a0 --- /dev/null +++ b/terratest/examples/roundrobin-weight1-ref-gslb.yaml @@ -0,0 +1,15 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: test-gslb + strategy: + type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too + dnsTtlSeconds: 5 + weight: + eu: 5 + us: 5 diff --git a/terratest/examples/roundrobin-weight1-ref-ingress.yaml b/terratest/examples/roundrobin-weight1-ref-ingress.yaml new file mode 100644 index 0000000000..98fd48ef92 --- /dev/null +++ b/terratest/examples/roundrobin-weight1-ref-ingress.yaml @@ -0,0 +1,39 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-gslb + labels: + app: test-gslb +spec: + ingressClassName: nginx + rules: + - host: terratest-notfound.cloud.example.com # This is the GSLB enabled host that clients would use + http: # This section mirrors the same structure as that of an Ingress resource and will be used verbatim when creating the corresponding Ingress resource that will match the GSLB host + paths: + - path: / + pathType: Prefix + backend: + service: + name: non-existing-app # Gslb should reflect NotFound status + port: + name: http + - host: terratest-unhealthy.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: unhealthy-app # Gslb should reflect Unhealthy status + port: + name: http + - host: terratest-roundrobin.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/examples/roundrobin_weight1.yaml b/terratest/examples/roundrobin-weight1.yaml similarity index 100% rename from terratest/examples/roundrobin_weight1.yaml rename to terratest/examples/roundrobin-weight1.yaml diff --git a/terratest/examples/roundrobin2-ref-gslb.yaml b/terratest/examples/roundrobin2-ref-gslb.yaml new file mode 100644 index 0000000000..84e0794c81 --- /dev/null +++ b/terratest/examples/roundrobin2-ref-gslb.yaml @@ -0,0 +1,12 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: roundrobin-test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: roundrobin-test-gslb + strategy: + type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too + dnsTtlSeconds: 5 diff --git a/terratest/examples/roundrobin2-ref-ingress.yaml b/terratest/examples/roundrobin2-ref-ingress.yaml new file mode 100644 index 0000000000..209ef8e28d --- /dev/null +++ b/terratest/examples/roundrobin2-ref-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: roundrobin-test-gslb + labels: + app: roundrobin-test-gslb +spec: + ingressClassName: nginx + rules: + - host: roundrobin-test.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/test/k8gb_abstract_full_roundrobin_test.go b/terratest/test/k8gb_abstract_full_roundrobin_test.go index 81d3cfd92d..72d61189ad 100644 --- a/terratest/test/k8gb_abstract_full_roundrobin_test.go +++ b/terratest/test/k8gb_abstract_full_roundrobin_test.go @@ -26,30 +26,35 @@ import ( "github.com/stretchr/testify/require" ) -func abstractTestFullRoundRobin(t *testing.T, n int) { +func abstractTestFullRoundRobin(t *testing.T, n int, host string, gslbPath string, ingressPath string) { if n < 2 || n > 8 { t.Logf("Use value of n that represents the number of clusters from interval [2,8]") t.FailNow() } t.Logf(fmt.Sprintf("Running TestFullRoundRobin for %d clusters", n)) tags := []string{"eu", "us", "cz", "af", "ru", "ap", "uk", "ca"} + testPrefix := "embedded ingress" + if ingressPath != "" { + testPrefix = "referenced ingress" + } var instances []*utils.Instance - const host = "roundrobin-test.cloud.example.com" - const gslbPath = "../examples/roundrobin2.yaml" - // start all the test apps on all the clusters for i := 0; i < n; i += 1 { - instance, er := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i). + workflow := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i). WithGslb(gslbPath, host). - WithTestApp(tags[i]). - Start() + WithTestApp(tags[i]) + if ingressPath != "" { + workflow = workflow.WithIngress(ingressPath) + } + + instance, er := workflow.Start() require.NoError(t, er) instances = append(instances, instance) defer instance.Kill() } var err error - t.Run(fmt.Sprintf("round-robin on %d concurrent clusters with podinfo running", n), func(t *testing.T) { + t.Run(fmt.Sprintf("%s round-robin on %d concurrent clusters with podinfo running", testPrefix, n), func(t *testing.T) { for _, ins := range instances { err = ins.WaitForAppIsRunning() require.NoError(t, err) @@ -61,13 +66,13 @@ func abstractTestFullRoundRobin(t *testing.T, n int) { for _, ins := range instances { workingTargets = append(workingTargets, ins.GetLocalTargets()...) } - t.Run(fmt.Sprintf("all %d clusters should be interconnected", n), func(t *testing.T) { + t.Run(fmt.Sprintf("%s all %d clusters should be interconnected", testPrefix, n), func(t *testing.T) { allShouldExpectTheseTargets(t, instances, workingTargets) }) // kill the apps on clusters one by one and expect less and less targets to be available for i, instance := range instances { - t.Run(fmt.Sprintf("kill podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) { + t.Run(fmt.Sprintf("%s kill podinfo on cluster %d (%s)", testPrefix, i+1, tags[i]), func(t *testing.T) { workingTargets = distinct(subtract(workingTargets, instance.GetLocalTargets())) t.Logf("New expected targets: %v", workingTargets) instance.StopTestApp() @@ -77,7 +82,7 @@ func abstractTestFullRoundRobin(t *testing.T, n int) { // start the test apps again on each cluster and check if the targets start appearing for i, instance := range instances { - t.Run(fmt.Sprintf("start podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo on cluster %d (%s)", testPrefix, i+1, tags[i]), func(t *testing.T) { instance.StartTestApp() workingTargets = distinct(append(workingTargets, instance.GetLocalTargets()...)) t.Logf("New expected targets: %v", workingTargets) diff --git a/terratest/test/k8gb_failover_playground_test.go b/terratest/test/k8gb_failover_playground_test.go index e5e3d19817..cbf46996de 100644 --- a/terratest/test/k8gb_failover_playground_test.go +++ b/terratest/test/k8gb_failover_playground_test.go @@ -22,6 +22,7 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ import ( + "fmt" "k8gbterratest/utils" "testing" @@ -33,20 +34,35 @@ import ( // see: https://github.com/k8gb-io/k8gb/blob/master/docs/local.md#failover func TestFailoverPlayground(t *testing.T) { const host = "playground-failover.cloud.example.com" - const gslbPath = "../examples/failover-playground.yaml" + abstractTestFailoverPlayground(t, host, "../examples/failover-playground.yaml", "") + abstractTestFailoverPlayground(t, host, "../examples/failover-playground-ref-gslb.yaml", "../examples/failover-playground-ref-ingress.yaml") +} + +func abstractTestFailoverPlayground(t *testing.T, host string, gslbPath string, ingressPath string) { + testPrefix := "embedded ingress" + if ingressPath != "" { + testPrefix = "referenced ingress" + } const euGeoTag = "eu" const usGeoTag = "us" - instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp(euGeoTag). - Start() + WithTestApp(euGeoTag) + if ingressPath != nil { + workflowEU = workflowEU.WithIngress(ingreessPath) + } + instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp(usGeoTag). - Start() + WithTestApp(usGeoTag) + if ingressPath != nil { + workflowUS = workflowUS.WithIngress(ingreessPath) + } + instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() @@ -63,7 +79,7 @@ func TestFailoverPlayground(t *testing.T) { assert.Equal(t, geoTag, httpResult.Message) } - t.Run("failover on two concurrent clusters with TestApp running", func(t *testing.T) { + t.Run(fmt.Sprintf("%s failover on two concurrent clusters with TestApp running", testPrefix), func(t *testing.T) { err = instanceEU.WaitForAppIsRunning() require.NoError(t, err) err = instanceUS.WaitForAppIsRunning() @@ -73,13 +89,13 @@ func TestFailoverPlayground(t *testing.T) { euLocalTargets := instanceEU.GetLocalTargets() usLocalTargets := instanceUS.GetLocalTargets() - t.Run("stop podinfo on eu cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s stop podinfo on eu cluster", testPrefix), func(t *testing.T) { instanceEU.StopTestApp() require.NoError(t, instanceEU.WaitForAppIsStopped()) actAndAssert(t.Name(), usGeoTag, usLocalTargets) }) - t.Run("start podinfo again on eu cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo again on eu cluster", testPrefix), func(t *testing.T) { instanceEU.StartTestApp() require.NoError(t, instanceEU.WaitForAppIsRunning()) actAndAssert(t.Name(), euGeoTag, euLocalTargets) diff --git a/terratest/test/k8gb_full_failover_test.go b/terratest/test/k8gb_full_failover_test.go index 0ad028f084..fd3f787f6c 100644 --- a/terratest/test/k8gb_full_failover_test.go +++ b/terratest/test/k8gb_full_failover_test.go @@ -29,23 +29,40 @@ import ( ) func TestFullFailover(t *testing.T) { - const host = "terratest-failover.cloud.example.com" - const gslbPath = "../examples/failover.yaml" + host := "terratest-failover.cloud.example.com" + abstractTestFullFailover(t, host, + "../examples/failover.yaml", "") + abstractTestFullFailover(t, host, + "../examples/failover-ref-gslb.yaml", "../examples/failover-ref-ingress.yaml") +} + +func abstractTestFullFailover(t *testing.T, host string, gslbPath string, ingressPath string) { + testPrefix := "embedded ingress" + if ingressPath != "" { + testPrefix = "referenced ingress" + } - instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp("eu"). - Start() + WithTestApp("eu") + if ingressPath != "" { + workflowEU = workflowEU.WithIngress(ingressPath) + } + instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp("us"). - Start() + WithTestApp("us") + if ingressPath != "" { + workflowUS = workflowUS.WithIngress(ingressPath) + } + instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() - t.Run("failover on two concurrent clusters with podinfo running", func(t *testing.T) { + t.Run(fmt.Sprintf("%s failover on two concurrent clusters with podinfo running" testPrefix), func(t *testing.T) { err = instanceEU.WaitForAppIsRunning() require.NoError(t, err) err = instanceUS.WaitForAppIsRunning() @@ -55,7 +72,7 @@ func TestFullFailover(t *testing.T) { euLocalTargets := instanceEU.GetLocalTargets() usLocalTargets := instanceUS.GetLocalTargets() - t.Run("kill podinfo on the second cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s kill podinfo on the second cluster", testPrefix), func(t *testing.T) { instanceUS.StopTestApp() err = instanceUS.WaitForExpected(euLocalTargets) require.NoError(t, err) @@ -63,7 +80,7 @@ func TestFullFailover(t *testing.T) { require.NoError(t, err) }) - t.Run("kill podinfo on the first cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s kill podinfo on the first cluster", testPrefix), func(t *testing.T) { instanceEU.StopTestApp() err = instanceEU.WaitForExpected([]string{}) require.NoError(t, err) @@ -71,7 +88,7 @@ func TestFullFailover(t *testing.T) { require.NoError(t, err) }) - t.Run("start podinfo on the second cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo on the second cluster", testPrefix), func(t *testing.T) { instanceUS.StartTestApp() err = instanceUS.WaitForExpected(usLocalTargets) require.NoError(t, err) @@ -79,7 +96,7 @@ func TestFullFailover(t *testing.T) { require.NoError(t, err) }) - t.Run("start podinfo on the first cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo on the first cluster", testPrefix), func(t *testing.T) { instanceEU.StartTestApp() err = instanceEU.WaitForExpected(euLocalTargets) require.NoError(t, err) diff --git a/terratest/test/k8gb_full_roundrobin_test.go b/terratest/test/k8gb_full_roundrobin_test.go index 317d2a634c..ee727d8097 100644 --- a/terratest/test/k8gb_full_roundrobin_test.go +++ b/terratest/test/k8gb_full_roundrobin_test.go @@ -26,5 +26,9 @@ import ( ) func TestFullRoundRobin(t *testing.T) { - abstractTestFullRoundRobin(t, settings.ClustersNumber) + host = "roundrobin-test.cloud.example.com" + abstractTestFullRoundRobin(t, settings.ClustersNumber, host, + "../examples/roundrobin2.yaml", "") + abstractTestFullRoundRobin(t, settings.ClustersNumber, host, + "../examples/roundrobin2-ref-gslb.yaml", "../examples/roundrobin2-ref-ingress.yaml") } diff --git a/terratest/test/k8gb_weight_test.go b/terratest/test/k8gb_weight_test.go index dbeb260bdc..150583fc45 100644 --- a/terratest/test/k8gb_weight_test.go +++ b/terratest/test/k8gb_weight_test.go @@ -28,21 +28,31 @@ import ( func TestWeightsExistsInLocalDNSEndpoint(t *testing.T) { t.Parallel() + abstractTestWeightsExistsInLocalDNSEndpoint(t, "../examples/roundrobin-weight1.yaml", "") + abstractTestWeightsExistsInLocalDNSEndpoint(t, "../examples/roundrobin-weight1-ref-gslb.yaml", "../examples/roundrobin-weight1-ref-ingress.yaml") +} + +func abstractTestWeightsExistsInLocalDNSEndpoint(t *testing.T, gslbPath string, ingressPath string) { const host = "terratest-roundrobin.cloud.example.com" const endpointDNSNameEU = "gslb-ns-eu-cloud.example.com" const endpointDNSNameUS = "gslb-ns-us-cloud.example.com" - const gslbPath = "../examples/roundrobin_weight1.yaml" - instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp("eu"). - Start() + WithTestApp("eu") + if ingressPath != "" { + workflowEU.WithIngress(ingressPath) + } + instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp("us"). - Start() + WithTestApp("us") + if ingressPath != "" { + workflowUS.WithIngress(ingressPath) + } + instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() diff --git a/terratest/utils/extensions.go b/terratest/utils/extensions.go index 629d7a5166..18e3ea8400 100644 --- a/terratest/utils/extensions.go +++ b/terratest/utils/extensions.go @@ -125,7 +125,15 @@ func (w *Workflow) WithIngress(path string) *Workflow { if path == "" { w.error = fmt.Errorf("empty ingress resource path") } - w.settings.ingressResourcePath = path + var err error + w.settings.ingressResourcePath, err = filepath.Abs(path) + if err != nil { + w.error = fmt.Errorf("reading %s; %s", path, err) + } + w.settings.ingressName, err = w.getManifestName(w.settings.ingressResourcePath) + if err != nil { + w.error = err + } return w } @@ -148,9 +156,6 @@ func (w *Workflow) WithGslb(path, host string) *Workflow { w.error = err } w.state.gslb.host = host - if err != nil { - w.error = err - } return w } @@ -211,12 +216,20 @@ func (w *Workflow) Start() (*Instance, error) { // gslb if w.settings.gslbResourcePath != "" { - w.t.Logf("Create ingress %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) - k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) - k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.state.gslb.name, 100, 1*time.Second) - ingress := k8s.GetIngress(w.t, w.k8sOptions, w.state.gslb.name) - require.Equal(w.t, ingress.Name, w.state.gslb.name) - w.settings.ingressName = w.state.gslb.name + if w.settings.ingressResourcePath == "" { + w.t.Logf("Create ingress %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) + k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.state.gslb.name, 100, 1*time.Second) + ingress := k8s.GetIngress(w.t, w.k8sOptions, w.state.gslb.name) + require.Equal(w.t, ingress.Name, w.state.gslb.name) + w.settings.ingressName = w.state.gslb.name + } else { + w.t.Logf("Create gslb %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) + w.t.Logf("Create ingress %s from %s", w.settings.ingressName, w.settings.ingressResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.ingressResourcePath) + k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.settings.ingressName, 100, 1*time.Second) + } } return &Instance{