Skip to content

Commit

Permalink
chore: Rename operation tests to upgrade tests (#1807)
Browse files Browse the repository at this point in the history
  • Loading branch information
@a-thaler
a-thaler authored Feb 5, 2025
1 parent 1456e32 commit f6384aa
Show file tree
Hide file tree
Showing 12 changed files with 197 additions and 55 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/pr-lifecycle.yml → .github/workflows/pr-upgrade.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: PR Lifecycle
name: PR Upgrade

on:
merge_group:
Expand Down Expand Up @@ -48,7 +48,7 @@ jobs:
- name: Run test on latest tag
shell: bash
run: |
bin/ginkgo run --junit-report=junit-report-latest-version.xml --tags e2e --flake-attempts=5 --label-filter="operational" -v test/e2e
bin/ginkgo run --junit-report=junit-report-latest-version.xml --tags e2e --flake-attempts=5 --label-filter="upgrade || operational" -v test/e2e
- name: Switch back to current revision
uses: actions/checkout@v4
Expand Down Expand Up @@ -82,7 +82,7 @@ jobs:
shell: bash
run: |
make install-tools # delete after the tools via go modules are released
bin/ginkgo run --junit-report=junit-report-current-version.xml --tags e2e --flake-attempts=5 --label-filter="operational" -v test/e2e
bin/ginkgo run --junit-report=junit-report-current-version.xml --tags e2e --flake-attempts=5 --label-filter="upgrade" -v test/e2e
- name: Finalize test
uses: "./.github/template/finalize-test"
Expand Down
150 changes: 146 additions & 4 deletions docs/contributor/assets/test-pyramid.drawio.svg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 2 additions & 2 deletions docs/contributor/testing.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ The roles and responsibilities during the STLC:
| ------------------------------------------------------------------------------------------------------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| Unit (located along with the individual source files) | Unit | It tests the individual units of application logic in isolation, focusing on the implementation correctness. |
| [E2E Acceptance](../../test/e2e) | Acceptance (high-level) | It tests the usability scenarios of Telemetry Manager in a cluster. This test suite assesses the functional correctness of Telemetry Controller. |
| E2E Operational | System | It validates the operational aspects of the module (successful module upgrades, deletions, etc.). |
| E2E Upgrade | System | It validates the operational aspects of the module (successful module upgrades, deletions, etc.). |
| [E2E integration](https://github.com/kyma-project/telemetry-manager/issues/261#issuecomment-1647336680) | Integration (high-level) | tests the Telemetry Module integration with 3rd party components and modules (with a focus on contract fulfilment). |

![E2E Test Suites](assets/e2e-test-suites.png)
Expand All @@ -43,7 +43,7 @@ Testing a new functionality encompasses two activities: capturing the acceptance

#### Testing of Existing Functionality

The regression test suite validates the correctness of the product's previously released functionalities using the "retest all" technique. It encompasses the end-to-end and operational test suites.
The regression test suite validates the correctness of the product's previously released functionalities using the "retest all" technique. It encompasses the end-to-end and upgrade test suites.

### Non-Functional Tests

Expand Down
12 changes: 6 additions & 6 deletions test/e2e/logs_basic_v1alpha1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
var objs []client.Object
objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())

backend := backend.New(mockNs, backend.SignalTypeLogs, backend.WithPersistentHostSecret(suite.IsOperational()))
backend := backend.New(mockNs, backend.SignalTypeLogs, backend.WithPersistentHostSecret(suite.IsUpgrade()))
logProducer := loggen.New(mockNs)
objs = append(objs, backend.K8sObjects()...)
objs = append(objs, logProducer.K8sObject())
Expand All @@ -45,7 +45,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
),
testutils.HTTPPort(backend.Port()),
)
if suite.IsOperational() {
if suite.IsUpgrade() {
pipelineBuilder.WithLabels(kitk8s.PersistentLabel)
}
logPipeline := pipelineBuilder.Build()
Expand All @@ -63,7 +63,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
})

It("Should have a running pipeline", Label(suite.LabelOperational), func() {
It("Should have a running pipeline", Label(suite.LabelUpgrade), func() {
assert.LogPipelineHealthy(ctx, k8sClient, pipelineName)
})

Expand All @@ -75,15 +75,15 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
assert.LogPipelineUnsupportedMode(ctx, k8sClient, pipelineName, false)
})

It("Should have a log backend running", Label(suite.LabelOperational), func() {
It("Should have a log backend running", Label(suite.LabelUpgrade), func() {
assert.DeploymentReady(ctx, k8sClient, types.NamespacedName{Namespace: mockNs, Name: backend.DefaultName})
})

It("Should have a log producer running", Label(suite.LabelOperational), func() {
It("Should have a log producer running", Label(suite.LabelUpgrade), func() {
assert.DeploymentReady(ctx, k8sClient, types.NamespacedName{Namespace: mockNs, Name: loggen.DefaultName})
})

It("Should have produced logs in the backend", Label(suite.LabelOperational), func() {
It("Should have produced logs in the backend", Label(suite.LabelUpgrade), func() {
assert.LogsDelivered(proxyClient, loggen.DefaultName, backendExportURL)
})
})
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/logs_otlp_output_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs, suite.LabelExperimental), Or
var objs []client.Object
objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())

backend := backend.New(mockNs, backend.SignalTypeLogsOtel, backend.WithPersistentHostSecret(suite.IsOperational()))
backend := backend.New(mockNs, backend.SignalTypeLogsOtel, backend.WithPersistentHostSecret(suite.IsUpgrade()))
logProducer := loggen.New(mockNs)
objs = append(objs, backend.K8sObjects()...)
objs = append(objs, logProducer.K8sObject())
Expand All @@ -52,7 +52,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs, suite.LabelExperimental), Or
hostSecretRef.Key,
),
)
if suite.IsOperational() {
if suite.IsUpgrade() {
pipelineBuilder.WithLabels(kitk8s.PersistentLabel)
}
logPipeline := pipelineBuilder.Build()
Expand Down
26 changes: 13 additions & 13 deletions test/e2e/metrics_basic_v1alpha1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),

objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())

backend := backend.New(mockNs, backend.SignalTypeMetrics, backend.WithPersistentHostSecret(suite.IsOperational()))
backend := backend.New(mockNs, backend.SignalTypeMetrics, backend.WithPersistentHostSecret(suite.IsUpgrade()))
objs = append(objs, backend.K8sObjects()...)
backendExportURL = backend.ExportURL(proxyClient)

Expand All @@ -49,7 +49,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),
hostSecretRef.Namespace,
hostSecretRef.Key,
))
if suite.IsOperational() {
if suite.IsUpgrade() {
metricPipelineBuilder.WithLabels(kitk8s.PersistentLabel)
}
metricPipeline := metricPipelineBuilder.Build()
Expand All @@ -70,11 +70,11 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),
Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
})

It("Should have a running metric gateway deployment", Label(suite.LabelOperational), func() {
It("Should have a running metric gateway deployment", Label(suite.LabelUpgrade), func() {
assert.DeploymentReady(ctx, k8sClient, kitkyma.MetricGatewayName)
})

It("Should reject scaling below minimum", Label(suite.LabelOperational), func() {
It("Should reject scaling below minimum", Label(suite.LabelUpgrade), func() {
var telemetry operatorv1alpha1.Telemetry
err := k8sClient.Get(ctx, kitkyma.TelemetryName, &telemetry)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -93,7 +93,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),
Expect(err).To(HaveOccurred())
})

It("Should scale up metric gateway replicas", Label(suite.LabelOperational), func() {
It("Should scale up metric gateway replicas", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var telemetry operatorv1alpha1.Telemetry
err := k8sClient.Get(ctx, kitkyma.TelemetryName, &telemetry)
Expand All @@ -115,7 +115,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(4)))
})

It("Should have 4 metric gateway replicas after scaling up", Label(suite.LabelOperational), func() {
It("Should have 4 metric gateway replicas after scaling up", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var deployment appsv1.Deployment
err := k8sClient.Get(ctx, kitkyma.MetricGatewayName, &deployment)
Expand All @@ -124,7 +124,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(4)))
})

It("Should scale down metric gateway replicas", Label(suite.LabelOperational), func() {
It("Should scale down metric gateway replicas", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var telemetry operatorv1alpha1.Telemetry
err := k8sClient.Get(ctx, kitkyma.TelemetryName, &telemetry)
Expand All @@ -146,7 +146,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(2)))
})

It("Should have 2 metric gateway replicas after scaling down", Label(suite.LabelOperational), func() {
It("Should have 2 metric gateway replicas after scaling down", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var deployment appsv1.Deployment
err := k8sClient.Get(ctx, kitkyma.MetricGatewayName, &deployment)
Expand All @@ -155,25 +155,25 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetB),
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(2)))
})

It("Should have a metrics backend running", Label(suite.LabelOperational), func() {
It("Should have a metrics backend running", Label(suite.LabelUpgrade), func() {
assert.DeploymentReady(ctx, k8sClient, types.NamespacedName{Name: backend.DefaultName, Namespace: mockNs})
assert.ServiceReady(ctx, k8sClient, types.NamespacedName{Name: backend.DefaultName, Namespace: mockNs})
})

It("Should have a running pipeline", Label(suite.LabelOperational), func() {
It("Should have a running pipeline", Label(suite.LabelUpgrade), func() {
assert.MetricPipelineHealthy(ctx, k8sClient, pipelineName)
})

It("Should deliver telemetrygen metrics", Label(suite.LabelOperational), func() {
It("Should deliver telemetrygen metrics", Label(suite.LabelUpgrade), func() {
assert.MetricsFromNamespaceDelivered(proxyClient, backendExportURL, mockNs, telemetrygen.MetricNames)
})

It("Should be able to get metric gateway metrics endpoint", Label(suite.LabelOperational), func() {
It("Should be able to get metric gateway metrics endpoint", Label(suite.LabelUpgrade), func() {
gatewayMetricsURL := proxyClient.ProxyURLForService(kitkyma.MetricGatewayMetricsService.Namespace, kitkyma.MetricGatewayMetricsService.Name, "metrics", ports.Metrics)
assert.EmitsOTelCollectorMetrics(proxyClient, gatewayMetricsURL)
})

It("Should have a working network policy", Label(suite.LabelOperational), func() {
It("Should have a working network policy", Label(suite.LabelUpgrade), func() {
var networkPolicy networkingv1.NetworkPolicy
Expect(k8sClient.Get(ctx, kitkyma.MetricGatewayNetworkPolicy, &networkPolicy)).To(Succeed())

Expand Down
2 changes: 1 addition & 1 deletion test/e2e/metrics_basic_v1beta1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics, suite.LabelExperimental),
assert.MetricPipelineHealthy(ctx, k8sClient, pipelineName)
})

It("Should deliver telemetrygen metrics", Label(suite.LabelOperational), func() {
It("Should deliver telemetrygen metrics", Label(suite.LabelUpgrade), func() {
assert.MetricsFromNamespaceDelivered(proxyClient, backendExportURL, mockNs, telemetrygen.MetricNames)
})

Expand Down
4 changes: 2 additions & 2 deletions test/e2e/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ var _ = BeforeSuite(func() {
Expect(err).NotTo(HaveOccurred())
Expect(k8sClient).NotTo(BeNil())

telemetryK8sObject = kitk8s.NewTelemetry("default", "kyma-system").Persistent(suite.IsOperational()).K8sObject()
telemetryK8sObject = kitk8s.NewTelemetry("default", "kyma-system").Persistent(suite.IsUpgrade()).K8sObject()
denyAllNetworkPolicyK8sObject := kitk8s.NewNetworkPolicy("deny-all-ingress-and-egress", kitkyma.SystemNamespaceName).K8sObject()
k8sObjects = []client.Object{
telemetryK8sObject,
Expand All @@ -82,7 +82,7 @@ var _ = BeforeSuite(func() {

var _ = AfterSuite(func() {
Expect(kitk8s.DeleteObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
if !suite.IsOperational() {
if !suite.IsUpgrade() {
Eventually(func(g Gomega) {
var validatingWebhookConfiguration admissionregistrationv1.ValidatingWebhookConfiguration
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: kitkyma.ValidatingWebhookName}, &validatingWebhookConfiguration)).Should(Succeed())
Expand Down
28 changes: 14 additions & 14 deletions test/e2e/traces_basic_v1alpha1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {

objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())

backend := backend.New(mockNs, backend.SignalTypeTraces, backend.WithPersistentHostSecret(suite.IsOperational()))
backend := backend.New(mockNs, backend.SignalTypeTraces, backend.WithPersistentHostSecret(suite.IsUpgrade()))
objs = append(objs, backend.K8sObjects()...)
backendExportURL = backend.ExportURL(proxyClient)

Expand All @@ -49,7 +49,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
hostSecretRef.Namespace,
hostSecretRef.Key,
))
if suite.IsOperational() {
if suite.IsUpgrade() {
tracePipelineBuilder.WithLabels(kitk8s.PersistentLabel)
}
tracePipeline := tracePipelineBuilder.Build()
Expand All @@ -71,11 +71,11 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
})

It("Should have a running trace gateway deployment", Label(suite.LabelOperational), func() {
It("Should have a running trace gateway deployment", Label(suite.LabelUpgrade), func() {
assert.DeploymentReady(ctx, k8sClient, kitkyma.TraceGatewayName)
})

It("Should have 2 trace gateway replicas", Label(suite.LabelOperational), func() {
It("Should have 2 trace gateway replicas", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var deployment appsv1.Deployment
err := k8sClient.Get(ctx, kitkyma.TraceGatewayName, &deployment)
Expand All @@ -84,7 +84,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(2)))
})

It("Should reject scaling below minimum", Label(suite.LabelOperational), func() {
It("Should reject scaling below minimum", Label(suite.LabelUpgrade), func() {
var telemetry operatorv1alpha1.Telemetry
err := k8sClient.Get(ctx, kitkyma.TelemetryName, &telemetry)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -103,7 +103,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
Expect(err).To(HaveOccurred())
})

It("Should scale up trace gateway replicas", Label(suite.LabelOperational), func() {
It("Should scale up trace gateway replicas", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var telemetry operatorv1alpha1.Telemetry
err := k8sClient.Get(ctx, kitkyma.TelemetryName, &telemetry)
Expand All @@ -125,7 +125,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(4)))
})

It("Should have 4 trace gateway replicas after scaling up", Label(suite.LabelOperational), func() {
It("Should have 4 trace gateway replicas after scaling up", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var deployment appsv1.Deployment
err := k8sClient.Get(ctx, kitkyma.TraceGatewayName, &deployment)
Expand All @@ -134,7 +134,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(4)))
})

It("Should scale down trace gateway replicas", Label(suite.LabelOperational), func() {
It("Should scale down trace gateway replicas", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var telemetry operatorv1alpha1.Telemetry
err := k8sClient.Get(ctx, kitkyma.TelemetryName, &telemetry)
Expand All @@ -156,7 +156,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(2)))
})

It("Should have 2 trace gateway replicas after scaling down", Label(suite.LabelOperational), func() {
It("Should have 2 trace gateway replicas after scaling down", Label(suite.LabelUpgrade), func() {
Eventually(func(g Gomega) int32 {
var deployment appsv1.Deployment
err := k8sClient.Get(ctx, kitkyma.TraceGatewayName, &deployment)
Expand All @@ -165,24 +165,24 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Equal(int32(2)))
})

It("Should have a trace backend running", Label(suite.LabelOperational), func() {
It("Should have a trace backend running", Label(suite.LabelUpgrade), func() {
assert.DeploymentReady(ctx, k8sClient, types.NamespacedName{Name: backend.DefaultName, Namespace: mockNs})
})

It("Should have a running pipeline", Label(suite.LabelOperational), func() {
It("Should have a running pipeline", Label(suite.LabelUpgrade), func() {
assert.TracePipelineHealthy(ctx, k8sClient, pipelineName)
})

It("Should deliver telemetrygen traces", Label(suite.LabelOperational), func() {
It("Should deliver telemetrygen traces", Label(suite.LabelUpgrade), func() {
assert.TracesFromNamespaceDelivered(proxyClient, backendExportURL, mockNs)
})

It("Should be able to get trace gateway metrics endpoint", Label(suite.LabelOperational), func() {
It("Should be able to get trace gateway metrics endpoint", Label(suite.LabelUpgrade), func() {
gatewayMetricsURL := proxyClient.ProxyURLForService(kitkyma.TraceGatewayMetricsService.Namespace, kitkyma.TraceGatewayMetricsService.Name, "metrics", ports.Metrics)
assert.EmitsOTelCollectorMetrics(proxyClient, gatewayMetricsURL)
})

It("Should have a working network policy", Label(suite.LabelOperational), func() {
It("Should have a working network policy", Label(suite.LabelUpgrade), func() {
var networkPolicy networkingv1.NetworkPolicy
Expect(k8sClient.Get(ctx, kitkyma.TraceGatewayNetworkPolicy, &networkPolicy)).To(Succeed())

Expand Down
2 changes: 1 addition & 1 deletion test/integration/istio/metrics_self_monitor_outage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelSelfMonitoringMetricsOutage), Orde
}

Context("Before deploying a metricpipeline", func() {
It("Should set scaling for metrics", Label(suite.LabelOperational), func() {
It("Should set scaling for metrics", Label(suite.LabelUpgrade), func() {
// retry until the Telemetry CR is updated correctly
Eventually(func() error {
var telemetry operatorv1alpha1.Telemetry
Expand Down
Loading

0 comments on commit f6384aa

Please sign in to comment.