Merge branch 'main' into add-cloud-provider-attributes-docs

kyma-project · Feb 4, 2025 · d62fd06 · d62fd06
2 parents 8ec25be + f9d10ac
commit d62fd06
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 32 deletions.
diff --git a/internal/otelcollector/config/metric/gateway/processors.go b/internal/otelcollector/config/metric/gateway/processors.go
@@ -240,13 +240,14 @@ func inputSourceEquals(inputSourceType metric.InputSourceType) string {
 }
 
 func otlpInputSource() string {
-	// When instrumentation scope is not set to
-	// io.kyma-project.telemetry/runtime or io.kyma-project.telemetry/prometheus or io.kyma-project.telemetry/istio
+	// When instrumentation scope is not set to any of the following values
+	// io.kyma-project.telemetry/runtime, io.kyma-project.telemetry/prometheus, io.kyma-project.telemetry/istio, and io.kyma-project.telemetry/kyma
 	// we assume the metric is being pushed directly to metrics gateway.
-	return fmt.Sprintf("not(%s or %s or %s)",
+	return fmt.Sprintf("not(%s or %s or %s or %s)",
 		ottlexpr.ScopeNameEquals(metric.InstrumentationScopeRuntime),
 		ottlexpr.ScopeNameEquals(metric.InstrumentationScopePrometheus),
 		ottlexpr.ScopeNameEquals(metric.InstrumentationScopeIstio),
+		ottlexpr.ScopeNameEquals(metric.InstrumentationScopeKyma),
 	)
 }
 

diff --git a/internal/otelcollector/config/metric/gateway/processors_test.go b/internal/otelcollector/config/metric/gateway/processors_test.go
@@ -144,7 +144,8 @@ func TestProcessors(t *testing.T) {
 		require.Equal(t,
 			"not(instrumentation_scope.name == \"io.kyma-project.telemetry/runtime\" or "+
 				"instrumentation_scope.name == \"io.kyma-project.telemetry/prometheus\" or "+
-				"instrumentation_scope.name == \"io.kyma-project.telemetry/istio\")",
+				"instrumentation_scope.name == \"io.kyma-project.telemetry/istio\" or "+
+				"instrumentation_scope.name == \"io.kyma-project.telemetry/kyma\")",
 			collectorConfig.Processors.DropIfInputSourceOTLP.Metrics.Metric[0],
 		)
 	})
@@ -190,7 +191,8 @@ func TestProcessors(t *testing.T) {
 
 		expectedCondition = "not(instrumentation_scope.name == \"io.kyma-project.telemetry/runtime\" or " +
 			"instrumentation_scope.name == \"io.kyma-project.telemetry/prometheus\" or " +
-			"instrumentation_scope.name == \"io.kyma-project.telemetry/istio\") and " +
+			"instrumentation_scope.name == \"io.kyma-project.telemetry/istio\" or " +
+			"instrumentation_scope.name == \"io.kyma-project.telemetry/kyma\") and " +
 			"resource.attributes[\"k8s.namespace.name\"] != nil and " +
 			"not((resource.attributes[\"k8s.namespace.name\"] == \"ns-1\" or resource.attributes[\"k8s.namespace.name\"] == \"ns-2\"))"
 		require.Equal(t, expectedCondition, namespaceFilters["filter/test-filter-by-namespace-otlp-input"].Metrics.Metric[0])
@@ -237,7 +239,8 @@ func TestProcessors(t *testing.T) {
 
 		expectedCondition = "not(instrumentation_scope.name == \"io.kyma-project.telemetry/runtime\" or " +
 			"instrumentation_scope.name == \"io.kyma-project.telemetry/prometheus\" or " +
-			"instrumentation_scope.name == \"io.kyma-project.telemetry/istio\") and " +
+			"instrumentation_scope.name == \"io.kyma-project.telemetry/istio\" or " +
+			"instrumentation_scope.name == \"io.kyma-project.telemetry/kyma\") and " +
 			"(resource.attributes[\"k8s.namespace.name\"] == \"ns-1\" or resource.attributes[\"k8s.namespace.name\"] == \"ns-2\")"
 		require.Equal(t, expectedCondition, namespaceFilters["filter/test-filter-by-namespace-otlp-input"].Metrics.Metric[0])
 	})

diff --git a/internal/otelcollector/config/metric/gateway/service.go b/internal/otelcollector/config/metric/gateway/service.go
@@ -29,13 +29,12 @@ func makeOutputPipelineServiceConfig(pipeline *telemetryv1alpha1.MetricPipeline)
 
 	input := pipeline.Spec.Input
 
+	processors = append(processors, "transform/set-instrumentation-scope-kyma")
 	processors = append(processors, makeInputSourceFiltersIDs(input)...)
 	processors = append(processors, makeNamespaceFiltersIDs(input, pipeline)...)
 	processors = append(processors, makeRuntimeResourcesFiltersIDs(input)...)
 	processors = append(processors, makeDiagnosticMetricFiltersIDs(input)...)
 
-	processors = append(processors, "transform/set-instrumentation-scope-kyma")
-
 	processors = append(processors, "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch")
 
 	return config.Pipeline{

diff --git a/internal/otelcollector/config/metric/gateway/service_test.go b/internal/otelcollector/config/metric/gateway/service_test.go
@@ -42,11 +42,11 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-if-input-source-otlp",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -78,9 +78,9 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-istio",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -112,10 +112,10 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-diagnostic-metrics-if-input-source-prometheus",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -147,10 +147,10 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-diagnostic-metrics-if-input-source-prometheus",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -182,9 +182,9 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-prometheus",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -216,10 +216,10 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-diagnostic-metrics-if-input-source-istio",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -251,10 +251,10 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-diagnostic-metrics-if-input-source-istio",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -286,10 +286,10 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -321,10 +321,46 @@ func TestService(t *testing.T) {
 
 			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
 			require.Equal(t, []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-runtime",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
+				"resource/insert-cluster-attributes",
+				"resource/delete-skip-enrichment-attribute",
+				"batch",
+			}, collectorConfig.Service.Pipelines["metrics/test-output"].Processors)
+			require.Equal(t, []string{"otlp/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Exporters)
+		})
+
+		t.Run("with otlp input and namespace filter", func(t *testing.T) {
+			collectorConfig, _, err := sut.Build(
+				ctx,
+				[]telemetryv1alpha1.MetricPipeline{
+					testutils.NewMetricPipelineBuilder().WithName("test").WithOTLPInput(true, testutils.IncludeNamespaces("test")).Build(),
+				},
+				BuildOptions{},
+			)
+			require.NoError(t, err)
+
+			require.Contains(t, collectorConfig.Service.Pipelines, "metrics/test-input")
+			require.Contains(t, collectorConfig.Service.Pipelines, "metrics/test-attributes-enrichment")
+			require.Contains(t, collectorConfig.Service.Pipelines, "metrics/test-output")
+
+			require.Equal(t, []string{"otlp", "singleton_receiver_creator/kymastats"}, collectorConfig.Service.Pipelines["metrics/test-input"].Receivers)
+			require.Equal(t, []string{"memory_limiter"}, collectorConfig.Service.Pipelines["metrics/test-input"].Processors)
+			require.Equal(t, []string{"routing/test"}, collectorConfig.Service.Pipelines["metrics/test-input"].Exporters)
+
+			require.Equal(t, []string{"routing/test"}, collectorConfig.Service.Pipelines["metrics/test-attributes-enrichment"].Receivers)
+			require.Equal(t, []string{"k8sattributes", "transform/resolve-service-name", "resource/drop-kyma-attributes"}, collectorConfig.Service.Pipelines["metrics/test-attributes-enrichment"].Processors)
+			require.Equal(t, []string{"forward/test"}, collectorConfig.Service.Pipelines["metrics/test-attributes-enrichment"].Exporters)
+
+			require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers)
+			require.Equal(t, []string{
 				"transform/set-instrumentation-scope-kyma",
+				"filter/drop-if-input-source-runtime",
+				"filter/drop-if-input-source-prometheus",
+				"filter/drop-if-input-source-istio",
+				"filter/test-filter-by-namespace-otlp-input",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -369,10 +405,10 @@ func TestService(t *testing.T) {
 
 		require.Equal(t, []string{"routing/test-1", "forward/test-1"}, collectorConfig.Service.Pipelines["metrics/test-1-output"].Receivers)
 		require.Equal(t, []string{
+			"transform/set-instrumentation-scope-kyma",
 			"filter/drop-if-input-source-prometheus",
 			"filter/drop-if-input-source-istio",
 			"filter/test-1-filter-by-namespace-runtime-input",
-			"transform/set-instrumentation-scope-kyma",
 			"resource/insert-cluster-attributes",
 			"resource/delete-skip-enrichment-attribute",
 			"batch",
@@ -394,11 +430,11 @@ func TestService(t *testing.T) {
 
 		require.Equal(t, []string{"routing/test-2", "forward/test-2"}, collectorConfig.Service.Pipelines["metrics/test-2-output"].Receivers)
 		require.Equal(t, []string{
+			"transform/set-instrumentation-scope-kyma",
 			"filter/drop-if-input-source-runtime",
 			"filter/drop-if-input-source-istio",
 			"filter/test-2-filter-by-namespace-prometheus-input",
 			"filter/drop-diagnostic-metrics-if-input-source-prometheus",
-			"transform/set-instrumentation-scope-kyma",
 			"resource/insert-cluster-attributes",
 			"resource/delete-skip-enrichment-attribute",
 			"batch",
@@ -420,10 +456,10 @@ func TestService(t *testing.T) {
 
 		require.Equal(t, []string{"routing/test-3", "forward/test-3"}, collectorConfig.Service.Pipelines["metrics/test-3-output"].Receivers)
 		require.Equal(t, []string{
+			"transform/set-instrumentation-scope-kyma",
 			"filter/drop-if-input-source-runtime",
 			"filter/drop-if-input-source-prometheus",
 			"filter/drop-diagnostic-metrics-if-input-source-istio",
-			"transform/set-instrumentation-scope-kyma",
 			"resource/insert-cluster-attributes",
 			"resource/delete-skip-enrichment-attribute",
 			"batch",
@@ -453,9 +489,9 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInput(true).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -469,10 +505,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputPodMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-pod-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -485,10 +521,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputContainerMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-container-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -501,10 +537,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputNodeMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-node-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -517,10 +553,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputVolumeMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-volume-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -533,10 +569,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputDeploymentMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-deployment-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -549,10 +585,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputDaemonSetMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-daemonset-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -565,10 +601,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputStatefulSetMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-statefulset-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -581,10 +617,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputJobMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-job-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",
@@ -604,6 +640,7 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				WithRuntimeInputJobMetrics(false).
 				Build(),
 			expectedProcessors: []string{
+				"transform/set-instrumentation-scope-kyma",
 				"filter/drop-if-input-source-prometheus",
 				"filter/drop-if-input-source-istio",
 				"filter/drop-runtime-pod-metrics",
@@ -614,7 +651,6 @@ func TestService_RuntimeResources_Enabled(t *testing.T) {
 				"filter/drop-runtime-daemonset-metrics",
 				"filter/drop-runtime-statefulset-metrics",
 				"filter/drop-runtime-job-metrics",
-				"transform/set-instrumentation-scope-kyma",
 				"resource/insert-cluster-attributes",
 				"resource/delete-skip-enrichment-attribute",
 				"batch",

diff --git a/internal/otelcollector/config/metric/gateway/testdata/config.yaml b/internal/otelcollector/config/metric/gateway/testdata/config.yaml
@@ -27,10 +27,10 @@ service:
                 - routing/test
                 - forward/test
             processors:
+                - transform/set-instrumentation-scope-kyma
                 - filter/drop-if-input-source-runtime
                 - filter/drop-if-input-source-prometheus
                 - filter/drop-if-input-source-istio
-                - transform/set-instrumentation-scope-kyma
                 - resource/insert-cluster-attributes
                 - resource/delete-skip-enrichment-attribute
                 - batch

diff --git a/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml b/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml
@@ -27,11 +27,11 @@ service:
                 - routing/test
                 - forward/test
             processors:
+                - transform/set-instrumentation-scope-kyma
                 - filter/drop-if-input-source-runtime
                 - filter/drop-if-input-source-prometheus
                 - filter/drop-if-input-source-istio
                 - filter/drop-if-input-source-otlp
-                - transform/set-instrumentation-scope-kyma
                 - resource/insert-cluster-attributes
                 - resource/delete-skip-enrichment-attribute
                 - batch
@@ -153,7 +153,7 @@ processors:
     filter/drop-if-input-source-otlp:
         metrics:
             metric:
-                - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or instrumentation_scope.name == "io.kyma-project.telemetry/prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio")
+                - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or instrumentation_scope.name == "io.kyma-project.telemetry/prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio" or instrumentation_scope.name == "io.kyma-project.telemetry/kyma")
     transform/resolve-service-name:
         error_mode: ignore
         metric_statements: