From f9d10ac6b12da347911ebfceeaee89eff0e17955 Mon Sep 17 00:00:00 2001 From: Hisar Balik Date: Tue, 4 Feb 2025 14:34:17 +0100 Subject: [PATCH] fix: Metric kyma.resource.status.state gets filtered-out by namespace filter (#1798) --- .../config/metric/gateway/processors.go | 7 +- .../config/metric/gateway/processors_test.go | 9 ++- .../config/metric/gateway/service.go | 3 +- .../config/metric/gateway/service_test.go | 78 ++++++++++++++----- .../metric/gateway/testdata/config.yaml | 2 +- .../testdata/config_otlp_disabled.yaml | 4 +- 6 files changed, 71 insertions(+), 32 deletions(-) diff --git a/internal/otelcollector/config/metric/gateway/processors.go b/internal/otelcollector/config/metric/gateway/processors.go index 42b73ce49..aabab89a0 100644 --- a/internal/otelcollector/config/metric/gateway/processors.go +++ b/internal/otelcollector/config/metric/gateway/processors.go @@ -240,13 +240,14 @@ func inputSourceEquals(inputSourceType metric.InputSourceType) string { } func otlpInputSource() string { - // When instrumentation scope is not set to - // io.kyma-project.telemetry/runtime or io.kyma-project.telemetry/prometheus or io.kyma-project.telemetry/istio + // When instrumentation scope is not set to any of the following values + // io.kyma-project.telemetry/runtime, io.kyma-project.telemetry/prometheus, io.kyma-project.telemetry/istio, and io.kyma-project.telemetry/kyma // we assume the metric is being pushed directly to metrics gateway. - return fmt.Sprintf("not(%s or %s or %s)", + return fmt.Sprintf("not(%s or %s or %s or %s)", ottlexpr.ScopeNameEquals(metric.InstrumentationScopeRuntime), ottlexpr.ScopeNameEquals(metric.InstrumentationScopePrometheus), ottlexpr.ScopeNameEquals(metric.InstrumentationScopeIstio), + ottlexpr.ScopeNameEquals(metric.InstrumentationScopeKyma), ) } diff --git a/internal/otelcollector/config/metric/gateway/processors_test.go b/internal/otelcollector/config/metric/gateway/processors_test.go index 8fdf8fa7b..0aa796508 100644 --- a/internal/otelcollector/config/metric/gateway/processors_test.go +++ b/internal/otelcollector/config/metric/gateway/processors_test.go @@ -144,7 +144,8 @@ func TestProcessors(t *testing.T) { require.Equal(t, "not(instrumentation_scope.name == \"io.kyma-project.telemetry/runtime\" or "+ "instrumentation_scope.name == \"io.kyma-project.telemetry/prometheus\" or "+ - "instrumentation_scope.name == \"io.kyma-project.telemetry/istio\")", + "instrumentation_scope.name == \"io.kyma-project.telemetry/istio\" or "+ + "instrumentation_scope.name == \"io.kyma-project.telemetry/kyma\")", collectorConfig.Processors.DropIfInputSourceOTLP.Metrics.Metric[0], ) }) @@ -190,7 +191,8 @@ func TestProcessors(t *testing.T) { expectedCondition = "not(instrumentation_scope.name == \"io.kyma-project.telemetry/runtime\" or " + "instrumentation_scope.name == \"io.kyma-project.telemetry/prometheus\" or " + - "instrumentation_scope.name == \"io.kyma-project.telemetry/istio\") and " + + "instrumentation_scope.name == \"io.kyma-project.telemetry/istio\" or " + + "instrumentation_scope.name == \"io.kyma-project.telemetry/kyma\") and " + "resource.attributes[\"k8s.namespace.name\"] != nil and " + "not((resource.attributes[\"k8s.namespace.name\"] == \"ns-1\" or resource.attributes[\"k8s.namespace.name\"] == \"ns-2\"))" require.Equal(t, expectedCondition, namespaceFilters["filter/test-filter-by-namespace-otlp-input"].Metrics.Metric[0]) @@ -237,7 +239,8 @@ func TestProcessors(t *testing.T) { expectedCondition = "not(instrumentation_scope.name == \"io.kyma-project.telemetry/runtime\" or " + "instrumentation_scope.name == \"io.kyma-project.telemetry/prometheus\" or " + - "instrumentation_scope.name == \"io.kyma-project.telemetry/istio\") and " + + "instrumentation_scope.name == \"io.kyma-project.telemetry/istio\" or " + + "instrumentation_scope.name == \"io.kyma-project.telemetry/kyma\") and " + "(resource.attributes[\"k8s.namespace.name\"] == \"ns-1\" or resource.attributes[\"k8s.namespace.name\"] == \"ns-2\")" require.Equal(t, expectedCondition, namespaceFilters["filter/test-filter-by-namespace-otlp-input"].Metrics.Metric[0]) }) diff --git a/internal/otelcollector/config/metric/gateway/service.go b/internal/otelcollector/config/metric/gateway/service.go index d0322b496..4f391fb88 100644 --- a/internal/otelcollector/config/metric/gateway/service.go +++ b/internal/otelcollector/config/metric/gateway/service.go @@ -29,13 +29,12 @@ func makeOutputPipelineServiceConfig(pipeline *telemetryv1alpha1.MetricPipeline) input := pipeline.Spec.Input + processors = append(processors, "transform/set-instrumentation-scope-kyma") processors = append(processors, makeInputSourceFiltersIDs(input)...) processors = append(processors, makeNamespaceFiltersIDs(input, pipeline)...) processors = append(processors, makeRuntimeResourcesFiltersIDs(input)...) processors = append(processors, makeDiagnosticMetricFiltersIDs(input)...) - processors = append(processors, "transform/set-instrumentation-scope-kyma") - processors = append(processors, "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch") return config.Pipeline{ diff --git a/internal/otelcollector/config/metric/gateway/service_test.go b/internal/otelcollector/config/metric/gateway/service_test.go index 3f35163c1..7861ff7e0 100644 --- a/internal/otelcollector/config/metric/gateway/service_test.go +++ b/internal/otelcollector/config/metric/gateway/service_test.go @@ -42,11 +42,11 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-if-input-source-otlp", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -78,9 +78,9 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-istio", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -112,10 +112,10 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-istio", "filter/drop-diagnostic-metrics-if-input-source-prometheus", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -147,10 +147,10 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-istio", "filter/drop-diagnostic-metrics-if-input-source-prometheus", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -182,9 +182,9 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-prometheus", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -216,10 +216,10 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-prometheus", "filter/drop-diagnostic-metrics-if-input-source-istio", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -251,10 +251,10 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-prometheus", "filter/drop-diagnostic-metrics-if-input-source-istio", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -286,10 +286,10 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -321,10 +321,46 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", + "resource/insert-cluster-attributes", + "resource/delete-skip-enrichment-attribute", + "batch", + }, collectorConfig.Service.Pipelines["metrics/test-output"].Processors) + require.Equal(t, []string{"otlp/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Exporters) + }) + + t.Run("with otlp input and namespace filter", func(t *testing.T) { + collectorConfig, _, err := sut.Build( + ctx, + []telemetryv1alpha1.MetricPipeline{ + testutils.NewMetricPipelineBuilder().WithName("test").WithOTLPInput(true, testutils.IncludeNamespaces("test")).Build(), + }, + BuildOptions{}, + ) + require.NoError(t, err) + + require.Contains(t, collectorConfig.Service.Pipelines, "metrics/test-input") + require.Contains(t, collectorConfig.Service.Pipelines, "metrics/test-attributes-enrichment") + require.Contains(t, collectorConfig.Service.Pipelines, "metrics/test-output") + + require.Equal(t, []string{"otlp", "singleton_receiver_creator/kymastats"}, collectorConfig.Service.Pipelines["metrics/test-input"].Receivers) + require.Equal(t, []string{"memory_limiter"}, collectorConfig.Service.Pipelines["metrics/test-input"].Processors) + require.Equal(t, []string{"routing/test"}, collectorConfig.Service.Pipelines["metrics/test-input"].Exporters) + + require.Equal(t, []string{"routing/test"}, collectorConfig.Service.Pipelines["metrics/test-attributes-enrichment"].Receivers) + require.Equal(t, []string{"k8sattributes", "transform/resolve-service-name", "resource/drop-kyma-attributes"}, collectorConfig.Service.Pipelines["metrics/test-attributes-enrichment"].Processors) + require.Equal(t, []string{"forward/test"}, collectorConfig.Service.Pipelines["metrics/test-attributes-enrichment"].Exporters) + + require.Equal(t, []string{"routing/test", "forward/test"}, collectorConfig.Service.Pipelines["metrics/test-output"].Receivers) + require.Equal(t, []string{ "transform/set-instrumentation-scope-kyma", + "filter/drop-if-input-source-runtime", + "filter/drop-if-input-source-prometheus", + "filter/drop-if-input-source-istio", + "filter/test-filter-by-namespace-otlp-input", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -369,10 +405,10 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test-1", "forward/test-1"}, collectorConfig.Service.Pipelines["metrics/test-1-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/test-1-filter-by-namespace-runtime-input", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -394,11 +430,11 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test-2", "forward/test-2"}, collectorConfig.Service.Pipelines["metrics/test-2-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-istio", "filter/test-2-filter-by-namespace-prometheus-input", "filter/drop-diagnostic-metrics-if-input-source-prometheus", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -420,10 +456,10 @@ func TestService(t *testing.T) { require.Equal(t, []string{"routing/test-3", "forward/test-3"}, collectorConfig.Service.Pipelines["metrics/test-3-output"].Receivers) require.Equal(t, []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-runtime", "filter/drop-if-input-source-prometheus", "filter/drop-diagnostic-metrics-if-input-source-istio", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -453,9 +489,9 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInput(true). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -469,10 +505,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputPodMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-pod-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -485,10 +521,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputContainerMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-container-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -501,10 +537,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputNodeMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-node-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -517,10 +553,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputVolumeMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-volume-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -533,10 +569,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputDeploymentMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-deployment-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -549,10 +585,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputDaemonSetMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-daemonset-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -565,10 +601,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputStatefulSetMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-statefulset-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -581,10 +617,10 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputJobMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-job-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", @@ -604,6 +640,7 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { WithRuntimeInputJobMetrics(false). Build(), expectedProcessors: []string{ + "transform/set-instrumentation-scope-kyma", "filter/drop-if-input-source-prometheus", "filter/drop-if-input-source-istio", "filter/drop-runtime-pod-metrics", @@ -614,7 +651,6 @@ func TestService_RuntimeResources_Enabled(t *testing.T) { "filter/drop-runtime-daemonset-metrics", "filter/drop-runtime-statefulset-metrics", "filter/drop-runtime-job-metrics", - "transform/set-instrumentation-scope-kyma", "resource/insert-cluster-attributes", "resource/delete-skip-enrichment-attribute", "batch", diff --git a/internal/otelcollector/config/metric/gateway/testdata/config.yaml b/internal/otelcollector/config/metric/gateway/testdata/config.yaml index 7dacfb6b5..02ed37ae6 100644 --- a/internal/otelcollector/config/metric/gateway/testdata/config.yaml +++ b/internal/otelcollector/config/metric/gateway/testdata/config.yaml @@ -27,10 +27,10 @@ service: - routing/test - forward/test processors: + - transform/set-instrumentation-scope-kyma - filter/drop-if-input-source-runtime - filter/drop-if-input-source-prometheus - filter/drop-if-input-source-istio - - transform/set-instrumentation-scope-kyma - resource/insert-cluster-attributes - resource/delete-skip-enrichment-attribute - batch diff --git a/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml b/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml index af1f71ffa..85a070c32 100644 --- a/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml +++ b/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml @@ -27,11 +27,11 @@ service: - routing/test - forward/test processors: + - transform/set-instrumentation-scope-kyma - filter/drop-if-input-source-runtime - filter/drop-if-input-source-prometheus - filter/drop-if-input-source-istio - filter/drop-if-input-source-otlp - - transform/set-instrumentation-scope-kyma - resource/insert-cluster-attributes - resource/delete-skip-enrichment-attribute - batch @@ -153,7 +153,7 @@ processors: filter/drop-if-input-source-otlp: metrics: metric: - - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or instrumentation_scope.name == "io.kyma-project.telemetry/prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio") + - not(instrumentation_scope.name == "io.kyma-project.telemetry/runtime" or instrumentation_scope.name == "io.kyma-project.telemetry/prometheus" or instrumentation_scope.name == "io.kyma-project.telemetry/istio" or instrumentation_scope.name == "io.kyma-project.telemetry/kyma") transform/resolve-service-name: error_mode: ignore metric_statements: