feat: Add cloud provider info to the gateways

config/rbac/role.yaml

@@ -11,6 +11,7 @@ rules:
   - ""
   resources:
   - secrets
+  - configmaps
   verbs:
   - get
   - list

docs/contributor/assets/multi_pipeline_gateway_with_connectors.yaml

@@ -31,7 +31,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-1
@@ -58,7 +58,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-2
@@ -85,7 +85,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-3
@@ -165,7 +165,7 @@ data:
                       name: k8s.pod.uid
                 - sources:
                     - from: connection
-        resource/insert-cluster-name:
+        resource/insert-cluster-attributes:
             attributes:
                 - action: insert
                   key: k8s.cluster.name

docs/contributor/assets/single_pipeline_gateway_with_connectors.yaml

@@ -31,7 +31,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-1
@@ -97,7 +97,7 @@ data:
                       name: k8s.pod.uid
                 - sources:
                     - from: connection
-        resource/insert-cluster-name:
+        resource/insert-cluster-attributes:
             attributes:
                 - action: insert
                   key: k8s.cluster.name

internal/otelcollector/config/gatewayprocs/k8s_attribute_proc.go

@@ -51,5 +51,25 @@ func extractLabels() []config.ExtractLabel {
 			Key:     "app",
 			TagName: "kyma.app_name",
 		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/region",
+			TagName: "cloud.region",
+		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/zone",
+			TagName: "cloud.availability_zone",
+		},
+		{
+			From:    "node",
+			Key:     "node.kubernetes.io/instance-type",
+			TagName: "host.type",
+		},
+		{
+			From:    "node",
+			Key:     "kubernetes.io/arch",
+			TagName: "host.arch",
+		},
 	}
 }

internal/otelcollector/config/gatewayprocs/k8s_attribute_proc_test.go

@@ -43,6 +43,26 @@ func TestK8sAttributesProcessorConfig(t *testing.T) {
 			Key:     "app",
 			TagName: "kyma.app_name",
 		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/region",
+			TagName: "cloud.region",
+		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/zone",
+			TagName: "cloud.availability_zone",
+		},
+		{
+			From:    "node",
+			Key:     "node.kubernetes.io/instance-type",
+			TagName: "host.type",
+		},
+		{
+			From:    "node",
+			Key:     "kubernetes.io/arch",
+			TagName: "host.arch",
+		},
 	}
 
 	config := K8sAttributesProcessorConfig()

internal/otelcollector/config/gatewayprocs/resource_procs.go

@@ -4,13 +4,30 @@ import (
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config"
 )
 
-func InsertClusterNameProcessorConfig() *config.ResourceProcessor {
+func InsertClusterAttributesProcessorConfig(clusterName, cloudProvider string) *config.ResourceProcessor {
+	if cloudProvider != "" {
+		return &config.ResourceProcessor{
+			Attributes: []config.AttributeAction{
+				{
+					Action: "insert",
+					Key:    "k8s.cluster.name",
+					Value:  clusterName,
+				},
+				{
+					Action: "insert",
+					Key:    "cloud.provider",
+					Value:  cloudProvider,
+				},
+			},
+		}
+	}
+
 	return &config.ResourceProcessor{
 		Attributes: []config.AttributeAction{
 			{
 				Action: "insert",
 				Key:    "k8s.cluster.name",
-				Value:  "${KUBERNETES_SERVICE_HOST}",
+				Value:  clusterName,
 			},
 		},
 	}

internal/otelcollector/config/gatewayprocs/resource_procs_test.go

@@ -15,11 +15,16 @@ func TestInsertClusterNameProcessorConfig(t *testing.T) {
 		{
 			Action: "insert",
 			Key:    "k8s.cluster.name",
-			Value:  "${KUBERNETES_SERVICE_HOST}",
+			Value:  "test-cluster",
+		},
+		{
+			Action: "insert",
+			Key:    "cloud.provider",
+			Value:  "test-cloud-provider",
 		},
 	}
 
-	config := InsertClusterNameProcessorConfig()
+	config := InsertClusterAttributesProcessorConfig("test-cluster", "test-cloud-provider")
 
 	require.ElementsMatch(expectedAttributeActions, config.Attributes, "Attributes should match")
 }

internal/otelcollector/config/log/gateway/config.go

@@ -19,9 +19,9 @@ type Receivers struct {
 type Processors struct {
 	config.BaseProcessors `yaml:",inline"`
 
-	K8sAttributes      *config.K8sAttributesProcessor `yaml:"k8sattributes,omitempty"`
-	InsertClusterName  *config.ResourceProcessor      `yaml:"resource/insert-cluster-name,omitempty"`
-	DropKymaAttributes *config.ResourceProcessor      `yaml:"resource/drop-kyma-attributes,omitempty"`
+	K8sAttributes           *config.K8sAttributesProcessor `yaml:"k8sattributes,omitempty"`
+	InsertClusterAttributes *config.ResourceProcessor      `yaml:"resource/insert-cluster-attributes,omitempty"`
+	DropKymaAttributes      *config.ResourceProcessor      `yaml:"resource/drop-kyma-attributes,omitempty"`
 }
 
 type Exporters map[string]Exporter

internal/otelcollector/config/log/gateway/config_builder.go

@@ -22,14 +22,19 @@ type Builder struct {
 	Reader client.Reader
 }
 
-func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.LogPipeline) (*Config, otlpexporter.EnvVars, error) {
+type BuildOptions struct {
+	ClusterName   string
+	CloudProvider string
+}
+
+func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.LogPipeline, opts BuildOptions) (*Config, otlpexporter.EnvVars, error) {
 	cfg := &Config{
 		Base: config.Base{
 			Service:    config.DefaultService(make(config.Pipelines)),
 			Extensions: config.DefaultExtensions(),
 		},
 		Receivers:  makeReceiversConfig(),
-		Processors: makeProcessorsConfig(),
+		Processors: makeProcessorsConfig(opts),
 		Exporters:  make(Exporters),
 	}
 
@@ -99,7 +104,7 @@ func makePipelineConfig(exporterIDs ...string) config.Pipeline {
 		Processors: []string{
 			"memory_limiter",
 			"k8sattributes",
-			"resource/insert-cluster-name",
+			"resource/insert-cluster-attributes",
 			"batch",
 		},
 		Exporters: exporterIDs,

internal/otelcollector/config/log/gateway/config_builder_test.go

@@ -26,7 +26,12 @@ func TestBuildConfig(t *testing.T) {
 	t.Run("otlp exporter endpoint", func(t *testing.T) {
 		collectorConfig, envVars, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput(testutils.OTLPEndpoint("http://localhost")).Build(),
-		})
+		},
+			BuildOptions{
+				ClusterName:   "test-cluster",
+				CloudProvider: "test-cloud-provider",
+			},
+		)
 		require.NoError(t, err)
 
 		const endpointEnvVar = "OTLP_ENDPOINT_TEST"
@@ -42,7 +47,10 @@ func TestBuildConfig(t *testing.T) {
 	})
 
 	t.Run("secure", func(t *testing.T) {
-		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput().Build()})
+		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput().Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 		require.NoError(t, err)
 		require.Contains(t, collectorConfig.Exporters, "otlp/test")
 
@@ -52,7 +60,10 @@ func TestBuildConfig(t *testing.T) {
 
 	t.Run("insecure", func(t *testing.T) {
 		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{
-			testutils.NewLogPipelineBuilder().WithName("test-insecure").WithOTLPOutput(testutils.OTLPEndpoint("http://localhost")).Build()})
+			testutils.NewLogPipelineBuilder().WithName("test-insecure").WithOTLPOutput(testutils.OTLPEndpoint("http://localhost")).Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 		require.NoError(t, err)
 		require.Contains(t, collectorConfig.Exporters, "otlp/test-insecure")
 
@@ -63,6 +74,9 @@ func TestBuildConfig(t *testing.T) {
 	t.Run("basic auth", func(t *testing.T) {
 		collectorConfig, envVars, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test-basic-auth").WithOTLPOutput(testutils.OTLPBasicAuth("user", "password")).Build(),
+		}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
 		})
 		require.NoError(t, err)
 		require.Contains(t, collectorConfig.Exporters, "otlp/test-basic-auth")
@@ -82,6 +96,9 @@ func TestBuildConfig(t *testing.T) {
 	t.Run("custom header", func(t *testing.T) {
 		collectorConfig, envVars, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test-custom-header").WithOTLPOutput(testutils.OTLPCustomHeader("Authorization", "TOKEN_VALUE", "Api-Token")).Build(),
+		}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
 		})
 		require.NoError(t, err)
 		require.Contains(t, collectorConfig.Exporters, "otlp/test-custom-header")
@@ -99,6 +116,9 @@ func TestBuildConfig(t *testing.T) {
 	t.Run("mtls", func(t *testing.T) {
 		collectorConfig, envVars, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test-mtls").WithOTLPOutput(testutils.OTLPClientTLSFromString("ca", "cert", "key")).Build(),
+		}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
 		})
 		require.NoError(t, err)
 		require.Contains(t, collectorConfig.Exporters, "otlp/test-mtls")
@@ -115,7 +135,10 @@ func TestBuildConfig(t *testing.T) {
 	})
 
 	t.Run("extensions", func(t *testing.T) {
-		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithOTLPOutput().Build()})
+		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithOTLPOutput().Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 		require.NoError(t, err)
 
 		require.NotEmpty(t, collectorConfig.Extensions.HealthCheck.Endpoint)
@@ -125,7 +148,10 @@ func TestBuildConfig(t *testing.T) {
 	})
 
 	t.Run("telemetry", func(t *testing.T) {
-		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithOTLPOutput().Build()})
+		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithOTLPOutput().Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 		require.NoError(t, err)
 
 		metricreaders := []config.MetricReader{
@@ -147,7 +173,10 @@ func TestBuildConfig(t *testing.T) {
 	})
 
 	t.Run("single pipeline queue size", func(t *testing.T) {
-		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput().Build()})
+		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput().Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 		require.NoError(t, err)
 		require.Equal(t, maxQueueSize, collectorConfig.Exporters["otlp/test"].OTLP.SendingQueue.QueueSize, "Pipeline should have the full queue size")
 	})
@@ -156,7 +185,10 @@ func TestBuildConfig(t *testing.T) {
 		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test-1").WithOTLPOutput().Build(),
 			testutils.NewLogPipelineBuilder().WithName("test-2").WithOTLPOutput().Build(),
-			testutils.NewLogPipelineBuilder().WithName("test-3").WithOTLPOutput().Build()})
+			testutils.NewLogPipelineBuilder().WithName("test-3").WithOTLPOutput().Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 
 		require.NoError(t, err)
 
@@ -167,15 +199,18 @@ func TestBuildConfig(t *testing.T) {
 	})
 
 	t.Run("single pipeline topology", func(t *testing.T) {
-		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput().Build()})
+		collectorConfig, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput().Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 		require.NoError(t, err)
 
 		require.Contains(t, collectorConfig.Service.Pipelines, "logs/test")
 		require.Contains(t, collectorConfig.Service.Pipelines["logs/test"].Receivers, "otlp")
 
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test"].Processors[0], "memory_limiter")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test"].Processors[1], "k8sattributes")
-		require.Equal(t, collectorConfig.Service.Pipelines["logs/test"].Processors[2], "resource/insert-cluster-name")
+		require.Equal(t, collectorConfig.Service.Pipelines["logs/test"].Processors[2], "resource/insert-cluster-attributes")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test"].Processors[3], "batch")
 
 		require.Contains(t, collectorConfig.Service.Pipelines["logs/test"].Exporters, "otlp/test")
@@ -184,7 +219,10 @@ func TestBuildConfig(t *testing.T) {
 	t.Run("multi pipeline topology", func(t *testing.T) {
 		collectorConfig, envVars, err := sut.Build(context.Background(), []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test-1").WithOTLPOutput().Build(),
-			testutils.NewLogPipelineBuilder().WithName("test-2").WithOTLPOutput().Build()})
+			testutils.NewLogPipelineBuilder().WithName("test-2").WithOTLPOutput().Build()}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
+		})
 		require.NoError(t, err)
 
 		require.Contains(t, collectorConfig.Exporters, "otlp/test-1")
@@ -195,15 +233,15 @@ func TestBuildConfig(t *testing.T) {
 		require.Contains(t, collectorConfig.Service.Pipelines["logs/test-1"].Receivers, "otlp")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-1"].Processors[0], "memory_limiter")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-1"].Processors[1], "k8sattributes")
-		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-1"].Processors[2], "resource/insert-cluster-name")
+		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-1"].Processors[2], "resource/insert-cluster-attributes")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-1"].Processors[3], "batch")
 
 		require.Contains(t, collectorConfig.Service.Pipelines, "logs/test-2")
 		require.Contains(t, collectorConfig.Service.Pipelines["logs/test-2"].Exporters, "otlp/test-2")
 		require.Contains(t, collectorConfig.Service.Pipelines["logs/test-2"].Receivers, "otlp")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-2"].Processors[0], "memory_limiter")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-2"].Processors[1], "k8sattributes")
-		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-2"].Processors[2], "resource/insert-cluster-name")
+		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-2"].Processors[2], "resource/insert-cluster-attributes")
 		require.Equal(t, collectorConfig.Service.Pipelines["logs/test-2"].Processors[3], "batch")
 
 		require.Contains(t, envVars, "OTLP_ENDPOINT_TEST_1")
@@ -213,6 +251,9 @@ func TestBuildConfig(t *testing.T) {
 	t.Run("marshaling", func(t *testing.T) {
 		config, _, err := sut.Build(context.Background(), []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test").WithOTLPOutput().Build(),
+		}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
 		})
 		require.NoError(t, err)
 
@@ -230,6 +271,9 @@ func TestBuildConfig(t *testing.T) {
 	t.Run("failed to make otlp exporter config", func(t *testing.T) {
 		_, _, err := sut.Build(ctx, []telemetryv1alpha1.LogPipeline{
 			testutils.NewLogPipelineBuilder().WithName("test-fail").WithOTLPOutput(testutils.OTLPBasicAuthFromSecret("nonexistent-secret", "default", "user", "password")).Build(),
+		}, BuildOptions{
+			ClusterName:   "test-cluster",
+			CloudProvider: "test-cloud-provider",
 		})
 		require.Error(t, err)
 		require.ErrorContains(t, err, "failed to make otlp exporter config")