feat: Add cloud provider info to the gateways (#1778)

kyma-project · Feb 3, 2025 · 9b9dcf8 · 9b9dcf8
1 parent 2550687
commit 9b9dcf8
Show file tree

Hide file tree

Showing 47 changed files with 812 additions and 172 deletions.
diff --git a/.k3d-kyma.yaml b/.k3d-kyma.yaml
@@ -12,3 +12,15 @@ registries:
     name: kyma
     hostPort: '5001'
 
+options:
+  k3s:
+    nodeLabels:
+      - label: topology.kubernetes.io/region=kyma-local
+        nodeFilters:
+          - server:*
+      - label: topology.kubernetes.io/zone=kyma-local
+        nodeFilters:
+          - server:*
+      - label: node.kubernetes.io/instance-type=local
+        nodeFilters:
+          - server:*
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -11,6 +11,7 @@ rules:
   - ""
   resources:
   - secrets
+  - configmaps
   verbs:
   - get
   - list

diff --git a/docs/contributor/assets/multi_pipeline_gateway_with_connectors.yaml b/docs/contributor/assets/multi_pipeline_gateway_with_connectors.yaml
@@ -31,7 +31,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-1
@@ -58,7 +58,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-2
@@ -85,7 +85,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-3
@@ -165,7 +165,7 @@ data:
                       name: k8s.pod.uid
                 - sources:
                     - from: connection
-        resource/insert-cluster-name:
+        resource/insert-cluster-attributes:
             attributes:
                 - action: insert
                   key: k8s.cluster.name

diff --git a/docs/contributor/assets/single_pipeline_gateway_with_connectors.yaml b/docs/contributor/assets/single_pipeline_gateway_with_connectors.yaml
@@ -31,7 +31,7 @@ data:
                     - filter/drop-if-input-source-runtime
                     - filter/drop-if-input-source-prometheus
                     - filter/drop-if-input-source-istio
-                    - resource/insert-cluster-name
+                    - resource/insert-cluster-attributes
                     - batch
                 exporters:
                     - otlp/load-test-1
@@ -97,7 +97,7 @@ data:
                       name: k8s.pod.uid
                 - sources:
                     - from: connection
-        resource/insert-cluster-name:
+        resource/insert-cluster-attributes:
             attributes:
                 - action: insert
                   key: k8s.cluster.name

diff --git a/internal/otelcollector/config/gatewayprocs/k8s_attribute_proc.go b/internal/otelcollector/config/gatewayprocs/k8s_attribute_proc.go
@@ -51,5 +51,25 @@ func extractLabels() []config.ExtractLabel {
 			Key:     "app",
 			TagName: "kyma.app_name",
 		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/region",
+			TagName: "cloud.region",
+		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/zone",
+			TagName: "cloud.availability_zone",
+		},
+		{
+			From:    "node",
+			Key:     "node.kubernetes.io/instance-type",
+			TagName: "host.type",
+		},
+		{
+			From:    "node",
+			Key:     "kubernetes.io/arch",
+			TagName: "host.arch",
+		},
 	}
 }
diff --git a/internal/otelcollector/config/gatewayprocs/k8s_attribute_proc_test.go b/internal/otelcollector/config/gatewayprocs/k8s_attribute_proc_test.go
@@ -43,6 +43,26 @@ func TestK8sAttributesProcessorConfig(t *testing.T) {
 			Key:     "app",
 			TagName: "kyma.app_name",
 		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/region",
+			TagName: "cloud.region",
+		},
+		{
+			From:    "node",
+			Key:     "topology.kubernetes.io/zone",
+			TagName: "cloud.availability_zone",
+		},
+		{
+			From:    "node",
+			Key:     "node.kubernetes.io/instance-type",
+			TagName: "host.type",
+		},
+		{
+			From:    "node",
+			Key:     "kubernetes.io/arch",
+			TagName: "host.arch",
+		},
 	}
 
 	config := K8sAttributesProcessorConfig()

diff --git a/internal/otelcollector/config/gatewayprocs/resource_procs.go b/internal/otelcollector/config/gatewayprocs/resource_procs.go
@@ -4,13 +4,30 @@ import (
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config"
 )
 
-func InsertClusterNameProcessorConfig() *config.ResourceProcessor {
+func InsertClusterAttributesProcessorConfig(clusterName, cloudProvider string) *config.ResourceProcessor {
+	if cloudProvider != "" {
+		return &config.ResourceProcessor{
+			Attributes: []config.AttributeAction{
+				{
+					Action: "insert",
+					Key:    "k8s.cluster.name",
+					Value:  clusterName,
+				},
+				{
+					Action: "insert",
+					Key:    "cloud.provider",
+					Value:  cloudProvider,
+				},
+			},
+		}
+	}
+
 	return &config.ResourceProcessor{
 		Attributes: []config.AttributeAction{
 			{
 				Action: "insert",
 				Key:    "k8s.cluster.name",
-				Value:  "${KUBERNETES_SERVICE_HOST}",
+				Value:  clusterName,
 			},
 		},
 	}

diff --git a/internal/otelcollector/config/gatewayprocs/resource_procs_test.go b/internal/otelcollector/config/gatewayprocs/resource_procs_test.go
@@ -15,11 +15,16 @@ func TestInsertClusterNameProcessorConfig(t *testing.T) {
 		{
 			Action: "insert",
 			Key:    "k8s.cluster.name",
-			Value:  "${KUBERNETES_SERVICE_HOST}",
+			Value:  "test-cluster",
+		},
+		{
+			Action: "insert",
+			Key:    "cloud.provider",
+			Value:  "test-cloud-provider",
 		},
 	}
 
-	config := InsertClusterNameProcessorConfig()
+	config := InsertClusterAttributesProcessorConfig("test-cluster", "test-cloud-provider")
 
 	require.ElementsMatch(expectedAttributeActions, config.Attributes, "Attributes should match")
 }

diff --git a/internal/otelcollector/config/log/gateway/config.go b/internal/otelcollector/config/log/gateway/config.go
@@ -19,9 +19,9 @@ type Receivers struct {
 type Processors struct {
 	config.BaseProcessors `yaml:",inline"`
 
-	K8sAttributes      *config.K8sAttributesProcessor `yaml:"k8sattributes,omitempty"`
-	InsertClusterName  *config.ResourceProcessor      `yaml:"resource/insert-cluster-name,omitempty"`
-	DropKymaAttributes *config.ResourceProcessor      `yaml:"resource/drop-kyma-attributes,omitempty"`
+	K8sAttributes           *config.K8sAttributesProcessor `yaml:"k8sattributes,omitempty"`
+	InsertClusterAttributes *config.ResourceProcessor      `yaml:"resource/insert-cluster-attributes,omitempty"`
+	DropKymaAttributes      *config.ResourceProcessor      `yaml:"resource/drop-kyma-attributes,omitempty"`
 }
 
 type Exporters map[string]Exporter

diff --git a/internal/otelcollector/config/log/gateway/config_builder.go b/internal/otelcollector/config/log/gateway/config_builder.go
@@ -22,14 +22,19 @@ type Builder struct {
 	Reader client.Reader
 }
 
-func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.LogPipeline) (*Config, otlpexporter.EnvVars, error) {
+type BuildOptions struct {
+	ClusterName   string
+	CloudProvider string
+}
+
+func (b *Builder) Build(ctx context.Context, pipelines []telemetryv1alpha1.LogPipeline, opts BuildOptions) (*Config, otlpexporter.EnvVars, error) {
 	cfg := &Config{
 		Base: config.Base{
 			Service:    config.DefaultService(make(config.Pipelines)),
 			Extensions: config.DefaultExtensions(),
 		},
 		Receivers:  makeReceiversConfig(),
-		Processors: makeProcessorsConfig(),
+		Processors: makeProcessorsConfig(opts),
 		Exporters:  make(Exporters),
 	}
 
@@ -99,7 +104,7 @@ func makePipelineConfig(exporterIDs ...string) config.Pipeline {
 		Processors: []string{
 			"memory_limiter",
 			"k8sattributes",
-			"resource/insert-cluster-name",
+			"resource/insert-cluster-attributes",
 			"batch",
 		},
 		Exporters: exporterIDs,
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,7 @@ rules: @@
       - ""
       resources:
       - secrets
+      - configmaps
       verbs:
       - get
       - list
@@ Expand Down @@