fix: Optimize Prometheus scrape target discovery (#1663)

kyma-project · Dec 10, 2024 · 502f50f · 502f50f
1 parent fbcb5b1
commit 502f50f
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 7 deletions.
diff --git a/hack/load-tests/metric-agent-test-setup.yaml b/hack/load-tests/metric-agent-test-setup.yaml
@@ -50,7 +50,7 @@ spec:
             - podAffinityTerm:
                 labelSelector:
                   matchExpressions:
-                    - key: app
+                    - key: app.kubernetes.io/name
                       operator: In
                       values:
                         - metric-agent-load-generator

diff --git a/internal/otelcollector/config/metric/agent/config.go b/internal/otelcollector/config/metric/agent/config.go
@@ -187,11 +187,17 @@ type StaticDiscoveryConfig struct {
 }
 
 type KubernetesDiscoveryConfig struct {
-	Role Role `yaml:"role"`
+	Role      Role                   `yaml:"role"`
+	Selectors []K8SDiscoverySelector `yaml:"selectors,omitempty"`
 }
 
 type Role string
 
+type K8SDiscoverySelector struct {
+	Role  Role   `yaml:"role"`
+	Field string `yaml:"field"`
+}
+
 const (
 	RoleEndpoints Role = "endpoints"
 	RolePod       Role = "pod"

diff --git a/internal/otelcollector/config/metric/agent/prometheus_receiver.go b/internal/otelcollector/config/metric/agent/prometheus_receiver.go
@@ -18,8 +18,9 @@ const (
 type AnnotatedResource string
 
 const (
-	AnnotatedPod     AnnotatedResource = "pod"
-	AnnotatedService AnnotatedResource = "service"
+	AnnotatedPod                   AnnotatedResource = "pod"
+	AnnotatedService               AnnotatedResource = "service"
+	PodNodeSelectorFieldExpression string            = "spec.nodeName=${MY_NODE_NAME}"
 )
 
 const (
@@ -37,7 +38,7 @@ func makePrometheusConfigForPods() *PrometheusReceiver {
 	scrapeConfig := ScrapeConfig{
 		ScrapeInterval:             scrapeInterval,
 		SampleLimit:                sampleLimit,
-		KubernetesDiscoveryConfigs: []KubernetesDiscoveryConfig{{Role: RolePod}},
+		KubernetesDiscoveryConfigs: makeDiscoveryConfigWithNodeSelector(RolePod),
 		JobName:                    appPodsJobName,
 		RelabelConfigs:             makePrometheusPodsRelabelConfigs(),
 	}
@@ -57,7 +58,7 @@ func makePrometheusConfigForServices(opts BuildOptions) *PrometheusReceiver {
 	baseScrapeConfig := ScrapeConfig{
 		ScrapeInterval:             scrapeInterval,
 		SampleLimit:                sampleLimit,
-		KubernetesDiscoveryConfigs: []KubernetesDiscoveryConfig{{Role: RoleEndpoints}},
+		KubernetesDiscoveryConfigs: makeDiscoveryConfigWithNodeSelector(RoleEndpoints),
 	}
 
 	httpScrapeConfig := baseScrapeConfig
@@ -146,7 +147,7 @@ func makePrometheusIstioConfig() *PrometheusReceiver {
 					SampleLimit:                sampleLimit,
 					MetricsPath:                "/stats/prometheus",
 					ScrapeInterval:             scrapeInterval,
-					KubernetesDiscoveryConfigs: []KubernetesDiscoveryConfig{{Role: RolePod}},
+					KubernetesDiscoveryConfigs: makeDiscoveryConfigWithNodeSelector(RolePod),
 					RelabelConfigs: []RelabelConfig{
 						keepIfRunningOnSameNode(NodeAffiliatedPod),
 						keepIfIstioProxy(),
@@ -291,3 +292,17 @@ func dropIfSchemeHTTPS() RelabelConfig {
 		Regex:        "(https)",
 	}
 }
+
+func makeDiscoveryConfigWithNodeSelector(role Role) []KubernetesDiscoveryConfig {
+	return []KubernetesDiscoveryConfig{
+		{
+			Role: role,
+			Selectors: []K8SDiscoverySelector{
+				{
+					Role:  RolePod,
+					Field: PodNodeSelectorFieldExpression,
+				},
+			},
+		},
+	}
+}
diff --git a/internal/otelcollector/config/metric/agent/testdata/config_istio_enabled.yaml b/internal/otelcollector/config/metric/agent/testdata/config_istio_enabled.yaml
@@ -175,6 +175,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: pod
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
     prometheus/app-services:
         config:
             scrape_configs:
@@ -223,6 +226,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: endpoints
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
                 - job_name: app-services-secure
                   sample_limit: 50000
                   scrape_interval: 30s
@@ -268,6 +274,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: endpoints
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
                   tls_config:
                     ca_file: /etc/istio-output-certs/root-cert.pem
                     cert_file: /etc/istio-output-certs/cert-chain.pem
@@ -299,6 +308,9 @@ receivers:
                       action: keep
                   kubernetes_sd_configs:
                     - role: pod
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
 processors:
     batch:
         send_batch_size: 1024

diff --git a/internal/otelcollector/config/metric/agent/testdata/config_istio_not_enabled.yaml b/internal/otelcollector/config/metric/agent/testdata/config_istio_not_enabled.yaml
@@ -164,6 +164,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: pod
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
     prometheus/app-services:
         config:
             scrape_configs:
@@ -212,6 +215,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: endpoints
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
 processors:
     batch:
         send_batch_size: 1024