feat(k8s-infra): introduce prometheus scraper using pod annotations

charts/k8s-infra/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: k8s-infra
 description: Helm chart for collecting metrics and logs in K8s
 type: application
-version: 0.11.20
+version: 0.12.0
 appVersion: "0.109.0"
 home: https://signoz.io
 icon: https://signoz.io/img/SigNozLogo-orange.svg

charts/k8s-infra/templates/NOTES.txt

@@ -1,7 +1,16 @@
+
+You have just deployed K8s-Infra chart:
+
+- otel-agent version: '{{ .Values.otelAgent.image.tag }}'
+- otel-deployment version: '{{ .Values.otelDeployment.image.tag }}'
+
+NOTE: If you enable Prometheus preset in this chart and have installed SigNoz helm chart in the same cluster,
+      please set otelCollectorMetrics.enabled to false in the signoz chart to avoid data duplication.
+
 {{- if not .Values.otelAgent.configMap.create }}
-[WARNING] "configMap" wil not be created and "config" will not take effect.
+[WARNING] otel-agent "configMap" will not be created and "otelAgent.config" will not take effect.
 {{ end }}
 
 {{- if not .Values.otelDeployment.configMap.create }}
-[WARNING] "configMap" wil not be created and "config" will not take effect.
+[WARNING] otel-deployment "configMap" will not be created and "otelDeployment.config" will not take effect.
 {{ end }}

charts/k8s-infra/templates/_config.tpl

@@ -71,6 +71,9 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- if .Values.presets.clusterMetrics.enabled }}
 {{- $config = (include "opentelemetry-collector.applyClusterMetricsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
+{{- if .Values.presets.prometheus.enabled }}
+{{- $config = (include "opentelemetry-collector.applyPrometheusConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
 {{- if .Values.presets.k8sEvents.enabled }}
 {{- $config = (include "opentelemetry-collector.applyK8sEventsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
@@ -83,6 +86,12 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- if or (eq (len (index (index $config.service.pipelines "metrics/internal") "receivers")) 0) (eq (len (index (index $config.service.pipelines "metrics/internal") "exporters")) 0) }}
 {{- $_ := unset $config.service.pipelines "metrics/internal" }}
 {{- end }}
+{{- if or (eq (len (index (index $config.service.pipelines "metrics/scraper") "receivers")) 0) (eq (len (index (index $config.service.pipelines "metrics/scraper") "exporters")) 0) }}
+{{- $_ := unset $config.service.pipelines "metrics/scraper" }}
+{{- end }}
+{{ if or (eq (len $config.service.pipelines.logs.receivers) 0) (eq (len $config.service.pipelines.logs.exporters) 0) }}
+{{- $_ := unset $config.service.pipelines "logs" }}
+{{- end }}
 {{- tpl (toYaml $config) . }}
 {{- end }}
 
@@ -100,6 +109,9 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "exporters" (prepend (index (index $config.service.pipelines "metrics/internal") "exporters") "logging" | uniq)  }}
 {{- end }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "exporters" (prepend (index (index $config.service.pipelines "metrics/scraper") "exporters") "logging" | uniq)  }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -128,6 +140,9 @@ exporters:
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "exporters" (prepend (index (index $config.service.pipelines "metrics/internal") "exporters") "otlp" | uniq)  }}
 {{- end }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "exporters" (prepend (index (index $config.service.pipelines "metrics/scraper") "exporters") "otlp" | uniq)  }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -189,6 +204,99 @@ receivers:
     {{- end }}
 {{- end }}
 
+{{- define "opentelemetry-collector.applyPrometheusConfig" -}}
+{{- $config := mustMergeOverwrite (include "opentelemetry-collector.prometheusConfig" .Values | fromYaml) .config }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "receivers" (append (index (index $config.service.pipelines "metrics/scraper") "receivers") "prometheus/scraper" | uniq)  }}
+{{- end }}
+{{- $config | toYaml }}
+{{- end }}
+
+{{- define "opentelemetry-collector.convertAnnotationToPrometheusMetaLabel" -}}
+{{- $name := . | lower -}}
+{{- $name = regexReplaceAll "\\." $name "_" -}}
+{{- $name := regexReplaceAll "/" $name "_" -}}
+{{- $name := regexReplaceAll "-" $name "_" -}}
+{{- $name := regexReplaceAll "[^a-z0-9_]" $name "_" -}}
+{{- $name -}}
+{{- end -}}
+
+{{- define "opentelemetry-collector.prometheusConfig" -}}
+{{- $annotationsPrefix := include "opentelemetry-collector.convertAnnotationToPrometheusMetaLabel" .Values.presets.prometheus.annotationsPrefix }}
+receivers:
+  prometheus/scraper:
+    config:
+      scrape_configs:
+        {{- if .Values.presets.prometheus.enabled }}
+        - job_name: "signoz-scraper"
+          scrape_interval: {{ .Values.presets.prometheus.scrapeInterval }}
+          kubernetes_sd_configs:
+            - role: pod
+              {{- if or .Values.presets.prometheus.namespaceScoped (len .Values.presets.prometheus.namespaces) }}
+              namespaces:
+                {{- if .Values.presets.prometheus.namespaceScoped }}
+                own_namespace: true
+                {{- end }}
+                {{- if .Values.presets.prometheus.namespaces }}
+                names: {{ toYaml .Values.presets.prometheus.namespaces | nindent 16 }}
+                {{- end }}
+              {{- end }}
+          relabel_configs:
+            - source_labels: [__meta_kubernetes_pod_annotation_{{ $annotationsPrefix }}_scrape]
+              action: keep
+              regex: true
+            - source_labels: [__meta_kubernetes_pod_annotation_{{ $annotationsPrefix }}_path]
+              action: replace
+              target_label: __metrics_path__
+              regex: (.+)
+            - source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_{{ $annotationsPrefix }}_port]
+              action: replace
+              separator: ":"
+              target_label: __address__
+            - target_label: job_name
+              replacement: signoz-scraper
+            {{- if .Values.presets.prometheus.includePodLabel }}
+            - action: labelmap
+              regex: __meta_kubernetes_pod_label_(.+)
+            {{- end }}
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+              action: replace
+              target_label: signoz_k8s_name
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+              action: replace
+              target_label: signoz_k8s_instance
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
+              action: replace
+              target_label: signoz_k8s_component
+            - source_labels: [__meta_kubernetes_namespace]
+              action: replace
+              target_label: k8s_namespace_name
+            - source_labels: [__meta_kubernetes_pod_name]
+              action: replace
+              target_label: k8s_pod_name
+            - source_labels: [__meta_kubernetes_pod_uid]
+              action: replace
+              target_label: k8s_pod_uid
+            {{- if .Values.presets.prometheus.includeContainerName }}
+            - source_labels: [__meta_kubernetes_pod_container_name]
+              action: replace
+              target_label: k8s_container_name
+            - source_labels: [__meta_kubernetes_pod_container_name]
+              regex: (.+)-init
+              action: drop
+            {{- end }}
+            - source_labels: [__meta_kubernetes_pod_node_name]
+              action: replace
+              target_label: k8s_node_name
+            - source_labels: [__meta_kubernetes_pod_ready]
+              action: replace
+              target_label: k8s_pod_ready
+            - source_labels: [__meta_kubernetes_pod_phase]
+              action: replace
+              target_label: k8s_pod_phase
+        {{- end }}
+{{- end }}
+
 {{- define "opentelemetry-collector.applyHostMetricsConfig" -}}
 {{- $config := mustMergeOverwrite (include "opentelemetry-collector.hostMetricsConfig" .Values | fromYaml) .config }}
 {{- if $config.service.pipelines.metrics }}
@@ -330,6 +438,9 @@ receivers:
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "processors" (prepend (index (index $config.service.pipelines "metrics/internal") "processors") "k8sattributes" | uniq) }}
 {{- end }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "processors" (prepend (index (index $config.service.pipelines "metrics/scraper") "processors") "k8sattributes" | uniq) }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -366,6 +477,9 @@ processors:
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "processors" (prepend (index (index $config.service.pipelines "metrics/internal") "processors") "resourcedetection" | uniq) }}
 {{- end }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "processors" (prepend (index (index $config.service.pipelines "metrics/scraper") "processors") "resourcedetection" | uniq) }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -383,6 +497,9 @@ processors:
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "processors" (prepend (index (index $config.service.pipelines "metrics/internal") "processors") "resourcedetection" | uniq) }}
 {{- end }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "processors" (prepend (index (index $config.service.pipelines "metrics/scraper") "processors") "resourcedetection" | uniq) }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 

charts/k8s-infra/templates/otel-agent/tests/test-connection.yaml

@@ -4,6 +4,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: "{{ include "otelAgent.fullname" . }}-test-connection"
+  namespace: {{ include "k8s-infra.namespace" . }}
   annotations:
     "helm.sh/hook": test
 spec:

charts/k8s-infra/templates/otel-deployment/tests/test-connection.yaml

@@ -4,6 +4,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: "{{ include "otelDeployment.fullname" . }}-test-connection"
+  namespace: {{ include "k8s-infra.namespace" . }}
   labels:
     {{- include "otelDeployment.labels" . | nindent 4 }}
   annotations:

charts/k8s-infra/tests/otel-deployment_prometheus_test.yaml

@@ -0,0 +1,143 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json
+suite: otel-deployment_prometheus_test.yaml
+templates:
+  - otel-deployment/configmap.yaml
+release:
+  name: k8s-infra
+  namespace: signoz
+set:
+  presets:
+    prometheus:
+      enabled: true
+      annotationsPrefix: "signoz.io"
+      scrapeInterval: 60s
+    loggingExporter:
+      enabled: true
+    otlpExporter:
+      enabled: false
+    logsCollection:
+      enabled: false
+    hostMetrics:
+      enabled: false
+    kubeletMetrics:
+      enabled: false
+    kubernetesAttributes:
+      enabled: false
+    clusterMetrics:
+      enabled: false
+    resourceDetection:
+      enabled: true
+    k8sEvents:
+      enabled: false
+tests:
+  - it: should have otel-deployment configuration in otel-deployment-config.yaml
+    asserts:
+      - exists:
+          path: data["otel-deployment-config.yaml"]
+  - it: should have prometheus/scraper receiver with signoz.io annotations prefix
+    asserts:
+      - equal:
+          path: data["otel-deployment-config.yaml"]
+          value: |-
+            exporters:
+              logging:
+                sampling_initial: 2
+                sampling_thereafter: 500
+                verbosity: basic
+            extensions:
+              health_check:
+                endpoint: 0.0.0.0:13133
+              pprof:
+                endpoint: localhost:1777
+              zpages:
+                endpoint: localhost:55679
+            processors:
+              batch:
+                send_batch_size: 10000
+                timeout: 1s
+              resourcedetection:
+                detectors:
+                - env
+                override: false
+                timeout: 2s
+            receivers:
+              prometheus/scraper:
+                config:
+                  scrape_configs:
+                  - job_name: signoz-scraper
+                    kubernetes_sd_configs:
+                    - role: pod
+                    relabel_configs:
+                    - action: keep
+                      regex: true
+                      source_labels:
+                      - __meta_kubernetes_pod_annotation_signoz_io_scrape
+                    - action: replace
+                      regex: (.+)
+                      source_labels:
+                      - __meta_kubernetes_pod_annotation_signoz_io_path
+                      target_label: __metrics_path__
+                    - action: replace
+                      separator: ':'
+                      source_labels:
+                      - __meta_kubernetes_pod_ip
+                      - __meta_kubernetes_pod_annotation_signoz_io_port
+                      target_label: __address__
+                    - replacement: signoz-scraper
+                      target_label: job_name
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_label_app_kubernetes_io_name
+                      target_label: signoz_k8s_name
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_label_app_kubernetes_io_instance
+                      target_label: signoz_k8s_instance
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_label_app_kubernetes_io_component
+                      target_label: signoz_k8s_component
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_namespace
+                      target_label: k8s_namespace_name
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_name
+                      target_label: k8s_pod_name
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_uid
+                      target_label: k8s_pod_uid
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_node_name
+                      target_label: k8s_node_name
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_ready
+                      target_label: k8s_pod_ready
+                    - action: replace
+                      source_labels:
+                      - __meta_kubernetes_pod_phase
+                      target_label: k8s_pod_phase
+                    scrape_interval: 60s
+            service:
+              extensions:
+              - health_check
+              - zpages
+              - pprof
+              pipelines:
+                metrics/scraper:
+                  exporters:
+                  - logging
+                  processors:
+                  - resourcedetection
+                  - batch
+                  receivers:
+                  - prometheus/scraper
+              telemetry:
+                logs:
+                  encoding: json
+                metrics:
+                  address: 0.0.0.0:8888

charts/k8s-infra/values.yaml

@@ -309,6 +309,24 @@ presets:
         enabled: true
       k8s.pod.status_reason:
         enabled: true
+  prometheus:
+    # -- Whether to enable metrics scraping using pod annotation
+    enabled: false
+    # -- Prefix for the pod annotations to be used for metrics scraping
+    annotationsPrefix: signoz.io
+    # -- How often to scrape metrics
+    scrapeInterval: 60s
+    # -- Whether to only scrape metrics from pods in the same namespace
+    namespaceScoped: false
+    # -- If set, only scrape metrics from pods in the specified namespaces
+    namespaces: []
+    # -- This will include all pod labels in the metrics, could potentially cause
+    # performance issues with large number of pods with many labels
+    includePodLabel: false
+    # -- This is not recommended in case of multiple containers or init containers in a pod
+    # Since it will create multiple timeseries for the same pod metrics with different container names
+    # containers with `-init` suffix in the name will be ignored
+    includeContainerName: false
   resourceDetection:
     enabled: true
     timeout: 2s
@@ -970,6 +988,10 @@ otelDeployment:
           receivers: []
           processors: [batch]
           exporters: []
+        metrics/scraper:
+          receivers: []
+          processors: [batch]
+          exporters: []
         logs:
           receivers: []
           processors: [batch]