From cdd93c9ea6fed1081d1fdf39f3cf0d519063a78d Mon Sep 17 00:00:00 2001 From: Semen Shestakov Date: Wed, 15 Nov 2023 17:49:44 +0200 Subject: [PATCH 1/6] SRE-101 Avoid duplication of metrics pipelines definition --- charts/agent/values.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml index 547b803..bfd26ad 100644 --- a/charts/agent/values.yaml +++ b/charts/agent/values.yaml @@ -380,14 +380,18 @@ config: receivers: [ otlp ] processors: [ memory_limiter, resource, resource/tempo, batch ] exporters: [ logging, otlp/global ] - metrics: - receivers: [ otlp ] - processors: [ memory_limiter, transform, resource, transform/metrics, batch/metrics ] + metrics: &metrics-pipeline exporters: [ logging, otlp/global ] - metrics/2: + processors: + - memory_limiter + - transform + - resource + - transform/metrics + - batch/metrics + receivers: [ otlp ] + metrics/debug: + <<: *metrics-pipeline receivers: [ prometheus ] - processors: [ memory_limiter, transform, resource, transform/metrics, batch/metrics ] - exporters: [ logging, otlp/global ] logs: receivers: [ otlp, filelog ] processors: [ filter/non-container, memory_limiter, k8sattributes, resource, attributes/loki, resource/loki-labels, resource/loki, transform, batch ] From 13f959ffd0891172666a94fd22cd780132cfe2e3 Mon Sep 17 00:00:00 2001 From: Semen Shestakov Date: Wed, 15 Nov 2023 18:28:04 +0200 Subject: [PATCH 2/6] SRE-101 Add pod label for logs --- charts/agent/values.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml index bfd26ad..7baa533 100644 --- a/charts/agent/values.yaml +++ b/charts/agent/values.yaml @@ -276,7 +276,15 @@ config: attributes: - action: insert key: loki.resource.labels - value: project, license, deployment_environment, service, namespace, container, aws_region + value: >- + aws_region, + container, + deployment_environment, + license, + namespace, + pod, + project, + service resource/loki: attributes: - action: insert From d05591ebc79b4b95a139cb277e7939debdff2630 Mon Sep 17 00:00:00 2001 From: Semen Shestakov Date: Wed, 15 Nov 2023 18:30:20 +0200 Subject: [PATCH 3/6] SRE-101 Fix no namespace label in logs --- charts/agent/values.yaml | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml index 7baa533..5c76910 100644 --- a/charts/agent/values.yaml +++ b/charts/agent/values.yaml @@ -285,12 +285,12 @@ config: pod, project, service - resource/loki: + resource/loki-format-logfmt: attributes: - action: insert key: loki.format value: logfmt - resource/loki-raw: + resource/loki-format-raw: attributes: - action: insert key: loki.format @@ -400,14 +400,31 @@ config: metrics/debug: <<: *metrics-pipeline receivers: [ prometheus ] - logs: - receivers: [ otlp, filelog ] - processors: [ filter/non-container, memory_limiter, k8sattributes, resource, attributes/loki, resource/loki-labels, resource/loki, transform, batch ] + logs/logfmt: &logs-pipeline exporters: [ logging, otlp/global ] - logs/raw: + processors: + - memory_limiter + - filter/non-container + - resource/loki-format-logfmt + - attributes/loki + - k8sattributes + - transform + - resource + - resource/loki-labels + - batch receivers: [ otlp, filelog ] - processors: [ filter/container, memory_limiter, k8sattributes, resource, attributes/loki, resource/loki-labels, resource/loki-raw, transform, batch ] - exporters: [ logging, otlp/global ] + logs/raw: + <<: *logs-pipeline + processors: + - memory_limiter + - filter/container + - resource/loki-format-raw + - attributes/loki + - k8sattributes + - transform + - resource + - resource/loki-labels + - batch prometheus: default: From da90c9c3e6c0a802a35c414591ef5c27df075fe2 Mon Sep 17 00:00:00 2001 From: Semen Shestakov Date: Wed, 15 Nov 2023 18:31:46 +0200 Subject: [PATCH 4/6] SRE-101 Clear redundant comments --- charts/agent/values.yaml | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml index 5c76910..3043e22 100644 --- a/charts/agent/values.yaml +++ b/charts/agent/values.yaml @@ -21,8 +21,6 @@ service: name: otelgrpc - port: 4318 name: otelhttp -# - port: 24224 -# name: fluentforward rbac: create: true @@ -62,6 +60,8 @@ securityContext: { } # runAsNonRoot: true # runAsUser: 1000 +affinity: {} +nodeSelector: {} resources: limits: cpu: 1 @@ -69,23 +69,7 @@ resources: requests: cpu: 500m memory: 0.5Gi - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: { } - -tolerations: [ ] - -affinity: { } - +tolerations: [] # required AWS SSM parameter store certificates certs: @@ -208,7 +192,6 @@ config: statements: # fix duplication: replace 127.0.0.1:8888 => host.name value - set(attributes["service.instance.id"], attributes["host.name"]) where attributes["service.name"] == "{{ .Values.info.name }}" - # k8sattributes processor to get the metadata from K8s k8sattributes: auth_type: "serviceAccount" passthrough: false @@ -234,18 +217,14 @@ config: - sources: - from: connection name: ip - # loki not support keys with dots - so replace it transform: metric_statements: - context: resource statements: -# - replace_all_patterns(attributes, "key", "k8s.(statefulset|deployment|daemonset|replicaset).name", "container") -# - replace_all_patterns(attributes, "key", "k8s.pod.name", "instance") - delete_key(attributes, "k8s.pod.uid") - delete_key(attributes, "k8s.container.restart_count") - delete_key(attributes, "k8s.container.name") - replace_all_patterns(attributes, "key", "k8s.node.name", "nodename") -# - replace_all_patterns(attributes, "key", "k8s.namespace.name", "namespace") log_statements: - context: resource statements: @@ -547,12 +526,10 @@ prometheus: - action: replace source_labels: [__meta_kubernetes_pod_container_name] target_label: container -# # but more importantly to follow grafana notion: / - action: replace replacement: "$$1" source_labels: [__meta_kubernetes_pod_container_name] target_label: job - # but more importantly to follow grafana notion: / - action: replace separator: "/" source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app_kubernetes_io_component] From ac86e6695a697886a5673e5fc4f77c15ad7b6e3e Mon Sep 17 00:00:00 2001 From: Semen Shestakov Date: Wed, 15 Nov 2023 18:36:04 +0200 Subject: [PATCH 5/6] agent: release 1.1.14 --- charts/agent/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/agent/Chart.yaml b/charts/agent/Chart.yaml index 74a9f92..8564e0b 100644 --- a/charts/agent/Chart.yaml +++ b/charts/agent/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.13 +version: 1.1.14 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 1e76ec53e0ab58d3f41232c37a9684c0a3257fe8 Mon Sep 17 00:00:00 2001 From: Semen Shestakov Date: Wed, 15 Nov 2023 19:06:03 +0200 Subject: [PATCH 6/6] SRE-101 Merge metrics pipelines --- charts/agent/values.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml index 3043e22..0ca0894 100644 --- a/charts/agent/values.yaml +++ b/charts/agent/values.yaml @@ -367,7 +367,7 @@ config: receivers: [ otlp ] processors: [ memory_limiter, resource, resource/tempo, batch ] exporters: [ logging, otlp/global ] - metrics: &metrics-pipeline + metrics: exporters: [ logging, otlp/global ] processors: - memory_limiter @@ -375,10 +375,7 @@ config: - resource - transform/metrics - batch/metrics - receivers: [ otlp ] - metrics/debug: - <<: *metrics-pipeline - receivers: [ prometheus ] + receivers: [ otlp, prometheus ] logs/logfmt: &logs-pipeline exporters: [ logging, otlp/global ] processors: