Merge pull request #36 from nfrigus/main

SRE-101 Fix no namespace label in logs
d7561985 · Nov 15, 2023 · a97d6e5 · a97d6e5
2 parents 725c66a + 1e76ec5
commit a97d6e5
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 42 deletions.
diff --git a/charts/agent/Chart.yaml b/charts/agent/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.13
+version: 1.1.14
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml
@@ -21,8 +21,6 @@ service:
       name: otelgrpc
     - port: 4318
       name: otelhttp
-#    - port: 24224
-#      name: fluentforward
 
 rbac:
   create: true
@@ -62,30 +60,16 @@ securityContext: { }
 # runAsNonRoot: true
 # runAsUser: 1000
 
+affinity: {}
+nodeSelector: {}
 resources:
   limits:
     cpu: 1
     memory: 1Gi
   requests:
     cpu: 500m
     memory: 0.5Gi
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
-
-nodeSelector: { }
-
-tolerations: [ ]
-
-affinity: { }
-
+tolerations: []
 
 # required AWS SSM parameter store certificates
 certs:
@@ -208,7 +192,6 @@ config:
           statements:
             # fix duplication: replace 127.0.0.1:8888 => host.name value
             - set(attributes["service.instance.id"], attributes["host.name"]) where attributes["service.name"] == "{{ .Values.info.name }}"
-    # k8sattributes processor to get the metadata from K8s
     k8sattributes:
       auth_type: "serviceAccount"
       passthrough: false
@@ -234,18 +217,14 @@ config:
         - sources:
             - from: connection
               name: ip
-    # loki not support keys with dots - so replace it
     transform:
       metric_statements:
         - context: resource
           statements:
-#            - replace_all_patterns(attributes, "key", "k8s.(statefulset|deployment|daemonset|replicaset).name", "container")
-#            - replace_all_patterns(attributes, "key", "k8s.pod.name", "instance")
             - delete_key(attributes, "k8s.pod.uid")
             - delete_key(attributes, "k8s.container.restart_count")
             - delete_key(attributes, "k8s.container.name")
             - replace_all_patterns(attributes, "key", "k8s.node.name", "nodename")
-#            - replace_all_patterns(attributes, "key", "k8s.namespace.name", "namespace")
       log_statements:
         - context: resource
           statements:
@@ -276,13 +255,21 @@ config:
       attributes:
         - action: insert
           key: loki.resource.labels
-          value: project, license, deployment_environment, service, namespace, container, aws_region
-    resource/loki:
+          value: >-
+            aws_region,
+            container,
+            deployment_environment,
+            license,
+            namespace,
+            pod,
+            project,
+            service
+    resource/loki-format-logfmt:
       attributes:
         - action: insert
           key: loki.format
           value: logfmt
-    resource/loki-raw:
+    resource/loki-format-raw:
       attributes:
         - action: insert
           key: loki.format
@@ -381,21 +368,39 @@ config:
         processors: [ memory_limiter, resource, resource/tempo, batch ]
         exporters: [ logging, otlp/global ]
       metrics:
-        receivers: [ otlp ]
-        processors: [ memory_limiter, transform, resource, transform/metrics, batch/metrics ]
         exporters: [ logging, otlp/global ]
-      metrics/2:
-        receivers: [ prometheus ]
-        processors: [ memory_limiter, transform, resource, transform/metrics,  batch/metrics ]
+        processors:
+          - memory_limiter
+          - transform
+          - resource
+          - transform/metrics
+          - batch/metrics
+        receivers: [ otlp, prometheus ]
+      logs/logfmt: &logs-pipeline
         exporters: [ logging, otlp/global ]
-      logs:
-        receivers: [  otlp, filelog ]
-        processors: [ filter/non-container, memory_limiter, k8sattributes, resource, attributes/loki, resource/loki-labels, resource/loki, transform, batch ]
-        exporters: [ logging, otlp/global ]
-      logs/raw:
+        processors:
+          - memory_limiter
+          - filter/non-container
+          - resource/loki-format-logfmt
+          - attributes/loki
+          - k8sattributes
+          - transform
+          - resource
+          - resource/loki-labels
+          - batch
         receivers: [ otlp, filelog ]
-        processors: [ filter/container, memory_limiter, k8sattributes, resource, attributes/loki, resource/loki-labels, resource/loki-raw, transform, batch ]
-        exporters: [ logging, otlp/global ]
+      logs/raw:
+        <<: *logs-pipeline
+        processors:
+          - memory_limiter
+          - filter/container
+          - resource/loki-format-raw
+          - attributes/loki
+          - k8sattributes
+          - transform
+          - resource
+          - resource/loki-labels
+          - batch
 
 prometheus:
     default:
@@ -518,12 +523,10 @@ prometheus:
             - action: replace
               source_labels: [__meta_kubernetes_pod_container_name]
               target_label: container
-#            # but more importantly to follow grafana notion: <ns>/<container_name>
             - action: replace
               replacement: "$$1"
               source_labels: [__meta_kubernetes_pod_container_name]
               target_label: job
-            # but more importantly to follow grafana notion: <ns>/<component_name>
             - action: replace
               separator: "/"
               source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app_kubernetes_io_component]