Update some configs and fixes the Pulp dashboard.

profiles/opentelemetry_dev/compose.yaml

@@ -49,9 +49,10 @@ services:
     volumes:
       - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/provisioning:/etc/grafana/provisioning
       - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/dashboards:/var/lib/grafana/dashboards
-      - type: bind
-        source: ./grafana/config.ini
-        target: /etc/grafana/config.ini
+      - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/config.ini:/etc/grafana/config.ini
+      # - type: bind
+      #   source: ./grafana/config.ini
+      #   target: /etc/grafana/config.ini
     ports:
       - "3000:3000"
 

profiles/opentelemetry_dev/grafana/dashboards/pulp_insights.json

@@ -1,35 +1,4 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_PROMETHEUS",
-      "label": "Prometheus",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
-  ],
-  "__elements": {},
-  "__requires": [
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "9.5.1"
-    },
-    {
-      "type": "datasource",
-      "id": "prometheus",
-      "name": "Prometheus",
-      "version": "1.0.0"
-    },
-    {
-      "type": "panel",
-      "id": "timeseries",
-      "name": "Time series",
-      "version": ""
-    }
-  ],
   "annotations": {
     "list": [
       {
@@ -49,21 +18,21 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": null,
   "links": [],
   "liveNow": false,
   "panels": [
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
+        "uid": "PBFA97CFB590B2093"
       },
       "fieldConfig": {
         "defaults": {
           "color": {
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -77,6 +46,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -135,11 +105,15 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
+          "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "http_server_duration_sum{http_status_code=\"200\"}",
+          "expr": "pulp_http_server_duration_milliseconds_count{http_status_code=\"200\"}",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
           "legendFormat": "__auto",
           "range": true,
-          "refId": "A"
+          "refId": "A",
+          "useBackend": false
         }
       ],
       "title": "200 Ok Requests",
@@ -148,42 +122,22 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
+        "uid": "PBFA97CFB590B2093"
       },
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
+            "mode": "thresholds"
           },
           "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
+            "fillOpacity": 80,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
               "tooltip": false,
               "viz": false
             },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
+            "lineWidth": 1
           },
           "mappings": [],
           "thresholds": {
@@ -198,7 +152,8 @@
                 "value": 80
               }
             ]
-          }
+          },
+          "unit": "ms"
         },
         "overrides": []
       },
@@ -210,15 +165,12 @@
       },
       "id": 1,
       "options": {
+        "bucketOffset": 0,
         "legend": {
           "calcs": [],
           "displayMode": "list",
           "placement": "bottom",
           "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
         }
       },
       "targets": [
@@ -227,27 +179,35 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
+          "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "histogram_quantile(0.99, sum by(le) (rate(http_server_duration_bucket{service_name=\"pulp-content\"}[5m])))",
-          "legendFormat": "__auto",
+          "exemplar": false,
+          "expr": "pulp_http_server_duration_milliseconds_bucket{exported_job=\"pulp-api\"}",
+          "format": "heatmap",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{le}}",
           "range": true,
-          "refId": "A"
+          "refId": "A",
+          "useBackend": false
         }
       ],
       "title": "Pulp Content Request Duration P99",
-      "type": "timeseries"
+      "type": "histogram"
     },
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
+        "uid": "PBFA97CFB590B2093"
       },
       "fieldConfig": {
         "defaults": {
           "color": {
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -261,6 +221,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -319,11 +280,15 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
+          "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "http_server_active_requests{service_name=\"pulp-content\"}",
+          "expr": "pulp_http_server_active_requests{exported_job=\"pulp-api\"}",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
           "legendFormat": "__auto",
           "range": true,
-          "refId": "A"
+          "refId": "A",
+          "useBackend": false
         }
       ],
       "title": "Pulp Content Active Requests",
@@ -332,19 +297,18 @@
   ],
   "refresh": "5s",
   "schemaVersion": 38,
-  "style": "dark",
   "tags": [],
   "templating": {
     "list": []
   },
   "time": {
-    "from": "now-6h",
+    "from": "now-1h",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "Pulp Insights",
   "uid": "e50bb9f2-372c-4e94-aa61-fe1f1554812c",
-  "version": 3,
+  "version": 1,
   "weekStart": ""
 }

profiles/opentelemetry_dev/grafana/provisioning/dashboards/all.yml

@@ -1,6 +1,9 @@
-- name: 'default'
-  org_id: 1
-  folder: ''
-  type: 'file'
-  options:
-    folder: '/var/lib/grafana/dashboards'
+apiVersion: 1
+
+providers:
+  - name: Default
+    # org_id: 1
+    # folder: ''
+    type: 'file'
+    options:
+      folder: '/var/lib/grafana/dashboards'

profiles/opentelemetry_dev/otel-collector/otel-collector-config.yaml

@@ -10,17 +10,18 @@ receivers:
 
 processors:
   batch:
-    timeout: 3s
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 128
 
 exporters:
   prometheus:
     endpoint: 0.0.0.0:8889
     send_timestamps: true
-    resource_to_telemetry_conversion:
-      enabled: true
+    namespace: pulp
   logging:
     verbosity: detailed
-  jaeger:
+  otlp/jaeger:
     endpoint: jaeger:14250
     tls:
       insecure: true
@@ -41,13 +42,13 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [batch]
-      exporters: [jaeger]
+      processors: [memory_limiter, batch]
+      exporters: [otlp/jaeger]
     metrics:
       receivers: [otlp]
-      processors: [batch]
+      processors: [memory_limiter, batch]
       exporters: [logging, prometheus]
     logs:
       receivers: [otlp]
-      processors: [batch]
+      processors: [memory_limiter, batch]
       exporters: [logging]