Changes some configs and add new graphs to the pulp dashboard

pulp · Nov 27, 2023 · 5e7c948 · 5e7c948
1 parent 8a13705
commit 5e7c948
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 101 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -36,9 +36,10 @@ jobs:
           echo "COMPOSE_INTERACTIVE_NO_CLI=1" >> $GITHUB_ENV
           #echo "OCI_ENV_PATH=${GITHUB_WORKSPACE}/oci_env/" >> $GITHUB_ENV
 
-      - name: clone pulpcore, pulp-openapi-generator
+      - name: clone pulpcore, pulp_file, pulp-openapi-generator
         run: |
           git clone --depth 1 https://github.com/pulp/pulpcore.git
+          git clone --depth 1 https://github.com/pulp/pulp_file.git
           git clone --depth 1 https://github.com/pulp/pulp-openapi-generator.git
 
       - name: Install podman compose

diff --git a/profiles/opentelemetry_dev/compose.yaml b/profiles/opentelemetry_dev/compose.yaml
@@ -7,20 +7,21 @@ services:
     container_name: jaeger
     image: jaegertracing/all-in-one:latest
     environment:
-      COLLECTOR_OTLP_ENABLED: "true"
+      # COLLECTOR_OTLP_ENABLED: "true"
       JAEGER_DISABLED: "true"
     ports:
-      - "16686:16686"
-      - "14250:14250"
-      - "14268:14268"
-      - "14269:14269"
+      - "16686"
+      - "14250"
+      - "14268"
+      - "14269"
+      - "4317"
 
   otel-collector:
     container_name: otel-collector
     image: otel/opentelemetry-collector
     command: [--config=/etc/otel-collector/otel-collector-config.yaml]
     volumes:
-      - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/otel-collector:/etc/otel-collector
+      - "{OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/otel-collector:/etc/otel-collector"
     ports:
       - "1888:1888"   # pprof extension
       - "8888:8888"   # Prometheus metrics exposed by the collector
@@ -37,7 +38,7 @@ services:
     container_name: prometheus
     image: prom/prometheus
     volumes:
-      - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/prometheus:/etc/prometheus
+      - "{OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/prometheus:/etc/prometheus"
     ports:
       - "9090:9090"
 
@@ -47,9 +48,9 @@ services:
       - prometheus
     restart: "always"
     volumes:
-      - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/provisioning:/etc/grafana/provisioning
-      - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/dashboards:/var/lib/grafana/dashboards
-      - {OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/config.ini:/etc/grafana/config.ini
+      - "{OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/provisioning:/etc/grafana/provisioning"
+      - "{OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/dashboards:/var/lib/grafana/dashboards"
+      - "{OCI_ENV_DIR}/profiles/{COMPOSE_PROFILE}/grafana/config.ini:/etc/grafana/config.ini"
       # - type: bind
       #   source: ./grafana/config.ini
       #   target: /etc/grafana/config.ini

diff --git a/profiles/opentelemetry_dev/grafana/dashboards/pulp_insights.json b/profiles/opentelemetry_dev/grafana/dashboards/pulp_insights.json
@@ -29,51 +29,23 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
+            "mode": "thresholds"
           },
           "mappings": [],
           "thresholds": {
-            "mode": "absolute",
+            "mode": "percentage",
             "steps": [
               {
                 "color": "green",
                 "value": null
               },
               {
-                "color": "red",
-                "value": 80
+                "color": "dark-red",
+                "value": 90
+              },
+              {
+                "color": "#EAB839",
+                "value": 95
               }
             ]
           }
@@ -86,38 +58,42 @@
         "x": 0,
         "y": 0
       },
-      "id": 3,
+      "id": 5,
       "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
         },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
       },
+      "pluginVersion": "10.2.2",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
+            "uid": "PBFA97CFB590B2093"
           },
           "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "pulp_http_server_duration_milliseconds_count{http_status_code=\"200\"}",
+          "editorMode": "code",
+          "expr": "(pulp_http_server_duration_milliseconds_count{exported_job=\"pulp-api\", http_status_code!~\"5.*\"} / pulp_http_server_duration_milliseconds_count{exported_job=\"pulp-api\"}) * 100",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
+          "instant": false,
           "legendFormat": "__auto",
           "range": true,
           "refId": "A",
           "useBackend": false
         }
       ],
-      "title": "200 Ok Requests",
-      "type": "timeseries"
+      "title": "Rate of != 500 requests",
+      "type": "gauge"
     },
     {
       "datasource": {
@@ -129,16 +105,6 @@
           "color": {
             "mode": "thresholds"
           },
-          "custom": {
-            "fillOpacity": 80,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineWidth": 1
-          },
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
@@ -152,27 +118,33 @@
                 "value": 80
               }
             ]
-          },
-          "unit": "ms"
+          }
         },
         "overrides": []
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 4,
         "x": 12,
         "y": 0
       },
-      "id": 1,
+      "id": 2,
       "options": {
-        "bucketOffset": 0,
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        }
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
       },
+      "pluginVersion": "10.2.2",
       "targets": [
         {
           "datasource": {
@@ -181,20 +153,17 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "exemplar": false,
-          "expr": "pulp_http_server_duration_milliseconds_bucket{exported_job=\"pulp-api\"}",
-          "format": "heatmap",
+          "expr": "pulp_http_server_active_requests{exported_job=\"pulp-api\"}",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{le}}",
+          "legendFormat": "__auto",
           "range": true,
           "refId": "A",
           "useBackend": false
         }
       ],
-      "title": "Pulp Content Request Duration P99",
-      "type": "histogram"
+      "title": "Pulp API Active Requests",
+      "type": "stat"
     },
     {
       "datasource": {
@@ -214,7 +183,7 @@
             "axisPlacement": "auto",
             "barAlignment": 0,
             "drawStyle": "line",
-            "fillOpacity": 0,
+            "fillOpacity": 5,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
@@ -223,6 +192,9 @@
             },
             "insertNulls": false,
             "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
             "lineWidth": 1,
             "pointSize": 5,
             "scaleDistribution": {
@@ -251,7 +223,8 @@
                 "value": 80
               }
             ]
-          }
+          },
+          "unit": "ms"
         },
         "overrides": []
       },
@@ -261,7 +234,7 @@
         "x": 0,
         "y": 8
       },
-      "id": 2,
+      "id": 6,
       "options": {
         "legend": {
           "calcs": [],
@@ -278,20 +251,58 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
+            "uid": "PBFA97CFB590B2093"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "pulp_http_server_active_requests{exported_job=\"pulp-api\"}",
+          "expr": "histogram_quantile(0.99, sum by(le) (rate(pulp_http_server_duration_milliseconds_bucket[$__rate_interval])))",
           "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "legendFormat": "__auto",
+          "includeNullMetadata": false,
+          "instant": false,
+          "interval": "2m",
+          "legendFormat": "P99",
           "range": true,
           "refId": "A",
           "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.95, sum by(le) (rate(pulp_http_server_duration_milliseconds_bucket[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "interval": "2m",
+          "legendFormat": "P95",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.5, sum by(le) (rate(pulp_http_server_duration_milliseconds_bucket[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "interval": "2m",
+          "legendFormat": "P50",
+          "range": true,
+          "refId": "C",
+          "useBackend": false
         }
       ],
-      "title": "Pulp Content Active Requests",
+      "title": "Pulp API P-Time Request Duration",
       "type": "timeseries"
     }
   ],
@@ -311,4 +322,4 @@
   "uid": "e50bb9f2-372c-4e94-aa61-fe1f1554812c",
   "version": 1,
   "weekStart": ""
-}
+}
diff --git a/profiles/opentelemetry_dev/otel-collector/otel-collector-config.yaml b/profiles/opentelemetry_dev/otel-collector/otel-collector-config.yaml
@@ -22,7 +22,7 @@ exporters:
   logging:
     verbosity: detailed
   otlp/jaeger:
-    endpoint: jaeger:14250
+    endpoint: jaeger:4317
     tls:
       insecure: true
 

diff --git a/profiles/opentelemetry_dev/prometheus/prometheus.yml b/profiles/opentelemetry_dev/prometheus/prometheus.yml
@@ -1,6 +1,6 @@
 # my global config
 global:
-  scrape_interval: 3s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
   # evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
   # scrape_timeout is set to the global default (10s).