NSQ mixin

nsq-mixin/.gitignore

@@ -0,0 +1,3 @@
+dashboards_out
+vendor
+prometheus_alerts.yaml

nsq-mixin/Makefile

@@ -0,0 +1,34 @@
+JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 1 --string-style s --comment-style s
+
+.PHONY: all
+all: build dashboards_out prometheus_alerts.yaml
+
+vendor: jsonnetfile.json
+	jb install
+
+.PHONY: build
+build: vendor
+
+.PHONY: fmt
+fmt:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+.PHONY: lint
+lint: build
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+	mixtool lint mixin.libsonnet
+
+dashboards_out: mixin.libsonnet config.libsonnet $(wildcard dashboards/*)
+	@mkdir -p dashboards_out
+	jsonnet -J vendor -m dashboards_out lib/dashboards.jsonnet
+
+prometheus_alerts.yaml: mixin.libsonnet lib/alerts.jsonnet alerts/*.libsonnet
+	jsonnet -J vendor -S lib/alerts.jsonnet > $@
+
+.PHONY: clean
+clean:
+	rm -rf dashboards_out prometheus_alerts.yaml

nsq-mixin/README.md

@@ -0,0 +1,156 @@
+# NSQ Mixin
+
+NSQ mixin is a set of configurable Grafana dashboards and alerts based on the metrics exported by the [NSQ statsd integration](https://nsq.io/components/nsqd.html#statsd--graphite-integration).
+In order to use it, you would need https://github.com/prometheus/statsd_exporter or https://grafana.com/docs/grafana-cloud/agent/ with statsd integration enabled.
+
+This NSQ mixin interesting features:
+1) Set of two dashboards:
+NSQ Topics - view metrics grouped by topics and channels
+![screenshot-0](https://user-images.githubusercontent.com/14870891/149538092-fa71774e-697d-45d7-a1e2-6f795977872e.png)
+NSQ Instances - view metrics  grouped by nsqd instances
+![screenshot-1](https://user-images.githubusercontent.com/14870891/149538111-bbbe2712-d3f6-47f0-8b1b-396061f46eb4.png)
+
+2) Dashboard links to quickly jump from one dashboard(view) to another.
+3) [Data links ](https://grafana.com/docs/grafana/latest/linking/data-links/#data-links) to contextually jump from instance view to topics view and vice versa
+![image](https://user-images.githubusercontent.com/14870891/149532730-6fdebd8d-204e-4962-861d-9c78af437afb.png)
+4) Alerts for topics and channels depth.
+
+## statsd_exporter/grafana agent configuration
+
+In the Grafana agent configuration file, the agent's statsd_exporter integration must be enabled and configured the following way:
+
+```yaml
+metrics:
+  wal_directory: /tmp/wal
+  configs:
+    - name: integrations
+      remote_write:
+        - url: http://cortex:9009/api/prom/push
+integrations:
+  statsd_exporter:
+    enabled: true
+    metric_relabel_configs:
+      - source_labels: [exported_job]
+        target_label: job
+        replacement: 'integrations/$1'
+      - source_labels: [exported_instance]
+        target_label: instance
+      - regex: (exported_instance|exported_job)
+        action: labeldrop
+    mapping_config:
+      defaults:
+        match_type: glob
+        glob_disable_ordering: false
+        ttl: 1m30s
+      mappings:
+       - match: "nsq.*.topic.*.channel.*.message_count"
+         name: "nsq_topic_channel_message_count"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+       - match: "nsq.*.topic.*.channel.*.requeue_count"
+         name: "nsq_topic_channel_requeue_count"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+       - match: "nsq.*.topic.*.channel.*.timeout_count"
+         name: "nsq_topic_channel_timeout_count"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+       - match: "nsq.*.topic.*.channel.*.*"
+         name: "nsq_topic_channel_${4}"
+         match_metric_type: gauge
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+      #nsq.<nsq_host>_<nsq_port>.topic.<topic_name>.backend_depth [gauge]
+       - match: "nsq.*.topic.*.message_count"
+         name: "nsq_topic_message_count"
+         help: Total number of messages for the topic
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+
+       - match: "nsq.*.topic.*.message_bytes"
+         name: "nsq_topic_message_bytes"
+         help: Total number of bytes of all messages
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+
+       - match: "nsq.*.topic.*.*" #depth, backend_depth and e2e_processing_latency_<percent>
+         name: "nsq_topic_${3}"
+         match_metric_type: gauge
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+      # mem
+      # nsq.<nsq_host>_<nsq_port>.mem.gc_runs
+       - match: "nsq.*.mem.gc_runs"
+         name: "nsq_mem_gc_runs"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+
+       - match: "nsq.*.mem.*"
+         name: "nsq_mem_${2}"
+         match_metric_type: gauge
+         labels:
+           instance: "$1"
+           job: "nsq"
+
+```
+
+## Generate config files
+
+You can manually generate dashboards, but first you should install some tools:
+
+```bash
+go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
+go install github.com/google/go-jsonnet/cmd/jsonnet@latest
+# or in brew: brew install go-jsonnet
+```
+
+For linting and formatting, you would also need `mixtool` and `jsonnetfmt` installed. If you
+have a working Go development environment, it's easiest to run the following:
+
+```bash
+go install github.com/monitoring-mixins/mixtool/cmd/mixtool@latest
+go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
+```
+
+The files in `dashboards_out` need to be imported
+into your Grafana server.  The exact details will be depending on your environment.
+
+`prometheus_alerts.yaml` needs to be imported into Prometheus.
+
+Edit `config.libsonnet` if required and then build JSON dashboard files for Grafana:
+
+```bash
+make
+```
+
+For more advanced uses of mixins, see
+https://github.com/monitoring-mixins/docs.

nsq-mixin/alerts/alerts.libsonnet

@@ -0,0 +1,43 @@
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'nsq',
+        rules: [
+          {
+            alert: 'NsqTopicDepthIncreasing',
+            expr: |||
+              sum by (topic) (nsq_topic_depth) > %(alertsCriticalTopicDepth)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'Topic depth is increasing.',
+              description: |||
+                Topic {{ $labels.topic }} depth is higher than %(alertsCriticalTopicDepth)s. The currect queue is {{ $value }}.
+              ||| % $._config,
+            },
+          },
+          {
+            alert: 'NsqChannelDepthIncreasing',
+            expr: |||
+              sum by (topic) (nsq_topic_channel_backend_depth) > %(alertsCriticalChannelDepth)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'Topic channel depth is increasing.',
+              description: |||
+                Channel {{ $labels.channel }} depth in topic {{ $labels.topic }} is higher than %(alertsCriticalChannelDepth)s. The currect queue is {{ $value }}.
+              ||| % $._config,
+            },
+          },
+        ],
+      },
+    ],
+  },
+}

nsq-mixin/config.libsonnet

@@ -0,0 +1,14 @@
+{
+  _config+:: {
+    local c = self,
+    dashboardNamePrefix: 'NSQ',
+    dashboardTags: ['nsq'],
+    dashboardPeriod: 'now-1h',
+    dashboardTimezone: 'default',
+    dashboardRefresh: '1m',
+
+    // alerts thresholds
+    alertsCriticalTopicDepth: '100',
+    alertsCriticalChannelDepth: '100',
+  },
+}

nsq-mixin/dashboards/dashboards.libsonnet

@@ -0,0 +1,2 @@
+(import 'topics.libsonnet') +
+(import 'instances.libsonnet')