NSQ mixin

nsq-mixin/.gitignore

@@ -0,0 +1,3 @@
+dashboards_out
+vendor
+prometheus_alerts.yaml

nsq-mixin/Makefile

@@ -0,0 +1,34 @@
+JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 1 --string-style s --comment-style s
+
+.PHONY: all
+all: build dashboards_out prometheus_alerts.yaml
+
+vendor: jsonnetfile.json
+	jb install
+
+.PHONY: build
+build: vendor
+
+.PHONY: fmt
+fmt:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+.PHONY: lint
+lint: build
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+	mixtool lint mixin.libsonnet
+
+dashboards_out: mixin.libsonnet config.libsonnet $(wildcard dashboards/*)
+	@mkdir -p dashboards_out
+	jsonnet -J vendor -m dashboards_out lib/dashboards.jsonnet
+
+prometheus_alerts.yaml: mixin.libsonnet lib/alerts.jsonnet alerts/*.libsonnet
+	jsonnet -J vendor -S lib/alerts.jsonnet > $@
+
+.PHONY: clean
+clean:
+	rm -rf dashboards_out prometheus_alerts.yaml

nsq-mixin/README.md

@@ -0,0 +1,156 @@
+# NSQ Mixin
+
+NSQ mixin is a set of configurable Grafana dashboards and alerts based on the metrics exported by the [NSQ statsd integration](https://nsq.io/components/nsqd.html#statsd--graphite-integration).
+In order to use it, you would need https://github.com/prometheus/statsd_exporter or https://grafana.com/docs/grafana-cloud/agent/ with statsd integration enabled.
+
+This NSQ mixin interesting features:
+1) Set of two dashboards:
+NSQ Topics - view metrics grouped by topics and channels
+![screenshot-0](https://storage.googleapis.com/grafanalabs-integration-assets/nsq/screenshots/screenshot0.png)
+NSQ Instances - view metrics  grouped by nsqd instances
+![screenshot-1](https://storage.googleapis.com/grafanalabs-integration-assets/nsq/screenshots/screenshot1.png)
+
+2) Dashboard links to quickly jump from one dashboard(view) to another.
+3) [Data links ](https://grafana.com/docs/grafana/latest/linking/data-links/#data-links) to contextually jump from instance view to topics view and vice versa
+![image](https://user-images.githubusercontent.com/14870891/149532730-6fdebd8d-204e-4962-861d-9c78af437afb.png)
+4) Alerts for topics and channels depth.
+
+## statsd_exporter/grafana agent configuration
+
+In the Grafana agent configuration file, the agent's statsd_exporter integration must be enabled and configured the following way:
+
+```yaml
+metrics:
+  wal_directory: /tmp/wal
+  configs:
+    - name: integrations
+      remote_write:
+        - url: http://cortex:9009/api/prom/push
+integrations:
+  statsd_exporter:
+    enabled: true
+    metric_relabel_configs:
+      - source_labels: [exported_job]
+        target_label: job
+        replacement: 'integrations/$1'
+      - source_labels: [exported_instance]
+        target_label: instance
+      - regex: (exported_instance|exported_job)
+        action: labeldrop
+    mapping_config:
+      defaults:
+        match_type: glob
+        glob_disable_ordering: false
+        ttl: 1m30s
+      mappings:
+       - match: "nsq.*.topic.*.channel.*.message_count"
+         name: "nsq_topic_channel_message_count"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+       - match: "nsq.*.topic.*.channel.*.requeue_count"
+         name: "nsq_topic_channel_requeue_count"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+       - match: "nsq.*.topic.*.channel.*.timeout_count"
+         name: "nsq_topic_channel_timeout_count"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+       - match: "nsq.*.topic.*.channel.*.*"
+         name: "nsq_topic_channel_${4}"
+         match_metric_type: gauge
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+           channel: "$3"
+
+      #nsq.<nsq_host>_<nsq_port>.topic.<topic_name>.backend_depth [gauge]
+       - match: "nsq.*.topic.*.message_count"
+         name: "nsq_topic_message_count"
+         help: Total number of messages for the topic
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+
+       - match: "nsq.*.topic.*.message_bytes"
+         name: "nsq_topic_message_bytes"
+         help: Total number of bytes of all messages
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+
+       - match: "nsq.*.topic.*.*" #depth, backend_depth and e2e_processing_latency_<percent>
+         name: "nsq_topic_${3}"
+         match_metric_type: gauge
+         labels:
+           instance: "$1"
+           job: "nsq"
+           topic: "$2"
+      # mem
+      # nsq.<nsq_host>_<nsq_port>.mem.gc_runs
+       - match: "nsq.*.mem.gc_runs"
+         name: "nsq_mem_gc_runs"
+         match_metric_type: counter
+         labels:
+           instance: "$1"
+           job: "nsq"
+
+       - match: "nsq.*.mem.*"
+         name: "nsq_mem_${2}"
+         match_metric_type: gauge
+         labels:
+           instance: "$1"
+           job: "nsq"
+
+```
+
+## Generate config files
+
+You can manually generate dashboards, but first you should install some tools:
+
+```bash
+go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
+go install github.com/google/go-jsonnet/cmd/jsonnet@latest
+# or in brew: brew install go-jsonnet
+```
+
+For linting and formatting, you would also need `mixtool` and `jsonnetfmt` installed. If you
+have a working Go development environment, it's easiest to run the following:
+
+```bash
+go install github.com/monitoring-mixins/mixtool/cmd/mixtool@latest
+go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
+```
+
+The files in `dashboards_out` need to be imported
+into your Grafana server.  The exact details will be depending on your environment.
+
+`prometheus_alerts.yaml` needs to be imported into Prometheus.
+
+Edit `config.libsonnet` if required and then build JSON dashboard files for Grafana:
+
+```bash
+make
+```
+
+For more advanced uses of mixins, see
+https://github.com/monitoring-mixins/docs.

nsq-mixin/alerts/alerts.libsonnet

@@ -0,0 +1,43 @@
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'nsq',
+        rules: [
+          {
+            alert: 'NsqTopicDepthIncreasing',
+            expr: |||
+              sum by (topic) (nsq_topic_depth) > %(alertsCriticalTopicDepth)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'Topic depth is increasing.',
+              description: |||
+                Topic {{ $labels.topic }} depth is higher than %(alertsCriticalTopicDepth)s. The currect queue is {{ $value }}.
+              ||| % $._config,
+            },
+          },
+          {
+            alert: 'NsqChannelDepthIncreasing',
+            expr: |||
+              sum by (topic) (nsq_topic_channel_backend_depth) > %(alertsCriticalChannelDepth)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'Topic channel depth is increasing.',
+              description: |||
+                Channel {{ $labels.channel }} depth in topic {{ $labels.topic }} is higher than %(alertsCriticalChannelDepth)s. The currect queue is {{ $value }}.
+              ||| % $._config,
+            },
+          },
+        ],
+      },
+    ],
+  },
+}

nsq-mixin/config.libsonnet

@@ -0,0 +1,14 @@
+{
+  _config+:: {
+    local c = self,
+    dashboardNamePrefix: 'NSQ',
+    dashboardTags: ['nsq'],
+    dashboardPeriod: 'now-1h',
+    dashboardTimezone: 'default',
+    dashboardRefresh: '1m',
+
+    // alerts thresholds
+    alertsCriticalTopicDepth: '100',
+    alertsCriticalChannelDepth: '100',
+  },
+}

nsq-mixin/dashboards/dashboards.libsonnet

@@ -0,0 +1,2 @@
+(import 'topics.libsonnet') +
+(import 'instances.libsonnet')

nsq-mixin/dashboards/instances.libsonnet

@@ -0,0 +1,213 @@
+local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libsonnet';
+local prometheus = grafana.prometheus;
+local nsqgrafonnet = import '../lib/nsqgrafonnet/nsqgrafonnet.libsonnet';
+
+{
+
+  grafanaDashboards+:: {
+
+    local nsqSelector = 'job=~"$job", instance=~"$instance"',
+    local nsqTopicSelector = nsqSelector + ',topic=~"$topic"',
+    local nsqChannelSelector = nsqTopicSelector + ',channel=~"$channel"',
+    local nsqTopicDataLinkQueryParams = '${datasource:queryparam}&${job:queryparam}&${__url_time_range}',
+    local nsqChannelDataLinkQueryParams = '${datasource:queryparam}&${job:queryparam}&${__url_time_range}',
+    local nsqHeapMemory =
+      nsqgrafonnet.timeseries.new(
+        'Heap memory (avg)',
+        description='Average memory usage if instance=All filter is selected'
+      )
+      .addTarget(prometheus.target(expr='avg by (job) (nsq_mem_heap_in_use_bytes{%s})' % nsqSelector, intervalFactor=2, legendFormat='heap in use'))
+      .addTarget(prometheus.target(expr='avg by (job) (nsq_mem_heap_idle_bytes{%s})' % nsqSelector, intervalFactor=2, legendFormat='heap idle bytes'))
+      .addTarget(prometheus.target(expr='avg by (job) (nsq_mem_heap_released_bytes{%s})' % nsqSelector, intervalFactor=2, legendFormat='heap bytes released'))
+      .withFillOpacity(5)
+      .withUnit('decbytes'),
+
+    local nsqHeapObjects =
+      nsqgrafonnet.timeseries.new('Heap objects')
+      .addTarget(prometheus.target(expr='nsq_mem_heap_objects{%s}' % nsqSelector, intervalFactor=2, legendFormat='{{ instance }}'))
+      .withFillOpacity(5)
+      .withUnit('short'),
+
+    local nsqNextGC =
+      nsqgrafonnet.timeseries.new(
+        'Next GC in bytes',
+        description='The number used bytes at which the runtime plans to perform the next garbage collection'
+      )
+      .addTarget(prometheus.target(expr='nsq_mem_next_gc_bytes{%s}' % nsqSelector, intervalFactor=2, legendFormat='{{ instance }}'))
+      .withFillOpacity(5)
+      .withUnit('decbytes'),
+
+    local nsqGCpause =
+      nsqgrafonnet.timeseries.new('GC pause time')
+      .addTarget(prometheus.target(expr='nsq_mem_gc_pause_usec_100{%s}' % nsqSelector, intervalFactor=2, legendFormat='p100 {{ instance }}', hide=true))
+      .addTarget(prometheus.target(expr='nsq_mem_gc_pause_usec_99{%s}' % nsqSelector, intervalFactor=2, legendFormat='p99 {{ instance }}', hide=false))
+      .addTarget(prometheus.target(expr='nsq_mem_gc_pause_usec_95{%s}' % nsqSelector, intervalFactor=2, legendFormat='p95 {{ instance }}', hide=true))
+      .withUnit('µs')
+      .withFillOpacity(5),
+
+    local nsqTopics =
+      nsqgrafonnet.timeseries.new('$topic topic messages')
+      .addTarget(prometheus.target(expr='avg by (instance) (rate(nsq_topic_message_count{%s}[$__rate_interval]))' % nsqTopicSelector, intervalFactor=1, legendFormat='{{ instance }} rps'))
+      .withUnit('reqps')
+      .addDataLink(
+        title='Show split by topics for ${__field.labels.instance}',
+        url='d/nsq-topics?var-instance=${__field.labels.instance}&%s' % nsqTopicDataLinkQueryParams
+      ),
+
+    local nsqTopicsDepth =
+      nsqgrafonnet.timeseries.new('$topic topic depth')
+      .addTarget(prometheus.target(expr='sum by (instance) (nsq_topic_depth{%s})' % nsqTopicSelector, intervalFactor=1, legendFormat='{{ instance }} depth'))
+      .addTarget(prometheus.target(expr='sum by (instance) (nsq_topic_backend_depth{%s})' % nsqTopicSelector, intervalFactor=1, legendFormat='{{ instance }} memory+disk depth'))
+      .withUnit('short')
+      .addDataLink(
+        title='Show split by topics for ${__field.labels.instance}',
+        url='d/nsq-topics?var-instance=${__field.labels.instance}&%s' % nsqTopicDataLinkQueryParams
+      ),
+
+    local nsqChannelClients =
+      nsqgrafonnet.timeseries.new('$channel channel clients')
+      .addTarget(prometheus.target(expr='sum by (instance) (nsq_topic_channel_clients{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }}'))
+      .addDataLink(
+        title='Show split by channels for ${__field.labels.instance}',
+        url='d/nsq-topics?var-instance=${__field.labels.instance}&%s' % nsqChannelDataLinkQueryParams
+      ),
+
+    local nsqChannelStats =
+      nsqgrafonnet.timeseries.new('$channel channel stats')
+      .addTarget(prometheus.target(expr='sum by (instance) (nsq_topic_channel_depth{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }} depth'))
+      .addTarget(prometheus.target(expr='sum by (instance) (nsq_topic_channel_backend_depth{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }} memory+disk depth'))
+      .addTarget(prometheus.target(expr='sum by (instance) (nsq_topic_channel_in_flight_count{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }} in-flight'))
+      .addTarget(prometheus.target(expr='avg by (instance) (rate(nsq_topic_channel_requeue_count{%s}[$__rate_interval]))' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }} requeue'))
+      .addTarget(prometheus.target(expr='avg by (instance) (rate(nsq_topic_channel_timeout_count{%s}[$__rate_interval]))' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }} timeout'))
+      .addTarget(prometheus.target(expr='sum by (instance) (nsq_topic_channel_deferred_count{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }} deferred'))
+      .withUnit('short')
+      .addDataLink(
+        title='Show split by channels for ${__field.labels.instance}',
+        url='d/nsq-topics?var-instance=${__field.labels.instance}&%s' % nsqChannelDataLinkQueryParams
+      ),
+
+    local nsqTopicsE2e =
+      nsqgrafonnet.timeseries.new('$topic topic end-to-end processing time')
+      .addTarget(prometheus.target(expr='avg by (instance) (nsq_topic_e2e_processing_latency_99{%s})' % nsqTopicSelector, intervalFactor=1, legendFormat='{{ instance }} p99'))
+      .addDataLink(
+        title='Show split by topics for ${__field.labels.instance}',
+        url='d/nsq-topics?var-instance=${__field.labels.instance}&%s' % nsqTopicDataLinkQueryParams
+      )
+      .withUnit('ns'),
+
+    local nsqChannelE2e =
+      nsqgrafonnet.timeseries.new('$channel channel end-to-end processing time')
+      .addTarget(prometheus.target(expr='avg by (instance) (nsq_topic_channel_e2e_processing_latency_99{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{ instance }} p99'))
+      .addDataLink(
+        title='Show split by channels for ${__field.labels.instance}',
+        url='d/nsq-topics?var-instance=${__field.labels.instance}&%s' % nsqChannelDataLinkQueryParams
+      )
+      .withUnit('ns'),
+
+    'nsq-instances.json':
+      grafana.dashboard.new(
+        '%s Instances' % $._config.dashboardNamePrefix,
+        time_from='%s' % $._config.dashboardPeriod,
+        editable=false,
+        tags=($._config.dashboardTags),
+        timezone='%s' % $._config.dashboardTimezone,
+        refresh='%s' % $._config.dashboardRefresh,
+        graphTooltip='shared_crosshair',
+        uid='nsq-instances'
+      )
+      .addLink(grafana.link.dashboards(
+        asDropdown=false,
+        title='Other NSQ dashboards',
+        includeVars=true,
+        keepTime=true,
+        tags=($._config.dashboardTags),
+      ))
+      .addTemplate(
+        {
+          current: {
+            text: 'Prometheus',
+            value: 'Prometheus',
+          },
+          hide: 0,
+          label: 'Data Source',
+          name: 'datasource',
+          options: [],
+          query: 'prometheus',
+          refresh: 1,
+          regex: '',
+          type: 'datasource',
+        }
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: 'job',
+          name: 'job',
+          includeAll: true,
+          allValue: '.+',
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_message_count, job)',
+          refresh: 1,
+          regex: '',
+          type: 'query',
+          datasource: '$datasource',
+        },
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: null,
+          name: 'instance',
+          includeAll: true,
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_message_count{job=~"$job"},instance)',
+          refresh: 2,
+          regex: '',
+          type: 'query',
+        },
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: null,
+          name: 'topic',
+          includeAll: true,
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_message_count{job=~"$job",instance=~"$instance"},topic)',
+          refresh: 2,
+          regex: '',
+          type: 'query',
+        },
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: null,
+          name: 'channel',
+          includeAll: true,
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_channel_message_count{job=~"$job",instance=~"$instance",topic=~"$topic"},channel)',
+          refresh: 2,
+          regex: '',
+          type: 'query',
+        },
+      )
+      .addPanel(grafana.row.new(title='Topics'), gridPos={ x: 0, y: 0, w: 0, h: 0 })
+      .addPanel(nsqTopicsE2e, gridPos={ x: 0, y: 0, w: 24, h: 6 })
+      .addPanel(nsqTopics, gridPos={ x: 0, y: 6, w: 12, h: 8 })
+      .addPanel(nsqTopicsDepth, gridPos={ x: 12, y: 6, w: 12, h: 8 })
+      .addPanel(grafana.row.new(title='Channels'), gridPos={ x: 0, y: 14, w: 0, h: 0 })
+      .addPanel(nsqChannelE2e, gridPos={ x: 0, y: 14, w: 24, h: 6 })
+      .addPanel(nsqChannelClients, gridPos={ x: 0, y: 20, w: 12, h: 8 })
+      .addPanel(nsqChannelStats, gridPos={ x: 12, y: 20, w: 12, h: 8 })
+      .addPanel(grafana.row.new(title='Memory'), gridPos={ x: 0, y: 28, w: 0, h: 0 })
+      .addPanel(nsqHeapMemory, gridPos={ x: 0, y: 28, w: 12, h: 8 })
+      .addPanel(nsqHeapObjects, gridPos={ x: 12, y: 28, w: 12, h: 8 })
+      .addPanel(nsqNextGC, gridPos={ x: 0, y: 36, w: 12, h: 8 })
+      .addPanel(nsqGCpause, gridPos={ x: 12, y: 36, w: 12, h: 8 }),
+  },
+}

nsq-mixin/dashboards/topics.libsonnet

@@ -0,0 +1,239 @@
+local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libsonnet';
+local prometheus = grafana.prometheus;
+local nsqgrafonnet = import '../lib/nsqgrafonnet/nsqgrafonnet.libsonnet';
+
+{
+
+  grafanaDashboards+:: {
+
+    local nsqSelector = 'job=~"$job", instance=~"$instance"',
+    local nsqTopicSelector = nsqSelector + ',topic=~"$topic"',
+    local nsqChannelSelector = nsqTopicSelector + ',channel=~"$channel"',
+    local nsqTopicDataLinkQueryParams = '${datasource:queryparam}&${job:queryparam}&$${__url_time_range}',
+    local nsqChannelDataLinkQueryParams = '${datasource:queryparam}&${job:queryparam}&${__url_time_range}',
+
+    local nsqTopics =
+      nsqgrafonnet.timeseries.new('Topic messages')
+      .addTarget(prometheus.target(expr='avg by (topic) (rate(nsq_topic_message_count{%s}[$__rate_interval]))' % nsqTopicSelector, intervalFactor=1, legendFormat='{{topic}} rps'))
+      .addTarget(prometheus.target(expr='sum by () (rate(nsq_topic_message_bytes{%s}[$__rate_interval]))' % nsqTopicSelector, intervalFactor=1, legendFormat='selected topics Bps rate'))
+      + {
+        fieldConfig+: {
+          overrides: [
+            {
+              matcher: {
+                id: 'byRegexp',
+                options: '.+ rps',
+              },
+              // must be in overrides to exlude bps sum rate
+              properties: [
+                {
+                  id: 'links',
+                  value: [
+                    {
+                      title: 'Show split by nsqd instances for ${__field.labels.topic} ',
+                      url: 'd/nsq-instances?var-topic=${__field.labels.topic}&%s' % nsqTopicDataLinkQueryParams,
+                    },
+                  ],
+                },
+
+              ],
+            },
+            {
+              matcher: {
+                id: 'byRegexp',
+                options: '.+ Bps rate',
+              },
+              properties: [
+                {
+                  id: 'custom.drawStyle',
+                  value: 'bars',
+                },
+                {
+                  id: 'custom.axisPlacement',
+                  value: 'right',
+                },
+                {
+                  id: 'custom.stacking',
+                  value: {
+                    mode: 'normal',
+                    group: 'A',
+                  },
+                },
+                {
+                  id: 'color',
+                  value: {
+                    mode: 'fixed',
+                    fixedColor: '#cef7913b',
+                  },
+                },
+                {
+                  id: 'unit',
+                  value: 'Bps',
+                },
+                {
+                  id: 'custom.barAlignment',
+                  value: -1,
+                },
+                {
+                  id: 'custom.fillOpacity',
+                  value: 10,
+                },
+              ],
+            },
+          ],
+        },
+      },
+
+    local nsqTopicsDepth =
+      nsqgrafonnet.timeseries.new('Topic depth')
+      .addTarget(prometheus.target(expr='sum without (instance) (nsq_topic_depth{%s})' % nsqTopicSelector, intervalFactor=1, legendFormat='{{topic}} depth'))
+      .addTarget(prometheus.target(expr='sum without (instance) (nsq_topic_backend_depth{%s})' % nsqTopicSelector, intervalFactor=1, legendFormat='{{topic}} memory+disk depth'))
+      .withUnit('short')
+      .addDataLink(
+        title='Show split by nsqd instances for ${__field.labels.topic}',
+        url='d/nsq-instances?var-topic=${__field.labels.topic}&%s' % nsqTopicDataLinkQueryParams
+      ),
+
+    local nsqChannelClients =
+      nsqgrafonnet.timeseries.new('Channel clients')
+      .addTarget(prometheus.target(expr='sum without (instance) (nsq_topic_channel_clients{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}}'))
+      .addDataLink(
+        title='Show split by nsqd instances for ${__field.labels.topic}/${__field.labels.channel}',
+        url='d/nsq-instances?var-topic=${__field.labels.topic}&var-channel=${__field.labels.channel}&%s' % nsqChannelDataLinkQueryParams
+      ),
+
+    local nsqChannelStats =
+      nsqgrafonnet.timeseries.new('Channel stats')
+      .addTarget(prometheus.target(expr='sum without (instance) (nsq_topic_channel_depth{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}} depth'))
+      .addTarget(prometheus.target(expr='sum without (instance) (nsq_topic_channel_backend_depth{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}} memory+disk depth'))
+      .addTarget(prometheus.target(expr='sum without (instance) (nsq_topic_channel_in_flight_count{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}} in-flight'))
+      .addTarget(prometheus.target(expr='avg without (instance) (rate(nsq_topic_channel_requeue_count{%s}[$__rate_interval]))' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}} requeue'))
+      .addTarget(prometheus.target(expr='avg without (instance) (rate(nsq_topic_channel_timeout_count{%s}[$__rate_interval]))' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}} timeout'))
+      .addTarget(prometheus.target(expr='sum without (instance) (nsq_topic_channel_deferred_count{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}} deferred'))
+      .withUnit('short')
+      .addDataLink(
+        title='Show split by nsqd instances for ${__field.labels.topic}/${__field.labels.channel}',
+        url='d/nsq-instances?var-topic=${__field.labels.topic}&var-channel=${__field.labels.channel}&%s' % nsqChannelDataLinkQueryParams
+      ),
+
+    local nsqTopicsE2e =
+      nsqgrafonnet.timeseries.new('Topic end-to-end processing time')
+      .addTarget(prometheus.target(expr='avg without (instance) (nsq_topic_e2e_processing_latency_99{%s})' % nsqTopicSelector, intervalFactor=1, legendFormat='{{topic}} p99'))
+      .addDataLink(
+        title='Show split by nsqd instances for ${__field.labels.topic}',
+        url='d/nsq-instances?var-topic=${__field.labels.topic}&%s' % nsqTopicDataLinkQueryParams
+      )
+      .withUnit('ns'),
+
+    local nsqChannelE2e =
+      nsqgrafonnet.timeseries.new('Channel end-to-end processing time')
+      .addTarget(prometheus.target(expr='avg without (instance) (nsq_topic_channel_e2e_processing_latency_99{%s})' % nsqChannelSelector, intervalFactor=1, legendFormat='{{topic}}/{{channel}} p99'))
+      .addDataLink(
+        title='Show split by nsqd instances for ${__field.labels.topic}/${__field.labels.channel}',
+        url='d/nsq-instances?var-topic=${__field.labels.topic}&var-channel=${__field.labels.channel}&%s' % nsqChannelDataLinkQueryParams
+      )
+      .withUnit('ns'),
+
+    'nsq-topics.json':
+      grafana.dashboard.new(
+        '%s Topics' % $._config.dashboardNamePrefix,
+        time_from='%s' % $._config.dashboardPeriod,
+        editable=false,
+        tags=($._config.dashboardTags),
+        timezone='%s' % $._config.dashboardTimezone,
+        refresh='%s' % $._config.dashboardRefresh,
+        graphTooltip='shared_crosshair',
+        uid='nsq-topics'
+      )
+      .addLink(grafana.link.dashboards(
+        asDropdown=false,
+        title='Other NSQ dashboards',
+        includeVars=true,
+        keepTime=true,
+        tags=($._config.dashboardTags),
+      ))
+      .addTemplate(
+        {
+          current: {
+            text: 'Prometheus',
+            value: 'Prometheus',
+          },
+          hide: 0,
+          label: 'Data Source',
+          name: 'datasource',
+          options: [],
+          query: 'prometheus',
+          refresh: 1,
+          regex: '',
+          type: 'datasource',
+        }
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: 'job',
+          name: 'job',
+          includeAll: true,
+          allValue: '.+',
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_message_count, job)',
+          refresh: 1,
+          regex: '',
+          type: 'query',
+          datasource: '$datasource',
+        },
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: null,
+          name: 'instance',
+          includeAll: true,
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_message_count{job=~"$job"},instance)',
+          refresh: 2,
+          regex: '',
+          type: 'query',
+        },
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: null,
+          name: 'topic',
+          includeAll: true,
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_message_count{job=~"$job",instance=~"$instance"},topic)',
+          refresh: 2,
+          regex: '',
+          type: 'query',
+        },
+      )
+      .addTemplate(
+        {
+          hide: 0,
+          label: null,
+          name: 'channel',
+          includeAll: true,
+          multi: true,
+          options: [],
+          query: 'label_values(nsq_topic_channel_message_count{job=~"$job",instance=~"$instance",topic=~"$topic"},channel)',
+          refresh: 2,
+          regex: '',
+          type: 'query',
+        },
+      )
+      .addPanel(grafana.row.new(title='Topics'), gridPos={ x: 0, y: 0, w: 0, h: 0 })
+      .addPanel(nsqTopicsE2e, gridPos={ x: 0, y: 0, w: 24, h: 6 })
+      .addPanel(nsqTopics, gridPos={ x: 0, y: 6, w: 12, h: 8 })
+      .addPanel(nsqTopicsDepth, gridPos={ x: 12, y: 6, w: 12, h: 8 })
+      .addPanel(grafana.row.new(title='Channels'), gridPos={ x: 0, y: 14, w: 0, h: 0 })
+      .addPanel(nsqChannelE2e, gridPos={ x: 0, y: 14, w: 24, h: 6 })
+      .addPanel(nsqChannelClients, gridPos={ x: 0, y: 20, w: 12, h: 8 })
+      .addPanel(nsqChannelStats, gridPos={ x: 12, y: 20, w: 12, h: 8 }),
+
+  },
+}

nsq-mixin/jsonnetfile.json

@@ -0,0 +1,15 @@
+{
+    "version": 1,
+    "dependencies": [
+      {
+        "source": {
+          "git": {
+            "remote": "https://github.com/grafana/grafonnet-lib.git",
+            "subdir": "grafonnet"
+          }
+        },
+        "version": "master"
+      }
+    ],
+    "legacyImports": false
+  }

nsq-mixin/lib/alerts.jsonnet

@@ -0,0 +1 @@
+std.manifestYamlDoc((import '../mixin.libsonnet').prometheusAlerts)

nsq-mixin/lib/dashboards.jsonnet

@@ -0,0 +1,6 @@
+local dashboards = (import '../mixin.libsonnet').grafanaDashboards;
+
+{
+  [name]: dashboards[name]
+  for name in std.objectFields(dashboards)
+}

nsq-mixin/lib/nsqgrafonnet/nsqgrafonnet.libsonnet

@@ -0,0 +1,3 @@
+{
+  timeseries:: import 'timeseries.libsonnet',
+}

nsq-mixin/lib/nsqgrafonnet/timeseries.libsonnet

@@ -0,0 +1,64 @@
+local grafana = import 'github.com/grafana/grafonnet-lib/grafonnet/grafana.libsonnet';
+local prometheus = grafana.prometheus;
+
+{
+  new(title, description=null)::
+    grafana.graphPanel.new(
+      title,
+      datasource='$datasource',
+      description=description
+    )
+    +
+    {
+      type: 'timeseries',
+      options+: {
+        tooltip: {
+          mode: 'multi',
+        },
+      },
+      fieldConfig+: {
+        defaults+: {
+          custom+: {
+            lineInterpolation: 'smooth',
+            fillOpacity: 0,
+            showPoints: 'never',
+          },
+        },
+      },
+
+      addDataLink(title, url):: self {
+
+        fieldConfig+: {
+          defaults+: {
+            links: [
+              {
+                title: title,
+                url: url,
+              },
+            ],
+          },
+        },
+      },
+
+      withUnit(unit):: self {
+
+        fieldConfig+: {
+          defaults+: {
+            unit: unit,
+          },
+        },
+      },
+
+      withFillOpacity(opacity):: self {
+        fieldConfig+: {
+          defaults+: {
+            custom+: {
+
+              fillOpacity: opacity,
+            },
+          },
+        },
+
+      },
+    },
+}

nsq-mixin/lib/rules.jsonnet

@@ -0,0 +1 @@
+std.manifestYamlDoc((import '../mixin.libsonnet').prometheusRules)

nsq-mixin/mixin.libsonnet

@@ -0,0 +1,3 @@
+(import 'dashboards/dashboards.libsonnet') +
+(import 'alerts/alerts.libsonnet') +
+(import 'config.libsonnet')