Merge pull request #636 from mendix/develop

Release 2023-05-25

buildpack/telemetry/metrics.py

@@ -45,12 +45,12 @@
             "values": ["mx.runtime.user.login"],
         },
         # Filter out irrelevant metrics to reduce
-        # the payload size passed to TSS
+        # the payload size passed to TSS/TFR
         # https://docs.mendix.com/refguide/metrics#filters
         {
             "type": "nameStartsWith",
             "result": "deny",
-            "values": ["commons.pool"],
+            "values": ["commons.pool", "jvm.buffer"],
         },
     ],
 }

buildpack/telemetry/telegraf.py

@@ -18,7 +18,7 @@
 from lib.m2ee.util import strtobool
 from jinja2 import Template
 
-from . import datadog, metrics, mx_java_agent, appdynamics, dynatrace
+from . import datadog, metrics, mx_java_agent, appdynamics, dynatrace, splunk
 
 NAMESPACE = "telegraf"
 DEPENDENCY = f"{NAMESPACE}.agent"
@@ -218,6 +218,26 @@ def _fix_metrics_registries_config(m2ee):
     )
 
 
+def _get_integration_usages():
+    """
+    Collects usage information of third-party integrations.
+    We emit these information via telegraf for internal monitoring
+    """
+    integration_usages = {}
+    checker_methods = {
+        "datadog": datadog.is_enabled,
+        "dynatrace": dynatrace.is_telegraf_enabled,
+        "appdynamics": appdynamics.appdynamics_used,
+        "splunk": splunk.is_splunk_enabled,
+    }
+
+    for integration, is_enabled in checker_methods.items():
+        if is_enabled():
+            integration_usages[integration] = 1
+
+    return integration_usages
+
+
 def update_config(m2ee, app_name):
     runtime_version = runtime.get_runtime_version()
     if not is_enabled(runtime_version) or not _is_installed():
@@ -266,6 +286,8 @@ def update_config(m2ee, app_name):
         telegraf_fileout_enabled=strtobool(
             os.getenv("TELEGRAF_FILEOUT_ENABLED", "false")
         ),
+        runtime_version=runtime_version,
+        integration_usages=_get_integration_usages(),
     )
 
     logging.debug("Writing Telegraf configuration file...")

etc/telegraf/telegraf.toml.j2

@@ -23,6 +23,34 @@
     hostname = "{{ hostname }}"
     omit_hostname = false
 
+# Integration usage monitoring
+{% for integration_name, is_enabled in integration_usages.items() %}
+[[inputs.exec]]
+    commands = ["echo {{ is_enabled }}"]
+    name_override = "integration_usage"
+    interval = "6h"
+    data_format = "value"
+    data_type = "integer"
+    [inputs.exec.tags]
+        app_name = "{{ app_name }}"
+        instance_index = "{{ cf_instance_index }}"
+        integration_name = "{{ integration_name }}"
+        internal_metrics = "true"
+{% endfor %}
+
+# Separate metric for runtime_version
+[[inputs.exec]]
+    commands = ["echo {{ runtime_version }}"]
+    name_override = "runtime_version"
+    interval = "24h"
+    data_format = "value"
+    data_type = "string"
+    [inputs.exec.tags]
+        app_name = "{{ app_name }}"
+        instance_index = "{{ cf_instance_index }}"
+        internal_metrics = "true"
+
+
 {% if statsd_port and not datadog_api_key %}
 # StatsD input for Mendix Java Agent
 [[inputs.statsd]]
@@ -261,6 +289,7 @@
     # Ignore any micrometer_metrics
     [outputs.datadog.tagdrop]
         micrometer_metrics = ["true"]
+        internal_metrics = ["true"]
 {% endif %}
 
 {% if http_outputs %}
@@ -283,6 +312,7 @@
     # from influxdb_listener plugin
     [outputs.http.tagdrop]
         micrometer_metrics = ["true"]
+        internal_metrics = ["true"]
 
 {% endfor %}
 {% endif %}
@@ -298,6 +328,7 @@
     # Ignore any micrometer_metrics
     [outputs.exec.tagdrop]
         micrometer_metrics = ["true"]
+        internal_metrics = ["true"]
 {% endif %}
 
 {% if dynatrace_enabled %}
@@ -312,9 +343,10 @@
 # The API token needs "Ingest metrics" scope permission.
     api_token = "{{ dynatrace_config['token'] }}"
 #
-# Ignore any micrometer_metrics
+# Ignore any metric with micrometer_metrics or internal_metrics tags, since they meant for our TSS/TFR metrics path
     [outputs.dynatrace.tagdrop]
         micrometer_metrics = ["true"]
+        internal_metrics = ["true"]
 #
 # Optional dimensions to be added to every metric
     [outputs.dynatrace.default_dimensions]
@@ -347,6 +379,8 @@
   app_name = "{{ app_name }}"
 
 [[outputs.http]]
+  alias = "mx-trends-metrics"
+
   ## URL is the address to send metrics to
   url = "{{ trends_storage_url }}"
 
@@ -363,6 +397,52 @@
   # tagexlude drops any non-relevant tags
   tagexclude = ["host"]
 
+  # Drop `mx_runtime_user_login` metrics
+  # We drop them as we don't want it to be part of
+  # the payload which is used for Cloud graphs
+  namedrop = ["mx_runtime_user_login"]
+
+  ## Additional HTTP headers
+  [outputs.http.headers]
+  Content-Type = "application/json"
+  # custom header field
+  Micrometer-Metrics = "true"
+
+  # Pass only those metrics that has below tag set
+  [outputs.http.tagpass]
+  micrometer_metrics = ["true"]
+  internal_metrics = ["true"]
+
+[[outputs.http]]
+  alias = "datalake-metrics"
+
+  # This http output is ONLY to send all the datalake metrics
+  # as a single payload. For instance;
+  #  `mx_runtime_user_login`
+  namepass = ["mx_runtime_user_login"]
+
+  ## URL is the address to send metrics to
+  url = "{{ trends_storage_url }}"
+
+  ## Timeout for HTTP message
+  timeout = "10s"
+
+  # Higher flush interval and batch size, so that we
+  # don't bombard TFR and DataLake with too many requests, but
+  # few requests with bigger payloads
+  metric_batch_size = 3000
+  flush_interval = "30s"
+
+  ## HTTP method, one of: "POST" or "PUT"
+  method = "POST"
+
+  ## Data format to output.
+  data_format = "json"
+  json_timestamp_units = "1ns"
+
+  # tagexlude drops any non-relevant tags
+  tagexclude = ["host"]
+
   ## Additional HTTP headers
   [outputs.http.headers]
   Content-Type = "application/json"
@@ -399,9 +479,15 @@
   # tagexlude drops any non-relevant tags
   tagexclude = ["host"]
 
+  # Drop `mx_runtime_user_login` metrics
+  # We drop them as we don't want it to be part of
+  # the payload which is used for Cloud graphs
+  namedrop = ["mx_runtime_user_login"]
+
   # Pass only those metrics that has below tag set
   [outputs.file.tagpass]
   micrometer_metrics = ["true"]
+  internal_metrics = ["true"]
 
 {% endif %}
 ####################################################################################

tests/integration/test_telegraf.py

@@ -93,7 +93,7 @@ def test_telegraph_config_for_micrometer(self):
 
         # Ensure the trends-storage-url is set
         output = self.run_on_container(
-            "cat {} | grep -A2 outputs.http".format(telegraf_config_path)
+            "cat {} | grep -A5 outputs.http".format(telegraf_config_path)
         )
         assert output is not None
         assert str(output).find("some-fake-url") >= 0