From dec828c3571bb1e4fc243b7764654a1d9626ff5b Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 31 Jan 2025 09:17:51 -0800 Subject: [PATCH] [Tracing] Implement initial weblog tests for the runtime metrics feature (#3878) --- manifests/cpp.yml | 1 + manifests/dotnet.yml | 3 +- manifests/golang.yml | 3 +- manifests/java.yml | 11 ++- manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 3 +- manifests/ruby.yml | 3 +- tests/test_config_consistency.py | 70 +++++++++++++++---- utils/_context/_scenarios/__init__.py | 8 +++ utils/build/docker/agent.Dockerfile | 1 + .../java/spring-boot-openliberty.Dockerfile | 1 - .../java/spring-boot-wildfly.Dockerfile | 1 + utils/interfaces/_agent.py | 12 ++++ 14 files changed, 101 insertions(+), 18 deletions(-) diff --git a/manifests/cpp.yml b/manifests/cpp.yml index 84f55829f9..bf39d60399 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -259,6 +259,7 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Empty: missing_feature Test_Config_RuntimeMetrics_Default: missing_feature Test_Config_RuntimeMetrics_Enabled: missing_feature + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: missing_feature Test_Config_UnifiedServiceTagging_CustomService: missing_feature Test_Config_UnifiedServiceTagging_Default: missing_feature test_distributed.py: diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 9b4ee0d979..f578832470 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -507,7 +507,8 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Default: v3.4.1 Test_Config_ObfuscationQueryStringRegexp_Empty: v3.4.1 Test_Config_RuntimeMetrics_Default: incomplete_test_app (test needs to account for dotnet runtime metrics) - Test_Config_RuntimeMetrics_Enabled: incomplete_test_app (test needs to account for dotnet runtime metrics) + Test_Config_RuntimeMetrics_Enabled: v2.0.0 + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: v2.0.0 Test_Config_UnifiedServiceTagging_CustomService: v3.3.0 Test_Config_UnifiedServiceTagging_Default: v3.3.0 test_data_integrity.py: diff --git a/manifests/golang.yml b/manifests/golang.yml index ff78eb7e7e..5b8c24e73a 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -637,7 +637,8 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Default: v1.67.0 Test_Config_ObfuscationQueryStringRegexp_Empty: v1.67.0 Test_Config_RuntimeMetrics_Default: incomplete_test_app (test needs to account for golang runtime metrics) - Test_Config_RuntimeMetrics_Enabled: incomplete_test_app (test needs to account for golang runtime metrics) + Test_Config_RuntimeMetrics_Enabled: incomplete_test_app (should be in v1.18.0 but we're not receiving series in tests) + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: incomplete_test_app (should be in v1.18.0 but we're not receiving series in tests) Test_Config_UnifiedServiceTagging_CustomService: v1.67.0 Test_Config_UnifiedServiceTagging_Default: v1.67.0 test_data_integrity.py: diff --git a/manifests/java.yml b/manifests/java.yml index 156b2df8b5..00717dc74c 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1724,7 +1724,16 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Default: v1.39.0 Test_Config_ObfuscationQueryStringRegexp_Empty: v1.39.0 Test_Config_RuntimeMetrics_Default: incomplete_test_app (test needs to account for java runtime metrics) - Test_Config_RuntimeMetrics_Enabled: incomplete_test_app (test needs to account for java runtime metrics) + Test_Config_RuntimeMetrics_Enabled: + '*': v0.64.0 + spring-boot-3-native: missing_feature (GraalVM. Tracing support only) + spring-boot-openliberty: incomplete_test_app (needs investigation to understand why test is failing) + spring-boot-wildfly: incomplete_test_app (needs investigation to understand why test is failing) + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: + '*': v0.64.0 + spring-boot-3-native: missing_feature (GraalVM. Tracing support only) + spring-boot-openliberty: incomplete_test_app (needs investigation to understand why test is failing) + spring-boot-wildfly: incomplete_test_app (needs investigation to understand why test is failing) Test_Config_UnifiedServiceTagging_CustomService: v1.39.0 Test_Config_UnifiedServiceTagging_Default: v1.39.0 test_data_integrity.py: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index e93ba7c908..82577f2a50 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -908,6 +908,7 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Empty: *ref_3_0_0 Test_Config_RuntimeMetrics_Default: *ref_3_0_0 Test_Config_RuntimeMetrics_Enabled: *ref_3_0_0 + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: missing_feature Test_Config_UnifiedServiceTagging_CustomService: *ref_5_25_0 Test_Config_UnifiedServiceTagging_Default: *ref_5_25_0 test_distributed.py: diff --git a/manifests/php.yml b/manifests/php.yml index 52b0509978..9800e5def2 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -490,6 +490,7 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Empty: v1.5.0 Test_Config_RuntimeMetrics_Default: missing_feature Test_Config_RuntimeMetrics_Enabled: missing_feature + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: missing_feature Test_Config_UnifiedServiceTagging_CustomService: v1.4.0 Test_Config_UnifiedServiceTagging_Default: v1.4.0 test_distributed.py: diff --git a/manifests/python.yml b/manifests/python.yml index 9545553727..cf2cfc948b 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -906,7 +906,8 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Configured: v2.0.0 Test_Config_ObfuscationQueryStringRegexp_Empty: v2.15.0 Test_Config_RuntimeMetrics_Default: incomplete_test_app (test needs to account for python runtime metrics) - Test_Config_RuntimeMetrics_Enabled: incomplete_test_app (test needs to account for python runtime metrics) + Test_Config_RuntimeMetrics_Enabled: incomplete_test_app (Python seems to send sketches instead of gauges/counters) + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: missing_feature Test_Config_UnifiedServiceTagging_CustomService: v2.0.0 Test_Config_UnifiedServiceTagging_Default: v2.0.0 test_data_integrity.py: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index a11008de44..66d7d23936 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -513,7 +513,8 @@ tests/: Test_Config_ObfuscationQueryStringRegexp_Default: bug (APMAPI-1013) Test_Config_ObfuscationQueryStringRegexp_Empty: missing_feature (environment variable is not supported) Test_Config_RuntimeMetrics_Default: incomplete_test_app (test needs to account for ruby runtime metrics) - Test_Config_RuntimeMetrics_Enabled: incomplete_test_app (test needs to account for ruby runtime metrics) + Test_Config_RuntimeMetrics_Enabled: missing_feature (should be in v0.44.0 but we're not receiving series in tests) + Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: missing_feature Test_Config_UnifiedServiceTagging_CustomService: v2.0.0 Test_Config_UnifiedServiceTagging_Default: v2.0.0 test_distributed.py: diff --git a/tests/test_config_consistency.py b/tests/test_config_consistency.py index 83aebef435..47ea9cfd88 100644 --- a/tests/test_config_consistency.py +++ b/tests/test_config_consistency.py @@ -4,12 +4,14 @@ import re import json +import time from utils import weblog, interfaces, scenarios, features, rfc, irrelevant, context, bug, missing_feature from utils.tools import logger # get the default log output stdout = interfaces.library_stdout if context.library != "dotnet" else interfaces.library_dotnet_managed runtime_metrics = {"nodejs": "runtime.node.mem.heap_total"} +runtime_metrics_langs = [".NET", "go", "nodejs", "python", "ruby"] log_injection_fields = {"nodejs": {"message": "msg"}} @@ -555,19 +557,63 @@ def test_log_injection_128bit_traceid_disabled(self): @scenarios.runtime_metrics_enabled @features.tracing_configuration_consistency class Test_Config_RuntimeMetrics_Enabled: - """Verify runtime metrics are enabled when DD_RUNTIME_METRICS_ENABLED=true""" + """Verify runtime metrics are enabled when DD_RUNTIME_METRICS_ENABLED=true and that they have the proper tags""" - # This test verifies runtime metrics by asserting the prescene of a metric in the dogstatsd endpoint - def test_config_runtimemetrics_enabled(self): - for data in interfaces.library.get_data("/dogstatsd/v2/proxy"): - lines = data["request"]["content"].split("\n") - metric_found = False - for line in lines: - if runtime_metrics[context.library.library] in line: - metric_found = True - break - assert metric_found, f"The metric {runtime_metrics[context.library.library]} was not found in any line" - break + def setup_main(self): + self.req = weblog.get("/") + + # Wait for 10s to allow the tracer to send runtime metrics on the default 10s interval + time.sleep(10) + + def test_main(self): + assert self.req.status_code == 200 + + runtime_metrics = [ + metric + for _, metric in interfaces.agent.get_metrics() + if metric["metric"].startswith("runtime.") or metric["metric"].startswith("jvm.") + ] + assert len(runtime_metrics) > 0 + + for metric in runtime_metrics: + tags = {tag.split(":")[0]: tag.split(":")[1] for tag in metric["tags"]} + assert tags.get("lang") in runtime_metrics_langs or tags.get("lang") is None + + # Test that Unified Service Tags are added to the runtime metrics + assert tags["service"] == "weblog" + assert tags["env"] == "system-tests" + assert tags["version"] == "1.0.0" + + # Test that DD_TAGS are added to the runtime metrics + # DD_TAGS=key1:val1,key2:val2 in default weblog containers + assert tags["key1"] == "val1" + assert tags["key2"] == "val2" + + +@scenarios.runtime_metrics_enabled +@features.tracing_configuration_consistency +class Test_Config_RuntimeMetrics_Enabled_WithRuntimeId: + """Verify runtime metrics are enabled when DD_RUNTIME_METRICS_ENABLED=true and that they have the runtime-id tag""" + + def setup_main(self): + self.req = weblog.get("/") + + # Wait for 10s to allow the tracer to send runtime metrics on the default 10s interval + time.sleep(10) + + def test_main(self): + assert self.req.status_code == 200 + + runtime_metrics = [ + metric + for _, metric in interfaces.agent.get_metrics() + if metric["metric"].startswith("runtime.") or metric["metric"].startswith("jvm.") + ] + assert len(runtime_metrics) > 0 + + for metric in runtime_metrics: + tags = {tag.split(":")[0]: tag.split(":")[1] for tag in metric["tags"]} + assert "runtime-id" in tags @rfc("https://docs.google.com/document/d/1kI-gTAKghfcwI7YzKhqRv2ExUstcHqADIWA4-TZ387o/edit#heading=h.8v16cioi7qxp") diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 62af784157..e1a86243b8 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -844,7 +844,15 @@ class _Scenarios: runtime_metrics_enabled = EndToEndScenario( "RUNTIME_METRICS_ENABLED", + # Add environment variable DD_DOGSTATSD_START_DELAY=0 to avoid the default 30s startup delay in the Java tracer. + # That delay is used in production to reduce the impact on startup and other side-effects on various application + # servers. These considerations do not apply to the system-tests environment so we can reduce it to 0s. + weblog_env={"DD_DOGSTATSD_START_DELAY": "0"}, runtime_metrics_enabled=True, + # Disable the proxy in between weblog and the agent so that we can send metrics (via UDP) to the agent. + # The mitmproxy can only proxy UDP traffic by doing a host-wide transparent proxy, but we currently + # via specific ports. As a result, with the proxy enabled all UDP traffic is being dropped. + use_proxy_for_weblog=False, doc="Test runtime metrics", ) diff --git a/utils/build/docker/agent.Dockerfile b/utils/build/docker/agent.Dockerfile index 6d73d96f80..b39208a366 100644 --- a/utils/build/docker/agent.Dockerfile +++ b/utils/build/docker/agent.Dockerfile @@ -10,6 +10,7 @@ RUN set -eux;\ # Datadog agent conf RUN echo '\ log_level: DEBUG\n\ +dogstatsd_non_local_traffic: true\n\ apm_config:\n\ apm_non_local_traffic: true\n\ trace_buffer: 5\n\ diff --git a/utils/build/docker/java/spring-boot-openliberty.Dockerfile b/utils/build/docker/java/spring-boot-openliberty.Dockerfile index e974cf9197..849b770986 100644 --- a/utils/build/docker/java/spring-boot-openliberty.Dockerfile +++ b/utils/build/docker/java/spring-boot-openliberty.Dockerfile @@ -22,7 +22,6 @@ COPY --from=build /binaries/SYSTEM_TESTS_LIBRARY_VERSION SYSTEM_TESTS_LIBRARY_VE COPY --from=build /app/target/myproject-0.0.1-SNAPSHOT.jar /app/app.jar COPY --from=build /dd-tracer/dd-java-agent.jar . -ENV DD_JMXFETCH_ENABLED=false ENV DD_TRACE_HEADER_TAGS='user-agent:http.request.headers.user-agent' # FIXME: Fails on APPSEC_BLOCKING, see APPSEC-51405 # ENV DD_TRACE_INTERNAL_EXIT_ON_FAILURE=true diff --git a/utils/build/docker/java/spring-boot-wildfly.Dockerfile b/utils/build/docker/java/spring-boot-wildfly.Dockerfile index 87947abdeb..cca2d0f347 100644 --- a/utils/build/docker/java/spring-boot-wildfly.Dockerfile +++ b/utils/build/docker/java/spring-boot-wildfly.Dockerfile @@ -26,6 +26,7 @@ COPY --from=build /dd-tracer/dd-java-agent.jar . COPY ./utils/build/docker/java/app.sh /app/app.sh RUN chmod +x /app/app.sh +ENV DD_APP_CUSTOMLOGMANAGER=true ENV DD_TRACE_HEADER_TAGS='user-agent:http.request.headers.user-agent' ENV DD_TRACE_INTERNAL_EXIT_ON_FAILURE=true ENV APP_EXTRA_ARGS="-Djboss.http.port=7777 -b=0.0.0.0" diff --git a/utils/interfaces/_agent.py b/utils/interfaces/_agent.py index fac292abec..a5b536c46b 100644 --- a/utils/interfaces/_agent.py +++ b/utils/interfaces/_agent.py @@ -143,6 +143,18 @@ def get_spans(self, request=None): def get_spans_list(self, request): return [span for _, span in self.get_spans(request)] + def get_metrics(self): + """Attempts to fetch the metrics the agent will submit to the backend.""" + + for data in self.get_data(path_filters="/api/v2/series"): + if "series" not in data["request"]["content"]: + raise ValueError("series property is missing in agent payload") + + content = data["request"]["content"]["series"] + + for point in content: + yield data, point + def get_dsm_data(self): return self.get_data(path_filters="/api/v0.1/pipeline_stats")