From 43820c8797aa2aac71e1e9a8f2a69f2e98231039 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 13:51:24 +0200 Subject: [PATCH 1/4] fix(asm): switch back to default if remote config stop sending the security rule file [backport 2.9] (#10053) Backport e3f90045c677465ad132daf55abd5f4776f3f856 from #10052 to 2.9. backporting https://github.com/DataDog/dd-trace-py/pull/10030 APPSEC-54105 (cherry picked from commit bc50e9cd69c4a21a101e11bf250a7904dc6b6937) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) Co-authored-by: Christophe Papazian <114495376+christophe-papazian@users.noreply.github.com> --- ddtrace/appsec/_processor.py | 25 ++++++++++++------- ddtrace/appsec/_remoteconfiguration.py | 17 ++++++++----- ...ix_rc_asm_dd_no_file-37e6f733583e334c.yaml | 4 +++ tests/appsec/appsec/test_processor.py | 2 +- .../appsec/appsec/test_remoteconfiguration.py | 12 +++++++++ 5 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 releasenotes/notes/fix_rc_asm_dd_no_file-37e6f733583e334c.yaml diff --git a/ddtrace/appsec/_processor.py b/ddtrace/appsec/_processor.py index 3af6f18547d..588d6a378ba 100644 --- a/ddtrace/appsec/_processor.py +++ b/ddtrace/appsec/_processor.py @@ -140,7 +140,7 @@ def _get_rate_limiter() -> RateLimiter: @dataclasses.dataclass(eq=False) class AppSecSpanProcessor(SpanProcessor): - rules: str = dataclasses.field(default_factory=get_rules) + rule_filename: str = dataclasses.field(default_factory=get_rules) obfuscation_parameter_key_regexp: bytes = dataclasses.field( default_factory=get_appsec_obfuscation_parameter_key_regexp ) @@ -159,28 +159,35 @@ def __post_init__(self) -> None: from ddtrace.appsec._ddwaf import DDWaf try: - with open(self.rules, "r") as f: - rules = json.load(f) + with open(self.rule_filename, "r") as f: + self._rules = json.load(f) except EnvironmentError as err: if err.errno == errno.ENOENT: - log.error("[DDAS-0001-03] ASM could not read the rule file %s. Reason: file does not exist", self.rules) + log.error( + "[DDAS-0001-03] ASM could not read the rule file %s. Reason: file does not exist", + self.rule_filename, + ) else: # TODO: try to log reasons - log.error("[DDAS-0001-03] ASM could not read the rule file %s.", self.rules) + log.error("[DDAS-0001-03] ASM could not read the rule file %s.", self.rule_filename) raise except JSONDecodeError: - log.error("[DDAS-0001-03] ASM could not read the rule file %s. Reason: invalid JSON file", self.rules) + log.error( + "[DDAS-0001-03] ASM could not read the rule file %s. Reason: invalid JSON file", self.rule_filename + ) raise except Exception: # TODO: try to log reasons - log.error("[DDAS-0001-03] ASM could not read the rule file %s.", self.rules) + log.error("[DDAS-0001-03] ASM could not read the rule file %s.", self.rule_filename) raise try: - self._ddwaf = DDWaf(rules, self.obfuscation_parameter_key_regexp, self.obfuscation_parameter_value_regexp) + self._ddwaf = DDWaf( + self._rules, self.obfuscation_parameter_key_regexp, self.obfuscation_parameter_value_regexp + ) if not self._ddwaf._handle or self._ddwaf.info.failed: stack_trace = "DDWAF.__init__: invalid rules\n ruleset: %s\nloaded:%s\nerrors:%s\n" % ( - rules, + self._rules, self._ddwaf.info.loaded, self._ddwaf.info.errors, ) diff --git a/ddtrace/appsec/_remoteconfiguration.py b/ddtrace/appsec/_remoteconfiguration.py index ca2db4f6710..23f30b335c4 100644 --- a/ddtrace/appsec/_remoteconfiguration.py +++ b/ddtrace/appsec/_remoteconfiguration.py @@ -91,8 +91,9 @@ def _add_rules_to_list(features: Mapping[str, Any], feature: str, message: str, if rules is not None: try: if ruleset.get(feature) is None: - ruleset[feature] = [] - ruleset[feature] += rules + ruleset[feature] = rules + else: + ruleset[feature] = ruleset[feature] + rules log.debug("Reloading Appsec %s: %s", message, str(rules)[:20]) except json.JSONDecodeError: log.error("ERROR Appsec %s: invalid JSON content from remote configuration", message) @@ -114,14 +115,18 @@ def _appsec_rules_data(features: Mapping[str, Any], test_tracer: Optional[Tracer if features and tracer._appsec_processor: ruleset = {} # type: dict[str, Optional[list[Any]]] - _add_rules_to_list(features, "rules_data", "rules data", ruleset) + if features.get("rules", None) == []: + # if rules is empty, we need to switch back to the default rules + ruleset = tracer._appsec_processor._rules.copy() or {} + _add_rules_to_list(features, "actions", "actions", ruleset) _add_rules_to_list(features, "custom_rules", "custom rules", ruleset) - _add_rules_to_list(features, "rules", "Datadog rules", ruleset) _add_rules_to_list(features, "exclusions", "exclusion filters", ruleset) + _add_rules_to_list(features, "exclusion_data", "exclusion data", ruleset) + _add_rules_to_list(features, "processors", "processors", ruleset) + _add_rules_to_list(features, "rules", "Datadog rules", ruleset) + _add_rules_to_list(features, "rules_data", "rules data", ruleset) _add_rules_to_list(features, "rules_override", "rules override", ruleset) _add_rules_to_list(features, "scanners", "scanners", ruleset) - _add_rules_to_list(features, "processors", "processors", ruleset) - _add_rules_to_list(features, "actions", "actions", ruleset) if ruleset: return tracer._appsec_processor._update_rules({k: v for k, v in ruleset.items() if v is not None}) diff --git a/releasenotes/notes/fix_rc_asm_dd_no_file-37e6f733583e334c.yaml b/releasenotes/notes/fix_rc_asm_dd_no_file-37e6f733583e334c.yaml new file mode 100644 index 00000000000..a3224a1df96 --- /dev/null +++ b/releasenotes/notes/fix_rc_asm_dd_no_file-37e6f733583e334c.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + ASM: This fix resolves an issue where the WAF could be disabled if the ASM_DD rule file was not found in Remote Config. diff --git a/tests/appsec/appsec/test_processor.py b/tests/appsec/appsec/test_processor.py index f01314037ca..967cbcebc34 100644 --- a/tests/appsec/appsec/test_processor.py +++ b/tests/appsec/appsec/test_processor.py @@ -79,7 +79,7 @@ def test_enable_custom_rules(): processor = AppSecSpanProcessor() assert processor.enabled - assert processor.rules == rules.RULES_GOOD_PATH + assert processor.rule_filename == rules.RULES_GOOD_PATH def test_ddwaf_ctx(tracer_appsec): diff --git a/tests/appsec/appsec/test_remoteconfiguration.py b/tests/appsec/appsec/test_remoteconfiguration.py index 223d9813468..91bf42fb638 100644 --- a/tests/appsec/appsec/test_remoteconfiguration.py +++ b/tests/appsec/appsec/test_remoteconfiguration.py @@ -1033,3 +1033,15 @@ def test_rc_rules_data_error_ddwaf(tracer): "rules": [{"invalid": mock.MagicMock()}], } assert not _appsec_rules_data(config, tracer) + + +def test_rules_never_empty(tracer): + with override_global_config(dict(_asm_enabled=True)): + tracer.configure(appsec_enabled=True, api_version="v0.4") + with mock.patch("ddtrace.appsec._processor.AppSecSpanProcessor._update_rules", autospec=True) as mock_update: + mock_update.reset_mock() + _appsec_rules_data({"rules": []}, tracer) + call = mock_update.mock_calls + args = call[-1][1][1] + assert "rules" in args + assert args["rules"], "empty rules should not be possible, it must switch to default." From bf946b7cbd53b777aadbbbb1e8d4c8c7b2cc0c60 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 13:27:04 +0100 Subject: [PATCH 2/4] fix(opentelemetry): resolve lazy sampling + distributed tracing bug [backport 2.9] (#10028) Backport 47aad1c2d7099c21ef4adfac52911e06edf2a213 from #9974 to 2.9. ## Description With [ddtrace v2.8.0 ](https://github.com/DataDog/dd-trace-py/commit/9707da19a243afec4a9cdaacdff21cadd95d9061) sampling decisions are no longer made when a root span is created. Instead sampling decisions are lazily evaluated when trace information is expected to leave a process. The ddtrace library sets a sampling decision on root spans when one of the following conditions are met: - an unsampled trace is about to be serialized and sent to the datadog agent [here]( https://github.com/DataDog/dd-trace-py/blob/08ed1994498a56d81b74083828bd7241c19c9613/ddtrace/_trace/processor/__init__.py#L143) - an unsampled trace is about to generate and send distributed tracing headers (via datadog's HttpPropagator) [here](https://github.com/DataDog/dd-trace-py/blob/08ed1994498a56d81b74083828bd7241c19c9613/ddtrace/propagation/http.py#L998) - a process is about to fork before a sampling decision is made [here](https://github.com/DataDog/dd-trace-py/blob/08ed1994498a56d81b74083828bd7241c19c9613/ddtrace/_trace/tracer.py#L370) Spans generated using OpenTelemetry API (via ddtrace TracerProvider) do not use the ddtrace HttpPropagator when propagating distributed traces. This leaves an edge case where the opentelemetry-api can propagate a distributed trace before a sampling decision is made. Since the default state of an opentelemetry span is [unsampled](https://github.com/open-telemetry/opentelemetry-python/blob/main/opentelemetry-api/src/opentelemetry/trace/span.py#L210), downstream services could receive a traceflag of `00` and drop spans that should have been kept. This could result in missing spans/incomplete traces in the Datadog UI. ## Fix This PR ensures that sampling decisions are ALWAYS made before a SpanContext is extracted from an OpenTelemetry span. Since `Span.get_span_context()` is the only mechanism to extract/propagate tracing information (ex: sampling decision, trace_id, span_id, etc.) from an OpenTelemetry Span, making a sampling decision here will ensure the OpenTelemetry API never propagates an undefined sampling decision. If `Span.get_span_context()` is never invoked, then OpenTelemetry spans will continue to be lazily sampled on serialization (just like Datadog spans). # TODO There are many cases where trace information from a Datadog span can escape a process before a sampling decision is made (ex: via threads, spawned processes, manual context propagation). For these scenarios we ask users to manually sample spans (via [these docs](https://ddtrace.readthedocs.io/en/stable/advanced_usage.html?highlight=sampling#distributed-tracing)). Ideally user's should NOT need to kno the internal workings of tracer sampling and they should not be required to call `tracer.sample(span)` in their applications to resolve missing span issues. Sampling decisions should ALWAYS be made when tracing internals access `Span.context.sampling_priority` for the first time. With this approach an invalid/undefined sampling priority is never returned. The Datadog Span Context will always return a consistent sampling decision. cc: @brettlangdon, @zacharycmontoya, @ZStriker19 ## Risk This change makes "lazy sampling" less "lazy" (for the OpenTelemetry API). Span tags and resource names set after `Span.get_span_context()` is called will NOT be used to make a sampling decision. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) Co-authored-by: Munir Abdinur Co-authored-by: Romain Komorn <136473744+romainkomorndatadog@users.noreply.github.com> --- ddtrace/opentelemetry/_span.py | 29 +++++++--- ...ing-decision-is-made-40ab760eada20b20.yaml | 5 ++ tests/opentelemetry/test_trace.py | 58 +++++++++++++++++-- ...race_with_flask_app[with_ddtrace_run].json | 55 +++++++++--------- ...sk_app[with_opentelemetry_instrument].json | 32 +++++----- 5 files changed, 125 insertions(+), 54 deletions(-) create mode 100644 releasenotes/notes/otel-ensure-sampling-decision-is-made-40ab760eada20b20.yaml diff --git a/ddtrace/opentelemetry/_span.py b/ddtrace/opentelemetry/_span.py index f4cf456bfc7..bd2b0dfe83b 100644 --- a/ddtrace/opentelemetry/_span.py +++ b/ddtrace/opentelemetry/_span.py @@ -10,6 +10,7 @@ from opentelemetry.trace.span import TraceState from ddtrace import config +from ddtrace import tracer as ddtracer from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE @@ -136,13 +137,27 @@ def kind(self): def get_span_context(self): # type: () -> SpanContext """Returns an OpenTelemetry SpanContext""" - ts = None - tf = TraceFlags.DEFAULT - if self._ddspan.context: - ts_str = w3c_tracestate_add_p(self._ddspan.context._tracestate, self._ddspan.span_id) - ts = TraceState.from_header([ts_str]) - if self._ddspan.context.sampling_priority and self._ddspan.context.sampling_priority > 0: - tf = TraceFlags.SAMPLED + if self._ddspan.context.sampling_priority is None: + # With the introduction of lazy sampling, spans are now sampled on serialization. With this change + # a spans trace flags could be propagated before a sampling + # decision is made. Since the default sampling decision is to unsample spans this can result + # in missing spans. To resolve this issue, a sampling decision must be made the first time + # the span context is accessed. + ddtracer.sample(self._ddspan._local_root or self._ddspan) + + if self._ddspan.context.sampling_priority is None: + tf = TraceFlags.DEFAULT + log.warning( + "Span context is missing a sampling decision, defaulting to unsampled: %s", str(self._ddspan.context) + ) + elif self._ddspan.context.sampling_priority > 0: + tf = TraceFlags.SAMPLED + else: + tf = TraceFlags.DEFAULT + + # Evaluate the tracestate header after the sampling decision has been made + ts_str = w3c_tracestate_add_p(self._ddspan.context._tracestate, self._ddspan.span_id) + ts = TraceState.from_header([ts_str]) return SpanContext(self._ddspan.trace_id, self._ddspan.span_id, False, tf, ts) diff --git a/releasenotes/notes/otel-ensure-sampling-decision-is-made-40ab760eada20b20.yaml b/releasenotes/notes/otel-ensure-sampling-decision-is-made-40ab760eada20b20.yaml new file mode 100644 index 00000000000..5c286f8a06e --- /dev/null +++ b/releasenotes/notes/otel-ensure-sampling-decision-is-made-40ab760eada20b20.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + opentelemetry: Resolves an edge case where distributed tracing headers could be generated before a sampling decision is made, + resulting in dropped spans in downstream services. diff --git a/tests/opentelemetry/test_trace.py b/tests/opentelemetry/test_trace.py index 3e4dba17aaf..6015fb4c299 100644 --- a/tests/opentelemetry/test_trace.py +++ b/tests/opentelemetry/test_trace.py @@ -1,5 +1,7 @@ import mock import opentelemetry +from opentelemetry.trace import set_span_in_context +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator import opentelemetry.version import pytest @@ -50,7 +52,7 @@ def test_otel_start_span_without_default_args(oteltracer): root = oteltracer.start_span("root-span") otel_span = oteltracer.start_span( "test-start-span", - context=opentelemetry.trace.set_span_in_context(root), + context=set_span_in_context(root), kind=opentelemetry.trace.SpanKind.CLIENT, attributes={"start_span_tag": "start_span_val"}, links=None, @@ -117,7 +119,7 @@ def test_otel_start_current_span_without_default_args(oteltracer): with oteltracer.start_as_current_span("root-span") as root: with oteltracer.start_as_current_span( "test-start-current-span-no-defualts", - context=opentelemetry.trace.set_span_in_context(root), + context=set_span_in_context(root), kind=opentelemetry.trace.SpanKind.SERVER, attributes={"start_current_span_tag": "start_cspan_val"}, links=[], @@ -138,6 +140,50 @@ def test_otel_start_current_span_without_default_args(oteltracer): otel_span.end() +def test_otel_get_span_context_sets_sampling_decision(oteltracer): + with oteltracer.start_span("otel-server") as otelspan: + # Sampling priority is not set on span creation + assert otelspan._ddspan.context.sampling_priority is None + # Ensure the sampling priority is always consistent with traceflags + span_context = otelspan.get_span_context() + # Sampling priority is evaluated when the SpanContext is first accessed + sp = otelspan._ddspan.context.sampling_priority + assert sp is not None + if sp > 0: + assert span_context.trace_flags == 1 + else: + assert span_context.trace_flags == 0 + # Ensure the sampling priority is always consistent + for _ in range(1000): + otelspan.get_span_context() + assert otelspan._ddspan.context.sampling_priority == sp + + +def test_distributed_trace_inject(oteltracer): # noqa:F811 + with oteltracer.start_as_current_span("test-otel-distributed-trace") as span: + headers = {} + TraceContextTextMapPropagator().inject(headers, set_span_in_context(span)) + sp = span.get_span_context() + assert headers["traceparent"] == f"00-{sp.trace_id:032x}-{sp.span_id:016x}-{sp.trace_flags:02x}" + assert headers["tracestate"] == sp.trace_state.to_header() + + +def test_distributed_trace_extract(oteltracer): # noqa:F811 + headers = { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + "tracestate": "congo=t61rcWkgMzE,dd=s:2", + } + context = TraceContextTextMapPropagator().extract(headers) + with oteltracer.start_as_current_span("test-otel-distributed-trace", context=context) as span: + sp = span.get_span_context() + assert sp.trace_id == int("0af7651916cd43dd8448eb211c80319c", 16) + assert span._ddspan.parent_id == int("b7ad6b7169203331", 16) + assert sp.trace_flags == 1 + assert sp.trace_state.get("congo") == "t61rcWkgMzE" + assert "s:2" in sp.trace_state.get("dd") + assert sp.is_remote is False + + @flaky(1717428664) @pytest.mark.parametrize( "flask_wsgi_application,flask_env_arg,flask_port,flask_command", @@ -164,10 +210,12 @@ def test_otel_start_current_span_without_default_args(oteltracer): "with_opentelemetry_instrument", ], ) -@pytest.mark.snapshot(ignores=["metrics.net.peer.port", "meta.traceparent", "meta.flask.version"]) +@pytest.mark.snapshot(ignores=["metrics.net.peer.port", "meta.traceparent", "meta.tracestate", "meta.flask.version"]) def test_distributed_trace_with_flask_app(flask_client, oteltracer): # noqa:F811 - with oteltracer.start_as_current_span("test-otel-distributed-trace"): - resp = flask_client.get("/otel") + with oteltracer.start_as_current_span("test-otel-distributed-trace") as span: + headers = {} + TraceContextTextMapPropagator().inject(headers, set_span_in_context(span)) + resp = flask_client.get("/otel", headers=headers) assert resp.text == "otel" assert resp.status_code == 200 diff --git a/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_ddtrace_run].json b/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_ddtrace_run].json index c05e181c129..132e962af9b 100644 --- a/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_ddtrace_run].json +++ b/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_ddtrace_run].json @@ -10,9 +10,9 @@ "error": 0, "meta": { "_dd.p.dm": "-0", - "_dd.p.tid": "655535db00000000", + "_dd.p.tid": "66a6956800000000", "language": "python", - "runtime-id": "b4ffa244c11343de919ca20a7e8eebcf" + "runtime-id": "0da930484bcf4b488bfd7bb9cfd9c4b6" }, "metrics": { "_dd.top_level": 1, @@ -35,11 +35,12 @@ "meta": { "_dd.base_service": "", "_dd.p.dm": "-0", - "_dd.p.tid": "655535db00000000", + "_dd.p.tid": "66a6956800000000", + "_dd.parent_id": "4f4e94b0f4d57229", "component": "flask", "flask.endpoint": "otel", "flask.url_rule": "/otel", - "flask.version": "2.1.3", + "flask.version": "1.1.4", "http.method": "GET", "http.route": "/otel", "http.status_code": "200", @@ -56,10 +57,10 @@ "_dd.top_level": 1, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 1, - "process_id": 7713 + "process_id": 31660 }, - "duration": 14813125, - "start": 1700083163141821300 + "duration": 4600000, + "start": 1722193256335520000 }, { "name": "flask.application", @@ -76,8 +77,8 @@ "flask.endpoint": "otel", "flask.url_rule": "/otel" }, - "duration": 14243208, - "start": 1700083163142061300 + "duration": 4219000, + "start": 1722193256335701000 }, { "name": "flask.try_trigger_before_first_request_functions", @@ -92,8 +93,8 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 14625, - "start": 1700083163142217383 + "duration": 12000, + "start": 1722193256335827000 }, { "name": "flask.preprocess_request", @@ -108,8 +109,8 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 20125, - "start": 1700083163142331717 + "duration": 13000, + "start": 1722193256335911000 }, { "name": "flask.dispatch_request", @@ -124,8 +125,8 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 13437750, - "start": 1700083163142415633 + "duration": 3646000, + "start": 1722193256335970000 }, { "name": "tests.opentelemetry.flask_app.otel", @@ -140,8 +141,8 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 13354542, - "start": 1700083163142480508 + "duration": 3585000, + "start": 1722193256336021000 }, { "name": "internal", @@ -155,8 +156,8 @@ "meta": { "_dd.base_service": "" }, - "duration": 52875, - "start": 1700083163155755800 + "duration": 36000, + "start": 1722193256339552000 }, { "name": "flask.process_response", @@ -171,8 +172,8 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 32375, - "start": 1700083163156004175 + "duration": 17000, + "start": 1722193256339703000 }, { "name": "flask.do_teardown_request", @@ -187,8 +188,8 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 31625, - "start": 1700083163156192133 + "duration": 14000, + "start": 1722193256339831000 }, { "name": "flask.do_teardown_appcontext", @@ -203,8 +204,8 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 9292, - "start": 1700083163156278258 + "duration": 10000, + "start": 1722193256339897000 }, { "name": "flask.response", @@ -219,6 +220,6 @@ "_dd.base_service": "", "component": "flask" }, - "duration": 311541, - "start": 1700083163156314342 + "duration": 183000, + "start": 1722193256339931000 }]] diff --git a/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_opentelemetry_instrument].json b/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_opentelemetry_instrument].json index e1fbd2196d3..daf39a83b8a 100644 --- a/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_opentelemetry_instrument].json +++ b/tests/snapshots/tests.opentelemetry.test_trace.test_distributed_trace_with_flask_app[with_opentelemetry_instrument].json @@ -10,23 +10,23 @@ "error": 0, "meta": { "_dd.p.dm": "-0", - "_dd.p.tid": "655535dc00000000", + "_dd.p.tid": "66a666b600000000", "language": "python", - "runtime-id": "b4ffa244c11343de919ca20a7e8eebcf" + "runtime-id": "4fe738d296c04c00aff7eb2a66389ad4" }, "metrics": { "_dd.top_level": 1, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 1, - "process_id": 7703 + "process_id": 75526 }, - "duration": 24467250, - "start": 1700083164080082009 + "duration": 3561000, + "start": 1722181302828129000 }, { "name": "server", "service": "", - "resource": "/otel", + "resource": "GET /otel", "trace_id": 0, "span_id": 2, "parent_id": 1, @@ -34,7 +34,8 @@ "error": 0, "meta": { "_dd.p.dm": "-0", - "_dd.p.tid": "655535dc00000000", + "_dd.p.tid": "66a666b600000000", + "_dd.parent_id": "ae6e2bf21738f62f", "http.flavor": "1.1", "http.host": "0.0.0.0:8001", "http.method": "GET", @@ -45,21 +46,22 @@ "http.target": "/otel", "http.user_agent": "python-requests/2.28.1", "language": "python", + "net.host.name": "0.0.0.0:8001", "net.peer.ip": "127.0.0.1", - "runtime-id": "f90fe90fc53a4388b02210639d156981", + "runtime-id": "0dacca250fe4471094dc593f4892b91f", "span.kind": "server", - "tracestate": "dd=s:1;t.dm:-0" + "tracestate": "dd=p:ae6e2bf21738f62f;s:1;t.dm:-0" }, "metrics": { "_dd.top_level": 1, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 1, "net.host.port": 8001, - "net.peer.port": 65508, - "process_id": 7721 + "net.peer.port": 62683, + "process_id": 75530 }, - "duration": 413250, - "start": 1700083164081574592 + "duration": 515000, + "start": 1722181302830460000 }, { "name": "internal", @@ -70,6 +72,6 @@ "parent_id": 2, "type": "", "error": 0, - "duration": 18250, - "start": 1700083164081867259 + "duration": 16000, + "start": 1722181302830847000 }]] From 51ed3d587022a6f71f8b7784184fee235cb44529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Fri, 2 Aug 2024 14:07:30 +0100 Subject: [PATCH 3/4] fix(internal): fix telemetry URL for EU [backport 2.9] (#10055) Backport 513f7e56b783f0b700b8baba9d5d76c6f57874b9 from #10021 to 2.9. This PR fixes incorrect URL for telemetry intake in EU that was causing missing telemetry data and SSL error log messages. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/internal/telemetry/writer.py | 2 +- ...sibility-fix-update_eu_telemetry_url-0642a6f665c75a0f.yaml | 4 ++++ tests/telemetry/test_writer.py | 4 +--- 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/ci_visibility-fix-update_eu_telemetry_url-0642a6f665c75a0f.yaml diff --git a/ddtrace/internal/telemetry/writer.py b/ddtrace/internal/telemetry/writer.py index 836daa5da74..fbc7eb34a48 100644 --- a/ddtrace/internal/telemetry/writer.py +++ b/ddtrace/internal/telemetry/writer.py @@ -117,7 +117,7 @@ def _get_agentless_telemetry_url(site: str): if site == "datad0g.com": return "https://all-http-intake.logs.datad0g.com" if site == "datadoghq.eu": - return "https://instrumentation-telemetry-intake.eu1.datadoghq.com" + return "https://instrumentation-telemetry-intake.datadoghq.eu" return f"https://instrumentation-telemetry-intake.{site}/" diff --git a/releasenotes/notes/ci_visibility-fix-update_eu_telemetry_url-0642a6f665c75a0f.yaml b/releasenotes/notes/ci_visibility-fix-update_eu_telemetry_url-0642a6f665c75a0f.yaml new file mode 100644 index 00000000000..d8a898e9806 --- /dev/null +++ b/releasenotes/notes/ci_visibility-fix-update_eu_telemetry_url-0642a6f665c75a0f.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + CI Visibility: fixes incorrect URL for telemetry intake in EU that was causing missing telemetry data and SSL error log messages. diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py index 18699170152..5781ec3761c 100644 --- a/tests/telemetry/test_writer.py +++ b/tests/telemetry/test_writer.py @@ -626,9 +626,7 @@ def test_telemetry_writer_agentless_setup_eu(): assert new_telemetry_writer._client._is_agentless is True assert new_telemetry_writer._client._is_disabled is False assert new_telemetry_writer._client._endpoint == "api/v2/apmtelemetry" - assert ( - new_telemetry_writer._client._telemetry_url == "https://instrumentation-telemetry-intake.eu1.datadoghq.com" - ) + assert new_telemetry_writer._client._telemetry_url == "https://instrumentation-telemetry-intake.datadoghq.eu" assert new_telemetry_writer._client._headers["dd-api-key"] == "foobarkey" From 0628471fefb0be7a58ad490efc0e6a5f93a10688 Mon Sep 17 00:00:00 2001 From: erikayasuda <153395705+erikayasuda@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:44:51 -0400 Subject: [PATCH 4/4] chore: add docs/ to apm-python code ownership [backport 2.9] (#10090) Manual backport of https://github.com/DataDog/dd-trace-py/pull/10041 ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5b8a97ce715..f8c79825ced 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -112,6 +112,8 @@ mypy.ini @DataDog/python-guild @DataDog/apm-core-pyt ddtrace/internal/_file_queue.py @DataDog/python-guild ddtrace/internal/_unpatched.py @DataDog/python-guild ddtrace/internal/compat.py @DataDog/python-guild @DataDog/apm-core-python +ddtrace/settings/config.py @DataDog/python-guild @DataDog/apm-sdk-api-python +docs/ @DataDog/python-guild tests/utils.py @DataDog/python-guild tests/.suitespec.json @DataDog/python-guild @DataDog/apm-core-python tests/suitespec.py @DataDog/python-guild @DataDog/apm-core-python