fix(tracing): only extract distributed headers if a trace is not already started

ddtrace/contrib/trace_utils.py

@@ -545,6 +545,18 @@ def activate_distributed_headers(tracer, int_config=None, request_headers=None,
     if override is False:
         return None
 
+    # Only extract and activate if we don't already have an activate span
+    # DEV: Only do this if there is an active Span, an active Context is fine to override
+    # DEV: Use _DD_TRACE_EXTRACT_IGNORE_ACTIVE_SPAN env var to override the default behavior
+    current_span = tracer.current_span()
+    if current_span and not config._extract_ignore_active_span:
+        log.debug(
+            "will not extract distributed headers, a Span(trace_id%d, span_id=%d) is already active",
+            current_span.trace_id,
+            current_span.span_id,
+        )
+        return
+
     if override or (int_config and distributed_tracing_enabled(int_config)):
         context = HTTPPropagator.extract(request_headers)
 

ddtrace/settings/config.py

@@ -494,6 +494,9 @@ def __init__(self):
 
         self._propagation_extract_first = asbool(os.getenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", False))
 
+        # When True any active span is ignored when extracting trace context from headers
+        self._extract_ignore_active_span = asbool(os.getenv("_DD_TRACE_EXTRACT_IGNORE_ACTIVE_SPAN", False))
+
         # Datadog tracer tags propagation
         x_datadog_tags_max_length = int(os.getenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", default=512))
         if x_datadog_tags_max_length < 0:

releasenotes/notes/fix-unncessary-header-extraction-c1facf2b30331afb.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    tracing: Fix scenarios when distributed tracing headers would be extracted and possibly activated when a trace was already started.

tests/contrib/wsgi/test_wsgi.py

@@ -7,6 +7,7 @@
 from ddtrace import config
 from ddtrace.contrib.wsgi import wsgi
 from tests.utils import override_config
+from tests.utils import override_global_config
 from tests.utils import override_http_config
 from tests.utils import snapshot
 
@@ -388,3 +389,77 @@ def test_schematization(ddtrace_run_python_code_in_subprocess, schema_version, s
         env["DD_SERVICE"] = service_name
     _, stderr, status, _ = ddtrace_run_python_code_in_subprocess(code, env=env)
     assert status == 0, stderr
+
+
+def test_distributed_tracing_existing_parent(tracer, test_spans):
+    """We should not parse and activate distributed context if there is already an active span"""
+
+    # Middleware that starts a root trace, but doesn't parse headers
+    def broken_middleware(app, tracer):
+        def middleware(environ, start_response):
+            with tracer.trace("broken_middleware"):
+                return app(environ, start_response)
+
+        return middleware
+
+    app = TestApp(broken_middleware(wsgi.DDWSGIMiddleware(application, tracer=tracer), tracer))
+    resp = app.get("/", headers={"X-Datadog-Parent-Id": "1234", "X-Datadog-Trace-Id": "4321"})
+
+    assert config.wsgi.distributed_tracing is True
+    assert resp.status == "200 OK"
+    assert resp.status_int == 200
+
+    spans = test_spans.pop()
+    assert len(spans) == 5
+
+    # The root should NOT inherit from distributed headers
+    root = spans[0]
+    assert root.name == "broken_middleware"
+    assert root.trace_id != 4321
+    assert root.parent_id != 1234
+
+    # The rest of the spans should inherit from the root
+    for span in spans[1:]:
+        assert span.trace_id == root.trace_id
+        if span.name == "wsgi.request":
+            assert span.parent_id == root.span_id
+
+
+def test_distributed_tracing_existing_parent_ff_enabled(tracer, test_spans):
+    """We should parse and activate distributed context even if there is already an active span"""
+
+    # Middleware that starts a root trace, but doesn't parse headers
+    def broken_middleware(app, tracer):
+        def middleware(environ, start_response):
+            with tracer.trace("broken_middleware"):
+                return app(environ, start_response)
+
+        return middleware
+
+    # DEV: Default is False (ignore distributed headers when there is an active span)
+    with override_global_config(dict(_extract_ignore_active_span=True)):
+        app = TestApp(broken_middleware(wsgi.DDWSGIMiddleware(application, tracer=tracer), tracer))
+        resp = app.get("/", headers={"X-Datadog-Parent-Id": "1234", "X-Datadog-Trace-Id": "4321"})
+
+    assert config.wsgi.distributed_tracing is True
+    assert resp.status == "200 OK"
+    assert resp.status_int == 200
+
+    spans = test_spans.pop()
+    assert len(spans) == 5
+
+    # The root should NOT inherit from distributed headers
+    root = spans[0]
+    assert root.name == "broken_middleware"
+    assert root.trace_id != 4321
+    assert root.parent_id != 1234
+
+    # The rest of the spans should inherit from distributed tracing headers
+    wsgi_request = spans[1]
+    assert wsgi_request.name == "wsgi.request"
+    assert wsgi_request.trace_id == 4321
+    assert wsgi_request.parent_id == 1234
+
+    for span in spans[3:]:
+        assert span.trace_id == wsgi_request.trace_id
+        assert span.parent_id != root.parent_id

tests/utils.py

@@ -148,6 +148,7 @@ def override_global_config(values):
         "_llmobs_enabled",
         "_llmobs_sample_rate",
         "_llmobs_ml_app",
+        "_extract_ignore_active_span",
     ]
 
     asm_config_keys = asm_config._asm_config_keys