From 22b5859f12b2bc038362aafdad0bd0016003c4a1 Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Thu, 16 Jan 2025 09:43:55 -0500 Subject: [PATCH 1/4] add telemetry for custom vs. autoinstrumented span creation --- ddtrace/llmobs/_integrations/base.py | 11 +++++++++++ ddtrace/llmobs/_llmobs.py | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/ddtrace/llmobs/_integrations/base.py b/ddtrace/llmobs/_integrations/base.py index 2e892904720..680f33ff589 100644 --- a/ddtrace/llmobs/_integrations/base.py +++ b/ddtrace/llmobs/_integrations/base.py @@ -16,6 +16,8 @@ from ddtrace.internal.dogstatsd import get_dogstatsd_client from ddtrace.internal.hostname import get_hostname from ddtrace.internal.logger import get_logger +from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT from ddtrace.internal.utils.formats import asbool from ddtrace.llmobs._constants import PARENT_ID_KEY from ddtrace.llmobs._constants import PROPAGATED_PARENT_ID_KEY @@ -134,6 +136,15 @@ def trace(self, pin: Pin, operation_id: str, submit_to_llmobs: bool = False, **k # in these cases to avoid conflicting with the later propagated tags. parent_id = _get_llmobs_parent_id(span) or "undefined" span._set_ctx_item(PARENT_ID_KEY, str(parent_id)) + telemetry_writer.add_count_metric( + namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + name="span.creation", + value=1, + tags=( + ("integration", self._integration_name), + ("autoinstrumented", "true"), + ), + ) return span @classmethod diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index b4f1dc1b2f6..aac63a31d94 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -525,6 +525,15 @@ def _start_span( model_provider: Optional[str] = None, ml_app: Optional[str] = None, ) -> Span: + telemetry_writer.add_count_metric( + namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + name="span.creation", + value=1, + tags=( + ("autoinstrumented", "false"), + ("kind", operation_kind), + ), + ) if name is None: name = operation_kind span = self.tracer.trace(name, resource=operation_kind, span_type=SpanTypes.LLM) From 7c037e89fe2ccbf587fc1ec3f7180a4753e3bc06 Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Thu, 16 Jan 2025 14:35:06 -0500 Subject: [PATCH 2/4] rename metric --- ddtrace/llmobs/_integrations/base.py | 2 +- ddtrace/llmobs/_llmobs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/llmobs/_integrations/base.py b/ddtrace/llmobs/_integrations/base.py index 680f33ff589..bfcc5407a06 100644 --- a/ddtrace/llmobs/_integrations/base.py +++ b/ddtrace/llmobs/_integrations/base.py @@ -138,7 +138,7 @@ def trace(self, pin: Pin, operation_id: str, submit_to_llmobs: bool = False, **k span._set_ctx_item(PARENT_ID_KEY, str(parent_id)) telemetry_writer.add_count_metric( namespace=TELEMETRY_APM_PRODUCT.LLMOBS, - name="span.creation", + name="span.start", value=1, tags=( ("integration", self._integration_name), diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index aac63a31d94..a6eb7914d2b 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -527,7 +527,7 @@ def _start_span( ) -> Span: telemetry_writer.add_count_metric( namespace=TELEMETRY_APM_PRODUCT.LLMOBS, - name="span.creation", + name="span.start", value=1, tags=( ("autoinstrumented", "false"), From f5f3d1c866a85efc272f159580c4297a94dae7d5 Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Tue, 21 Jan 2025 10:10:00 -0500 Subject: [PATCH 3/4] fix namespace argument to add count metric --- ddtrace/llmobs/_integrations/base.py | 4 ++-- ddtrace/llmobs/_llmobs.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ddtrace/llmobs/_integrations/base.py b/ddtrace/llmobs/_integrations/base.py index bfcc5407a06..94310e5328a 100644 --- a/ddtrace/llmobs/_integrations/base.py +++ b/ddtrace/llmobs/_integrations/base.py @@ -17,7 +17,7 @@ from ddtrace.internal.hostname import get_hostname from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer -from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.utils.formats import asbool from ddtrace.llmobs._constants import PARENT_ID_KEY from ddtrace.llmobs._constants import PROPAGATED_PARENT_ID_KEY @@ -137,7 +137,7 @@ def trace(self, pin: Pin, operation_id: str, submit_to_llmobs: bool = False, **k parent_id = _get_llmobs_parent_id(span) or "undefined" span._set_ctx_item(PARENT_ID_KEY, str(parent_id)) telemetry_writer.add_count_metric( - namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + namespace=TELEMETRY_NAMESPACE.MLOBS, name="span.start", value=1, tags=( diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index a6eb7914d2b..0c336e29bcd 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -28,6 +28,7 @@ from ddtrace.internal.service import ServiceStatusError from ddtrace.internal.telemetry import telemetry_writer from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import parse_tags_str @@ -526,7 +527,7 @@ def _start_span( ml_app: Optional[str] = None, ) -> Span: telemetry_writer.add_count_metric( - namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + namespace=TELEMETRY_NAMESPACE.MLOBS, name="span.start", value=1, tags=( From 9099e10bbb447ff4d6cc649fdba7af1768bb179c Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Fri, 24 Jan 2025 14:33:43 -0500 Subject: [PATCH 4/4] remove unnecessary import --- ddtrace/llmobs/_llmobs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index e6a9eb8cfd5..768f4bdb292 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -28,7 +28,6 @@ from ddtrace.internal.telemetry import telemetry_writer from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE -from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import parse_tags_str from ddtrace.llmobs import _constants as constants