From 806df5d31ce990dbffd95149fa6e11873595e5c9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 20 Jan 2025 20:36:27 -0800 Subject: [PATCH] (Feat) `datadog_llm_observability` callback - emit `request_tags` on logs (#7883) * dd - emit tags on llm obs payload * dd - show requester tags on traces * test_get_datadog_tags * _get_datadog_tags * fix dd POD_NAME * test_get_datadog_tags --- litellm/integrations/datadog/datadog.py | 33 ++++++++++++-- .../integrations/datadog/datadog_llm_obs.py | 3 ++ litellm/proxy/proxy_config.yaml | 3 ++ litellm/types/integrations/datadog_llm_obs.py | 1 + tests/logging_callback_tests/test_datadog.py | 45 +++++++++++++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index a1ee81291732..e3cebb016a3d 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -274,7 +274,9 @@ def create_datadog_logging_payload( dd_payload = DatadogPayload( ddsource=self._get_datadog_source(), - ddtags=self._get_datadog_tags(), + ddtags=self._get_datadog_tags( + standard_logging_object=standard_logging_object + ), hostname=self._get_datadog_hostname(), message=json_payload, service=self._get_datadog_service(), @@ -444,8 +446,33 @@ def _create_v0_logging_payload( return dd_payload @staticmethod - def _get_datadog_tags(): - return f"env:{os.getenv('DD_ENV', 'unknown')},service:{os.getenv('DD_SERVICE', 'litellm')},version:{os.getenv('DD_VERSION', 'unknown')},HOSTNAME:{DataDogLogger._get_datadog_hostname()},POD_NAME:{os.getenv('POD_NAME', 'unknown')}" + def _get_datadog_tags( + standard_logging_object: Optional[StandardLoggingPayload] = None, + ) -> str: + """ + Get the datadog tags for the request + + DD tags need to be as follows: + - tags: ["user_handle:dog@gmail.com", "app_version:1.0.0"] + """ + base_tags = { + "env": os.getenv("DD_ENV", "unknown"), + "service": os.getenv("DD_SERVICE", "litellm"), + "version": os.getenv("DD_VERSION", "unknown"), + "HOSTNAME": DataDogLogger._get_datadog_hostname(), + "POD_NAME": os.getenv("POD_NAME", "unknown"), + } + + tags = [f"{k}:{v}" for k, v in base_tags.items()] + + if standard_logging_object: + _request_tags: List[str] = ( + standard_logging_object.get("request_tags", []) or [] + ) + request_tags = [f"request_tag:{tag}" for tag in _request_tags] + tags.extend(request_tags) + + return ",".join(tags) @staticmethod def _get_datadog_source(): diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 18744fa6ee96..e4e074bab78e 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -159,6 +159,9 @@ def create_llm_obs_payload( start_ns=int(start_time.timestamp() * 1e9), duration=int((end_time - start_time).total_seconds() * 1e9), metrics=metrics, + tags=[ + self._get_datadog_tags(standard_logging_object=standard_logging_payload) + ], ) def _get_response_messages(self, response_obj: Any) -> List[Any]: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 868aec66cef2..49e8dd6934b6 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -19,3 +19,6 @@ model_list: general_settings: store_prompts_in_spend_logs: true + +litellm_settings: + callbacks: ["datadog_llm_observability"] diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index 91ea8c257581..9298b157d27f 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -40,6 +40,7 @@ class LLMObsPayload(TypedDict): start_ns: int duration: int metrics: LLMMetrics + tags: List class DDSpanAttributes(TypedDict): diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index 2d3cb36046d0..2a7549b4738a 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -532,3 +532,48 @@ async def test_datadog_non_serializable_messages(): # Check that the non-serializable objects were converted to strings assert isinstance(dict_payload["messages"][0]["content"], str) assert isinstance(dict_payload["response"]["choices"][0]["message"]["content"], str) + + +def test_get_datadog_tags(): + """Test the _get_datadog_tags static method with various inputs""" + # Test with no standard_logging_object and default env vars + base_tags = DataDogLogger._get_datadog_tags() + assert "env:" in base_tags + assert "service:" in base_tags + assert "version:" in base_tags + assert "POD_NAME:" in base_tags + assert "HOSTNAME:" in base_tags + + # Test with custom env vars + test_env = { + "DD_ENV": "production", + "DD_SERVICE": "custom-service", + "DD_VERSION": "1.0.0", + "HOSTNAME": "test-host", + "POD_NAME": "pod-123", + } + with patch.dict(os.environ, test_env): + custom_tags = DataDogLogger._get_datadog_tags() + assert "env:production" in custom_tags + assert "service:custom-service" in custom_tags + assert "version:1.0.0" in custom_tags + assert "HOSTNAME:test-host" in custom_tags + assert "POD_NAME:pod-123" in custom_tags + + # Test with standard_logging_object containing request_tags + standard_logging_obj = create_standard_logging_payload() + standard_logging_obj["request_tags"] = ["tag1", "tag2"] + + tags_with_request = DataDogLogger._get_datadog_tags(standard_logging_obj) + assert "request_tag:tag1" in tags_with_request + assert "request_tag:tag2" in tags_with_request + + # Test with empty request_tags + standard_logging_obj["request_tags"] = [] + tags_empty_request = DataDogLogger._get_datadog_tags(standard_logging_obj) + assert "request_tag:" not in tags_empty_request + + # Test with None request_tags + standard_logging_obj["request_tags"] = None + tags_none_request = DataDogLogger._get_datadog_tags(standard_logging_obj) + assert "request_tag:" not in tags_none_request