(docs) Document StandardLoggingPayload Spec (#7201)

* add slp spec to docs

* docs slp

* test slp enforcement

docs/my-website/docs/proxy/logging.md

@@ -113,90 +113,7 @@ Removes any field with `user_api_key_*` from metadata.
 
 Found under `kwargs["standard_logging_object"]`. This is a standard payload, logged for every response.
 
-```python
-
-class StandardLoggingPayload(TypedDict):
-    id: str
-    trace_id: str  # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
-    call_type: str
-    response_cost: float
-    response_cost_failure_debug_info: Optional[
-        StandardLoggingModelCostFailureDebugInformation
-    ]
-    status: StandardLoggingPayloadStatus
-    total_tokens: int
-    prompt_tokens: int
-    completion_tokens: int
-    startTime: float # Note: making this camelCase was a mistake, everything should be snake case
-    endTime: float
-    completionStartTime: float # time the first token of the LLM response is returned (for streaming responses)
-    response_time: float # time the LLM takes to respond (for streaming uses time to first token)
-    model_map_information: StandardLoggingModelInformation
-    model: str
-    model_id: Optional[str]
-    model_group: Optional[str]
-    api_base: str
-    metadata: StandardLoggingMetadata
-    cache_hit: Optional[bool]
-    cache_key: Optional[str]
-    saved_cache_cost: float
-    request_tags: list
-    end_user: Optional[str]
-    requester_ip_address: Optional[str]
-    messages: Optional[Union[str, list, dict]]
-    response: Optional[Union[str, list, dict]]
-    error_str: Optional[str]
-    model_parameters: dict
-    hidden_params: StandardLoggingHiddenParams
-
-class StandardLoggingHiddenParams(TypedDict):
-    model_id: Optional[str]
-    cache_key: Optional[str]
-    api_base: Optional[str]
-    response_cost: Optional[str]
-    additional_headers: Optional[StandardLoggingAdditionalHeaders]
-
-class StandardLoggingAdditionalHeaders(TypedDict, total=False):
-    x_ratelimit_limit_requests: int
-    x_ratelimit_limit_tokens: int
-    x_ratelimit_remaining_requests: int
-    x_ratelimit_remaining_tokens: int
-
-class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
-    """
-    Specific metadata k,v pairs logged to integration for easier cost tracking
-    """
-
-    spend_logs_metadata: Optional[
-        dict
-    ]  # special param to log k,v pairs to spendlogs for a call
-    requester_ip_address: Optional[str]
-    requester_metadata: Optional[dict]
-
-class StandardLoggingModelInformation(TypedDict):
-    model_map_key: str
-    model_map_value: Optional[ModelInfo]
-  
-
-StandardLoggingPayloadStatus = Literal["success", "failure"]
-
-class StandardLoggingModelCostFailureDebugInformation(TypedDict, total=False):
-    """
-    Debug information, if cost tracking fails.
-
-    Avoid logging sensitive information like response or optional params
-    """
-
-    error_str: Required[str]
-    traceback_str: Required[str]
-    model: str
-    cache_hit: Optional[bool]
-    custom_llm_provider: Optional[str]
-    base_model: Optional[str]
-    call_type: str
-    custom_pricing: Optional[bool]
-```
-
+[👉 **Standard Logging Payload Specification**](./logging_spec)
 
 ## Langfuse
 

docs/my-website/docs/proxy/logging_spec.md

@@ -0,0 +1,114 @@
+
+# StandardLoggingPayload Specification
+
+Found under `kwargs["standard_logging_object"]`. This is a standard payload, logged for every successful and failed response.
+
+## StandardLoggingPayload
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `id` | `str` | Unique identifier |
+| `trace_id` | `str` | Trace multiple LLM calls belonging to same overall request |
+| `call_type` | `str` | Type of call |
+| `response_cost` | `float` | Cost of the response in USD ($) |
+| `response_cost_failure_debug_info` | `StandardLoggingModelCostFailureDebugInformation` | Debug information if cost tracking fails |
+| `status` | `StandardLoggingPayloadStatus` | Status of the payload |
+| `total_tokens` | `int` | Total number of tokens |
+| `prompt_tokens` | `int` | Number of prompt tokens |
+| `completion_tokens` | `int` | Number of completion tokens |
+| `startTime` | `float` | Start time of the call |
+| `endTime` | `float` | End time of the call |
+| `completionStartTime` | `float` | Time to first token for streaming requests |
+| `response_time` | `float` | Total response time. If streaming, this is the time to first token |
+| `model_map_information` | `StandardLoggingModelInformation` | Model mapping information |
+| `model` | `str` | Model name sent in request |
+| `model_id` | `Optional[str]` | Model ID of the deployment used |
+| `model_group` | `Optional[str]` | `model_group` used for the request |
+| `api_base` | `str` | LLM API base URL |
+| `metadata` | `StandardLoggingMetadata` | Metadata information |
+| `cache_hit` | `Optional[bool]` | Whether cache was hit |
+| `cache_key` | `Optional[str]` | Optional cache key |
+| `saved_cache_cost` | `float` | Cost saved by cache |
+| `request_tags` | `list` | List of request tags |
+| `end_user` | `Optional[str]` | Optional end user identifier |
+| `requester_ip_address` | `Optional[str]` | Optional requester IP address |
+| `messages` | `Optional[Union[str, list, dict]]` | Messages sent in the request |
+| `response` | `Optional[Union[str, list, dict]]` | LLM response |
+| `error_str` | `Optional[str]` | Optional error string |
+| `error_information` | `Optional[StandardLoggingPayloadErrorInformation]` | Optional error information |
+| `model_parameters` | `dict` | Model parameters |
+| `hidden_params` | `StandardLoggingHiddenParams` | Hidden parameters |
+
+## StandardLoggingUserAPIKeyMetadata
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `user_api_key_hash` | `Optional[str]` | Hash of the litellm virtual key |
+| `user_api_key_alias` | `Optional[str]` | Alias of the API key |
+| `user_api_key_org_id` | `Optional[str]` | Organization ID associated with the key |
+| `user_api_key_team_id` | `Optional[str]` | Team ID associated with the key |
+| `user_api_key_user_id` | `Optional[str]` | User ID associated with the key |
+| `user_api_key_team_alias` | `Optional[str]` | Team alias associated with the key |
+
+## StandardLoggingMetadata
+
+Inherits from `StandardLoggingUserAPIKeyMetadata` and adds:
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `spend_logs_metadata` | `Optional[dict]` | Key-value pairs for spend logging |
+| `requester_ip_address` | `Optional[str]` | Requester's IP address |
+| `requester_metadata` | `Optional[dict]` | Additional requester metadata |
+
+## StandardLoggingAdditionalHeaders
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `x_ratelimit_limit_requests` | `int` | Rate limit for requests |
+| `x_ratelimit_limit_tokens` | `int` | Rate limit for tokens |
+| `x_ratelimit_remaining_requests` | `int` | Remaining requests in rate limit |
+| `x_ratelimit_remaining_tokens` | `int` | Remaining tokens in rate limit |
+
+## StandardLoggingHiddenParams
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `model_id` | `Optional[str]` | Optional model ID |
+| `cache_key` | `Optional[str]` | Optional cache key |
+| `api_base` | `Optional[str]` | Optional API base URL |
+| `response_cost` | `Optional[str]` | Optional response cost |
+| `additional_headers` | `Optional[StandardLoggingAdditionalHeaders]` | Additional headers |
+
+## StandardLoggingModelInformation
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `model_map_key` | `str` | Model map key |
+| `model_map_value` | `Optional[ModelInfo]` | Optional model information |
+
+## StandardLoggingModelCostFailureDebugInformation
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `error_str` | `str` | Error string |
+| `traceback_str` | `str` | Traceback string |
+| `model` | `str` | Model name |
+| `cache_hit` | `Optional[bool]` | Whether cache was hit |
+| `custom_llm_provider` | `Optional[str]` | Optional custom LLM provider |
+| `base_model` | `Optional[str]` | Optional base model |
+| `call_type` | `str` | Call type |
+| `custom_pricing` | `Optional[bool]` | Whether custom pricing was used |
+
+## StandardLoggingPayloadErrorInformation
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `error_code` | `Optional[str]` | Optional error code (eg. "429") |
+| `error_class` | `Optional[str]` | Optional error class (eg. "RateLimitError") |
+| `llm_provider` | `Optional[str]` | LLM provider that returned the error (eg. "openai")` |
+
+## StandardLoggingPayloadStatus
+
+A literal type with two possible values:
+- `"success"`
+- `"failure"`

docs/my-website/sidebars.js

@@ -102,7 +102,7 @@ const sidebars = {
         {
           type: "category",
           label: "Logging, Alerting, Metrics",
-          items: ["proxy/logging", "proxy/team_logging","proxy/alerting", "proxy/prometheus",],
+          items: ["proxy/logging", "proxy/logging_spec", "proxy/team_logging","proxy/alerting", "proxy/prometheus"],
         },
         {
           type: "category",

tests/documentation_tests/test_standard_logging_payload.py

@@ -0,0 +1,79 @@
+import os
+import re
+import sys
+
+from typing import get_type_hints
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+
+from litellm.types.utils import StandardLoggingPayload
+
+
+def get_all_fields(type_dict, prefix=""):
+    """Recursively get all fields from TypedDict and its nested types"""
+    fields = set()
+
+    # Get type hints for the TypedDict
+    hints = get_type_hints(type_dict)
+
+    for field_name, field_type in hints.items():
+        full_field_name = f"{prefix}{field_name}" if prefix else field_name
+        fields.add(full_field_name)
+
+        # Check if the field type is another TypedDict we should process
+        if hasattr(field_type, "__annotations__"):
+            nested_fields = get_all_fields(field_type)
+            fields.update(nested_fields)
+    return fields
+
+
+def test_standard_logging_payload_documentation():
+    # Get all fields from StandardLoggingPayload and its nested types
+    all_fields = get_all_fields(StandardLoggingPayload)
+
+    print("All fields in StandardLoggingPayload: ")
+    for _field in all_fields:
+        print(_field)
+
+    # Read the documentation
+    docs_path = "../../docs/my-website/docs/proxy/logging_spec.md"
+
+    try:
+        with open(docs_path, "r", encoding="utf-8") as docs_file:
+            content = docs_file.read()
+
+            # Extract documented fields from the table
+            doc_field_pattern = re.compile(r"\|\s*`([^`]+?)`\s*\|")
+            documented_fields = set(doc_field_pattern.findall(content))
+
+            # Clean up documented fields (remove whitespace)
+            documented_fields = {field.strip() for field in documented_fields}
+
+            # Clean up documented fields (remove whitespace)
+            documented_fields = {field.strip() for field in documented_fields}
+            print("\n\nDocumented fields: ")
+            for _field in documented_fields:
+                print(_field)
+
+            # Compare and find undocumented fields
+            undocumented_fields = all_fields - documented_fields
+
+            print("\n\nUndocumented fields: ")
+            for _field in undocumented_fields:
+                print(_field)
+
+            if undocumented_fields:
+                raise Exception(
+                    f"\nFields not documented in 'StandardLoggingPayload': {undocumented_fields}"
+                )
+
+            print(
+                f"All {len(all_fields)} fields are documented in 'StandardLoggingPayload'"
+            )
+
+    except FileNotFoundError:
+        raise Exception(
+            f"Documentation file not found at {docs_path}. Please ensure the documentation exists."
+        )