diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index 08ebf8b28c..5cb5ab3af2 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -56,12 +56,32 @@ litellm_settings: **Step 2**: Set Required env variables for datadog +#### Direct API + +Send logs directly to Datadog API: + ```shell DD_API_KEY="5f2d0f310***********" # your datadog API Key DD_SITE="us5.datadoghq.com" # your datadog base url DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source. use to differentiate dev vs. prod deployments ``` +#### Via DataDog Agent + +Send logs through a local DataDog agent (useful for containerized environments): + +```shell +DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent +DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) +DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) +DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source +``` + +When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: +- Centralized log shipping in containerized environments +- Reducing direct API calls from multiple services +- Leveraging agent-side processing and filtering + **Step 3**: Start the proxy, make a test request Start proxy @@ -169,8 +189,10 @@ LiteLLM supports customizing the following Datadog environment variables | Environment Variable | Description | Default Value | Required | |---------------------|-------------|---------------|----------| -| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes | -| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes | +| `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* | +| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* | +| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | +| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | | `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | | `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | | `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | @@ -178,3 +200,6 @@ LiteLLM supports customizing the following Datadog environment variables | `HOSTNAME` | Hostname tag for your logs | "" | ❌ No | | `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | +\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required +\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required + diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 128b5978d2..4404bf6898 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -460,6 +460,8 @@ router_settings: | DD_BASE_URL | Base URL for Datadog integration | DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration | _DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration +| DD_AGENT_HOST | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API +| DD_AGENT_PORT | Port of DataDog agent for log intake. Default is 10518 | DD_API_KEY | API key for Datadog integration | DD_SITE | Site URL for Datadog (e.g., datadoghq.com) | DD_SOURCE | Source identifier for Datadog logs diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 0c62667f74..46e1a2c201 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -17,7 +17,6 @@ import asyncio import datetime import os import traceback -from litellm._uuid import uuid from datetime import datetime as datetimeObj from typing import Any, Dict, List, Optional, Union @@ -26,6 +25,7 @@ from httpx import Response import litellm from litellm._logging import verbose_logger +from litellm._uuid import uuid from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, @@ -60,17 +60,19 @@ class DataDogLogger( """ Initializes the datadog logger, checks if the correct env variables are set - Required environment variables: + Required environment variables (Direct API): `DD_API_KEY` - your datadog api key `DD_SITE` - your datadog site, example = `"us5.datadoghq.com"` + + Optional environment variables (DataDog Agent): + `DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` + `DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) + + Note: If DD_AGENT_HOST is set, logs will be sent to the agent instead of directly to DataDog API. + In this case, DD_API_KEY and DD_SITE are not required (agent handles authentication). """ try: verbose_logger.debug("Datadog: in init datadog logger") - # check if the correct env variables are set - if os.getenv("DD_API_KEY", None) is None: - raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>") - if os.getenv("DD_SITE", None) is None: - raise Exception("DD_SITE is not set in .env, set 'DD_SITE=<>") ######################################################### # Handle datadog_params set as litellm.datadog_params @@ -81,21 +83,16 @@ class DataDogLogger( self.async_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) - self.DD_API_KEY = os.getenv("DD_API_KEY") - self.intake_url = ( - f"https://http-intake.logs.{os.getenv('DD_SITE')}/api/v2/logs" - ) - - ################################### - # OPTIONAL -only used for testing - dd_base_url: Optional[str] = ( - os.getenv("_DATADOG_BASE_URL") - or os.getenv("DATADOG_BASE_URL") - or os.getenv("DD_BASE_URL") - ) - if dd_base_url is not None: - self.intake_url = f"{dd_base_url}/api/v2/logs" - ################################### + + # Configure DataDog endpoint (Agent or Direct API) + dd_agent_host = os.getenv("DD_AGENT_HOST") + if dd_agent_host: + self._configure_dd_agent(dd_agent_host=dd_agent_host) + else: + self._configure_dd_direct_api() + + # Optional override for testing + self._apply_dd_base_url_override() self.sync_client = _get_httpx_client() asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() @@ -123,6 +120,47 @@ class DataDogLogger( dict_datadog_params = DatadogInitParams(**litellm.datadog_params).model_dump() return dict_datadog_params + def _configure_dd_agent(self, dd_agent_host: str) -> None: + """ + Configure DataDog Agent for log forwarding + + Args: + dd_agent_host: Hostname or IP of DataDog agent + """ + dd_agent_port = os.getenv("DD_AGENT_PORT", "10518") # default port for logs + self.intake_url = f"http://{dd_agent_host}:{dd_agent_port}/api/v2/logs" + self.DD_API_KEY = os.getenv("DD_API_KEY") # Optional when using agent + verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}") + + def _configure_dd_direct_api(self) -> None: + """ + Configure direct DataDog API connection + + Raises: + Exception: If required environment variables are not set + """ + if os.getenv("DD_API_KEY", None) is None: + raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>") + if os.getenv("DD_SITE", None) is None: + raise Exception("DD_SITE is not set in .env, set 'DD_SITE=<>") + + self.DD_API_KEY = os.getenv("DD_API_KEY") + self.intake_url = ( + f"https://http-intake.logs.{os.getenv('DD_SITE')}/api/v2/logs" + ) + + def _apply_dd_base_url_override(self) -> None: + """ + Apply base URL override for testing purposes + """ + dd_base_url: Optional[str] = ( + os.getenv("_DATADOG_BASE_URL") + or os.getenv("DATADOG_BASE_URL") + or os.getenv("DD_BASE_URL") + ) + if dd_base_url is not None: + self.intake_url = f"{dd_base_url}/api/v2/logs" + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): """ Async Log success events to Datadog @@ -226,12 +264,16 @@ class DataDogLogger( end_time=end_time, ) + # Build headers + headers = {} + # Add API key if available (required for direct API, optional for agent) + if self.DD_API_KEY: + headers["DD-API-KEY"] = self.DD_API_KEY + response = self.sync_client.post( url=self.intake_url, json=dd_payload, # type: ignore - headers={ - "DD-API-KEY": self.DD_API_KEY, - }, + headers=headers, ) response.raise_for_status() @@ -342,14 +384,21 @@ class DataDogLogger( from litellm.litellm_core_utils.safe_json_dumps import safe_dumps compressed_data = gzip.compress(safe_dumps(data).encode("utf-8")) + + # Build headers + headers = { + "Content-Encoding": "gzip", + "Content-Type": "application/json", + } + + # Add API key if available (required for direct API, optional for agent) + if self.DD_API_KEY: + headers["DD-API-KEY"] = self.DD_API_KEY + response = await self.async_client.post( url=self.intake_url, data=compressed_data, # type: ignore - headers={ - "DD-API-KEY": self.DD_API_KEY, - "Content-Encoding": "gzip", - "Content-Type": "application/json", - }, + headers=headers, ) return response diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index db6f193327..5870395533 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -628,4 +628,30 @@ async def test_datadog_message_redaction(): finally: # Clean up litellm.datadog_params = None - litellm.callbacks = [] \ No newline at end of file + litellm.callbacks = [] + + +def test_datadog_agent_configuration(): + """ + Test that DataDog logger correctly configures agent endpoint when DD_AGENT_HOST is set + """ + test_env = { + "DD_AGENT_HOST": "localhost", + "DD_AGENT_PORT": "10518", + } + + # Remove DD_SITE and DD_API_KEY to verify they're not required for agent mode + env_to_remove = ["DD_SITE", "DD_API_KEY"] + + with patch.dict(os.environ, test_env, clear=False): + for key in env_to_remove: + os.environ.pop(key, None) + + with patch("asyncio.create_task"): + dd_logger = DataDogLogger() + + # Verify agent endpoint is configured correctly + assert dd_logger.intake_url == "http://localhost:10518/api/v2/logs", f"Expected agent URL, got {dd_logger.intake_url}" + + # Verify DD_API_KEY is optional (can be None) + assert dd_logger.DD_API_KEY is None or isinstance(dd_logger.DD_API_KEY, str) \ No newline at end of file