From 10e769a5e4108b669fb1a39ee01c4bb660d524e2 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 25 Feb 2026 22:03:51 +0530 Subject: [PATCH 01/13] feat: ability to trace metrics --- docs/my-website/docs/observability/datadog.md | 60 ++ litellm/__init__.py | 756 +++++++++++++----- litellm/integrations/callback_configs.json | 23 +- .../integrations/datadog/datadog_metrics.py | 288 +++++++ .../custom_logger_registry.py | 2 + litellm/litellm_core_utils/litellm_logging.py | 30 +- litellm/proxy/common_utils/callback_utils.py | 8 +- .../health_endpoints/_health_endpoints.py | 16 + litellm/types/integrations/datadog_metrics.py | 19 + .../datadog/test_datadog_metrics.py | 272 +++++++ 10 files changed, 1245 insertions(+), 229 deletions(-) create mode 100644 litellm/integrations/datadog/datadog_metrics.py create mode 100644 litellm/types/integrations/datadog_metrics.py create mode 100644 tests/test_litellm/integrations/datadog/test_datadog_metrics.py diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index 9385b0020c..ef48594cbf 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem'; LiteLLM Supports logging to the following Datdog Integrations: - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/) - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/) +- `datadog_metrics` [Datadog Custom Metrics](#datadog-custom-metrics) - `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management) - `ddtrace-run` [Datadog Tracing](#datadog-tracing) @@ -168,6 +169,65 @@ On the Datadog LLM Observability page, you should see that both input messages a +## Datadog Custom Metrics + +| Feature | Details | +|---------|---------| +| **What is logged** | Latency metrics, request counts by status code | +| **Events** | Success + Failure | +| **Product Link** | [Datadog Metrics](https://docs.datadoghq.com/metrics/) | + +Publishes the following metrics to Datadog via the `/api/v2/series` endpoint: + +| Metric | Type | Description | +|--------|------|-------------| +| `litellm.request.total_latency` | Gauge | End-to-end request latency (seconds) | +| `litellm.llm_api.latency` | Gauge | Time spent waiting for the LLM provider response (seconds) | +| `litellm.llm_api.request_count` | Count | Request count, tagged with status code | + +Using `total_latency` and `llm_api.latency`, you can derive **internal latency** = `total_latency - llm_api.latency`. + +All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, `POD_NAME`, `provider`, `model_name`, `model_group`, `team`, `status_code`. + +**Step 1**: Create a `config.yaml` file + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["datadog_metrics"] + failure_callback: ["datadog_metrics"] +``` + +**Step 2**: Set required env variables + +```shell +DD_API_KEY="your-api-key" +DD_SITE="us5.datadoghq.com" # your datadog site +``` + +**Step 3**: Start the proxy and make a test request + +```shell +litellm --config config.yaml +``` + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}] +}' +``` + +**Step 4**: View metrics in Datadog Metrics Explorer + +Navigate to **Metrics > Explorer** in Datadog and search for `litellm.request.total_latency`, `litellm.llm_api.latency`, or `litellm.llm_api.request_count`. + ## Datadog Cloud Cost Management | Feature | Details | diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e42f2c1ea..41f5b0ff42 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -75,6 +75,7 @@ from litellm.constants import ( ) import httpx import dotenv + # register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" @@ -106,6 +107,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "otel", "datadog", "datadog_llm_observability", + "datadog_metrics", "galileo", "braintrust", "arize", @@ -147,7 +149,9 @@ _known_custom_logger_compatible_callbacks: List = list( get_args(_custom_logger_compatible_callbacks_literal) ) callbacks: List[ - Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded + Union[ + Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger" + ] # CustomLogger is lazy-loaded ] = [] callback_settings: Dict[str, Dict[str, Any]] = {} initialized_langfuse_clients: int = 0 @@ -157,42 +161,50 @@ prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. -gcs_pub_sub_use_v1: Optional[bool] = ( - False # if you want to use v1 gcs pubsub logged payload -) -generic_api_use_v1: Optional[bool] = ( - False # if you want to use v1 generic api logged payload -) +gcs_pub_sub_use_v1: Optional[ + bool +] = False # if you want to use v1 gcs pubsub logged payload +generic_api_use_v1: Optional[ + bool +] = False # if you want to use v1 generic api logged payload argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded +_async_input_callback: List[ + Union[str, Callable, "CustomLogger"] +] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded +_async_success_callback: List[ + Union[str, Callable, "CustomLogger"] +] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded +_async_failure_callback: List[ + Union[str, Callable, "CustomLogger"] +] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False -standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it +standard_logging_payload_excluded_fields: Optional[ + List[str] +] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False -add_user_information_to_llm_headers: Optional[bool] = ( - None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers -) +add_user_information_to_llm_headers: Optional[ + bool +] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# -email: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -token: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) +email: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +token: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) @@ -251,9 +263,9 @@ use_client: bool = False ssl_verify: Union[str, bool] = True ssl_security_level: Optional[str] = None ssl_certificate: Optional[str] = None -ssl_ecdh_curve: Optional[str] = ( - None # Set to 'X25519' to disable PQC and improve performance -) +ssl_ecdh_curve: Optional[ + str +] = None # Set to 'X25519' to disable PQC and improve performance disable_streaming_logging: bool = False disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False @@ -303,24 +315,20 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) -caching: bool = ( - False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -caching_with_models: bool = ( - False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -cache: Optional["Cache"] = ( - None # cache object <- use this - https://docs.litellm.ai/docs/caching -) +caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +cache: Optional[ + "Cache" +] = None # cache object <- use this - https://docs.litellm.ai/docs/caching default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None model_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers -budget_duration: Optional[str] = ( - None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). -) +budget_duration: Optional[ + str +] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) @@ -329,9 +337,7 @@ forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} -add_function_to_prompt: bool = ( - False # if function calling not supported by api, append function call details to system prompt -) +add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' @@ -378,9 +384,7 @@ prometheus_emit_stream_label: bool = False disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) -disable_copilot_system_to_assistant: bool = ( - False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. -) +disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. public_mcp_servers: Optional[List[str]] = None public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None @@ -399,17 +403,13 @@ if TYPE_CHECKING: ######## Networking Settings ######## -use_aiohttp_transport: bool = ( - True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. -) +use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) -force_ipv4: bool = ( - False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. -) +force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. network_mock: bool = False # When True, use mock transport — no real network calls ####### STOP SEQUENCE LIMIT ####### @@ -424,13 +424,13 @@ context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 allow_dynamic_callback_disabling: bool = True -num_retries_per_request: Optional[int] = ( - None # for the request overall (incl. fallbacks + model retries) -) +num_retries_per_request: Optional[ + int +] = None # for the request overall (incl. fallbacks + model retries) ####### SECRET MANAGERS ##################### -secret_manager_client: Optional[Any] = ( - None # list of instantiated key management clients - e.g. azure kv, infisical, etc. -) +secret_manager_client: Optional[ + Any +] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc. _google_kms_resource_name: Optional[str] = None _key_management_system: Optional["KeyManagementSystem"] = None # Note: KeyManagementSettings must be eagerly imported because _key_management_settings @@ -443,12 +443,12 @@ output_parse_pii: bool = False from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map model_cost = get_model_cost_map(url=model_cost_map_url) -cost_discount_config: Dict[str, float] = ( - {} -) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount -cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = ( - {} -) # Provider-specific or global cost margins. Examples: +cost_discount_config: Dict[ + str, float +] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount +cost_margin_config: Dict[ + str, Union[float, Dict[str, float]] +] = {} # Provider-specific or global cost margins. Examples: # Percentage: {"openai": 0.10} = 10% margin # Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request # Global: {"global": 0.05} = 5% global margin on all providers @@ -1107,10 +1107,12 @@ openai_video_generation_models = ["sora-2"] # Import KeyManagementSettings here (before utils import) because _key_management_settings # is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) from litellm.types.secret_managers.main import KeyManagementSettings + _key_management_settings: KeyManagementSettings = KeyManagementSettings() # client must be imported immediately as it's used as a decorator at function definition time from .utils import client + # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time @@ -1139,6 +1141,7 @@ from .llms.topaz.common_utils import TopazModelInfo # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo + # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) # All remaining configs are now lazy loaded - see _lazy_imports_registry.py @@ -1220,6 +1223,7 @@ from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * from .responses.main import * + # Interactions API is available as litellm.interactions module # Usage: litellm.interactions.create(), litellm.interactions.get(), etc. from . import interactions @@ -1278,12 +1282,12 @@ from . import rag from .types.llms.custom_llm import CustomLLMItem custom_provider_map: List[CustomLLMItem] = [] -_custom_providers: List[str] = ( - [] -) # internal helper util, used to track names of custom providers -disable_hf_tokenizer_download: Optional[bool] = ( - None # disable huggingface tokenizer download. Defaults to openai clk100 -) +_custom_providers: List[ + str +] = [] # internal helper util, used to track names of custom providers +disable_hf_tokenizer_download: Optional[ + bool +] = None # disable huggingface tokenizer download. Defaults to openai clk100 global_disable_no_log_param: bool = False ### CLI UTILITIES ### @@ -1322,128 +1326,318 @@ if TYPE_CHECKING: from litellm.caching.caching import Cache # Type stubs for lazy-loaded configs to help mypy - from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig - from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig - from .llms.galadriel.chat.transformation import GaladrielChatConfig as GaladrielChatConfig + from .llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig as AmazonConverseConfig, + ) + from .llms.openai_like.chat.handler import ( + OpenAILikeChatConfig as OpenAILikeChatConfig, + ) + from .llms.galadriel.chat.transformation import ( + GaladrielChatConfig as GaladrielChatConfig, + ) from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig - from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig as AzureAnthropicConfig + from .llms.azure_ai.anthropic.transformation import ( + AzureAnthropicConfig as AzureAnthropicConfig, + ) from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig - from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig + from .llms.compactifai.chat.transformation import ( + CompactifAIChatConfig as CompactifAIChatConfig, + ) from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig - from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig - from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig - from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig + from .llms.aiohttp_openai.chat.transformation import ( + AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig, + ) + from .llms.huggingface.chat.transformation import ( + HuggingFaceChatConfig as HuggingFaceChatConfig, + ) + from .llms.huggingface.embedding.transformation import ( + HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig, + ) from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig from .llms.maritalk import MaritalkConfig as MaritalkConfig - from .llms.openrouter.chat.transformation import OpenrouterConfig as OpenrouterConfig + from .llms.openrouter.chat.transformation import ( + OpenrouterConfig as OpenrouterConfig, + ) from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig - from .llms.anthropic.completion.transformation import AnthropicTextConfig as AnthropicTextConfig + from .llms.anthropic.completion.transformation import ( + AnthropicTextConfig as AnthropicTextConfig, + ) from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig from .llms.triton.completion.transformation import TritonConfig as TritonConfig - from .llms.triton.completion.transformation import TritonGenerateConfig as TritonGenerateConfig - from .llms.triton.completion.transformation import TritonInferConfig as TritonInferConfig - from .llms.triton.embedding.transformation import TritonEmbeddingConfig as TritonEmbeddingConfig - from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig as HuggingFaceRerankConfig - from .llms.databricks.chat.transformation import DatabricksConfig as DatabricksConfig - from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig as DatabricksEmbeddingConfig + from .llms.triton.completion.transformation import ( + TritonGenerateConfig as TritonGenerateConfig, + ) + from .llms.triton.completion.transformation import ( + TritonInferConfig as TritonInferConfig, + ) + from .llms.triton.embedding.transformation import ( + TritonEmbeddingConfig as TritonEmbeddingConfig, + ) + from .llms.huggingface.rerank.transformation import ( + HuggingFaceRerankConfig as HuggingFaceRerankConfig, + ) + from .llms.databricks.chat.transformation import ( + DatabricksConfig as DatabricksConfig, + ) + from .llms.databricks.embed.transformation import ( + DatabricksEmbeddingConfig as DatabricksEmbeddingConfig, + ) from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig - from .llms.cohere.rerank.transformation import CohereRerankConfig as CohereRerankConfig - from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config as CohereRerankV2Config - from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig as AzureAIRerankConfig - from .llms.infinity.rerank.transformation import InfinityRerankConfig as InfinityRerankConfig - from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig as JinaAIRerankConfig - from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig as DeepinfraRerankConfig - from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig as HostedVLLMRerankConfig - from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig as NvidiaNimRerankConfig - from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig as NvidiaNimRankingConfig - from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig - from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig - from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig - from .llms.watsonx.rerank.transformation import IBMWatsonXRerankConfig as IBMWatsonXRerankConfig + from .llms.cohere.rerank.transformation import ( + CohereRerankConfig as CohereRerankConfig, + ) + from .llms.cohere.rerank_v2.transformation import ( + CohereRerankV2Config as CohereRerankV2Config, + ) + from .llms.azure_ai.rerank.transformation import ( + AzureAIRerankConfig as AzureAIRerankConfig, + ) + from .llms.infinity.rerank.transformation import ( + InfinityRerankConfig as InfinityRerankConfig, + ) + from .llms.jina_ai.rerank.transformation import ( + JinaAIRerankConfig as JinaAIRerankConfig, + ) + from .llms.deepinfra.rerank.transformation import ( + DeepinfraRerankConfig as DeepinfraRerankConfig, + ) + from .llms.hosted_vllm.rerank.transformation import ( + HostedVLLMRerankConfig as HostedVLLMRerankConfig, + ) + from .llms.nvidia_nim.rerank.transformation import ( + NvidiaNimRerankConfig as NvidiaNimRerankConfig, + ) + from .llms.nvidia_nim.rerank.ranking_transformation import ( + NvidiaNimRankingConfig as NvidiaNimRankingConfig, + ) + from .llms.vertex_ai.rerank.transformation import ( + VertexAIRerankConfig as VertexAIRerankConfig, + ) + from .llms.fireworks_ai.rerank.transformation import ( + FireworksAIRerankConfig as FireworksAIRerankConfig, + ) + from .llms.voyage.rerank.transformation import ( + VoyageRerankConfig as VoyageRerankConfig, + ) + from .llms.watsonx.rerank.transformation import ( + IBMWatsonXRerankConfig as IBMWatsonXRerankConfig, + ) from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig - from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig as TogetherAITextCompletionConfig - from .llms.cloudflare.chat.transformation import CloudflareChatConfig as CloudflareChatConfig + from .llms.together_ai.completion.transformation import ( + TogetherAITextCompletionConfig as TogetherAITextCompletionConfig, + ) + from .llms.cloudflare.chat.transformation import ( + CloudflareChatConfig as CloudflareChatConfig, + ) from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig - from .llms.sagemaker.completion.transformation import SagemakerConfig as SagemakerConfig - from .llms.sagemaker.chat.transformation import SagemakerChatConfig as SagemakerChatConfig + from .llms.sagemaker.completion.transformation import ( + SagemakerConfig as SagemakerConfig, + ) + from .llms.sagemaker.chat.transformation import ( + SagemakerChatConfig as SagemakerChatConfig, + ) from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig - from .llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig as AnthropicMessagesConfig - from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig + from .llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig as AnthropicMessagesConfig, + ) + from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig, + ) from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as VertexGeminiConfig - from .llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig - from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig as VertexAIAnthropicConfig - from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import VertexAILlama3Config as VertexAILlama3Config - from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import VertexAIAi21Config as VertexAIAi21Config - from .llms.bedrock.chat.invoke_handler import AmazonCohereChatConfig as AmazonCohereChatConfig - from .llms.bedrock.common_utils import AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig - from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import AmazonAI21Config as AmazonAI21Config - from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import AmazonInvokeNovaConfig as AmazonInvokeNovaConfig - from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import AmazonQwen2Config as AmazonQwen2Config - from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import AmazonQwen3Config as AmazonQwen3Config - from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import AmazonAnthropicConfig as AmazonAnthropicConfig - from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig - from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import AmazonCohereConfig as AmazonCohereConfig - from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig - from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config - from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig - from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig - from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig - from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig - from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig - from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig - from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig as AmazonStabilityConfig - from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config as AmazonStability3Config - from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig as AmazonNovaCanvasConfig - from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config as AmazonTitanG1Config - from .llms.bedrock.embed.amazon_titan_multimodal_transformation import AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config - from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig as CohereV2ChatConfig - from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig - from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig - from .llms.bedrock.embed.amazon_nova_transformation import AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig - from .llms.openai.openai import OpenAIConfig as OpenAIConfig, MistralEmbeddingConfig as MistralEmbeddingConfig - from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig as OpenAIImageVariationConfig - from .llms.deepgram.audio_transcription.transformation import DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig - from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig - from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig as VertexGeminiConfig, + ) + from .llms.gemini.chat.transformation import ( + GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig, + ) + from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( + VertexAIAnthropicConfig as VertexAIAnthropicConfig, + ) + from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( + VertexAILlama3Config as VertexAILlama3Config, + ) + from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import ( + VertexAIAi21Config as VertexAIAi21Config, + ) + from .llms.bedrock.chat.invoke_handler import ( + AmazonCohereChatConfig as AmazonCohereChatConfig, + ) + from .llms.bedrock.common_utils import ( + AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import ( + AmazonAI21Config as AmazonAI21Config, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import ( + AmazonInvokeNovaConfig as AmazonInvokeNovaConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import ( + AmazonQwen2Config as AmazonQwen2Config, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( + AmazonQwen3Config as AmazonQwen3Config, + ) + from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import ( + AmazonAnthropicConfig as AmazonAnthropicConfig, + ) + from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import ( + AmazonCohereConfig as AmazonCohereConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import ( + AmazonLlamaConfig as AmazonLlamaConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import ( + AmazonDeepSeekR1Config as AmazonDeepSeekR1Config, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import ( + AmazonMistralConfig as AmazonMistralConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( + AmazonMoonshotConfig as AmazonMoonshotConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import ( + AmazonTitanConfig as AmazonTitanConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( + AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig, + ) + from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig as AmazonInvokeConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig, + ) + from .llms.bedrock.image_generation.amazon_stability1_transformation import ( + AmazonStabilityConfig as AmazonStabilityConfig, + ) + from .llms.bedrock.image_generation.amazon_stability3_transformation import ( + AmazonStability3Config as AmazonStability3Config, + ) + from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( + AmazonNovaCanvasConfig as AmazonNovaCanvasConfig, + ) + from .llms.bedrock.embed.amazon_titan_g1_transformation import ( + AmazonTitanG1Config as AmazonTitanG1Config, + ) + from .llms.bedrock.embed.amazon_titan_multimodal_transformation import ( + AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config, + ) + from .llms.cohere.chat.v2_transformation import ( + CohereV2ChatConfig as CohereV2ChatConfig, + ) + from .llms.bedrock.embed.cohere_transformation import ( + BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig, + ) + from .llms.bedrock.embed.twelvelabs_marengo_transformation import ( + TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig, + ) + from .llms.bedrock.embed.amazon_nova_transformation import ( + AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig, + ) + from .llms.openai.openai import ( + OpenAIConfig as OpenAIConfig, + MistralEmbeddingConfig as MistralEmbeddingConfig, + ) + from .llms.openai.image_variations.transformation import ( + OpenAIImageVariationConfig as OpenAIImageVariationConfig, + ) + from .llms.deepgram.audio_transcription.transformation import ( + DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig, + ) + from .llms.topaz.image_variations.transformation import ( + TopazImageVariationConfig as TopazImageVariationConfig, + ) + from litellm.llms.openai.completion.transformation import ( + OpenAITextCompletionConfig as OpenAITextCompletionConfig, + ) from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig from .llms.a2a.chat.transformation import A2AConfig as A2AConfig - from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig - from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig - from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig as InfinityEmbeddingConfig - from .llms.azure_ai.chat.transformation import AzureAIStudioConfig as AzureAIStudioConfig + from .llms.voyage.embedding.transformation import ( + VoyageEmbeddingConfig as VoyageEmbeddingConfig, + ) + from .llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig, + ) + from .llms.infinity.embedding.transformation import ( + InfinityEmbeddingConfig as InfinityEmbeddingConfig, + ) + from .llms.azure_ai.chat.transformation import ( + AzureAIStudioConfig as AzureAIStudioConfig, + ) from .llms.mistral.chat.transformation import MistralConfig as MistralConfig - from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig - from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig - from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig - from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig - from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig - from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig - from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig - from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig - from .llms.databricks.responses.transformation import DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig - from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig - from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config - from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig - from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig as BaseSkillsAPIConfig - from .llms.gradient_ai.chat.transformation import GradientAIConfig as GradientAIConfig + from .llms.openai.responses.transformation import ( + OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig, + ) + from .llms.azure.responses.transformation import ( + AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig, + ) + from .llms.azure.responses.o_series_transformation import ( + AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig, + ) + from .llms.xai.responses.transformation import ( + XAIResponsesAPIConfig as XAIResponsesAPIConfig, + ) + from .llms.litellm_proxy.responses.transformation import ( + LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, + ) + from .llms.volcengine.responses.transformation import ( + VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig, + ) + from .llms.manus.responses.transformation import ( + ManusResponsesAPIConfig as ManusResponsesAPIConfig, + ) + from .llms.perplexity.responses.transformation import ( + PerplexityResponsesConfig as PerplexityResponsesConfig, + ) + from .llms.databricks.responses.transformation import ( + DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig, + ) + from .llms.gemini.interactions.transformation import ( + GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, + ) + from .llms.openai.chat.o_series_transformation import ( + OpenAIOSeriesConfig as OpenAIOSeriesConfig, + OpenAIOSeriesConfig as OpenAIO1Config, + ) + from .llms.anthropic.skills.transformation import ( + AnthropicSkillsConfig as AnthropicSkillsConfig, + ) + from .llms.base_llm.skills.transformation import ( + BaseSkillsAPIConfig as BaseSkillsAPIConfig, + ) + from .llms.gradient_ai.chat.transformation import ( + GradientAIConfig as GradientAIConfig, + ) from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig - from .llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config as OpenAIGPT5Config - from .llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig - from .llms.openai.transcriptions.gpt_transformation import OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig - from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig + from .llms.openai.chat.gpt_5_transformation import ( + OpenAIGPT5Config as OpenAIGPT5Config, + ) + from .llms.openai.transcriptions.whisper_transformation import ( + OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig, + ) + from .llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig, + ) + from .llms.openai.chat.gpt_audio_transformation import ( + OpenAIGPTAudioConfig as OpenAIGPTAudioConfig, + ) from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig - from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig + from .llms.nvidia_nim.embed import ( + NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig, + ) # Type stubs for lazy-loaded config instances openaiOSeriesConfig: OpenAIOSeriesConfig @@ -1455,21 +1649,47 @@ if TYPE_CHECKING: # Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig - from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig - from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig - from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig - from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config as _AzureOpenAIO1Config - from .llms.perplexity.chat.transformation import PerplexityChatConfig as _PerplexityChatConfig + from .llms.deepseek.chat.transformation import ( + DeepSeekChatConfig as _DeepSeekChatConfig, + ) + from .llms.sap.chat.transformation import ( + GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig, + ) + from .llms.sap.embed.transformation import ( + GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig, + ) + from .llms.azure.chat.o_series_transformation import ( + AzureOpenAIO1Config as _AzureOpenAIO1Config, + ) + from .llms.perplexity.chat.transformation import ( + PerplexityChatConfig as _PerplexityChatConfig, + ) from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig - from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig as _IBMWatsonXChatConfig - from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig as _IBMWatsonXAIConfig - from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig + from .llms.watsonx.chat.transformation import ( + IBMWatsonXChatConfig as _IBMWatsonXChatConfig, + ) + from .llms.watsonx.completion.transformation import ( + IBMWatsonXAIConfig as _IBMWatsonXAIConfig, + ) + from .llms.litellm_proxy.chat.transformation import ( + LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig, + ) from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig - from .llms.llamafile.chat.transformation import LlamafileChatConfig as _LlamafileChatConfig - from .llms.lm_studio.chat.transformation import LMStudioChatConfig as _LMStudioChatConfig - from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig - from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig + from .llms.llamafile.chat.transformation import ( + LlamafileChatConfig as _LlamafileChatConfig, + ) + from .llms.lm_studio.chat.transformation import ( + LMStudioChatConfig as _LMStudioChatConfig, + ) + from .llms.lm_studio.embed.transformation import ( + LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, + ) + from .llms.watsonx.embed.transformation import ( + IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig, + ) + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig as _VertexGeminiConfig, + ) # Type stubs for lazy-loaded config classes (to help mypy understand types) VLLMConfig: Type[_VLLMConfig] @@ -1489,55 +1709,122 @@ if TYPE_CHECKING: IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig - from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig + from .llms.featherless_ai.chat.transformation import ( + FeatherlessAIConfig as FeatherlessAIConfig, + ) from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig - from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig - from .llms.fireworks_ai.chat.transformation import FireworksAIConfig as FireworksAIConfig - from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig as FireworksAITextCompletionConfig - from .llms.fireworks_ai.audio_transcription.transformation import FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig - from .llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig - from .llms.friendliai.chat.transformation import FriendliaiChatConfig as FriendliaiChatConfig - from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig as JinaAIEmbeddingConfig + from .llms.sambanova.embedding.transformation import ( + SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig, + ) + from .llms.fireworks_ai.chat.transformation import ( + FireworksAIConfig as FireworksAIConfig, + ) + from .llms.fireworks_ai.completion.transformation import ( + FireworksAITextCompletionConfig as FireworksAITextCompletionConfig, + ) + from .llms.fireworks_ai.audio_transcription.transformation import ( + FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig, + ) + from .llms.fireworks_ai.embed.fireworks_ai_transformation import ( + FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig, + ) + from .llms.friendliai.chat.transformation import ( + FriendliaiChatConfig as FriendliaiChatConfig, + ) + from .llms.jina_ai.embedding.transformation import ( + JinaAIEmbeddingConfig as JinaAIEmbeddingConfig, + ) from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig - from .llms.volcengine.chat.transformation import VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig - from .llms.codestral.completion.transformation import CodestralTextCompletionConfig as CodestralTextCompletionConfig - from .llms.azure.azure import AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig + from .llms.volcengine.chat.transformation import ( + VolcEngineChatConfig as VolcEngineChatConfig, + VolcEngineChatConfig as VolcEngineConfig, + ) + from .llms.codestral.completion.transformation import ( + CodestralTextCompletionConfig as CodestralTextCompletionConfig, + ) + from .llms.azure.azure import ( + AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig, + ) from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig - from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig as AzureOpenAIConfig - from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config - from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig - from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig - from .llms.hosted_vllm.embedding.transformation import HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig - from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig - from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig - from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig + from .llms.azure.chat.gpt_transformation import ( + AzureOpenAIConfig as AzureOpenAIConfig, + ) + from .llms.azure.chat.gpt_5_transformation import ( + AzureOpenAIGPT5Config as AzureOpenAIGPT5Config, + ) + from .llms.azure.completion.transformation import ( + AzureOpenAITextConfig as AzureOpenAITextConfig, + ) + from .llms.hosted_vllm.chat.transformation import ( + HostedVLLMChatConfig as HostedVLLMChatConfig, + ) + from .llms.hosted_vllm.embedding.transformation import ( + HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig, + ) + from .llms.github_copilot.chat.transformation import ( + GithubCopilotConfig as GithubCopilotConfig, + ) + from .llms.github_copilot.responses.transformation import ( + GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig, + ) + from .llms.github_copilot.embedding.transformation import ( + GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig, + ) from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig - from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig + from .llms.chatgpt.responses.transformation import ( + ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig, + ) from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig - from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig + from .llms.gigachat.embedding.transformation import ( + GigaChatEmbeddingConfig as GigaChatEmbeddingConfig, + ) from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig - from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig - from .llms.moonshot.chat.transformation import MoonshotChatConfig as MoonshotChatConfig - from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig as DockerModelRunnerChatConfig + from .llms.dashscope.chat.transformation import ( + DashScopeChatConfig as DashScopeChatConfig, + ) + from .llms.moonshot.chat.transformation import ( + MoonshotChatConfig as MoonshotChatConfig, + ) + from .llms.docker_model_runner.chat.transformation import ( + DockerModelRunnerChatConfig as DockerModelRunnerChatConfig, + ) from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig - from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig as LambdaAIChatConfig - from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig as HyperbolicChatConfig - from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig as VercelAIGatewayConfig - from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig as OVHCloudChatConfig - from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig - from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig as CometAPIEmbeddingConfig - from .llms.lemonade.chat.transformation import LemonadeChatConfig as LemonadeChatConfig - from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig - from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig as AmazonNovaChatConfig + from .llms.lambda_ai.chat.transformation import ( + LambdaAIChatConfig as LambdaAIChatConfig, + ) + from .llms.hyperbolic.chat.transformation import ( + HyperbolicChatConfig as HyperbolicChatConfig, + ) + from .llms.vercel_ai_gateway.chat.transformation import ( + VercelAIGatewayConfig as VercelAIGatewayConfig, + ) + from .llms.ovhcloud.chat.transformation import ( + OVHCloudChatConfig as OVHCloudChatConfig, + ) + from .llms.ovhcloud.embedding.transformation import ( + OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig, + ) + from .llms.cometapi.embed.transformation import ( + CometAPIEmbeddingConfig as CometAPIEmbeddingConfig, + ) + from .llms.lemonade.chat.transformation import ( + LemonadeChatConfig as LemonadeChatConfig, + ) + from .llms.snowflake.embedding.transformation import ( + SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig, + ) + from .llms.amazon_nova.chat.transformation import ( + AmazonNovaChatConfig as AmazonNovaChatConfig, + ) from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( @@ -1598,6 +1885,7 @@ if TYPE_CHECKING: # Bedrock tool name mappings instance (lazy-loaded) from litellm.caching.caching import InMemoryCache + bedrock_tool_name_mappings: InMemoryCache # Azure exception class (lazy-loaded) @@ -1616,11 +1904,15 @@ if TYPE_CHECKING: from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams # Logging callback manager class and instance (lazy-loaded) - from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + from litellm.litellm_core_utils.logging_callback_manager import ( + LoggingCallbackManager, + ) + logging_callback_manager: LoggingCallbackManager # provider_list is lazy-loaded from litellm.types.utils import LlmProviders + provider_list: List[Union[LlmProviders, str]] # Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block @@ -1645,7 +1937,10 @@ def __getattr__(name: str) -> Any: global _async_client_cleanup_registered # Register async client cleanup on first access (only once) if not _async_client_cleanup_registered: - from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup + from litellm.llms.custom_httpx.async_client_cleanup import ( + register_async_client_cleanup, + ) + register_async_client_cleanup() _async_client_cleanup_registered = True @@ -1662,36 +1957,45 @@ def __getattr__(name: str) -> Any: # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "encoding" not in _globals: from .main import encoding as _encoding + _globals["encoding"] = _encoding return _globals["encoding"] # Lazy load bedrock_tool_name_mappings instance if name == "bedrock_tool_name_mappings": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "bedrock_tool_name_mappings" not in _globals: - from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings + from .llms.bedrock.chat.invoke_handler import ( + bedrock_tool_name_mappings as _bedrock_tool_name_mappings, + ) + _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings return _globals["bedrock_tool_name_mappings"] # Lazy load AzureOpenAIError exception class if name == "AzureOpenAIError": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "AzureOpenAIError" not in _globals: from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError + _globals["AzureOpenAIError"] = _AzureOpenAIError return _globals["AzureOpenAIError"] # Lazy load openaiOSeriesConfig instance if name == "openaiOSeriesConfig": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() if "openaiOSeriesConfig" not in _globals: # Import the config class and instantiate it @@ -1709,6 +2013,7 @@ def __getattr__(name: str) -> Any: } if name in _config_instances: from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() if name not in _globals: # Import the config class and instantiate it @@ -1723,17 +2028,20 @@ def __getattr__(name: str) -> Any: # Lazy load provider_list if name == "provider_list": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "provider_list" not in _globals: # LlmProviders is eagerly imported above, so we can import it directly from litellm.types.utils import LlmProviders + _globals["provider_list"] = list(LlmProviders) return _globals["provider_list"] # Lazy load priority_reservation_settings instance if name == "priority_reservation_settings": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "priority_reservation_settings" not in _globals: @@ -1745,6 +2053,7 @@ def __getattr__(name: str) -> Any: # Lazy load logging_callback_manager instance if name == "logging_callback_manager": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "logging_callback_manager" not in _globals: @@ -1756,19 +2065,41 @@ def __getattr__(name: str) -> Any: # Lazy load _service_logger module if name == "_service_logger": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "_service_logger" not in _globals: # Import the module lazily import litellm._service_logger + _globals["_service_logger"] = litellm._service_logger return _globals["_service_logger"] # Lazy load evals module functions - if name in ["acreate_eval", "alist_evals", "aget_eval", "aupdate_eval", "adelete_eval", "acancel_eval", - "create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval", - "acreate_run", "alist_runs", "aget_run", "acancel_run", "adelete_run", - "create_run", "list_runs", "get_run", "cancel_run", "delete_run"]: + if name in [ + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", + "create_eval", + "list_evals", + "get_eval", + "update_eval", + "delete_eval", + "cancel_eval", + "acreate_run", + "alist_runs", + "aget_run", + "acancel_run", + "adelete_run", + "create_run", + "list_runs", + "get_run", + "cancel_run", + "delete_run", + ]: from litellm.evals.main import ( acreate_eval, alist_evals, @@ -1793,6 +2124,7 @@ def __getattr__(name: str) -> Any: cancel_run, delete_run, ) + return locals()[name] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 6a003b8c49..c2b0c4ddce 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -83,6 +83,27 @@ }, "description": "Datadog Logging Integration" }, + { + "id": "datadog_metrics", + "displayName": "Datadog Metrics", + "logo": "datadog.png", + "supports_key_team_logging": false, + "dynamic_params": { + "dd_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Datadog API key for authentication", + "required": true + }, + "dd_site": { + "type": "text", + "ui_name": "Site", + "description": "Datadog site URL (e.g., us5.datadoghq.com)", + "required": true + } + }, + "description": "Datadog Custom Metrics Integration" + }, { "id": "datadog_cost_management", "displayName": "Datadog Cost Management", @@ -434,4 +455,4 @@ }, "description": "SQS Queue (AWS) Logging Integration" } -] \ No newline at end of file +] diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py new file mode 100644 index 0000000000..ca3e4aa5a1 --- /dev/null +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -0,0 +1,288 @@ +import asyncio +import os +import time +from datetime import datetime +from typing import List, Optional, Union + +from litellm._logging import verbose_logger +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus +from litellm.types.integrations.datadog_metrics import ( + DatadogMetricPoint, + DatadogMetricSeries, + DatadogMetricsPayload, +) +from litellm.types.utils import StandardLoggingPayload + + +class DatadogMetricsLogger(CustomBatchLogger): + def __init__(self, **kwargs): + self.dd_api_key = os.getenv("DD_API_KEY") + self.dd_app_key = os.getenv("DD_APP_KEY") + self.dd_site = os.getenv("DD_SITE", "datadoghq.com") + + if not self.dd_api_key: + verbose_logger.warning( + "Datadog Metrics: DD_API_KEY is required. Integration will not work." + ) + + self.upload_url = f"https://api.{self.dd_site}/api/v2/series" + + self.async_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.LoggingCallback + ) + + # Initialize lock + self.flush_lock = asyncio.Lock() + + # Check if flush_lock is already in kwargs to avoid double passing + kwargs["flush_lock"] = self.flush_lock + + # Send metrics more quickly to datadog (every 5 seconds) + if "flush_interval" not in kwargs: + kwargs["flush_interval"] = 5 + + super().__init__(**kwargs) + + # Start periodic flush task + asyncio.create_task(self.periodic_flush()) + + def _extract_tags( + self, + log: StandardLoggingPayload, + status_code: Optional[Union[str, int]] = None, + ) -> List[str]: + """ + Builds the list of tags for a Datadog metric point + """ + from litellm.integrations.datadog.datadog_handler import ( + get_datadog_env, + get_datadog_hostname, + get_datadog_pod_name, + get_datadog_service, + ) + + # Base tags + tags = [ + f"env:{get_datadog_env()}", + f"service:{get_datadog_service()}", + f"version:{os.getenv('DD_VERSION', 'unknown')}", + f"HOSTNAME:{get_datadog_hostname()}", + f"POD_NAME:{get_datadog_pod_name()}", + ] + + # Add metric-specific tags + if provider := log.get("custom_llm_provider"): + tags.append(f"provider:{provider}") + + if model := log.get("model"): + tags.append(f"model_name:{model}") + + if model_group := log.get("model_group"): + tags.append(f"model_group:{model_group}") + + if status_code is not None: + tags.append(f"status_code:{status_code}") + + # Extract team tag + metadata = log.get("metadata", {}) or {} + team_tag = ( + metadata.get("user_api_key_team_alias") + or metadata.get("team_alias") # type: ignore + or metadata.get("user_api_key_team_id") + or metadata.get("team_id") # type: ignore + ) + + if team_tag: + tags.append(f"team:{team_tag}") + + return tags + + def _add_metrics_from_log( + self, + log: StandardLoggingPayload, + kwargs: dict, + status_code: Union[str, int] = "200", + ): + """ + Extracts latencies and appends Datadog metric series to the queue + """ + tags = self._extract_tags(log, status_code=status_code) + + # We record metrics with the end_time as the timestamp for the point + end_time_dt = kwargs.get("end_time") or datetime.now() + timestamp = int(end_time_dt.timestamp()) + + # 1. Total Request Latency Metric (End to End) + start_time_dt = kwargs.get("start_time") + if start_time_dt and end_time_dt: + total_duration = (end_time_dt - start_time_dt).total_seconds() + series_total_latency: DatadogMetricSeries = { + "metric": "litellm.request.total_latency", + "type": 3, # gauge + "points": [{"timestamp": timestamp, "value": total_duration}], + "tags": tags, + } + self.log_queue.append(series_total_latency) + + # 2. LLM API Latency Metric (Provider alone) + api_call_start_time = kwargs.get("api_call_start_time") + if api_call_start_time and end_time_dt: + llm_api_duration = (end_time_dt - api_call_start_time).total_seconds() + series_llm_latency: DatadogMetricSeries = { + "metric": "litellm.llm_api.latency", + "type": 3, # gauge + "points": [{"timestamp": timestamp, "value": llm_api_duration}], + "tags": tags, + } + self.log_queue.append(series_llm_latency) + + # 3. Request Count / Status Code + series_count: DatadogMetricSeries = { + "metric": "litellm.llm_api.request_count", + "type": 1, # count + "points": [{"timestamp": timestamp, "value": 1.0}], + "tags": tags, + } + self.log_queue.append(series_count) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + try: + standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object", None + ) + + if standard_logging_object is None: + return + + self._add_metrics_from_log( + log=standard_logging_object, kwargs=kwargs, status_code="200" + ) + + if len(self.log_queue) >= self.batch_size: + await self.flush_queue() + + except Exception as e: + verbose_logger.exception( + f"Datadog Metrics: Error in async_log_success_event: {str(e)}" + ) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + try: + standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object", None + ) + + if standard_logging_object is None: + return + + # Extract status code from error information + status_code = "500" # default + error_information = ( + standard_logging_object.get("error_information", {}) or {} + ) + if "error_code" in error_information and error_information["error_code"] is not None: # type: ignore + status_code = str(error_information["error_code"]) # type: ignore + + self._add_metrics_from_log( + log=standard_logging_object, kwargs=kwargs, status_code=status_code + ) + + if len(self.log_queue) >= self.batch_size: + await self.flush_queue() + + except Exception as e: + verbose_logger.exception( + f"Datadog Metrics: Error in async_log_failure_event: {str(e)}" + ) + + async def async_send_batch(self): + if not self.log_queue: + return + + try: + # We must only send the current batch, so copy and clear log queue + batch = self.log_queue.copy() + # Note: CustomBatchLogger clears queue in flush_queue, but we'll manually copy what we need + + payload_data: DatadogMetricsPayload = {"series": batch} + + await self._upload_to_datadog(payload_data) + + except Exception as e: + verbose_logger.exception( + f"Datadog Metrics: Error in async_send_batch: {str(e)}" + ) + + async def _upload_to_datadog(self, payload: DatadogMetricsPayload): + if not self.dd_api_key: + return + + headers = { + "Content-Type": "application/json", + "DD-API-KEY": self.dd_api_key, + } + + if self.dd_app_key: + headers["DD-APPLICATION-KEY"] = self.dd_app_key + + import gzip + + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + + json_data = safe_dumps(payload) + compressed_data = gzip.compress(json_data.encode("utf-8")) + headers["Content-Encoding"] = "gzip" + + response = await self.async_client.post( + self.upload_url, content=compressed_data, headers=headers # type: ignore + ) + + response.raise_for_status() + + verbose_logger.debug( + f"Datadog Metrics: Uploaded {len(payload['series'])} metric points. Status: {response.status_code}" + ) + + async def async_health_check(self) -> IntegrationHealthCheckStatus: + """ + Check if the service is healthy + """ + try: + # Send a test metric point to Datadog + test_metric_point: DatadogMetricPoint = { + "timestamp": int(time.time()), + "value": 1.0, + } + test_metric_series: DatadogMetricSeries = { + "metric": "litellm.health_check", + "type": 3, # Gauge + "points": [test_metric_point], + "tags": ["env:health_check"], + } + + payload_data: DatadogMetricsPayload = {"series": [test_metric_series]} + + await self._upload_to_datadog(payload_data) + + return IntegrationHealthCheckStatus( + status="healthy", + error_message=None, + ) + except Exception as e: + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message=str(e), + ) + + async def get_request_response_payload( + self, + request_id: str, + start_time_utc: Optional[datetime], + end_time_utc: Optional[datetime], + ) -> Optional[dict]: + pass diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index fc73701ea9..2d483f7861 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -20,6 +20,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger +from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger from litellm.integrations.deepeval import DeepEvalLogger from litellm.integrations.dotprompt import DotpromptManager from litellm.integrations.focus.focus_logger import FocusLogger @@ -66,6 +67,7 @@ class CustomLoggerRegistry: "prometheus": PrometheusLogger, "datadog": DataDogLogger, "datadog_llm_observability": DataDogLLMObsLogger, + "datadog_metrics": DatadogMetricsLogger, "gcs_bucket": GCSBucketLogger, "opik": OpikLogger, "argilla": ArgillaLogger, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0601e7e845..98f573d471 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -134,6 +134,7 @@ from ..integrations.azure_storage.azure_storage import AzureBlobStorageLogger from ..integrations.custom_prompt_management import CustomPromptManagement from ..integrations.datadog.datadog import DataDogLogger from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger +from ..integrations.datadog.datadog_metrics import DatadogMetricsLogger from ..integrations.dotprompt import DotpromptManager from ..integrations.dynamodb import DyanmoDBLogger from ..integrations.galileo import GalileoObserve @@ -1653,9 +1654,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload( - logging_result, start_time, end_time - ) + ] = self._build_standard_logging_payload(logging_result, start_time, end_time) if ( standard_logging_payload := self.model_call_details.get( @@ -2518,9 +2517,7 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload( - result, start_time, end_time - ) + ] = self._build_standard_logging_payload(result, start_time, end_time) # print standard logging payload if ( @@ -3665,6 +3662,14 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _datadog_llm_obs_logger = DataDogLLMObsLogger() _in_memory_loggers.append(_datadog_llm_obs_logger) return _datadog_llm_obs_logger # type: ignore + elif logging_integration == "datadog_metrics": + for callback in _in_memory_loggers: + if isinstance(callback, DatadogMetricsLogger): + return callback # type: ignore + + _datadog_metrics_logger = DatadogMetricsLogger() + _in_memory_loggers.append(_datadog_metrics_logger) + return _datadog_metrics_logger # type: ignore elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4214,6 +4219,10 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 for callback in _in_memory_loggers: if isinstance(callback, DataDogLLMObsLogger): return callback + elif logging_integration == "datadog_metrics": + for callback in _in_memory_loggers: + if isinstance(callback, DatadogMetricsLogger): + return callback elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4695,9 +4704,11 @@ class StandardLoggingPayloadSetup: ).model_dump() if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): - return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - _raw - ).model_dump() + return ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _raw + ).model_dump() + ) return _raw if isinstance(_raw, Usage): return _raw.model_dump() @@ -5543,4 +5554,3 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: model_parameters={"stream": True}, hidden_params=hidden_params, ) - diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 62ca6dc2ae..9ecae363ed 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -390,9 +390,7 @@ def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]: ) if "applied_policies" in _metadata: - headers["x-litellm-applied-policies"] = ",".join( - _metadata["applied_policies"] - ) + headers["x-litellm-applied-policies"] = ",".join(_metadata["applied_policies"]) if "policy_sources" in _metadata: sources = _metadata["policy_sources"] @@ -449,9 +447,7 @@ def add_policy_to_applied_policies_header( request_data["metadata"] = _metadata -def add_policy_sources_to_metadata( - request_data: Dict, policy_sources: Dict[str, str] -): +def add_policy_sources_to_metadata(request_data: Dict, policy_sources: Dict[str, str]): """ Store policy match reasons in metadata for x-litellm-policy-sources header. diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index f3bed3656f..48e3e8c0ad 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -226,6 +226,7 @@ async def health_services_endpoint( # noqa: PLR0915 "custom_callback_api", "langsmith", "datadog", + "datadog_metrics", "datadog_llm_observability", "generic_api", "arize", @@ -280,6 +281,21 @@ async def health_services_endpoint( # noqa: PLR0915 else "Datadog is healthy" ), } + elif service == "datadog_metrics": + from litellm.integrations.datadog.datadog_metrics import ( + DatadogMetricsLogger, + ) + + datadog_metrics_logger = DatadogMetricsLogger() + response = await datadog_metrics_logger.async_health_check() + return { + "status": response["status"], + "message": ( + response["error_message"] + if response["status"] == "unhealthy" + else "Datadog Metrics is healthy" + ), + } elif service == "arize": from litellm.integrations.arize.arize import ArizeLogger diff --git a/litellm/types/integrations/datadog_metrics.py b/litellm/types/integrations/datadog_metrics.py new file mode 100644 index 0000000000..4b21881ed5 --- /dev/null +++ b/litellm/types/integrations/datadog_metrics.py @@ -0,0 +1,19 @@ +from typing import List + +from typing_extensions import TypedDict + + +class DatadogMetricPoint(TypedDict): + timestamp: int # Unix epoch seconds + value: float # The metric value + + +class DatadogMetricSeries(TypedDict): + metric: str + type: int # 1=count, 2=rate, 3=gauge, distribution is submitted as type=3, but distributions use a different endpoint /api/v1/distribution_points, wait actually according to DD /api/v2/series: 0=unspecified, 1=count, 2=rate, 3=gauge. For histogram/distribution we use type 3 or 1. + points: List[DatadogMetricPoint] + tags: List[str] + + +class DatadogMetricsPayload(TypedDict): + series: List[DatadogMetricSeries] diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py new file mode 100644 index 0000000000..1008a4f78c --- /dev/null +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -0,0 +1,272 @@ +import os +import time +from datetime import datetime, timedelta +from unittest.mock import AsyncMock + +import pytest +from httpx import Response + +from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger +from litellm.types.utils import StandardLoggingPayload + + +@pytest.fixture +def clean_env(): + """Set test env vars and restore originals after test.""" + keys = ["DD_API_KEY", "DD_APP_KEY", "DD_SITE", "DD_ENV", "DD_SERVICE", "DD_VERSION"] + originals = {k: os.environ.get(k) for k in keys} + + os.environ["DD_API_KEY"] = "test_api_key" + os.environ["DD_APP_KEY"] = "test_app_key" + os.environ["DD_SITE"] = "test.datadoghq.com" + os.environ["DD_ENV"] = "test-env" + os.environ["DD_SERVICE"] = "test-service" + os.environ["DD_VERSION"] = "1.0.0" + + yield + + for k, v in originals.items(): + if v is not None: + os.environ[k] = v + elif k in os.environ: + del os.environ[k] + + +@pytest.mark.asyncio +async def test_init(clean_env): + """Test initialization sets up clients and url correctly.""" + logger = DatadogMetricsLogger() + assert logger.dd_api_key == "test_api_key" + assert logger.dd_site == "test.datadoghq.com" + assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" + + +@pytest.mark.asyncio +async def test_extract_tags(clean_env): + """Test tag extraction from a StandardLoggingPayload.""" + logger = DatadogMetricsLogger() + + payload = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + model_group="gpt-4", + metadata={"user_api_key_team_alias": "test-team"}, + ) + + tags = logger._extract_tags(log=payload, status_code="200") + + assert "env:test-env" in tags + assert "service:test-service" in tags + assert "version:1.0.0" in tags + assert "provider:openai" in tags + assert "model_name:gpt-4o" in tags + assert "model_group:gpt-4" in tags + assert "status_code:200" in tags + assert "team:test-team" in tags + + +@pytest.mark.asyncio +async def test_extract_tags_no_team(clean_env): + """Test tag extraction when no team info is present.""" + logger = DatadogMetricsLogger() + + payload = StandardLoggingPayload( + custom_llm_provider="anthropic", + model="claude-3-sonnet", + ) + + tags = logger._extract_tags(log=payload, status_code="500") + + assert "provider:anthropic" in tags + assert "model_name:claude-3-sonnet" in tags + assert "status_code:500" in tags + assert not any(tag.startswith("team:") for tag in tags) + + +@pytest.mark.asyncio +async def test_add_metrics_from_log(clean_env): + """Test that _add_metrics_from_log appends the correct metric series to the queue.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + start_time = now - timedelta(seconds=2) + api_call_start_time = now - timedelta(seconds=1) + + payload = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + ) + + kwargs = { + "start_time": start_time, + "api_call_start_time": api_call_start_time, + "end_time": now, + } + + logger._add_metrics_from_log(log=payload, kwargs=kwargs, status_code="200") + + # Should have 3 series: total_latency, llm_api_latency, request_count + assert len(logger.log_queue) == 3 + + metrics = {s["metric"]: s for s in logger.log_queue} + + # Total latency ~2s + total = metrics["litellm.request.total_latency"] + assert total["type"] == 3 # gauge + assert abs(total["points"][0]["value"] - 2.0) < 0.1 + + # LLM API latency ~1s + llm = metrics["litellm.llm_api.latency"] + assert llm["type"] == 3 # gauge + assert abs(llm["points"][0]["value"] - 1.0) < 0.1 + + # Request count + count = metrics["litellm.llm_api.request_count"] + assert count["type"] == 1 # count + assert count["points"][0]["value"] == 1.0 + assert "status_code:200" in count["tags"] + + +@pytest.mark.asyncio +async def test_async_log_success_event(clean_env): + """Test that success events are added to the queue.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + start_time = now - timedelta(seconds=1) + + await logger.async_log_success_event( + kwargs={ + "standard_logging_object": StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + ), + "start_time": start_time, + "end_time": now, + }, + response_obj=None, + start_time=start_time, + end_time=now, + ) + + # At least request_count and total_latency + assert len(logger.log_queue) >= 2 + + +@pytest.mark.asyncio +async def test_async_log_success_event_no_standard_logging_object(clean_env): + """Test that events without standard_logging_object are skipped.""" + logger = DatadogMetricsLogger(batch_size=100) + + await logger.async_log_success_event( + kwargs={}, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert len(logger.log_queue) == 0 + + +@pytest.mark.asyncio +async def test_async_log_failure_event_extracts_status_code(clean_env): + """Test that failure events extract the error status code.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + start_time = now - timedelta(seconds=1) + + await logger.async_log_failure_event( + kwargs={ + "standard_logging_object": StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + error_information={"error_code": "429"}, + ), + "start_time": start_time, + "end_time": now, + }, + response_obj=None, + start_time=start_time, + end_time=now, + ) + + count_series = next( + (s for s in logger.log_queue if s["metric"] == "litellm.llm_api.request_count"), + None, + ) + assert count_series is not None + assert "status_code:429" in count_series["tags"] + + +@pytest.mark.asyncio +async def test_async_log_failure_event_default_status_code(clean_env): + """Test that failure events default to 500 when no error_code is present.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + + await logger.async_log_failure_event( + kwargs={ + "standard_logging_object": StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + ), + "start_time": now, + "end_time": now, + }, + response_obj=None, + start_time=now, + end_time=now, + ) + + count_series = next( + (s for s in logger.log_queue if s["metric"] == "litellm.llm_api.request_count"), + None, + ) + assert count_series is not None + assert "status_code:500" in count_series["tags"] + + +@pytest.mark.asyncio +async def test_async_send_batch(clean_env): + """Test that async_send_batch uploads metrics to Datadog.""" + logger = DatadogMetricsLogger() + logger.async_client = AsyncMock() + logger.async_client.post.return_value = Response(202, json={"status": "ok"}) + + # Manually add a metric series to the queue + logger.log_queue = [ + { + "metric": "litellm.request.total_latency", + "type": 3, + "points": [{"timestamp": int(time.time()), "value": 1.5}], + "tags": ["env:test"], + } + ] + + await logger.async_send_batch() + + assert logger.async_client.post.called + call_args = logger.async_client.post.call_args + assert call_args[0][0] == "https://api.test.datadoghq.com/api/v2/series" + + # Verify gzip + JSON payload + import gzip + import json + + compressed = call_args[1]["content"] + payload = json.loads(gzip.decompress(compressed).decode("utf-8")) + assert len(payload["series"]) == 1 + assert payload["series"][0]["metric"] == "litellm.request.total_latency" + + +@pytest.mark.asyncio +async def test_async_send_batch_empty_queue(clean_env): + """Test that async_send_batch does nothing when queue is empty.""" + logger = DatadogMetricsLogger() + logger.async_client = AsyncMock() + + await logger.async_send_batch() + + assert not logger.async_client.post.called From cd60e3d4e0add4e3a84ce6c5f284a4863b8a9cad Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 25 Feb 2026 22:27:51 +0530 Subject: [PATCH 02/13] fix: req changes --- docs/my-website/docs/observability/datadog.md | 2 +- litellm/integrations/datadog/datadog_metrics.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index ef48594cbf..e83cfcbafe 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -193,7 +193,7 @@ All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, ```yaml model_list: - - model_name: gpt-3.5-turbo + - model_name: gpt-3.5-turbo litellm_params: model: gpt-3.5-turbo litellm_settings: diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index ca3e4aa5a1..43dd066260 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -1,4 +1,5 @@ import asyncio +import gzip import os import time from datetime import datetime @@ -6,6 +7,7 @@ from typing import List, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -39,8 +41,9 @@ class DatadogMetricsLogger(CustomBatchLogger): # Initialize lock self.flush_lock = asyncio.Lock() - # Check if flush_lock is already in kwargs to avoid double passing - kwargs["flush_lock"] = self.flush_lock + # Only set flush_lock if not already provided by caller + if "flush_lock" not in kwargs: + kwargs["flush_lock"] = self.flush_lock # Send metrics more quickly to datadog (every 5 seconds) if "flush_interval" not in kwargs: @@ -230,10 +233,6 @@ class DatadogMetricsLogger(CustomBatchLogger): if self.dd_app_key: headers["DD-APPLICATION-KEY"] = self.dd_app_key - import gzip - - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - json_data = safe_dumps(payload) compressed_data = gzip.compress(json_data.encode("utf-8")) headers["Content-Encoding"] = "gzip" From c08ef3f96a8441ec79c0d4fa61ffa02702000885 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Wed, 25 Feb 2026 22:28:59 +0530 Subject: [PATCH 03/13] Update docs/my-website/docs/observability/datadog.md Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> From 94be31a8169496c96e378cfc70bc2d7a7f89efd5 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 26 Feb 2026 11:30:33 +0530 Subject: [PATCH 04/13] fix req changes --- litellm/__init__.py | 757 +++++------------- .../integrations/datadog/datadog_metrics.py | 25 +- litellm/litellm_core_utils/litellm_logging.py | 35 +- .../health_endpoints/_health_endpoints.py | 2 +- litellm/types/integrations/datadog_metrics.py | 2 +- 5 files changed, 247 insertions(+), 574 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 41f5b0ff42..4531e01760 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -75,7 +75,6 @@ from litellm.constants import ( ) import httpx import dotenv - # register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" @@ -106,8 +105,8 @@ _custom_logger_compatible_callbacks_literal = Literal[ "prometheus", "otel", "datadog", - "datadog_llm_observability", "datadog_metrics", + "datadog_llm_observability", "galileo", "braintrust", "arize", @@ -149,9 +148,7 @@ _known_custom_logger_compatible_callbacks: List = list( get_args(_custom_logger_compatible_callbacks_literal) ) callbacks: List[ - Union[ - Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger" - ] # CustomLogger is lazy-loaded + Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded ] = [] callback_settings: Dict[str, Dict[str, Any]] = {} initialized_langfuse_clients: int = 0 @@ -161,50 +158,42 @@ prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. -gcs_pub_sub_use_v1: Optional[ - bool -] = False # if you want to use v1 gcs pubsub logged payload -generic_api_use_v1: Optional[ - bool -] = False # if you want to use v1 generic api logged payload +gcs_pub_sub_use_v1: Optional[bool] = ( + False # if you want to use v1 gcs pubsub logged payload +) +generic_api_use_v1: Optional[bool] = ( + False # if you want to use v1 generic api logged payload +) argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[ - Union[str, Callable, "CustomLogger"] -] = ( # CustomLogger is lazy-loaded +_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_success_callback: List[ - Union[str, Callable, "CustomLogger"] -] = ( # CustomLogger is lazy-loaded +_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[ - Union[str, Callable, "CustomLogger"] -] = ( # CustomLogger is lazy-loaded +_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False -standard_logging_payload_excluded_fields: Optional[ - List[str] -] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it +standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False -add_user_information_to_llm_headers: Optional[ - bool -] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers +add_user_information_to_llm_headers: Optional[bool] = ( + None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers +) store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# -email: Optional[ - str -] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -token: Optional[ - str -] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +email: Optional[str] = ( + None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +token: Optional[str] = ( + None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) @@ -263,9 +252,9 @@ use_client: bool = False ssl_verify: Union[str, bool] = True ssl_security_level: Optional[str] = None ssl_certificate: Optional[str] = None -ssl_ecdh_curve: Optional[ - str -] = None # Set to 'X25519' to disable PQC and improve performance +ssl_ecdh_curve: Optional[str] = ( + None # Set to 'X25519' to disable PQC and improve performance +) disable_streaming_logging: bool = False disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False @@ -315,20 +304,24 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) -caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -cache: Optional[ - "Cache" -] = None # cache object <- use this - https://docs.litellm.ai/docs/caching +caching: bool = ( + False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +caching_with_models: bool = ( + False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +cache: Optional["Cache"] = ( + None # cache object <- use this - https://docs.litellm.ai/docs/caching +) default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None model_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers -budget_duration: Optional[ - str -] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). +budget_duration: Optional[str] = ( + None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). +) default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) @@ -337,7 +330,9 @@ forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} -add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt +add_function_to_prompt: bool = ( + False # if function calling not supported by api, append function call details to system prompt +) client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' @@ -384,7 +379,9 @@ prometheus_emit_stream_label: bool = False disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) -disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. +disable_copilot_system_to_assistant: bool = ( + False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. +) public_mcp_servers: Optional[List[str]] = None public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None @@ -403,13 +400,17 @@ if TYPE_CHECKING: ######## Networking Settings ######## -use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. +use_aiohttp_transport: bool = ( + True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. +) aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) -force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +force_ipv4: bool = ( + False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +) network_mock: bool = False # When True, use mock transport — no real network calls ####### STOP SEQUENCE LIMIT ####### @@ -424,13 +425,13 @@ context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 allow_dynamic_callback_disabling: bool = True -num_retries_per_request: Optional[ - int -] = None # for the request overall (incl. fallbacks + model retries) +num_retries_per_request: Optional[int] = ( + None # for the request overall (incl. fallbacks + model retries) +) ####### SECRET MANAGERS ##################### -secret_manager_client: Optional[ - Any -] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc. +secret_manager_client: Optional[Any] = ( + None # list of instantiated key management clients - e.g. azure kv, infisical, etc. +) _google_kms_resource_name: Optional[str] = None _key_management_system: Optional["KeyManagementSystem"] = None # Note: KeyManagementSettings must be eagerly imported because _key_management_settings @@ -443,12 +444,12 @@ output_parse_pii: bool = False from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map model_cost = get_model_cost_map(url=model_cost_map_url) -cost_discount_config: Dict[ - str, float -] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount -cost_margin_config: Dict[ - str, Union[float, Dict[str, float]] -] = {} # Provider-specific or global cost margins. Examples: +cost_discount_config: Dict[str, float] = ( + {} +) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount +cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = ( + {} +) # Provider-specific or global cost margins. Examples: # Percentage: {"openai": 0.10} = 10% margin # Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request # Global: {"global": 0.05} = 5% global margin on all providers @@ -1107,12 +1108,10 @@ openai_video_generation_models = ["sora-2"] # Import KeyManagementSettings here (before utils import) because _key_management_settings # is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) from litellm.types.secret_managers.main import KeyManagementSettings - _key_management_settings: KeyManagementSettings = KeyManagementSettings() # client must be imported immediately as it's used as a decorator at function definition time from .utils import client - # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time @@ -1141,7 +1140,6 @@ from .llms.topaz.common_utils import TopazModelInfo # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo - # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) # All remaining configs are now lazy loaded - see _lazy_imports_registry.py @@ -1223,7 +1221,6 @@ from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * from .responses.main import * - # Interactions API is available as litellm.interactions module # Usage: litellm.interactions.create(), litellm.interactions.get(), etc. from . import interactions @@ -1282,12 +1279,12 @@ from . import rag from .types.llms.custom_llm import CustomLLMItem custom_provider_map: List[CustomLLMItem] = [] -_custom_providers: List[ - str -] = [] # internal helper util, used to track names of custom providers -disable_hf_tokenizer_download: Optional[ - bool -] = None # disable huggingface tokenizer download. Defaults to openai clk100 +_custom_providers: List[str] = ( + [] +) # internal helper util, used to track names of custom providers +disable_hf_tokenizer_download: Optional[bool] = ( + None # disable huggingface tokenizer download. Defaults to openai clk100 +) global_disable_no_log_param: bool = False ### CLI UTILITIES ### @@ -1326,318 +1323,128 @@ if TYPE_CHECKING: from litellm.caching.caching import Cache # Type stubs for lazy-loaded configs to help mypy - from .llms.bedrock.chat.converse_transformation import ( - AmazonConverseConfig as AmazonConverseConfig, - ) - from .llms.openai_like.chat.handler import ( - OpenAILikeChatConfig as OpenAILikeChatConfig, - ) - from .llms.galadriel.chat.transformation import ( - GaladrielChatConfig as GaladrielChatConfig, - ) + from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig + from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig + from .llms.galadriel.chat.transformation import GaladrielChatConfig as GaladrielChatConfig from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig - from .llms.azure_ai.anthropic.transformation import ( - AzureAnthropicConfig as AzureAnthropicConfig, - ) + from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig as AzureAnthropicConfig from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig - from .llms.compactifai.chat.transformation import ( - CompactifAIChatConfig as CompactifAIChatConfig, - ) + from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig - from .llms.aiohttp_openai.chat.transformation import ( - AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig, - ) - from .llms.huggingface.chat.transformation import ( - HuggingFaceChatConfig as HuggingFaceChatConfig, - ) - from .llms.huggingface.embedding.transformation import ( - HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig, - ) + from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig + from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig + from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig from .llms.maritalk import MaritalkConfig as MaritalkConfig - from .llms.openrouter.chat.transformation import ( - OpenrouterConfig as OpenrouterConfig, - ) + from .llms.openrouter.chat.transformation import OpenrouterConfig as OpenrouterConfig from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig - from .llms.anthropic.completion.transformation import ( - AnthropicTextConfig as AnthropicTextConfig, - ) + from .llms.anthropic.completion.transformation import AnthropicTextConfig as AnthropicTextConfig from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig from .llms.triton.completion.transformation import TritonConfig as TritonConfig - from .llms.triton.completion.transformation import ( - TritonGenerateConfig as TritonGenerateConfig, - ) - from .llms.triton.completion.transformation import ( - TritonInferConfig as TritonInferConfig, - ) - from .llms.triton.embedding.transformation import ( - TritonEmbeddingConfig as TritonEmbeddingConfig, - ) - from .llms.huggingface.rerank.transformation import ( - HuggingFaceRerankConfig as HuggingFaceRerankConfig, - ) - from .llms.databricks.chat.transformation import ( - DatabricksConfig as DatabricksConfig, - ) - from .llms.databricks.embed.transformation import ( - DatabricksEmbeddingConfig as DatabricksEmbeddingConfig, - ) + from .llms.triton.completion.transformation import TritonGenerateConfig as TritonGenerateConfig + from .llms.triton.completion.transformation import TritonInferConfig as TritonInferConfig + from .llms.triton.embedding.transformation import TritonEmbeddingConfig as TritonEmbeddingConfig + from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig as HuggingFaceRerankConfig + from .llms.databricks.chat.transformation import DatabricksConfig as DatabricksConfig + from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig as DatabricksEmbeddingConfig from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig - from .llms.cohere.rerank.transformation import ( - CohereRerankConfig as CohereRerankConfig, - ) - from .llms.cohere.rerank_v2.transformation import ( - CohereRerankV2Config as CohereRerankV2Config, - ) - from .llms.azure_ai.rerank.transformation import ( - AzureAIRerankConfig as AzureAIRerankConfig, - ) - from .llms.infinity.rerank.transformation import ( - InfinityRerankConfig as InfinityRerankConfig, - ) - from .llms.jina_ai.rerank.transformation import ( - JinaAIRerankConfig as JinaAIRerankConfig, - ) - from .llms.deepinfra.rerank.transformation import ( - DeepinfraRerankConfig as DeepinfraRerankConfig, - ) - from .llms.hosted_vllm.rerank.transformation import ( - HostedVLLMRerankConfig as HostedVLLMRerankConfig, - ) - from .llms.nvidia_nim.rerank.transformation import ( - NvidiaNimRerankConfig as NvidiaNimRerankConfig, - ) - from .llms.nvidia_nim.rerank.ranking_transformation import ( - NvidiaNimRankingConfig as NvidiaNimRankingConfig, - ) - from .llms.vertex_ai.rerank.transformation import ( - VertexAIRerankConfig as VertexAIRerankConfig, - ) - from .llms.fireworks_ai.rerank.transformation import ( - FireworksAIRerankConfig as FireworksAIRerankConfig, - ) - from .llms.voyage.rerank.transformation import ( - VoyageRerankConfig as VoyageRerankConfig, - ) - from .llms.watsonx.rerank.transformation import ( - IBMWatsonXRerankConfig as IBMWatsonXRerankConfig, - ) + from .llms.cohere.rerank.transformation import CohereRerankConfig as CohereRerankConfig + from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config as CohereRerankV2Config + from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig as AzureAIRerankConfig + from .llms.infinity.rerank.transformation import InfinityRerankConfig as InfinityRerankConfig + from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig as JinaAIRerankConfig + from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig as DeepinfraRerankConfig + from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig as HostedVLLMRerankConfig + from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig as NvidiaNimRerankConfig + from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig as NvidiaNimRankingConfig + from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig + from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig + from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig + from .llms.watsonx.rerank.transformation import IBMWatsonXRerankConfig as IBMWatsonXRerankConfig from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig - from .llms.together_ai.completion.transformation import ( - TogetherAITextCompletionConfig as TogetherAITextCompletionConfig, - ) - from .llms.cloudflare.chat.transformation import ( - CloudflareChatConfig as CloudflareChatConfig, - ) + from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig as TogetherAITextCompletionConfig + from .llms.cloudflare.chat.transformation import CloudflareChatConfig as CloudflareChatConfig from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig - from .llms.sagemaker.completion.transformation import ( - SagemakerConfig as SagemakerConfig, - ) - from .llms.sagemaker.chat.transformation import ( - SagemakerChatConfig as SagemakerChatConfig, - ) + from .llms.sagemaker.completion.transformation import SagemakerConfig as SagemakerConfig + from .llms.sagemaker.chat.transformation import SagemakerChatConfig as SagemakerChatConfig from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig - from .llms.anthropic.experimental_pass_through.messages.transformation import ( - AnthropicMessagesConfig as AnthropicMessagesConfig, - ) - from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( - AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig, - ) + from .llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig as AnthropicMessagesConfig + from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig as VertexGeminiConfig, - ) - from .llms.gemini.chat.transformation import ( - GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig, - ) - from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( - VertexAIAnthropicConfig as VertexAIAnthropicConfig, - ) - from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( - VertexAILlama3Config as VertexAILlama3Config, - ) - from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import ( - VertexAIAi21Config as VertexAIAi21Config, - ) - from .llms.bedrock.chat.invoke_handler import ( - AmazonCohereChatConfig as AmazonCohereChatConfig, - ) - from .llms.bedrock.common_utils import ( - AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import ( - AmazonAI21Config as AmazonAI21Config, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import ( - AmazonInvokeNovaConfig as AmazonInvokeNovaConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import ( - AmazonQwen2Config as AmazonQwen2Config, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( - AmazonQwen3Config as AmazonQwen3Config, - ) - from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import ( - AmazonAnthropicConfig as AmazonAnthropicConfig, - ) - from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( - AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import ( - AmazonCohereConfig as AmazonCohereConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import ( - AmazonLlamaConfig as AmazonLlamaConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import ( - AmazonDeepSeekR1Config as AmazonDeepSeekR1Config, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import ( - AmazonMistralConfig as AmazonMistralConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( - AmazonMoonshotConfig as AmazonMoonshotConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import ( - AmazonTitanConfig as AmazonTitanConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( - AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig, - ) - from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( - AmazonInvokeConfig as AmazonInvokeConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( - AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig, - ) - from .llms.bedrock.image_generation.amazon_stability1_transformation import ( - AmazonStabilityConfig as AmazonStabilityConfig, - ) - from .llms.bedrock.image_generation.amazon_stability3_transformation import ( - AmazonStability3Config as AmazonStability3Config, - ) - from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( - AmazonNovaCanvasConfig as AmazonNovaCanvasConfig, - ) - from .llms.bedrock.embed.amazon_titan_g1_transformation import ( - AmazonTitanG1Config as AmazonTitanG1Config, - ) - from .llms.bedrock.embed.amazon_titan_multimodal_transformation import ( - AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config, - ) - from .llms.cohere.chat.v2_transformation import ( - CohereV2ChatConfig as CohereV2ChatConfig, - ) - from .llms.bedrock.embed.cohere_transformation import ( - BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig, - ) - from .llms.bedrock.embed.twelvelabs_marengo_transformation import ( - TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig, - ) - from .llms.bedrock.embed.amazon_nova_transformation import ( - AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig, - ) - from .llms.openai.openai import ( - OpenAIConfig as OpenAIConfig, - MistralEmbeddingConfig as MistralEmbeddingConfig, - ) - from .llms.openai.image_variations.transformation import ( - OpenAIImageVariationConfig as OpenAIImageVariationConfig, - ) - from .llms.deepgram.audio_transcription.transformation import ( - DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig, - ) - from .llms.topaz.image_variations.transformation import ( - TopazImageVariationConfig as TopazImageVariationConfig, - ) - from litellm.llms.openai.completion.transformation import ( - OpenAITextCompletionConfig as OpenAITextCompletionConfig, - ) + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as VertexGeminiConfig + from .llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig + from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig as VertexAIAnthropicConfig + from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import VertexAILlama3Config as VertexAILlama3Config + from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import VertexAIAi21Config as VertexAIAi21Config + from .llms.bedrock.chat.invoke_handler import AmazonCohereChatConfig as AmazonCohereChatConfig + from .llms.bedrock.common_utils import AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig + from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import AmazonAI21Config as AmazonAI21Config + from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import AmazonInvokeNovaConfig as AmazonInvokeNovaConfig + from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import AmazonQwen2Config as AmazonQwen2Config + from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import AmazonQwen3Config as AmazonQwen3Config + from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import AmazonAnthropicConfig as AmazonAnthropicConfig + from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig + from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import AmazonCohereConfig as AmazonCohereConfig + from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig + from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config + from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig + from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig + from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig + from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig + from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig + from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig + from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig as AmazonStabilityConfig + from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config as AmazonStability3Config + from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig as AmazonNovaCanvasConfig + from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config as AmazonTitanG1Config + from .llms.bedrock.embed.amazon_titan_multimodal_transformation import AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config + from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig as CohereV2ChatConfig + from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig + from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig + from .llms.bedrock.embed.amazon_nova_transformation import AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig + from .llms.openai.openai import OpenAIConfig as OpenAIConfig, MistralEmbeddingConfig as MistralEmbeddingConfig + from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig as OpenAIImageVariationConfig + from .llms.deepgram.audio_transcription.transformation import DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig + from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig + from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig from .llms.a2a.chat.transformation import A2AConfig as A2AConfig - from .llms.voyage.embedding.transformation import ( - VoyageEmbeddingConfig as VoyageEmbeddingConfig, - ) - from .llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig, - ) - from .llms.infinity.embedding.transformation import ( - InfinityEmbeddingConfig as InfinityEmbeddingConfig, - ) - from .llms.azure_ai.chat.transformation import ( - AzureAIStudioConfig as AzureAIStudioConfig, - ) + from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig + from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig + from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig as InfinityEmbeddingConfig + from .llms.azure_ai.chat.transformation import AzureAIStudioConfig as AzureAIStudioConfig from .llms.mistral.chat.transformation import MistralConfig as MistralConfig - from .llms.openai.responses.transformation import ( - OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig, - ) - from .llms.azure.responses.transformation import ( - AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig, - ) - from .llms.azure.responses.o_series_transformation import ( - AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig, - ) - from .llms.xai.responses.transformation import ( - XAIResponsesAPIConfig as XAIResponsesAPIConfig, - ) - from .llms.litellm_proxy.responses.transformation import ( - LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, - ) - from .llms.volcengine.responses.transformation import ( - VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig, - ) - from .llms.manus.responses.transformation import ( - ManusResponsesAPIConfig as ManusResponsesAPIConfig, - ) - from .llms.perplexity.responses.transformation import ( - PerplexityResponsesConfig as PerplexityResponsesConfig, - ) - from .llms.databricks.responses.transformation import ( - DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig, - ) - from .llms.gemini.interactions.transformation import ( - GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, - ) - from .llms.openai.chat.o_series_transformation import ( - OpenAIOSeriesConfig as OpenAIOSeriesConfig, - OpenAIOSeriesConfig as OpenAIO1Config, - ) - from .llms.anthropic.skills.transformation import ( - AnthropicSkillsConfig as AnthropicSkillsConfig, - ) - from .llms.base_llm.skills.transformation import ( - BaseSkillsAPIConfig as BaseSkillsAPIConfig, - ) - from .llms.gradient_ai.chat.transformation import ( - GradientAIConfig as GradientAIConfig, - ) + from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig + from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig + from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig + from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig + from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig + from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig + from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig + from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig + from .llms.databricks.responses.transformation import DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig + from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig + from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config + from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig + from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig as BaseSkillsAPIConfig + from .llms.gradient_ai.chat.transformation import GradientAIConfig as GradientAIConfig from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig - from .llms.openai.chat.gpt_5_transformation import ( - OpenAIGPT5Config as OpenAIGPT5Config, - ) - from .llms.openai.transcriptions.whisper_transformation import ( - OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig, - ) - from .llms.openai.transcriptions.gpt_transformation import ( - OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig, - ) - from .llms.openai.chat.gpt_audio_transformation import ( - OpenAIGPTAudioConfig as OpenAIGPTAudioConfig, - ) + from .llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config as OpenAIGPT5Config + from .llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig + from .llms.openai.transcriptions.gpt_transformation import OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig + from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig - from .llms.nvidia_nim.embed import ( - NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig, - ) + from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig # Type stubs for lazy-loaded config instances openaiOSeriesConfig: OpenAIOSeriesConfig @@ -1649,47 +1456,21 @@ if TYPE_CHECKING: # Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig - from .llms.deepseek.chat.transformation import ( - DeepSeekChatConfig as _DeepSeekChatConfig, - ) - from .llms.sap.chat.transformation import ( - GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig, - ) - from .llms.sap.embed.transformation import ( - GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig, - ) - from .llms.azure.chat.o_series_transformation import ( - AzureOpenAIO1Config as _AzureOpenAIO1Config, - ) - from .llms.perplexity.chat.transformation import ( - PerplexityChatConfig as _PerplexityChatConfig, - ) + from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig + from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig + from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig + from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config as _AzureOpenAIO1Config + from .llms.perplexity.chat.transformation import PerplexityChatConfig as _PerplexityChatConfig from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig - from .llms.watsonx.chat.transformation import ( - IBMWatsonXChatConfig as _IBMWatsonXChatConfig, - ) - from .llms.watsonx.completion.transformation import ( - IBMWatsonXAIConfig as _IBMWatsonXAIConfig, - ) - from .llms.litellm_proxy.chat.transformation import ( - LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig, - ) + from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig as _IBMWatsonXChatConfig + from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig as _IBMWatsonXAIConfig + from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig - from .llms.llamafile.chat.transformation import ( - LlamafileChatConfig as _LlamafileChatConfig, - ) - from .llms.lm_studio.chat.transformation import ( - LMStudioChatConfig as _LMStudioChatConfig, - ) - from .llms.lm_studio.embed.transformation import ( - LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, - ) - from .llms.watsonx.embed.transformation import ( - IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig, - ) - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig as _VertexGeminiConfig, - ) + from .llms.llamafile.chat.transformation import LlamafileChatConfig as _LlamafileChatConfig + from .llms.lm_studio.chat.transformation import LMStudioChatConfig as _LMStudioChatConfig + from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig + from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig # Type stubs for lazy-loaded config classes (to help mypy understand types) VLLMConfig: Type[_VLLMConfig] @@ -1709,122 +1490,55 @@ if TYPE_CHECKING: IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig - from .llms.featherless_ai.chat.transformation import ( - FeatherlessAIConfig as FeatherlessAIConfig, - ) + from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig - from .llms.sambanova.embedding.transformation import ( - SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig, - ) - from .llms.fireworks_ai.chat.transformation import ( - FireworksAIConfig as FireworksAIConfig, - ) - from .llms.fireworks_ai.completion.transformation import ( - FireworksAITextCompletionConfig as FireworksAITextCompletionConfig, - ) - from .llms.fireworks_ai.audio_transcription.transformation import ( - FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig, - ) - from .llms.fireworks_ai.embed.fireworks_ai_transformation import ( - FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig, - ) - from .llms.friendliai.chat.transformation import ( - FriendliaiChatConfig as FriendliaiChatConfig, - ) - from .llms.jina_ai.embedding.transformation import ( - JinaAIEmbeddingConfig as JinaAIEmbeddingConfig, - ) + from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig + from .llms.fireworks_ai.chat.transformation import FireworksAIConfig as FireworksAIConfig + from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig as FireworksAITextCompletionConfig + from .llms.fireworks_ai.audio_transcription.transformation import FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig + from .llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig + from .llms.friendliai.chat.transformation import FriendliaiChatConfig as FriendliaiChatConfig + from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig as JinaAIEmbeddingConfig from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig - from .llms.volcengine.chat.transformation import ( - VolcEngineChatConfig as VolcEngineChatConfig, - VolcEngineChatConfig as VolcEngineConfig, - ) - from .llms.codestral.completion.transformation import ( - CodestralTextCompletionConfig as CodestralTextCompletionConfig, - ) - from .llms.azure.azure import ( - AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig, - ) + from .llms.volcengine.chat.transformation import VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig + from .llms.codestral.completion.transformation import CodestralTextCompletionConfig as CodestralTextCompletionConfig + from .llms.azure.azure import AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig - from .llms.azure.chat.gpt_transformation import ( - AzureOpenAIConfig as AzureOpenAIConfig, - ) - from .llms.azure.chat.gpt_5_transformation import ( - AzureOpenAIGPT5Config as AzureOpenAIGPT5Config, - ) - from .llms.azure.completion.transformation import ( - AzureOpenAITextConfig as AzureOpenAITextConfig, - ) - from .llms.hosted_vllm.chat.transformation import ( - HostedVLLMChatConfig as HostedVLLMChatConfig, - ) - from .llms.hosted_vllm.embedding.transformation import ( - HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig, - ) - from .llms.github_copilot.chat.transformation import ( - GithubCopilotConfig as GithubCopilotConfig, - ) - from .llms.github_copilot.responses.transformation import ( - GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig, - ) - from .llms.github_copilot.embedding.transformation import ( - GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig, - ) + from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig as AzureOpenAIConfig + from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config + from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig + from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig + from .llms.hosted_vllm.embedding.transformation import HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig + from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig + from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig + from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig - from .llms.chatgpt.responses.transformation import ( - ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig, - ) + from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig - from .llms.gigachat.embedding.transformation import ( - GigaChatEmbeddingConfig as GigaChatEmbeddingConfig, - ) + from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig - from .llms.dashscope.chat.transformation import ( - DashScopeChatConfig as DashScopeChatConfig, - ) - from .llms.moonshot.chat.transformation import ( - MoonshotChatConfig as MoonshotChatConfig, - ) - from .llms.docker_model_runner.chat.transformation import ( - DockerModelRunnerChatConfig as DockerModelRunnerChatConfig, - ) + from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig + from .llms.moonshot.chat.transformation import MoonshotChatConfig as MoonshotChatConfig + from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig as DockerModelRunnerChatConfig from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig - from .llms.lambda_ai.chat.transformation import ( - LambdaAIChatConfig as LambdaAIChatConfig, - ) - from .llms.hyperbolic.chat.transformation import ( - HyperbolicChatConfig as HyperbolicChatConfig, - ) - from .llms.vercel_ai_gateway.chat.transformation import ( - VercelAIGatewayConfig as VercelAIGatewayConfig, - ) - from .llms.ovhcloud.chat.transformation import ( - OVHCloudChatConfig as OVHCloudChatConfig, - ) - from .llms.ovhcloud.embedding.transformation import ( - OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig, - ) - from .llms.cometapi.embed.transformation import ( - CometAPIEmbeddingConfig as CometAPIEmbeddingConfig, - ) - from .llms.lemonade.chat.transformation import ( - LemonadeChatConfig as LemonadeChatConfig, - ) - from .llms.snowflake.embedding.transformation import ( - SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig, - ) - from .llms.amazon_nova.chat.transformation import ( - AmazonNovaChatConfig as AmazonNovaChatConfig, - ) + from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig as LambdaAIChatConfig + from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig as HyperbolicChatConfig + from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig as VercelAIGatewayConfig + from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig as OVHCloudChatConfig + from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig + from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig as CometAPIEmbeddingConfig + from .llms.lemonade.chat.transformation import LemonadeChatConfig as LemonadeChatConfig + from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig + from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig as AmazonNovaChatConfig from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( @@ -1885,7 +1599,6 @@ if TYPE_CHECKING: # Bedrock tool name mappings instance (lazy-loaded) from litellm.caching.caching import InMemoryCache - bedrock_tool_name_mappings: InMemoryCache # Azure exception class (lazy-loaded) @@ -1904,15 +1617,11 @@ if TYPE_CHECKING: from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams # Logging callback manager class and instance (lazy-loaded) - from litellm.litellm_core_utils.logging_callback_manager import ( - LoggingCallbackManager, - ) - + from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager logging_callback_manager: LoggingCallbackManager # provider_list is lazy-loaded from litellm.types.utils import LlmProviders - provider_list: List[Union[LlmProviders, str]] # Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block @@ -1937,10 +1646,7 @@ def __getattr__(name: str) -> Any: global _async_client_cleanup_registered # Register async client cleanup on first access (only once) if not _async_client_cleanup_registered: - from litellm.llms.custom_httpx.async_client_cleanup import ( - register_async_client_cleanup, - ) - + from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup register_async_client_cleanup() _async_client_cleanup_registered = True @@ -1957,45 +1663,36 @@ def __getattr__(name: str) -> Any: # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "encoding" not in _globals: from .main import encoding as _encoding - _globals["encoding"] = _encoding return _globals["encoding"] # Lazy load bedrock_tool_name_mappings instance if name == "bedrock_tool_name_mappings": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "bedrock_tool_name_mappings" not in _globals: - from .llms.bedrock.chat.invoke_handler import ( - bedrock_tool_name_mappings as _bedrock_tool_name_mappings, - ) - + from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings return _globals["bedrock_tool_name_mappings"] # Lazy load AzureOpenAIError exception class if name == "AzureOpenAIError": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "AzureOpenAIError" not in _globals: from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError - _globals["AzureOpenAIError"] = _AzureOpenAIError return _globals["AzureOpenAIError"] # Lazy load openaiOSeriesConfig instance if name == "openaiOSeriesConfig": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() if "openaiOSeriesConfig" not in _globals: # Import the config class and instantiate it @@ -2013,7 +1710,6 @@ def __getattr__(name: str) -> Any: } if name in _config_instances: from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() if name not in _globals: # Import the config class and instantiate it @@ -2028,20 +1724,17 @@ def __getattr__(name: str) -> Any: # Lazy load provider_list if name == "provider_list": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "provider_list" not in _globals: # LlmProviders is eagerly imported above, so we can import it directly from litellm.types.utils import LlmProviders - _globals["provider_list"] = list(LlmProviders) return _globals["provider_list"] # Lazy load priority_reservation_settings instance if name == "priority_reservation_settings": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "priority_reservation_settings" not in _globals: @@ -2053,7 +1746,6 @@ def __getattr__(name: str) -> Any: # Lazy load logging_callback_manager instance if name == "logging_callback_manager": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "logging_callback_manager" not in _globals: @@ -2065,41 +1757,19 @@ def __getattr__(name: str) -> Any: # Lazy load _service_logger module if name == "_service_logger": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "_service_logger" not in _globals: # Import the module lazily import litellm._service_logger - _globals["_service_logger"] = litellm._service_logger return _globals["_service_logger"] # Lazy load evals module functions - if name in [ - "acreate_eval", - "alist_evals", - "aget_eval", - "aupdate_eval", - "adelete_eval", - "acancel_eval", - "create_eval", - "list_evals", - "get_eval", - "update_eval", - "delete_eval", - "cancel_eval", - "acreate_run", - "alist_runs", - "aget_run", - "acancel_run", - "adelete_run", - "create_run", - "list_runs", - "get_run", - "cancel_run", - "delete_run", - ]: + if name in ["acreate_eval", "alist_evals", "aget_eval", "aupdate_eval", "adelete_eval", "acancel_eval", + "create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval", + "acreate_run", "alist_runs", "aget_run", "acancel_run", "adelete_run", + "create_run", "list_runs", "get_run", "cancel_run", "delete_run"]: from litellm.evals.main import ( acreate_eval, alist_evals, @@ -2124,7 +1794,6 @@ def __getattr__(name: str) -> Any: cancel_run, delete_run, ) - return locals()[name] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index 43dd066260..a15918bc6d 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -7,6 +7,12 @@ from typing import List, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.datadog.datadog_handler import ( + get_datadog_env, + get_datadog_hostname, + get_datadog_pod_name, + get_datadog_service, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -22,7 +28,7 @@ from litellm.types.utils import StandardLoggingPayload class DatadogMetricsLogger(CustomBatchLogger): - def __init__(self, **kwargs): + def __init__(self, start_periodic_flush: bool = True, **kwargs): self.dd_api_key = os.getenv("DD_API_KEY") self.dd_app_key = os.getenv("DD_APP_KEY") self.dd_site = os.getenv("DD_SITE", "datadoghq.com") @@ -51,8 +57,9 @@ class DatadogMetricsLogger(CustomBatchLogger): super().__init__(**kwargs) - # Start periodic flush task - asyncio.create_task(self.periodic_flush()) + # Start periodic flush task only if instructed + if start_periodic_flush: + asyncio.create_task(self.periodic_flush()) def _extract_tags( self, @@ -62,13 +69,6 @@ class DatadogMetricsLogger(CustomBatchLogger): """ Builds the list of tags for a Datadog metric point """ - from litellm.integrations.datadog.datadog_handler import ( - get_datadog_env, - get_datadog_hostname, - get_datadog_pod_name, - get_datadog_service, - ) - # Base tags tags = [ f"env:{get_datadog_env()}", @@ -188,8 +188,9 @@ class DatadogMetricsLogger(CustomBatchLogger): error_information = ( standard_logging_object.get("error_information", {}) or {} ) - if "error_code" in error_information and error_information["error_code"] is not None: # type: ignore - status_code = str(error_information["error_code"]) # type: ignore + error_code = error_information.get("error_code") # type: ignore + if error_code is not None: + status_code = str(error_code) self._add_metrics_from_log( log=standard_logging_object, kwargs=kwargs, status_code=status_code diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 98f573d471..9a6c9c09f2 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -133,8 +133,8 @@ from ..integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger from ..integrations.azure_storage.azure_storage import AzureBlobStorageLogger from ..integrations.custom_prompt_management import CustomPromptManagement from ..integrations.datadog.datadog import DataDogLogger -from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger from ..integrations.datadog.datadog_metrics import DatadogMetricsLogger +from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger from ..integrations.dotprompt import DotpromptManager from ..integrations.dynamodb import DyanmoDBLogger from ..integrations.galileo import GalileoObserve @@ -1654,7 +1654,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload(logging_result, start_time, end_time) + ] = self._build_standard_logging_payload( + logging_result, start_time, end_time + ) if ( standard_logging_payload := self.model_call_details.get( @@ -2517,7 +2519,9 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload(result, start_time, end_time) + ] = self._build_standard_logging_payload( + result, start_time, end_time + ) # print standard logging payload if ( @@ -3658,10 +3662,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _datadog_logger = DataDogLogger() _in_memory_loggers.append(_datadog_logger) return _datadog_logger # type: ignore - elif logging_integration == "datadog_llm_observability": - _datadog_llm_obs_logger = DataDogLLMObsLogger() - _in_memory_loggers.append(_datadog_llm_obs_logger) - return _datadog_llm_obs_logger # type: ignore elif logging_integration == "datadog_metrics": for callback in _in_memory_loggers: if isinstance(callback, DatadogMetricsLogger): @@ -3670,6 +3670,10 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _datadog_metrics_logger = DatadogMetricsLogger() _in_memory_loggers.append(_datadog_metrics_logger) return _datadog_metrics_logger # type: ignore + elif logging_integration == "datadog_llm_observability": + _datadog_llm_obs_logger = DataDogLLMObsLogger() + _in_memory_loggers.append(_datadog_llm_obs_logger) + return _datadog_llm_obs_logger # type: ignore elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4215,14 +4219,14 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 for callback in _in_memory_loggers: if isinstance(callback, DataDogLogger): return callback - elif logging_integration == "datadog_llm_observability": - for callback in _in_memory_loggers: - if isinstance(callback, DataDogLLMObsLogger): - return callback elif logging_integration == "datadog_metrics": for callback in _in_memory_loggers: if isinstance(callback, DatadogMetricsLogger): return callback + elif logging_integration == "datadog_llm_observability": + for callback in _in_memory_loggers: + if isinstance(callback, DataDogLLMObsLogger): + return callback elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4704,11 +4708,9 @@ class StandardLoggingPayloadSetup: ).model_dump() if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): - return ( - ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - _raw - ).model_dump() - ) + return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _raw + ).model_dump() return _raw if isinstance(_raw, Usage): return _raw.model_dump() @@ -5554,3 +5556,4 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: model_parameters={"stream": True}, hidden_params=hidden_params, ) + diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 48e3e8c0ad..83891c90e1 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -286,7 +286,7 @@ async def health_services_endpoint( # noqa: PLR0915 DatadogMetricsLogger, ) - datadog_metrics_logger = DatadogMetricsLogger() + datadog_metrics_logger = DatadogMetricsLogger(start_periodic_flush=False) response = await datadog_metrics_logger.async_health_check() return { "status": response["status"], diff --git a/litellm/types/integrations/datadog_metrics.py b/litellm/types/integrations/datadog_metrics.py index 4b21881ed5..7269a77cc2 100644 --- a/litellm/types/integrations/datadog_metrics.py +++ b/litellm/types/integrations/datadog_metrics.py @@ -10,7 +10,7 @@ class DatadogMetricPoint(TypedDict): class DatadogMetricSeries(TypedDict): metric: str - type: int # 1=count, 2=rate, 3=gauge, distribution is submitted as type=3, but distributions use a different endpoint /api/v1/distribution_points, wait actually according to DD /api/v2/series: 0=unspecified, 1=count, 2=rate, 3=gauge. For histogram/distribution we use type 3 or 1. + type: int # 0=unspecified, 1=count, 2=rate, 3=gauge points: List[DatadogMetricPoint] tags: List[str] From 218383f739e2654b9055d08b3638fcf90bbb499e Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 26 Feb 2026 11:45:28 +0530 Subject: [PATCH 05/13] fix: req changes --- litellm/integrations/datadog/datadog_metrics.py | 1 + litellm/proxy/health_endpoints/_health_endpoints.py | 11 ++++++++++- litellm/types/integrations/datadog_metrics.py | 5 +++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index a15918bc6d..a22efd6600 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -150,6 +150,7 @@ class DatadogMetricsLogger(CustomBatchLogger): "type": 1, # count "points": [{"timestamp": timestamp, "value": 1.0}], "tags": tags, + "interval": self.flush_interval, } self.log_queue.append(series_count) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 83891c90e1..f37eb344d3 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -285,8 +285,17 @@ async def health_services_endpoint( # noqa: PLR0915 from litellm.integrations.datadog.datadog_metrics import ( DatadogMetricsLogger, ) + from litellm.litellm_core_utils.litellm_logging import ( + get_custom_logger_compatible_class, + ) - datadog_metrics_logger = DatadogMetricsLogger(start_periodic_flush=False) + datadog_metrics_logger = get_custom_logger_compatible_class( + "datadog_metrics" + ) + if datadog_metrics_logger is None: + datadog_metrics_logger = DatadogMetricsLogger( + start_periodic_flush=False + ) response = await datadog_metrics_logger.async_health_check() return { "status": response["status"], diff --git a/litellm/types/integrations/datadog_metrics.py b/litellm/types/integrations/datadog_metrics.py index 7269a77cc2..4c980cdee6 100644 --- a/litellm/types/integrations/datadog_metrics.py +++ b/litellm/types/integrations/datadog_metrics.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from typing_extensions import TypedDict @@ -8,11 +8,12 @@ class DatadogMetricPoint(TypedDict): value: float # The metric value -class DatadogMetricSeries(TypedDict): +class DatadogMetricSeries(TypedDict, total=False): metric: str type: int # 0=unspecified, 1=count, 2=rate, 3=gauge points: List[DatadogMetricPoint] tags: List[str] + interval: Optional[int] # Required for count (type=1) and rate (type=2) metrics class DatadogMetricsPayload(TypedDict): From 9b7b987b35902f4e9a26a1d6c57688f4504de47a Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Fri, 27 Feb 2026 15:15:57 +0530 Subject: [PATCH 06/13] fix: req changes --- litellm/integrations/datadog/datadog_metrics.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index a22efd6600..70576d2d15 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -209,16 +209,14 @@ class DatadogMetricsLogger(CustomBatchLogger): if not self.log_queue: return + batch = self.log_queue.copy() + payload_data: DatadogMetricsPayload = {"series": batch} + try: - # We must only send the current batch, so copy and clear log queue - batch = self.log_queue.copy() - # Note: CustomBatchLogger clears queue in flush_queue, but we'll manually copy what we need - - payload_data: DatadogMetricsPayload = {"series": batch} - await self._upload_to_datadog(payload_data) - except Exception as e: + # Re-insert failed batch so next flush retries + self.log_queue.extend(batch) verbose_logger.exception( f"Datadog Metrics: Error in async_send_batch: {str(e)}" ) From 9e4cf0e4df174004b9f872fca9f3ac12157c3c4d Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 15:59:36 +0530 Subject: [PATCH 07/13] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../integrations/datadog/test_datadog_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index 1008a4f78c..c851f5fdea 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -35,9 +35,9 @@ def clean_env(): @pytest.mark.asyncio async def test_init(clean_env): """Test initialization sets up clients and url correctly.""" - logger = DatadogMetricsLogger() - assert logger.dd_api_key == "test_api_key" - assert logger.dd_site == "test.datadoghq.com" +async def test_init(clean_env): + """Test initialization sets up clients and url correctly.""" + logger = DatadogMetricsLogger(start_periodic_flush=False) assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" From 9d3e97a2d8d33849ed6fead42a1e689e1f096f3f Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 15:59:47 +0530 Subject: [PATCH 08/13] Update litellm/integrations/datadog/datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/integrations/datadog/datadog_metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index 70576d2d15..fcf40701e2 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -215,11 +215,10 @@ class DatadogMetricsLogger(CustomBatchLogger): try: await self._upload_to_datadog(payload_data) except Exception as e: - # Re-insert failed batch so next flush retries - self.log_queue.extend(batch) verbose_logger.exception( f"Datadog Metrics: Error in async_send_batch: {str(e)}" ) + raise async def _upload_to_datadog(self, payload: DatadogMetricsPayload): if not self.dd_api_key: From 8dc8e826ba6d0dd97bf5c170be5d2d367faaab18 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:21:41 +0530 Subject: [PATCH 09/13] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../test_litellm/integrations/datadog/test_datadog_metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index c851f5fdea..d5b6cce972 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -35,8 +35,7 @@ def clean_env(): @pytest.mark.asyncio async def test_init(clean_env): """Test initialization sets up clients and url correctly.""" -async def test_init(clean_env): - """Test initialization sets up clients and url correctly.""" + logger = DatadogMetricsLogger(start_periodic_flush=False) logger = DatadogMetricsLogger(start_periodic_flush=False) assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" From 44dd13994157d4ce479f68e7fa3a4084e202aabb Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:21:58 +0530 Subject: [PATCH 10/13] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/integrations/datadog/test_datadog_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index d5b6cce972..3b396590e2 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -43,7 +43,7 @@ async def test_init(clean_env): @pytest.mark.asyncio async def test_extract_tags(clean_env): """Test tag extraction from a StandardLoggingPayload.""" - logger = DatadogMetricsLogger() + logger = DatadogMetricsLogger(start_periodic_flush=False) payload = StandardLoggingPayload( custom_llm_provider="openai", From 04fda9ec4ac9e28b6b73d0a3ee82a51cb6aaf98f Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:28:33 +0530 Subject: [PATCH 11/13] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/integrations/datadog/test_datadog_metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index 3b396590e2..f9d7969461 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -36,7 +36,6 @@ def clean_env(): async def test_init(clean_env): """Test initialization sets up clients and url correctly.""" logger = DatadogMetricsLogger(start_periodic_flush=False) - logger = DatadogMetricsLogger(start_periodic_flush=False) assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" From b6a53fb174f832c1dc0690a1350d77e158f5e793 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:28:48 +0530 Subject: [PATCH 12/13] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/integrations/datadog/test_datadog_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index f9d7969461..35e2f537cc 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -66,7 +66,7 @@ async def test_extract_tags(clean_env): @pytest.mark.asyncio async def test_extract_tags_no_team(clean_env): """Test tag extraction when no team info is present.""" - logger = DatadogMetricsLogger() + logger = DatadogMetricsLogger(start_periodic_flush=False) payload = StandardLoggingPayload( custom_llm_provider="anthropic", From 24aa8bac09d32d5ca7dd04df9c43b02af35b09bd Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Sat, 28 Feb 2026 16:31:05 +0530 Subject: [PATCH 13/13] fix req changes test case --- .../datadog/test_datadog_metrics.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index 35e2f537cc..757c558c29 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta from unittest.mock import AsyncMock import pytest -from httpx import Response +from httpx import Request, Response from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger from litellm.types.utils import StandardLoggingPayload @@ -84,7 +84,7 @@ async def test_extract_tags_no_team(clean_env): @pytest.mark.asyncio async def test_add_metrics_from_log(clean_env): """Test that _add_metrics_from_log appends the correct metric series to the queue.""" - logger = DatadogMetricsLogger(batch_size=100) + logger = DatadogMetricsLogger(batch_size=100, start_periodic_flush=False) now = datetime.now() start_time = now - timedelta(seconds=2) @@ -128,7 +128,7 @@ async def test_add_metrics_from_log(clean_env): @pytest.mark.asyncio async def test_async_log_success_event(clean_env): """Test that success events are added to the queue.""" - logger = DatadogMetricsLogger(batch_size=100) + logger = DatadogMetricsLogger(batch_size=100, start_periodic_flush=False) now = datetime.now() start_time = now - timedelta(seconds=1) @@ -154,7 +154,7 @@ async def test_async_log_success_event(clean_env): @pytest.mark.asyncio async def test_async_log_success_event_no_standard_logging_object(clean_env): """Test that events without standard_logging_object are skipped.""" - logger = DatadogMetricsLogger(batch_size=100) + logger = DatadogMetricsLogger(batch_size=100, start_periodic_flush=False) await logger.async_log_success_event( kwargs={}, @@ -169,7 +169,7 @@ async def test_async_log_success_event_no_standard_logging_object(clean_env): @pytest.mark.asyncio async def test_async_log_failure_event_extracts_status_code(clean_env): """Test that failure events extract the error status code.""" - logger = DatadogMetricsLogger(batch_size=100) + logger = DatadogMetricsLogger(batch_size=100, start_periodic_flush=False) now = datetime.now() start_time = now - timedelta(seconds=1) @@ -200,7 +200,7 @@ async def test_async_log_failure_event_extracts_status_code(clean_env): @pytest.mark.asyncio async def test_async_log_failure_event_default_status_code(clean_env): """Test that failure events default to 500 when no error_code is present.""" - logger = DatadogMetricsLogger(batch_size=100) + logger = DatadogMetricsLogger(batch_size=100, start_periodic_flush=False) now = datetime.now() @@ -229,9 +229,12 @@ async def test_async_log_failure_event_default_status_code(clean_env): @pytest.mark.asyncio async def test_async_send_batch(clean_env): """Test that async_send_batch uploads metrics to Datadog.""" - logger = DatadogMetricsLogger() + logger = DatadogMetricsLogger(start_periodic_flush=False) logger.async_client = AsyncMock() - logger.async_client.post.return_value = Response(202, json={"status": "ok"}) + mock_request = Request("POST", "https://api.test.datadoghq.com/api/v2/series") + logger.async_client.post.return_value = Response( + 202, json={"status": "ok"}, request=mock_request + ) # Manually add a metric series to the queue logger.log_queue = [ @@ -262,7 +265,7 @@ async def test_async_send_batch(clean_env): @pytest.mark.asyncio async def test_async_send_batch_empty_queue(clean_env): """Test that async_send_batch does nothing when queue is empty.""" - logger = DatadogMetricsLogger() + logger = DatadogMetricsLogger(start_periodic_flush=False) logger.async_client = AsyncMock() await logger.async_send_batch()