diff --git a/.circleci/config.yml b/.circleci/config.yml index e49dca6faa..faf43ff0b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1291,6 +1291,7 @@ jobs: pip install jinja2 pip install "tokenizers==0.20.0" pip install "uvloop==0.21.0" + pip install "fastuuid==0.12.0" pip install jsonschema - setup_litellm_enterprise_pip - run: diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index f41a5291e5..8e0f1dfe7e 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -14,4 +14,5 @@ google-cloud-iam==2.19.1 fastapi-sso==0.16.0 uvloop==0.21.0 mcp==1.10.1 # for MCP server -semantic_router==0.1.10 # for auto-routing with litellm \ No newline at end of file +semantic_router==0.1.10 # for auto-routing with litellm +fastuuid==0.12.0 \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index 89c3c34bd7..766b7a1db6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,4 @@ tests *.tgz log.txt docker/Dockerfile.* +*.whl diff --git a/.gitignore b/.gitignore index ed8c88c899..547734ddce 100644 --- a/.gitignore +++ b/.gitignore @@ -95,4 +95,5 @@ test.py litellm_config.yaml .cursor .vscode/launch.json -litellm/proxy/to_delete_loadtest_work/* \ No newline at end of file +*.whl +litellm/proxy/to_delete_loadtest_work/* diff --git a/docs/my-website/docs/providers/cometapi.md b/docs/my-website/docs/providers/cometapi.md new file mode 100644 index 0000000000..1245bacfad --- /dev/null +++ b/docs/my-website/docs/providers/cometapi.md @@ -0,0 +1,144 @@ +# CometAPI +LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models. + +## Authentication + +To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering. + +## Usage + +Set your CometAPI key as an environment variable and use the completion function: + +```python +import os +from litellm import completion + +# Set API key +os.environ["COMETAPI_KEY"] = "your_comet_api_key_here" + +# Define messages +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Method 1: Using environment variable (recommended) +response = completion( + model="cometapi/gpt-5", + messages=messages +) + +print(response.choices[0].message.content) +``` + +### Alternative Usage - Explicit API Key + +You can also pass the API key explicitly: + +```python +import os +from litellm import completion + +# Define messages +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Method 2: Explicitly passing API key +response = completion( + model="cometapi/gpt-4o", + messages=messages, + api_key="your_comet_api_key_here" +) + +print(response.choices[0].message.content) +``` + +## Usage - Streaming + +Just set `stream=True` when calling completion: + +```python +import os +from litellm import completion + +os.environ["COMETAPI_KEY"] = "your_comet_api_key_here" + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +response = completion( + model="cometapi/gpt-5", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk.choices[0].delta.content or "", end="") +``` + +## Usage - Async Streaming + +For async streaming, use `acompletion`: + +```python +from litellm import acompletion +import asyncio, os, traceback + +async def completion_call(): + try: + os.environ["COMETAPI_KEY"] = "your_comet_api_key_here" + + print("test acompletion + streaming") + response = await acompletion( + model="cometapi/chatgpt-4o-latest", + messages=[{"content": "Hello, how are you?", "role": "user"}], + stream=True + ) + print(f"response: {response}") + async for chunk in response: + print(chunk) + except: + print(f"error occurred: {traceback.format_exc()}") + pass + +# Run the async function +await completion_call() +``` + +## CometAPI Models + +CometAPI offers access to 500+ AI models through a unified API. Some popular models include: + +| Model Name | Function Call | +|------------|---------------| +| cometapi/gpt-5 | `completion('cometapi/gpt-5', messages)` | +| cometapi/gpt-5-mini | `completion('cometapi/gpt-5-mini', messages)` | +| cometapi/gpt-5-nano | `completion('cometapi/gpt-5-nano', messages)` | +| cometapi/gpt-oss-20b | `completion('cometapi/gpt-oss-20b', messages)` | +| cometapi/gpt-oss-120b | `completion('cometapi/gpt-oss-120b', messages)` | +| cometapi/chatgpt-4o-latest | `completion('cometapi/chatgpt-4o-latest', messages)` | + +For a complete list of available models, visit the [CometAPI Models page](https://www.cometapi.com/model/). + +## Environment Variables + +| Variable | Description | Required | +|----------|-------------|----------| +| `COMETAPI_KEY` | Your CometAPI API key | Yes | + +## Error Handling + +```python +import os +from litellm import completion + +try: + os.environ["COMETAPI_KEY"] = "your_comet_api_key_here" + + messages = [{"content": "Hello, how are you?", "role": "user"}] + + response = completion( + model="cometapi/gpt-5", + messages=messages + ) + + print(response.choices[0].message.content) + +except Exception as e: + print(f"Error: {e}") +``` diff --git a/docs/my-website/docs/proxy/guardrails/noma_security.md b/docs/my-website/docs/proxy/guardrails/noma_security.md new file mode 100644 index 0000000000..3a50841d65 --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/noma_security.md @@ -0,0 +1,299 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Noma Security + +Use [Noma Security](https://noma.security/) to protect your LLM applications with comprehensive AI content moderation and safety guardrails. + +## Quick Start + +### 1. Define Guardrails on your LiteLLM config.yaml + +Define your guardrails under the `guardrails` section: + +```yaml showLineNumbers title="litellm config.yaml" +model_list: + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "noma-guard" + litellm_params: + guardrail: noma + mode: "during_call" + api_key: os.environ/NOMA_API_KEY + api_base: os.environ/NOMA_API_BASE + - guardrail_name: "noma-pre-guard" + litellm_params: + guardrail: noma + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + api_base: os.environ/NOMA_API_BASE +``` + +#### Supported values for `mode` + +- `pre_call` Run **before** LLM call, on **input** +- `post_call` Run **after** LLM call, on **input & output** +- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel with the LLM call. Response not returned until guardrail check completes + +### 2. Start LiteLLM Gateway + +```shell +litellm --config config.yaml --detailed_debug +``` + +### 3. Test request + + + + +Expect this to fail since the request contains harmful content: + +```shell showLineNumbers title="Curl Request" +curl -i http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Tell me how to hack into someone's email account"} + ] + }' +``` + +Expected response on failure: + +```json +{ + "error": { + "message": "{\n \"error\": \"Request blocked by Noma guardrail\",\n \"details\": {\n \"prompt\": {\n \"harmfulContent\": {\n \"result\": true,\n \"confidence\": 0.95\n }\n }\n }\n }", + "type": "None", + "param": "None", + "code": "400" + } +} +``` + + + + + +```shell showLineNumbers title="Curl Request" +curl -i http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "What is the capital of France?"} + ] + }' +``` + +Expected response: + +```json +{ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } +} +``` + + + + +## Supported Params + +```yaml +guardrails: + - guardrail_name: "noma-guard" + litellm_params: + guardrail: noma + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + api_base: os.environ/NOMA_API_BASE + ### OPTIONAL ### + # application_id: "my-app" + # monitor_mode: false + # block_failures: true +``` + +### Required Parameters + +- **`api_key`**: Your Noma Security API key (set as `os.environ/NOMA_API_KEY` in YAML config) + +### Optional Parameters + +- **`api_base`**: Noma API base URL (defaults to `https://api.noma.security/`) +- **`application_id`**: Your application identifier (defaults to `"litellm"`) +- **`monitor_mode`**: If `true`, logs violations without blocking (defaults to `false`) +- **`block_failures`**: If `true`, blocks requests when guardrail API failures occur (defaults to `true`) + +## Environment Variables + +You can set these environment variables instead of hardcoding values in your config: + +```shell +export NOMA_API_KEY="your-api-key-here" +export NOMA_API_BASE="https://api.noma.security/" # Optional +export NOMA_APPLICATION_ID="my-app" # Optional +export NOMA_MONITOR_MODE="false" # Optional +export NOMA_BLOCK_FAILURES="true" # Optional +``` + +## Advanced Configuration + +### Monitor Mode + +Use monitor mode to test your guardrails without blocking requests: + +```yaml +guardrails: + - guardrail_name: "noma-monitor" + litellm_params: + guardrail: noma + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + monitor_mode: true # Log violations but don't block +``` + +### Handling API Failures + +Control behavior when the Noma API is unavailable: + +```yaml +guardrails: + - guardrail_name: "noma-failopen" + litellm_params: + guardrail: noma + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + block_failures: false # Allow requests to proceed if guardrail API fails +``` + +### Multiple Guardrails + +Apply different configurations for input and output: + +```yaml +guardrails: + - guardrail_name: "noma-strict-input" + litellm_params: + guardrail: noma + mode: "pre_call" + api_key: os.environ/NOMA_API_KEY + block_failures: true + + - guardrail_name: "noma-monitor-output" + litellm_params: + guardrail: noma + mode: "post_call" + api_key: os.environ/NOMA_API_KEY + monitor_mode: true +``` + +## ✨ Pass Additional Parameters + +Use `extra_body` to pass additional parameters to the Noma Security API call, such as dynamically setting the application ID for specific requests. + + + + +```python +import openai +client = openai.OpenAI( + api_key="your-api-key", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "Hello, how are you?"}], + extra_body={ + "guardrails": { + "noma-guard": { + "extra_body": { + "application_id": "my-specific-app-id" + } + } + } + } +) +``` + + + + +```shell +curl 'http://0.0.0.0:4000/v1/chat/completions' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "guardrails": { + "noma-guard": { + "extra_body": { + "application_id": "my-specific-app-id" + } + } + } +}' +``` + + + +This allows you to override the default `application_id` parameter for specific requests, which is useful for tracking usage across different applications or components. + +## Response Details + +When content is blocked, Noma provides detailed information about the violations as JSON inside the `message` field, with the following structure: + +```json +{ + "error": "Request blocked by Noma guardrail", + "details": { + "prompt": { + "harmfulContent": { + "result": true, + "confidence": 0.95 + }, + "sensitiveData": { + "email": { + "result": true, + "entities": ["user@example.com"] + } + }, + "bannedTopics": { + "violence": { + "result": true, + "confidence": 0.88 + } + } + } + } +} +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 7827649d2d..9af8a9b6f6 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -40,6 +40,7 @@ const sidebars = { "proxy/guardrails/guardrails_ai", "proxy/guardrails/lakera_ai", "proxy/guardrails/model_armor", + "proxy/guardrails/noma_security", "proxy/guardrails/openai_moderation", "proxy/guardrails/pangea", "proxy/guardrails/pillar_security", diff --git a/litellm/_logging.py b/litellm/_logging.py index 73902d2fc5..1cf2a49832 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -4,21 +4,41 @@ import os import sys from datetime import datetime from logging import Formatter - set_verbose = False +def __strtobool(val: str) -> bool: + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ('y', 'yes', 't', 'true', 'on', '1'): + return True + elif val in ('n', 'no', 'f', 'false', 'off', '0'): + return False + else: + raise ValueError(f"invalid truth value {val!r}") + if set_verbose is True: logging.warning( "`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs." ) -json_logs = bool(os.getenv("JSON_LOGS", False)) + +json_logs = __strtobool(os.getenv("JSON_LOGS", "False")) # Create a handler for the logger (you may need to adapt this based on your needs) log_level = os.getenv("LITELLM_LOG", "DEBUG") numeric_level: str = getattr(logging, log_level.upper()) handler = logging.StreamHandler() handler.setLevel(numeric_level) +log_file = os.getenv("LITELLM_LOG_FILE", "") +file_handler = None +if log_file: + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(numeric_level) class JsonFormatter(Formatter): def __init__(self): super(JsonFormatter, self).__init__() @@ -40,6 +60,7 @@ class JsonFormatter(Formatter): return json.dumps(json_record) +json_formatter = JsonFormatter() # Function to set up exception handlers for JSON logging def _setup_json_exception_handlers(formatter): @@ -89,8 +110,10 @@ def _setup_json_exception_handlers(formatter): # Create a formatter and set it for the handler if json_logs: - handler.setFormatter(JsonFormatter()) - _setup_json_exception_handlers(JsonFormatter()) + handler.setFormatter(json_formatter) + if file_handler: + file_handler.setFormatter(json_formatter) + _setup_json_exception_handlers(json_formatter) else: formatter = logging.Formatter( "\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s", @@ -98,11 +121,18 @@ else: ) handler.setFormatter(formatter) + if file_handler: + file_handler.setFormatter(formatter) verbose_proxy_logger = logging.getLogger("LiteLLM Proxy") verbose_router_logger = logging.getLogger("LiteLLM Router") verbose_logger = logging.getLogger("LiteLLM") +# Set logger levels +verbose_proxy_logger.setLevel(numeric_level) +verbose_router_logger.setLevel(numeric_level) +verbose_logger.setLevel(numeric_level) + # Add the handler to the logger verbose_router_logger.addHandler(handler) verbose_proxy_logger.addHandler(handler) @@ -125,6 +155,13 @@ def _suppress_loggers(): # Call the suppression function _suppress_loggers() +if file_handler: + verbose_router_logger.addHandler(file_handler) + verbose_proxy_logger.addHandler(file_handler) + verbose_logger.addHandler(file_handler) + + + ALL_LOGGERS = [ logging.getLogger(), verbose_logger, @@ -153,10 +190,10 @@ def _turn_on_json(): - Adds a JSON formatter to all loggers """ handler = logging.StreamHandler() - handler.setFormatter(JsonFormatter()) + handler.setFormatter(json_formatter) _initialize_loggers_with_handler(handler) # Set up exception handlers - _setup_json_exception_handlers(JsonFormatter()) + _setup_json_exception_handlers(json_formatter) def _turn_on_debug(): diff --git a/litellm/constants.py b/litellm/constants.py index 72cc52fdf2..78d5e5760d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -638,18 +638,55 @@ featherless_ai_models: set = set([ ]) nebius_models: set = set([ + # deepseek models + "deepseek-ai/DeepSeek-R1-0528", + "deepseek-ai/DeepSeek-V3-0324", + "deepseek-ai/DeepSeek-V3", + "deepseek-ai/DeepSeek-R1", + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + # google models + "google/gemma-2-2b-it", + "google/gemma-2-9b-it-fast", + # llama models + "meta-llama/Llama-3.3-70B-Instruct", + "meta-llama/Meta-Llama-3.1-70B-Instruct", + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "meta-llama/Meta-Llama-3.1-405B-Instruct", + "NousResearch/Hermes-3-Llama-405B", + # microsoft models + "microsoft/phi-4", + # mistral models + "mistralai/Mistral-Nemo-Instruct-2407", + "mistralai/Devstral-Small-2505", + # moonshot models + "moonshotai/Kimi-K2-Instruct", + # nvidia models + "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1", + "nvidia/Llama-3_3-Nemotron-Super-49B-v1", + # openai models + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + # qwen models + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + "Qwen/Qwen3-235B-A22B-Instruct-2507", "Qwen/Qwen3-235B-A22B", - "Qwen/Qwen3-30B-A3B-fast", + "Qwen/Qwen3-30B-A3B", "Qwen/Qwen3-32B", "Qwen/Qwen3-14B", - "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1", - "deepseek-ai/DeepSeek-V3-0324", - "deepseek-ai/DeepSeek-V3-0324-fast", - "deepseek-ai/DeepSeek-R1", - "deepseek-ai/DeepSeek-R1-fast", - "meta-llama/Llama-3.3-70B-Instruct-fast", - "Qwen/Qwen2.5-32B-Instruct-fast", - "Qwen/Qwen2.5-Coder-32B-Instruct-fast", + "Qwen/Qwen3-4B-fast", + "Qwen/Qwen2.5-Coder-7B", + "Qwen/Qwen2.5-Coder-32B-Instruct", + "Qwen/Qwen2.5-72B-Instruct", + "Qwen/QwQ-32B", + "Qwen/Qwen3-30B-A3B-Thinking-2507", + "Qwen/Qwen3-30B-A3B-Instruct-2507", + # zai models + "zai-org/GLM-4.5", + "zai-org/GLM-4.5-Air", + # other models + "aaditya/Llama3-OpenBioLLM-70B", + "ProdeusUnity/Stellar-Odyssey-12b-v0.0", + "all-hands/openhands-lm-32b-v0.1", ]) dashscope_models: set = set([ diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 4df944edba..fab2c1e76e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -529,7 +529,7 @@ def _get_count_function( encoding = tiktoken.get_encoding("cl100k_base") def count_tokens(text: str) -> int: - return len(encoding.encode(text)) + return len(encoding.encode(text, disallowed_special=())) else: raise ValueError("Unsupported tokenizer type") diff --git a/litellm/main.py b/litellm/main.py index 6102fe3ccc..70f5512550 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1256,6 +1256,7 @@ def completion( # type: ignore # noqa: PLR0915 additional_drop_params=kwargs.get("additional_drop_params"), remove_sensitive_keys=True, add_provider_specific_params=True, + provider_config=provider_config, ) if litellm.add_function_to_prompt and optional_params.get( diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py new file mode 100644 index 0000000000..dc3e4d9768 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py @@ -0,0 +1,36 @@ +from typing import TYPE_CHECKING + +from litellm.types.guardrails import SupportedGuardrailIntegrations + +from .noma import NomaGuardrail + +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + + +def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): + import litellm + + _noma_callback = NomaGuardrail( + guardrail_name=guardrail.get("guardrail_name", ""), + api_key=litellm_params.api_key, + api_base=litellm_params.api_base, + application_id=litellm_params.application_id, + monitor_mode=litellm_params.monitor_mode, + block_failures=litellm_params.block_failures, + event_hook=litellm_params.mode, + default_on=litellm_params.default_on, + ) + litellm.logging_callback_manager.add_litellm_callback(_noma_callback) + + return _noma_callback + + +guardrail_initializer_registry = { + SupportedGuardrailIntegrations.NOMA.value: initialize_guardrail, +} + + +guardrail_class_registry = { + SupportedGuardrailIntegrations.NOMA.value: NomaGuardrail, +} diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py new file mode 100644 index 0000000000..ed5929f056 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -0,0 +1,403 @@ +# +-------------------------------------------------------------+ +# +# Noma Security Guardrail Integration for LiteLLM +# https://noma.security +# +# +-------------------------------------------------------------+ + +import copy +import os +from typing import Any, Dict, Literal, Optional, Union +from urllib.parse import urljoin + +from fastapi import HTTPException + +import litellm +from litellm import DualCache, ModelResponse +from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import EmbeddingResponse, ImageResponse + + +class NomaBlockedMessage(HTTPException): + """Exception raised when Noma guardrail blocks a message""" + + def __init__(self, classification_response: dict): + classification = self._filter_triggered_classifications(classification_response) + super().__init__( + status_code=400, + detail={ + "error": "Request blocked by Noma guardrail", + "details": classification, + }, + ) + + def _filter_triggered_classifications( + self, + response_dict: dict, + ) -> dict: + """Filter and return only triggered classifications""" + filtered_response = copy.deepcopy(response_dict) + + # Filter prompt classifications if present + if filtered_response.get("prompt"): + filtered_response["prompt"] = self.filter_classification_object( + filtered_response["prompt"] + ) + + # Filter response classifications if present + if filtered_response.get("response"): + filtered_response["response"] = self.filter_classification_object( + filtered_response["response"] + ) + + return filtered_response + + def filter_classification_object( + self, + classification_obj: dict, + ) -> dict: + """Filter classification object to only include triggered items""" + if not classification_obj: + return {} + + result = {} + + for key, value in classification_obj.items(): + if value is None: + continue + + if key in [ + "allowedTopics", + "bannedTopics", + "topicGuardrails", + ] and isinstance(value, dict): + filtered_topics = {} + for topic, topic_result in value.items(): + if self._is_result_true(topic_result): + filtered_topics[topic] = topic_result + + if filtered_topics: + result[key] = filtered_topics + + elif key == "sensitiveData" and isinstance(value, dict): + filtered_sensitive = {} + for data_type, data_result in value.items(): + if self._is_result_true(data_result): + filtered_sensitive[data_type] = data_result + + if filtered_sensitive: + result[key] = filtered_sensitive + + elif isinstance(value, dict) and "result" in value: + if self._is_result_true(value): + result[key] = value + + return result + + def _is_result_true(self, result_obj: Optional[Dict[str, Any]]) -> bool: + """ + Check if a result object has a "result" field that is True. + + Args: + result_obj: A dictionary that may contain a "result" field + + Returns: + True if the "result" field exists and is True, False otherwise + """ + if not result_obj or not isinstance(result_obj, dict): + return False + + return result_obj.get("result") is True + + +class NomaGuardrail(CustomGuardrail): + """ + Noma Security Guardrail for LiteLLM + + This guardrail integrates with Noma Security's AI-DR API to provide + content moderation and safety checks for LLM inputs and outputs. + """ + + _DEFAULT_API_BASE = "https://api.noma.security/" + _AIDR_ENDPOINT = "/ai-dr/v1/prompt/scan/aggregate" + + def __init__( + self, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + application_id: Optional[str] = None, + monitor_mode: Optional[bool] = None, + block_failures: Optional[bool] = None, + **kwargs, + ): + self.async_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) + self.api_key = api_key or os.environ.get("NOMA_API_KEY") + self.api_base = api_base or os.environ.get( + "NOMA_API_BASE", NomaGuardrail._DEFAULT_API_BASE + ) + self.application_id = application_id or os.environ.get( + "NOMA_APPLICATION_ID", "litellm" + ) + + if monitor_mode is None: + self.monitor_mode = ( + os.environ.get("NOMA_MONITOR_MODE", "false").lower() == "true" + ) + else: + self.monitor_mode = monitor_mode + + if block_failures is None: + self.block_failures = ( + os.environ.get("NOMA_BLOCK_FAILURES", "true").lower() == "true" + ) + else: + self.block_failures = block_failures + + super().__init__(**kwargs) + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: Literal[ + "completion", + "text_completion", + "embeddings", + "image_generation", + "moderation", + "audio_transcription", + "pass_through_endpoint", + "rerank", + "mcp_call", + ], + ) -> Optional[Union[Exception, str, dict]]: + verbose_proxy_logger.debug("Running Noma pre-call hook") + + if ( + self.should_run_guardrail( + data=data, event_type=GuardrailEventHooks.pre_call + ) + is False + ): + return data + + try: + return await self._check_user_message(data, user_api_key_dict) + except NomaBlockedMessage: + raise + except Exception as e: + verbose_proxy_logger.error(f"Noma pre-call hook failed: {str(e)}") + + if self.block_failures and not self.monitor_mode: + raise + return data + + async def async_moderation_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + call_type: Literal[ + "completion", + "embeddings", + "image_generation", + "moderation", + "audio_transcription", + "responses", + "mcp_call", + ], + ) -> Union[Exception, str, dict, None]: + event_type: GuardrailEventHooks = GuardrailEventHooks.during_call + if self.should_run_guardrail(data=data, event_type=event_type) is not True: + return data + + try: + return await self._check_user_message(data, user_api_key_dict) + except NomaBlockedMessage: + raise + except Exception as e: + verbose_proxy_logger.error(f"Noma moderation hook failed: {str(e)}") + + if self.block_failures and not self.monitor_mode: + raise + return data + + async def async_post_call_success_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + response: Union[Any, ModelResponse, EmbeddingResponse, ImageResponse], + ): + event_type: GuardrailEventHooks = GuardrailEventHooks.post_call + if self.should_run_guardrail(data=data, event_type=event_type) is not True: + return response + + try: + return await self._check_llm_response(data, response, user_api_key_dict) + except NomaBlockedMessage: + raise + except Exception as e: + verbose_proxy_logger.error(f"Noma post-call hook failed: {str(e)}") + if self.block_failures and not self.monitor_mode: + raise + return response + + async def _check_user_message( + self, + request_data: dict, + user_auth: UserAPIKeyAuth, + ) -> Union[Exception, str, dict, None]: + """Check user message for policy violations""" + extra_data = self.get_guardrail_dynamic_request_body_params(request_data) + + user_message = await self._extract_user_message(request_data) + if not user_message: + return request_data + + payload = {"request": {"text": user_message}} + response_json = await self._call_noma_api( + payload=payload, + llm_request_id=None, + request_data=request_data, + user_auth=user_auth, + extra_data=extra_data, + ) + await self._check_verdict("user", user_message, response_json) + + return request_data + + async def _check_llm_response( + self, + request_data: dict, + response: Union[Any, ModelResponse, EmbeddingResponse, ImageResponse], + user_auth: UserAPIKeyAuth, + ) -> Union[Exception, ModelResponse, Any]: + """Check LLM response for policy violations""" + extra_data = self.get_guardrail_dynamic_request_body_params(request_data) + + if not isinstance(response, litellm.ModelResponse): + return response + + content = None + for choice in response.choices: + if isinstance(choice, litellm.Choices) and choice.message.content: + content = choice.message.content + break + + if not content or not isinstance(content, str): + return response + + payload = {"response": {"text": content}} + + response_json = await self._call_noma_api( + payload=payload, + llm_request_id=response.id, + request_data=request_data, + user_auth=user_auth, + extra_data=extra_data, + ) + await self._check_verdict("assistant", content, response_json) + + return response + + async def _extract_user_message(self, data: dict) -> Optional[str]: + """Extract the last user message from request data""" + messages = data.get("messages", []) + if not messages: + return None + + # Get the last user message + user_messages = [msg for msg in messages if msg.get("role") == "user"] + if not user_messages: + return None + + last_user_message = user_messages[-1].get("content", "") + if not last_user_message or not isinstance(last_user_message, str): + return None + + return last_user_message + + async def _call_noma_api( + self, + payload: dict, + llm_request_id: Optional[str], + request_data: dict, + user_auth: UserAPIKeyAuth, + extra_data: dict, + ) -> dict: + call_id = request_data.get("litellm_call_id") + headers = { + "X-Noma-AIDR-Application-ID": self.application_id, + **({"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}), + **({"X-Noma-Request-ID": call_id} if call_id else {}), + } + endpoint = urljoin( + self.api_base or "https://api.noma.security/", NomaGuardrail._AIDR_ENDPOINT + ) + + response = await self.async_handler.post( + endpoint, + headers=headers, + json={ + **payload, + "context": { + "applicationId": extra_data.get("application_id") + or request_data.get("metadata", {}) + .get("headers", {}) + .get("x-noma-application-id"), + "ipAddress": request_data.get("metadata", {}).get( + "requester_ip_address", None + ), + "userId": user_auth.user_email + if user_auth.user_email + else user_auth.user_id, + "sessionId": call_id, + "requestId": llm_request_id, + }, + }, + ) + response.raise_for_status() + + return response.json() + + async def _check_verdict( + self, + type: Literal["user", "assistant"], + message: str, + response_json: dict, + ) -> None: + """ + Check the verdict from the Noma API and raise an exception if needed + """ + if not response_json.get("verdict", True): + msg = str.format( + "Noma guardrail blocked {type} message: {message}", + type=type, + message=message, + ) + + if self.monitor_mode: + verbose_proxy_logger.warning(msg) + else: + verbose_proxy_logger.debug(msg) + original_response = response_json.get("originalResponse", {}) + raise NomaBlockedMessage(original_response) + else: + msg = str.format( + "Noma guardrail allowed {type} message: {message}", + type=type, + message=message, + ) + if self.monitor_mode: + verbose_proxy_logger.info(msg) + else: + verbose_proxy_logger.debug(msg) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index fd18484a89..f31f304bda 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -40,6 +40,7 @@ class SupportedGuardrailIntegrations(Enum): AZURE_TEXT_MODERATIONS = "azure/text_moderations" MODEL_ARMOR = "model_armor" OPENAI_MODERATION = "openai_moderation" + NOMA = "noma" class Role(Enum): SYSTEM = "system" @@ -359,6 +360,23 @@ class PillarGuardrailConfigModel(BaseModel): ) +class NomaGuardrailConfigModel(BaseModel): + """Configuration parameters for the Noma Security guardrail""" + + application_id: Optional[str] = Field( + default=None, + description="Application ID for Noma Security. Defaults to 'litellm' if not provided", + ) + monitor_mode: Optional[bool] = Field( + default=None, + description="If True, logs violations without blocking. Defaults to False if not provided", + ) + block_failures: Optional[bool] = Field( + default=None, + description="If True, blocks requests on API failures. Defaults to True if not provided", + ) + + class BaseLitellmParams(BaseModel): # works for new and patch update guardrails api_key: Optional[str] = Field( default=None, description="API key for the guardrail service" @@ -445,6 +463,7 @@ class LitellmParams( LakeraV2GuardrailConfigModel, LassoGuardrailConfigModel, PillarGuardrailConfigModel, + NomaGuardrailConfigModel, BaseLitellmParams, ): guardrail: str = Field(description="The type of guardrail integration to use") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index fad46a8b9f..30d46d61c9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,6 +1,5 @@ import json import time -import uuid from enum import Enum from typing import ( TYPE_CHECKING, @@ -14,6 +13,7 @@ from typing import ( Union, ) +import fastuuid as uuid from aiohttp import FormData from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import FileTypes # type: ignore diff --git a/litellm/utils.py b/litellm/utils.py index aa3c00735e..40a1438b3f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3088,6 +3088,7 @@ def pre_process_non_default_params( model: str, remove_sensitive_keys: bool = False, add_provider_specific_params: bool = False, + provider_config: Optional[BaseConfig] = None, ) -> dict: """ Pre-process non-default params to a standardized format @@ -3103,14 +3104,6 @@ def pre_process_non_default_params( additional_endpoint_specific_params=["messages"], ) - provider_config: Optional[BaseConfig] = None - if custom_llm_provider is not None and custom_llm_provider in [ - provider.value for provider in LlmProviders - ]: - provider_config = ProviderConfigManager.get_provider_chat_config( - model=model, provider=LlmProviders(custom_llm_provider) - ) - if "response_format" in non_default_params: if provider_config is not None: non_default_params[ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 104b5bb793..b7297f0719 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -11887,6 +11887,63 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/openai/gpt-5-mini": { + "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_tool_choice": true, + "supports_reasoning": true + }, + "openrouter/openai/gpt-5-nano": { + "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 5e-09, + "litellm_provider": "openrouter", + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_tool_choice": true, + "supports_reasoning": true + }, + "openrouter/openai/gpt-5-chat": { + "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_tool_choice": false, + "supports_reasoning": true + }, "openrouter/openai/gpt-oss-20b": { "max_tokens": 32768, "max_input_tokens": 131072, @@ -15171,236 +15228,16 @@ "litellm_provider": "ollama", "mode": "completion" }, - "deepinfra/deepseek-ai/DeepSeek-V3": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 3.8e-07, - "output_cost_per_token": 8.9e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Phind/Phind-CodeLlama-34B-v2": { + "deepinfra/Austism/chronos-hermes-13b-v2": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 6e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 1.5e-08, - "output_cost_per_token": 2e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/google/gemma-2-9b-it": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 3e-08, - "output_cost_per_token": 6e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Qwen/Qwen2-7B-Instruct": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 5.5e-08, - "output_cost_per_token": 5.5e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Qwen/QVQ-72B-Preview": { - "max_tokens": 32000, - "max_input_tokens": 32000, - "max_output_tokens": 32000, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 2.3e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/microsoft/Phi-4-multimodal-instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 1e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/mistralai/Devstral-Small-2507": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 2.8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/microsoft/WizardLM-2-7B": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 5.5e-08, - "output_cost_per_token": 5.5e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/meta-llama/Llama-3.2-90B-Vision-Instruct": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 3.5e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/mistralai/Mistral-Small-3.2-24B-Instruct-2506": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 1e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 2.8e-07, - "output_cost_per_token": 8.8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 2.4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/anthropic/claude-3-7-sonnet-latest": { - "max_tokens": 200000, - "max_input_tokens": 200000, - "max_output_tokens": 200000, - "input_cost_per_token": 3.3e-06, - "output_cost_per_token": 1.65e-05, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/cognitivecomputations/dolphin-2.9.1-llama-3-70b": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 3.5e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/Qwen/Qwen2.5-Coder-32B-Instruct": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 6e-08, - "output_cost_per_token": 1.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/Qwen/Qwen3-235B-A22B": { - "max_tokens": 40960, - "max_input_tokens": 40960, - "max_output_tokens": 40960, "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 6e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/deepseek-ai/DeepSeek-V3-0324-Turbo": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/microsoft/WizardLM-2-8x22B": { - "max_tokens": 65536, - "max_input_tokens": 65536, - "max_output_tokens": 65536, - "input_cost_per_token": 4.8e-07, - "output_cost_per_token": 4.8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/Sao10K/L3-8B-Lunaris-v1-Turbo": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 2e-08, - "output_cost_per_token": 5e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/meta-llama/Llama-Guard-4-12B": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, "deepinfra/Gryphe/MythoMax-L2-13b": { "max_tokens": 4096, "max_input_tokens": 4096, @@ -15411,72 +15248,32 @@ "mode": "chat", "supports_tool_choice": true }, - "deepinfra/meta-llama/Llama-3.2-1B-Instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 5e-09, - "output_cost_per_token": 1e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/google/gemma-2-27b-it": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 2.7e-07, - "output_cost_per_token": 2.7e-07, + "deepinfra/Gryphe/MythoMax-L2-13b-turbo": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": false }, - "deepinfra/Qwen/Qwen2.5-VL-32B-Instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 6e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/nvidia/Llama-3.1-Nemotron-70B-Instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 1.2e-07, - "output_cost_per_token": 3e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/mistralai/Mixtral-8x22B-Instruct-v0.1": { - "max_tokens": 65536, - "max_input_tokens": 65536, - "max_output_tokens": 65536, - "input_cost_per_token": 6.5e-07, - "output_cost_per_token": 6.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Qwen/Qwen2.5-7B-Instruct": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 4e-08, + "deepinfra/KoboldAI/LLaMA2-13B-Tiefighter": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1e-07, "output_cost_per_token": 1e-07, "litellm_provider": "deepinfra", "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": true }, - "deepinfra/google/gemini-1.5-flash-8b": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 3.75e-08, - "output_cost_per_token": 1.5e-07, + "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true @@ -15491,6 +15288,376 @@ "mode": "chat", "supports_tool_choice": true }, + "deepinfra/NovaSky-AI/Sky-T1-32B-Preview": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Phind/Phind-CodeLlama-34B-v2": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/QVQ-72B-Preview": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Qwen/QwQ-32B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/QwQ-32B-Preview": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Qwen/Qwen2-72B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen2-7B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 5.5e-08, + "output_cost_per_token": 5.5e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen2.5-72B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3.9e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen2.5-7B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Qwen/Qwen2.5-Coder-32B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Qwen/Qwen2.5-Coder-7B": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 5e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Qwen/Qwen2.5-VL-32B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-14B": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "max_output_tokens": 40960, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-235B-A22B": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "max_output_tokens": 40960, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-30B-A3B": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "max_output_tokens": 40960, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 2.9e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-32B": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "max_output_tokens": 40960, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/Sao10K/L3-70B-Euryale-v2.1": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Sao10K/L3-8B-Lunaris-v1": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 6e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Sao10K/L3-8B-Lunaris-v1-Turbo": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 5e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Sao10K/L3.1-70B-Euryale-v2.2": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 7.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 7.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/allenai/olmOCR-7B-0725-FP8": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/anthropic/claude-3-7-sonnet-latest": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "input_cost_per_token": 3.3e-06, + "output_cost_per_token": 1.65e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/anthropic/claude-4-opus": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "input_cost_per_token": 1.65e-05, + "output_cost_per_token": 8.25e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/anthropic/claude-4-sonnet": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "input_cost_per_token": 3.3e-06, + "output_cost_per_token": 1.65e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/bigcode/starcoder2-15b-instruct-v0.1": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/cognitivecomputations/dolphin-2.6-mixtral-8x7b": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/cognitivecomputations/dolphin-2.9.1-llama-3-70b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/deepinfra/airoboros-70b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 9e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-Prover-V2-671B": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.18e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/deepseek-ai/DeepSeek-R1": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 2.15e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-R1-0528": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.15e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -15501,22 +15668,182 @@ "mode": "chat", "supports_tool_choice": false }, - "deepinfra/meta-llama/Llama-Guard-3-8B": { + "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 5.5e-08, - "output_cost_per_token": 5.5e-08, + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-R1-Turbo": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-V3": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 8.9e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-V3-0324": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 2.8e-07, + "output_cost_per_token": 8.8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-V3-0324-Turbo": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/deepseek-ai/DeepSeek-V3.1": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1e-06, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": false }, - "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, + "deepinfra/google/codegemma-7b-it": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/google/gemini-1.5-flash": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemini-1.5-flash-8b": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "input_cost_per_token": 3.75e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemini-2.0-flash-001": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemini-2.5-flash": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "input_cost_per_token": 2.1e-07, + "output_cost_per_token": 1.75e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemini-2.5-pro": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "input_cost_per_token": 8.75e-07, + "output_cost_per_token": 7e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemma-1.1-7b-it": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 7e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemma-2-27b-it": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 2.7e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/google/gemma-2-9b-it": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 6e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/google/gemma-3-12b-it": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, "input_cost_per_token": 5e-08, - "output_cost_per_token": 8e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemma-3-27b-it": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.7e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/google/gemma-3-4b-it": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 4e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true @@ -15531,6 +15858,16 @@ "mode": "chat", "supports_tool_choice": false }, + "deepinfra/mattshumer/Reflection-Llama-3.1-70B": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, "deepinfra/meta-llama/Llama-2-13b-chat-hf": { "max_tokens": 4096, "max_input_tokens": 4096, @@ -15541,182 +15878,32 @@ "mode": "chat", "supports_tool_choice": true }, - "deepinfra/anthropic/claude-4-opus": { - "max_tokens": 200000, - "max_input_tokens": 200000, - "max_output_tokens": 200000, - "input_cost_per_token": 1.65e-05, - "output_cost_per_token": 8.25e-05, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/openchat/openchat-3.6-8b": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 5.5e-08, - "output_cost_per_token": 5.5e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/google/gemma-3-27b-it": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 9e-08, - "output_cost_per_token": 1.7e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Austism/chronos-hermes-13b-v2": { + "deepinfra/meta-llama/Llama-2-70b-chat-hf": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 1.3e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Sao10K/L3.1-70B-Euryale-v2.2": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 6.5e-07, - "output_cost_per_token": 7.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/Qwen/QwQ-32B-Preview": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 1.2e-07, - "output_cost_per_token": 1.8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/anthropic/claude-4-sonnet": { - "max_tokens": 200000, - "max_input_tokens": 200000, - "max_output_tokens": 200000, - "input_cost_per_token": 3.3e-06, - "output_cost_per_token": 1.65e-05, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/microsoft/Phi-3-medium-4k-instruct": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 1.4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/mattshumer/Reflection-Llama-3.1-70B": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 3.5e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/openchat/openchat_3.5": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 5.5e-08, - "output_cost_per_token": 5.5e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 6.5e-07, - "output_cost_per_token": 7.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 2.3e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/deepseek-ai/DeepSeek-V3.1": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 3.2e-07, - "output_cost_per_token": 1.15e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/Qwen/Qwen2.5-Coder-7B": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 2.5e-08, - "output_cost_per_token": 5e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/cognitivecomputations/dolphin-2.6-mixtral-8x7b": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 2.4e-07, - "output_cost_per_token": 2.4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/meta-llama/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 8e-07, + "input_cost_per_token": 6.4e-07, "output_cost_per_token": 8e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/deepseek-ai/DeepSeek-Prover-V2-671B": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 5e-07, - "output_cost_per_token": 2.18e-06, + "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 4.9e-08, + "output_cost_per_token": 4.9e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": false }, - "deepinfra/zai-org/GLM-4.5": { + "deepinfra/meta-llama/Llama-3.2-1B-Instruct": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 5.5e-07, - "output_cost_per_token": 2e-06, + "input_cost_per_token": 5e-09, + "output_cost_per_token": 1e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true @@ -15731,276 +15918,36 @@ "mode": "chat", "supports_tool_choice": true }, - "deepinfra/meta-llama/Meta-Llama-3-70B-Instruct": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 3e-07, + "deepinfra/meta-llama/Llama-3.2-90B-Vision-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/meta-llama/Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.3e-07, "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/google/gemini-1.5-flash": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 3e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/KoboldAI/LLaMA2-13B-Tiefighter": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/google/gemini-2.5-pro": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 8.75e-07, - "output_cost_per_token": 7e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Qwen/Qwen3-30B-A3B": { - "max_tokens": 40960, - "max_input_tokens": 40960, - "max_output_tokens": 40960, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 2.9e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Qwen/QwQ-32B": { + "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 1.5e-07, + "input_cost_per_token": 3.8e-08, + "output_cost_per_token": 1.2e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/moonshotai/Kimi-K2-Instruct": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 5e-07, - "output_cost_per_token": 2e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Sao10K/L3-70B-Euryale-v2.1": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 7e-07, - "output_cost_per_token": 8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/microsoft/phi-4-reasoning-plus": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 3.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/google/gemma-3-12b-it": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 1e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/google/gemini-2.5-flash": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 2.1e-07, - "output_cost_per_token": 1.75e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/deepseek-ai/DeepSeek-R1": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 4.5e-07, - "output_cost_per_token": 2.15e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/mistralai/Mistral-7B-Instruct-v0.3": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 2.8e-08, - "output_cost_per_token": 5.4e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 1.2e-07, - "output_cost_per_token": 3.9e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/Qwen/Qwen3-14B": { - "max_tokens": 40960, - "max_input_tokens": 40960, - "max_output_tokens": 40960, - "input_cost_per_token": 6e-08, - "output_cost_per_token": 2.4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/allenai/olmOCR-7B-0725-FP8": { - "max_tokens": 16384, - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "input_cost_per_token": 2.7e-07, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 4e-07, - "output_cost_per_token": 1.6e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/microsoft/phi-4": { - "max_tokens": 16384, - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 1.4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 7e-07, - "output_cost_per_token": 8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/zai-org/GLM-4.5-Air": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 1.1e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 1.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/openai/gpt-oss-120b": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 9e-08, - "output_cost_per_token": 4.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/google/codegemma-7b-it": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": { - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 2e-08, - "output_cost_per_token": 4e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/openbmb/MiniCPM-Llama3-V-2_5": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 3.4e-07, - "output_cost_per_token": 3.4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/bigcode/starcoder2-15b-instruct-v0.1": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 1.5e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { "max_tokens": 1048576, "max_input_tokens": 1048576, @@ -16011,6 +15958,16 @@ "mode": "chat", "supports_tool_choice": true }, + "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-Turbo": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "max_tokens": 327680, "max_input_tokens": 327680, @@ -16021,32 +15978,62 @@ "mode": "chat", "supports_tool_choice": true }, - "deepinfra/google/gemini-2.0-flash-001": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 1e-07, + "deepinfra/meta-llama/Llama-Guard-3-8B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5.5e-08, + "output_cost_per_token": 5.5e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/meta-llama/Llama-Guard-4-12B": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/meta-llama/Meta-Llama-3-70B-Instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 3e-07, "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/Gryphe/MythoMax-L2-13b-turbo": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 1.3e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/google/gemma-1.1-7b-it": { + "deepinfra/meta-llama/Meta-Llama-3-8B-Instruct": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 7e-08, - "output_cost_per_token": 7e-08, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 6e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/meta-llama/Meta-Llama-3.1-405B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true @@ -16071,97 +16058,37 @@ "mode": "chat", "supports_tool_choice": true }, - "deepinfra/Qwen/Qwen3-32B": { - "max_tokens": 40960, - "max_input_tokens": 40960, - "max_output_tokens": 40960, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, + "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-08, + "output_cost_per_token": 2e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 6e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/meta-llama/Llama-2-70b-chat-hf": { + "deepinfra/microsoft/Phi-3-medium-4k-instruct": { "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 6.4e-07, - "output_cost_per_token": 8e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/nvidia/Nemotron-4-340B-Instruct": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 4.2e-06, - "output_cost_per_token": 4.2e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 5e-07, - "output_cost_per_token": 2.15e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/deepseek-ai/DeepSeek-R1-Turbo": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/NovaSky-AI/Sky-T1-32B-Preview": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 1.2e-07, - "output_cost_per_token": 1.8e-07, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 1.4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": false }, - "deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 6e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/mistralai/Mistral-Small-3.1-24B-Instruct-2503": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, + "deepinfra/microsoft/Phi-4-multimodal-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": false }, - "deepinfra/mistralai/Mistral-7B-Instruct-v0.1": { + "deepinfra/microsoft/WizardLM-2-7B": { "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, @@ -16169,64 +16096,64 @@ "output_cost_per_token": 5.5e-08, "litellm_provider": "deepinfra", "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/microsoft/WizardLM-2-8x22B": { + "max_tokens": 65536, + "max_input_tokens": 65536, + "max_output_tokens": 65536, + "input_cost_per_token": 4.8e-07, + "output_cost_per_token": 4.8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/microsoft/phi-4": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 1.4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", "supports_tool_choice": true }, - "deepinfra/Qwen/Qwen2-72B-Instruct": { + "deepinfra/microsoft/phi-4-reasoning-plus": { "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 3.5e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true - }, - "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-Turbo": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 5e-07, - "output_cost_per_token": 5e-07, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": false }, - "deepinfra/Sao10K/L3-8B-Lunaris-v1": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 3e-08, - "output_cost_per_token": 6e-08, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": false - }, - "deepinfra/deepinfra/airoboros-70b": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "input_cost_per_token": 7e-07, - "output_cost_per_token": 9e-07, + "deepinfra/mistralai/Devstral-Small-2505": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/google/gemma-3-4b-it": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 2e-08, - "output_cost_per_token": 4e-08, + "deepinfra/mistralai/Devstral-Small-2507": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/meta-llama/Meta-Llama-3-8B-Instruct": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 3e-08, - "output_cost_per_token": 6e-08, + "deepinfra/mistralai/Mistral-7B-Instruct-v0.1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 5.5e-08, + "output_cost_per_token": 5.5e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true @@ -16241,35 +16168,115 @@ "mode": "chat", "supports_tool_choice": false }, - "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 3.8e-08, - "output_cost_per_token": 1.2e-07, + "deepinfra/mistralai/Mistral-7B-Instruct-v0.3": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 2.8e-08, + "output_cost_per_token": 5.4e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/mistralai/Devstral-Small-2505": { + "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 8e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/mistralai/Mistral-Small-3.1-24B-Instruct-2503": { "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 6e-08, - "output_cost_per_token": 1.2e-07, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/mistralai/Mistral-Small-3.2-24B-Instruct-2506": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true }, - "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { + "deepinfra/mistralai/Mixtral-8x22B-Instruct-v0.1": { + "max_tokens": 65536, + "max_input_tokens": 65536, + "max_output_tokens": 65536, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 6.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 2.4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/moonshotai/Kimi-K2-Instruct": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 4.9e-08, - "output_cost_per_token": 4.9e-08, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2e-06, "litellm_provider": "deepinfra", "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": true + }, + "deepinfra/nvidia/Llama-3.1-Nemotron-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/nvidia/Nemotron-4-340B-Instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 4.2e-06, + "output_cost_per_token": 4.2e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/openai/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true }, "deepinfra/openai/gpt-oss-20b": { "max_tokens": 131072, @@ -16281,6 +16288,56 @@ "mode": "chat", "supports_tool_choice": true }, + "deepinfra/openbmb/MiniCPM-Llama3-V-2_5": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 3.4e-07, + "output_cost_per_token": 3.4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/openchat/openchat-3.6-8b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 5.5e-08, + "output_cost_per_token": 5.5e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": false + }, + "deepinfra/openchat/openchat_3.5": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 5.5e-08, + "output_cost_per_token": 5.5e-08, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/zai-org/GLM-4.5": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, + "deepinfra/zai-org/GLM-4.5-Air": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.1e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_tool_choice": true + }, "perplexity/codellama-34b-instruct": { "max_tokens": 16384, "max_input_tokens": 16384, diff --git a/poetry.lock b/poetry.lock index 66bdc2a4ff..29d1a87708 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1250,6 +1250,41 @@ lz4 = ["lz4"] snappy = ["cramjam"] zstandard = ["zstandard"] +[[package]] +name = "fastuuid" +version = "0.12.0" +description = "Python bindings to Rust's UUID library." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "fastuuid-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:22a900ef0956aacf862b460e20541fdae2d7c340594fe1bd6fdcb10d5f0791a9"}, + {file = "fastuuid-0.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0302f5acf54dc75de30103025c5a95db06d6c2be36829043a0aa16fc170076bc"}, + {file = "fastuuid-0.12.0-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:7946b4a310cfc2d597dcba658019d72a2851612a2cebb949d809c0e2474cf0a6"}, + {file = "fastuuid-0.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:a1b6764dd42bf0c46c858fb5ade7b7a3d93b7a27485a7a5c184909026694cd88"}, + {file = "fastuuid-0.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2bced35269315d16fe0c41003f8c9d63f2ee16a59295d90922cad5e6a67d0418"}, + {file = "fastuuid-0.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82106e4b0a24f4f2f73c88f89dadbc1533bb808900740ca5db9bbb17d3b0c824"}, + {file = "fastuuid-0.12.0-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:4db1bc7b8caa1d7412e1bea29b016d23a8d219131cff825b933eb3428f044dca"}, + {file = "fastuuid-0.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:07afc8e674e67ac3d35a608c68f6809da5fab470fb4ef4469094fdb32ba36c51"}, + {file = "fastuuid-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:328694a573fe9dce556b0b70c9d03776786801e028d82f0b6d9db1cb0521b4d1"}, + {file = "fastuuid-0.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02acaea2c955bb2035a7d8e7b3fba8bd623b03746ae278e5fa932ef54c702f9f"}, + {file = "fastuuid-0.12.0-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:ed9f449cba8cf16cced252521aee06e633d50ec48c807683f21cc1d89e193eb0"}, + {file = "fastuuid-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:0df2ea4c9db96fd8f4fa38d0e88e309b3e56f8fd03675a2f6958a5b082a0c1e4"}, + {file = "fastuuid-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7fe2407316a04ee8f06d3dbc7eae396d0a86591d92bafe2ca32fce23b1145786"}, + {file = "fastuuid-0.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9b31dd488d0778c36f8279b306dc92a42f16904cba54acca71e107d65b60b0c"}, + {file = "fastuuid-0.12.0-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:b19361ee649365eefc717ec08005972d3d1eb9ee39908022d98e3bfa9da59e37"}, + {file = "fastuuid-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:8fc66b11423e6f3e1937385f655bedd67aebe56a3dcec0cb835351cfe7d358c9"}, + {file = "fastuuid-0.12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7b15c54d300279ab20a9cc0579ada9c9f80d1bc92997fc61fb7bf3103d7cb26b"}, + {file = "fastuuid-0.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:458f1bc3ebbd76fdb89ad83e6b81ccd3b2a99fa6707cd3650b27606745cfb170"}, + {file = "fastuuid-0.12.0-cp38-cp38-manylinux_2_34_x86_64.whl", hash = "sha256:a8f0f83fbba6dc44271a11b22e15838641b8c45612cdf541b4822a5930f6893c"}, + {file = "fastuuid-0.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:7cfd2092253d3441f6a8c66feff3c3c009da25a5b3da82bc73737558543632be"}, + {file = "fastuuid-0.12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9303617e887429c193d036d47d0b32b774ed3618431123e9106f610d601eb57e"}, + {file = "fastuuid-0.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8790221325b376e1122e95f865753ebf456a9fb8faf0dca4f9bf7a3ff620e413"}, + {file = "fastuuid-0.12.0-cp39-cp39-manylinux_2_34_x86_64.whl", hash = "sha256:e4b12d3e23515e29773fa61644daa660ceb7725e05397a986c2109f512579a48"}, + {file = "fastuuid-0.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:e41656457c34b5dcb784729537ea64c7d9bbaf7047b480c6c6a64c53379f455a"}, + {file = "fastuuid-0.12.0.tar.gz", hash = "sha256:d0bd4e5b35aad2826403f4411937c89e7c88857b1513fe10f696544c03e9bd8e"}, +] + [[package]] name = "filelock" version = "3.16.1" @@ -6541,4 +6576,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "17a23611c832b757244c5b5dfd3a6eadae4699602a823587ea115513bfca8e4d" +content-hash = "f41e6359109c5c52dba2a28f301b04030d865265f408974082b390bf45568a01" diff --git a/pyproject.toml b/pyproject.toml index 7c3f24c89c..2b9650a152 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.76.0" +version = "1.76.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -20,6 +20,7 @@ Documentation = "https://docs.litellm.ai" [tool.poetry.dependencies] python = ">=3.8.1,<4.0, !=3.9.7" +fastuuid = ">=0.12.0" httpx = ">=0.23.0" openai = ">=1.99.5" python-dotenv = ">=0.2.0" @@ -155,7 +156,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.76.0" +version = "1.76.1" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index aab643e78d..2d31819dc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep uvicorn==0.29.0 # server dep gunicorn==23.0.0 # server dep +fastuuid==0.12.0 # for uuid4 uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls redis==5.2.1 # redis caching @@ -23,7 +24,7 @@ async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging prometheus_client==0.20.0 # for /metrics endpoint on proxy ddtrace==2.19.0 # for advanced DD tracing / profiling -orjson==3.10.12 # fast /embedding responses +orjson==3.11.2 # fast /embedding responses polars==1.31.0 # for data processing apscheduler==3.10.4 # for resetting budget in background fastapi-sso==0.16.0 # admin UI, SSO diff --git a/test_profile_mock_response.py.lprof b/test_profile_mock_response.py.lprof new file mode 100644 index 0000000000..b99ff91148 Binary files /dev/null and b/test_profile_mock_response.py.lprof differ diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index ac8a00d850..dd0f03a8a3 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -3,9 +3,23 @@ import importlib import os import sys +import tempfile +import random +import string import pytest +# Set up a temporary log directory and file BEFORE importing litellm +temp_dir = tempfile.mkdtemp(prefix="litellm_test_") +test_log_file = os.path.join(temp_dir, "test_litellm.log") + +# Store original log file for cleanup +orig_log_file = os.getenv("LITELLM_LOG_FILE") + +# Set environment variables to use temporary files BEFORE importing litellm +os.environ["LITELLM_LOG_FILE"] = test_log_file + +# Import litellm after setting up the environment sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path @@ -13,6 +27,61 @@ import asyncio import litellm +@pytest.fixture(scope="function") +def temp_log_file(): + """ + Creates a temporary log file in /tmp/litellm.log for testing. + Returns the path to the temporary log file and cleans it up after the test. + """ + # Generate a random number for the log file + random_number = ''.join(random.choices(string.digits, k=8)) + log_file_path = f"/tmp/litellm{random_number}.log" + + # Set the environment variable for litellm to use this temporary log file + original_log_file = os.environ.get("LITELLM_LOG_FILE") + os.environ["LITELLM_LOG_FILE"] = log_file_path + + yield log_file_path + + # Cleanup: Restore original environment variable and remove the temporary file + if original_log_file is not None: + os.environ["LITELLM_LOG_FILE"] = original_log_file + else: + os.environ.pop("LITELLM_LOG_FILE", None) + + # Remove the temporary log file if it exists + if os.path.exists(log_file_path): + try: + os.remove(log_file_path) + except OSError: + pass # Ignore errors if file can't be removed + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_temp_log_dir(): + """ + Cleans up the temporary log directory created at module import time. + This runs once per test session after all tests are complete. + """ + yield + + if orig_log_file is not None: + os.environ["LITELLM_LOG_FILE"] = orig_log_file + else: + os.environ.pop("LITELLM_LOG_FILE", None) + + # Cleanup: Remove the temporary directory created at module import time + if os.path.exists(temp_dir): + try: + # Remove the test log file first + if os.path.exists(test_log_file): + os.remove(test_log_file) + + # Remove the temporary directory + import shutil + shutil.rmtree(temp_dir, ignore_errors=True) + except OSError: + pass # Ignore errors if cleanup fails @pytest.fixture(scope="session") def event_loop(): @@ -25,7 +94,6 @@ def event_loop(): - @pytest.fixture(scope="function", autouse=True) def setup_and_teardown(): """ @@ -77,3 +145,4 @@ def pytest_collection_modifyitems(config, items): # Reorder the items list items[:] = custom_logger_tests + other_tests + diff --git a/tests/test_litellm/integrations/test_braintrust_logging.py b/tests/test_litellm/integrations/test_braintrust_logging.py index cca13b4e9e..cb227148ed 100644 --- a/tests/test_litellm/integrations/test_braintrust_logging.py +++ b/tests/test_litellm/integrations/test_braintrust_logging.py @@ -44,17 +44,20 @@ class TestBraintrustLogger(unittest.TestCase): BraintrustLogger(api_key=None) self.assertIn("Missing keys=['BRAINTRUST_API_KEY']", str(context.exception)) - @patch('litellm.integrations.braintrust_logging.global_braintrust_sync_http_handler') - def test_log_success_event_with_default_span_name(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.HTTPHandler') + def test_log_success_event_with_default_span_name(self, MockHTTPHandler): """Test log_success_event uses default span name when not provided.""" + # Mock HTTP response + mock_response = Mock() + mock_response.json.return_value = {"id": "test-project-id"} + mock_http_handler = Mock() + mock_http_handler.post.return_value = mock_response + MockHTTPHandler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - mock_response = Mock() - mock_response.json.return_value = {"id": "test-project-id"} - mock_http_handler.post.return_value = mock_response - # Create a mock response object message_mock = Mock() message_mock.json = Mock(return_value={"content": "test"}) @@ -62,6 +65,8 @@ class TestBraintrustLogger(unittest.TestCase): choice_mock = Mock() choice_mock.message = message_mock choice_mock.dict = Mock(return_value={"message": {"content": "test"}}) + # Mock the __getitem__ to support response_obj["choices"][0]["message"] + choice_mock.__getitem__ = Mock(return_value=message_mock) response_obj = Mock(spec=litellm.ModelResponse) response_obj.choices = [choice_mock] @@ -90,17 +95,20 @@ class TestBraintrustLogger(unittest.TestCase): json_data = call_args.kwargs['json'] self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion') - @patch('litellm.integrations.braintrust_logging.global_braintrust_sync_http_handler') - def test_log_success_event_with_custom_span_name(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.HTTPHandler') + def test_log_success_event_with_custom_span_name(self, MockHTTPHandler): """Test log_success_event uses custom span name when provided.""" + # Mock HTTP response + mock_response = Mock() + mock_response.json.return_value = {"id": "test-project-id"} + mock_http_handler = Mock() + mock_http_handler.post.return_value = mock_response + MockHTTPHandler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - mock_response = Mock() - mock_response.json.return_value = {"id": "test-project-id"} - mock_http_handler.post.return_value = mock_response - # Create a mock response object message_mock = Mock() message_mock.json = Mock(return_value={"content": "test"}) @@ -108,6 +116,7 @@ class TestBraintrustLogger(unittest.TestCase): choice_mock = Mock() choice_mock.message = message_mock choice_mock.dict = Mock(return_value={"message": {"content": "test"}}) + choice_mock.__getitem__ = Mock(return_value=message_mock) response_obj = Mock(spec=litellm.ModelResponse) response_obj.choices = [choice_mock] @@ -135,17 +144,20 @@ class TestBraintrustLogger(unittest.TestCase): json_data = call_args.kwargs['json'] self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Custom Operation') - @patch('litellm.integrations.braintrust_logging.global_braintrust_http_handler') - async def test_async_log_success_event_with_default_span_name(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.get_async_httpx_client') + async def test_async_log_success_event_with_default_span_name(self, mock_get_http_handler): """Test async_log_success_event uses default span name when not provided.""" + # Mock async HTTP response + mock_response = Mock() + mock_response.json.return_value = {"id": "test-project-id"} + mock_http_handler = MagicMock() + mock_http_handler.post = MagicMock(return_value=mock_response) + mock_get_http_handler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - mock_response = Mock() - mock_response.json.return_value = {"id": "test-project-id"} - mock_http_handler.post = MagicMock(return_value=mock_response) - # Create a mock response object message_mock = Mock() message_mock.json = Mock(return_value={"content": "test"}) @@ -153,6 +165,7 @@ class TestBraintrustLogger(unittest.TestCase): choice_mock = Mock() choice_mock.message = message_mock choice_mock.dict = Mock(return_value={"message": {"content": "test"}}) + choice_mock.__getitem__ = Mock(return_value=message_mock) response_obj = Mock(spec=litellm.ModelResponse) response_obj.choices = [choice_mock] @@ -180,17 +193,20 @@ class TestBraintrustLogger(unittest.TestCase): json_data = call_args.kwargs['json'] self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion') - @patch('litellm.integrations.braintrust_logging.global_braintrust_http_handler') - async def test_async_log_success_event_with_custom_span_name(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.get_async_httpx_client') + async def test_async_log_success_event_with_custom_span_name(self, mock_get_http_handler): """Test async_log_success_event uses custom span name when provided.""" + # Mock async HTTP response + mock_response = Mock() + mock_response.json.return_value = {"id": "test-project-id"} + mock_http_handler = MagicMock() + mock_http_handler.post = MagicMock(return_value=mock_response) + mock_get_http_handler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - mock_response = Mock() - mock_response.json.return_value = {"id": "test-project-id"} - mock_http_handler.post = MagicMock(return_value=mock_response) - # Create a mock response object message_mock = Mock() message_mock.json = Mock(return_value={"content": "test"}) @@ -198,6 +214,7 @@ class TestBraintrustLogger(unittest.TestCase): choice_mock = Mock() choice_mock.message = message_mock choice_mock.dict = Mock(return_value={"message": {"content": "test"}}) + choice_mock.__getitem__ = Mock(return_value=message_mock) response_obj = Mock(spec=litellm.ModelResponse) response_obj.choices = [choice_mock] @@ -225,17 +242,20 @@ class TestBraintrustLogger(unittest.TestCase): json_data = call_args.kwargs['json'] self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Async Custom Operation') - @patch('litellm.integrations.braintrust_logging.global_braintrust_sync_http_handler') - def test_span_name_with_multiple_metadata_fields(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.HTTPHandler') + def test_span_name_with_multiple_metadata_fields(self, MockHTTPHandler): """Test that span_name works correctly alongside other metadata fields.""" + # Mock HTTP response + mock_response = Mock() + mock_response.json.return_value = {"id": "test-project-id"} + mock_http_handler = Mock() + mock_http_handler.post.return_value = mock_response + MockHTTPHandler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - mock_response = Mock() - mock_response.json.return_value = {"id": "test-project-id"} - mock_http_handler.post.return_value = mock_response - # Create a mock response object message_mock = Mock() message_mock.json = Mock(return_value={"content": "test"}) @@ -243,6 +263,7 @@ class TestBraintrustLogger(unittest.TestCase): choice_mock = Mock() choice_mock.message = message_mock choice_mock.dict = Mock(return_value={"message": {"content": "test"}}) + choice_mock.__getitem__ = Mock(return_value=message_mock) response_obj = Mock(spec=litellm.ModelResponse) response_obj.choices = [choice_mock] diff --git a/tests/test_litellm/integrations/test_braintrust_span_name.py b/tests/test_litellm/integrations/test_braintrust_span_name.py index d3d98ea70a..10e512fc0c 100644 --- a/tests/test_litellm/integrations/test_braintrust_span_name.py +++ b/tests/test_litellm/integrations/test_braintrust_span_name.py @@ -11,16 +11,18 @@ from litellm.integrations.braintrust_logging import BraintrustLogger class TestBraintrustSpanName(unittest.TestCase): """Test custom span_name functionality in Braintrust logging.""" - @patch('litellm.integrations.braintrust_logging.global_braintrust_sync_http_handler') - def test_default_span_name(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.HTTPHandler') + def test_default_span_name(self, MockHTTPHandler): """Test that default span name is 'Chat Completion' when not provided.""" + # Mock HTTP response + mock_http_handler = Mock() + mock_http_handler.post.return_value = Mock() + MockHTTPHandler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - # Mock HTTP response - mock_http_handler.post.return_value = Mock() - # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", @@ -52,16 +54,18 @@ class TestBraintrustSpanName(unittest.TestCase): json_data = call_args.kwargs['json'] self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion') - @patch('litellm.integrations.braintrust_logging.global_braintrust_sync_http_handler') - def test_custom_span_name(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.HTTPHandler') + def test_custom_span_name(self, MockHTTPHandler): """Test that custom span name is used when provided in metadata.""" + # Mock HTTP response + mock_http_handler = Mock() + mock_http_handler.post.return_value = Mock() + MockHTTPHandler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - # Mock HTTP response - mock_http_handler.post.return_value = Mock() - # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", @@ -93,16 +97,18 @@ class TestBraintrustSpanName(unittest.TestCase): json_data = call_args.kwargs['json'] self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Custom Operation') - @patch('litellm.integrations.braintrust_logging.global_braintrust_sync_http_handler') - def test_span_name_with_other_metadata(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.HTTPHandler') + def test_span_name_with_other_metadata(self, MockHTTPHandler): """Test that span_name works alongside other metadata fields.""" + # Mock HTTP response + mock_http_handler = Mock() + mock_http_handler.post.return_value = Mock() + MockHTTPHandler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - # Mock HTTP response - mock_http_handler.post.return_value = Mock() - # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", @@ -153,16 +159,18 @@ class TestBraintrustSpanName(unittest.TestCase): # Span name should be in span_attributes, not in metadata self.assertIn('span_name', event_metadata) # span_name is also kept in metadata - @patch('litellm.integrations.braintrust_logging.global_braintrust_http_handler') - async def test_async_custom_span_name(self, mock_http_handler): + @patch('litellm.integrations.braintrust_logging.get_async_httpx_client') + async def test_async_custom_span_name(self, mock_get_http_handler): """Test async logging with custom span name.""" + # Mock async HTTP response + mock_http_handler = MagicMock() + mock_http_handler.post = MagicMock(return_value=Mock()) + mock_get_http_handler.return_value = mock_http_handler + # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - # Mock async HTTP response - mock_http_handler.post = MagicMock(return_value=Mock()) - # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 71ee367bde..5d17ea3dc3 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -451,6 +451,7 @@ def test_img_url_token_counter(img_url): def test_token_encode_disallowed_special(): encode(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>") + token_counter(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>") def test_token_counter(): diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py new file mode 100644 index 0000000000..aeea5f81b1 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py @@ -0,0 +1,498 @@ +import os +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +import litellm +from litellm import ModelResponse +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.guardrail_hooks.noma import ( + NomaGuardrail, + initialize_guardrail, +) +from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage +from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 +from litellm.types.utils import Choices, Message + + +@pytest.fixture +def noma_guardrail(): + """Create a NomaGuardrail instance for testing""" + return NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + application_id="test-app", + monitor_mode=False, + block_failures=True, + guardrail_name="test-noma-guardrail", + event_hook="pre_call", + default_on=True, + ) + + +@pytest.fixture +def mock_user_api_key_dict(): + """Create a mock UserAPIKeyAuth object""" + return UserAPIKeyAuth( + user_id="test-user-id", + user_email="test@example.com", + key_name="test-key", + key_alias=None, + team_id=None, + team_alias=None, + user_role=None, + api_key="test-api-key", + permissions={}, + models=[], + spend=0.0, + max_budget=None, + soft_budget=None, + tpm_limit=None, + rpm_limit=None, + parallel_request_limit=None, + metadata={}, + max_parallel_requests=None, + allowed_cache_controls=[], + model_spend={}, + model_max_budget={}, + ) + + +@pytest.fixture +def mock_request_data(): + """Create mock request data""" + return { + "messages": [ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello, how are you?"}, + ], + "litellm_call_id": "test-call-id", + "metadata": {"requester_ip_address": "192.168.1.1"}, + } + + +class TestNomaGuardrailConfiguration: + """Test configuration and initialization of Noma guardrail""" + + def test_init_with_config(self): + """Test initializing Noma guardrail via init_guardrails_v2""" + with patch.dict( + os.environ, + { + "NOMA_API_KEY": "test-api-key", + "NOMA_API_BASE": "https://api.test.noma.security/", + }, + ): + init_guardrails_v2( + all_guardrails=[ + { + "guardrail_name": "noma-pre-guard", + "litellm_params": { + "guardrail": "noma", + "mode": "pre_call", + "application_id": "test-app", + "monitor_mode": False, + "block_failures": True, + }, + } + ], + config_file_path="", + ) + + def test_init_with_env_vars(self): + """Test initialization with environment variables""" + with patch.dict( + os.environ, + { + "NOMA_API_KEY": "env-api-key", + "NOMA_API_BASE": "https://env.api.noma.security/", + "NOMA_APPLICATION_ID": "env-app-id", + "NOMA_MONITOR_MODE": "true", + "NOMA_BLOCK_FAILURES": "false", + }, + ): + guardrail = NomaGuardrail() + assert guardrail.api_key == "env-api-key" + assert guardrail.api_base == "https://env.api.noma.security/" + assert guardrail.application_id == "env-app-id" + assert guardrail.monitor_mode is True + assert guardrail.block_failures is False + + def test_init_with_params_override_env(self): + """Test that constructor params override environment variables""" + with patch.dict( + os.environ, + { + "NOMA_API_KEY": "env-api-key", + "NOMA_MONITOR_MODE": "true", + }, + ): + guardrail = NomaGuardrail( + api_key="param-api-key", + monitor_mode=False, + ) + assert guardrail.api_key == "param-api-key" + assert guardrail.monitor_mode is False + + def test_initialize_guardrail_function(self): + """Test the initialize_guardrail function""" + from litellm.types.guardrails import Guardrail, LitellmParams + + litellm_params = LitellmParams( + guardrail="noma", + mode="pre_call", + api_key="test-key", + api_base="https://test.api/", + application_id="test-app", + monitor_mode=True, + block_failures=False, + ) + + guardrail = Guardrail( + guardrail_name="test-guardrail", + litellm_params=litellm_params, + ) + + with patch("litellm.logging_callback_manager.add_litellm_callback") as mock_add: + result = initialize_guardrail(litellm_params, guardrail) + + assert isinstance(result, NomaGuardrail) + assert result.api_key == "test-key" + assert result.api_base == "https://test.api/" + assert result.application_id == "test-app" + assert result.monitor_mode is True + assert result.block_failures is False + mock_add.assert_called_once_with(result) + + +class TestNomaBlockedMessage: + """Test the NomaBlockedMessage exception class""" + + def test_blocked_message_basic(self): + """Test basic blocked message creation""" + response = { + "verdict": False, + "prompt": { + "harmfulContent": {"result": True, "confidence": 0.9}, + "code": {"result": False, "confidence": 0.1}, + }, + } + + exception = NomaBlockedMessage(response) + assert exception.status_code == 400 + assert exception.detail["error"] == "Request blocked by Noma guardrail" + assert "harmfulContent" in exception.detail["details"]["prompt"] + assert "code" not in exception.detail["details"]["prompt"] + + def test_blocked_message_with_sensitive_data(self): + """Test blocked message with sensitive data detection""" + response = { + "verdict": False, + "prompt": { + "sensitiveData": { + "email": {"result": True, "entities": ["test@example.com"]}, + "phone": {"result": False}, + }, + }, + } + + exception = NomaBlockedMessage(response) + assert "email" in exception.detail["details"]["prompt"]["sensitiveData"] + assert "phone" not in exception.detail["details"]["prompt"]["sensitiveData"] + + def test_blocked_message_with_topics(self): + """Test blocked message with topic guardrails""" + response = { + "verdict": False, + "prompt": { + "bannedTopics": { + "violence": {"result": True, "confidence": 0.95}, + "politics": {"result": False, "confidence": 0.2}, + }, + }, + } + + exception = NomaBlockedMessage(response) + assert "violence" in exception.detail["details"]["prompt"]["bannedTopics"] + assert "politics" not in exception.detail["details"]["prompt"]["bannedTopics"] + + +class TestNomaGuardrailHooks: + """Test the guardrail hook methods""" + + @pytest.mark.asyncio + async def test_pre_call_hook_allowed( + self, noma_guardrail, mock_user_api_key_dict, mock_request_data + ): + """Test pre-call hook when content is allowed""" + mock_response = MagicMock() + mock_response.json.return_value = {"verdict": True} + mock_response.raise_for_status = MagicMock() + + with patch.object( + noma_guardrail.async_handler, "post", return_value=mock_response + ) as mock_post: + result = await noma_guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key_dict, + cache=MagicMock(), + data=mock_request_data, + call_type="completion", + ) + + assert result == mock_request_data + mock_post.assert_called_once() + + # Verify API call details + call_args = mock_post.call_args + assert call_args[0][0].endswith("/ai-dr/v1/prompt/scan/aggregate") + assert call_args[1]["headers"]["X-Noma-AIDR-Application-ID"] == "test-app" + assert call_args[1]["headers"]["Authorization"] == "Bearer test-api-key" + assert call_args[1]["json"]["request"]["text"] == "Hello, how are you?" + + @pytest.mark.asyncio + async def test_pre_call_hook_blocked( + self, noma_guardrail, mock_user_api_key_dict, mock_request_data + ): + """Test pre-call hook when content is blocked""" + mock_response = MagicMock() + mock_response.json.return_value = { + "verdict": False, + "originalResponse": { + "prompt": {"harmfulContent": {"result": True, "confidence": 0.9}} + }, + } + mock_response.raise_for_status = MagicMock() + + with patch.object( + noma_guardrail.async_handler, "post", return_value=mock_response + ): + with pytest.raises(NomaBlockedMessage) as exc_info: + await noma_guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key_dict, + cache=MagicMock(), + data=mock_request_data, + call_type="completion", + ) + + assert exc_info.value.status_code == 400 + assert "harmfulContent" in exc_info.value.detail["details"]["prompt"] + + @pytest.mark.asyncio + async def test_pre_call_hook_monitor_mode( + self, mock_user_api_key_dict, mock_request_data + ): + """Test pre-call hook in monitor mode (logs but doesn't block)""" + guardrail = NomaGuardrail( + api_key="test-key", + monitor_mode=True, + guardrail_name="test-guardrail", + event_hook="pre_call", + default_on=True, + ) + + mock_response = MagicMock() + mock_response.json.return_value = { + "verdict": False, + "originalResponse": {"prompt": {"harmfulContent": {"result": True}}}, + } + mock_response.raise_for_status = MagicMock() + + with patch.object(guardrail.async_handler, "post", return_value=mock_response): + # Should not raise exception in monitor mode + result = await guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key_dict, + cache=MagicMock(), + data=mock_request_data, + call_type="completion", + ) + + assert result == mock_request_data + + @pytest.mark.asyncio + async def test_post_call_success_hook( + self, noma_guardrail, mock_user_api_key_dict, mock_request_data + ): + """Test post-call success hook""" + # Create a mock ModelResponse + response = ModelResponse( + id="test-response-id", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="I'm doing well, thank you!", role="assistant" + ), + ) + ], + created=1234567890, + model="gpt-3.5-turbo", + object="chat.completion", + system_fingerprint=None, + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + ) + + mock_api_response = MagicMock() + mock_api_response.json.return_value = {"verdict": True} + mock_api_response.raise_for_status = MagicMock() + + # Update guardrail to use post_call event hook + noma_guardrail.event_hook = "post_call" + + with patch.object( + noma_guardrail.async_handler, "post", return_value=mock_api_response + ) as mock_post: + result = await noma_guardrail.async_post_call_success_hook( + data=mock_request_data, + user_api_key_dict=mock_user_api_key_dict, + response=response, + ) + + assert result == response + mock_post.assert_called_once() + + # Verify API call details + call_args = mock_post.call_args + assert ( + call_args[1]["json"]["response"]["text"] == "I'm doing well, thank you!" + ) + assert call_args[1]["json"]["context"]["requestId"] == "test-response-id" + + @pytest.mark.asyncio + async def test_moderation_hook( + self, noma_guardrail, mock_user_api_key_dict, mock_request_data + ): + """Test moderation hook (during_call)""" + # Update guardrail to use during_call event hook + noma_guardrail.event_hook = "during_call" + + mock_response = MagicMock() + mock_response.json.return_value = {"verdict": True} + mock_response.raise_for_status = MagicMock() + + with patch.object( + noma_guardrail.async_handler, "post", return_value=mock_response + ): + result = await noma_guardrail.async_moderation_hook( + data=mock_request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion", + ) + + assert result == mock_request_data + + @pytest.mark.asyncio + async def test_api_failure_handling( + self, noma_guardrail, mock_user_api_key_dict, mock_request_data + ): + with patch.object( + noma_guardrail.async_handler, + "post", + side_effect=httpx.HTTPStatusError( + "API Error", request=MagicMock(), response=MagicMock(status_code=500) + ), + ): + with pytest.raises(httpx.HTTPStatusError): + await noma_guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key_dict, + cache=MagicMock(), + data=mock_request_data, + call_type="completion", + ) + + @pytest.mark.asyncio + async def test_api_failure_no_block( + self, mock_user_api_key_dict, mock_request_data + ): + guardrail = NomaGuardrail( + api_key="test-key", + block_failures=False, + guardrail_name="test-guardrail", + event_hook="pre_call", + default_on=True, + ) + + with patch.object( + guardrail.async_handler, + "post", + side_effect=httpx.HTTPStatusError( + "API Error", request=MagicMock(), response=MagicMock(status_code=500) + ), + ): + result = await guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key_dict, + cache=MagicMock(), + data=mock_request_data, + call_type="completion", + ) + + assert result == mock_request_data + + def test_extract_user_message(self, noma_guardrail): + data = { + "messages": [ + {"role": "system", "content": "System prompt"}, + {"role": "user", "content": "First user message"}, + {"role": "assistant", "content": "Assistant response"}, + {"role": "user", "content": "Second user message"}, + ] + } + + import asyncio + + message = asyncio.run(noma_guardrail._extract_user_message(data)) + assert message == "Second user message" + + data = {"messages": [{"role": "system", "content": "System prompt"}]} + message = asyncio.run(noma_guardrail._extract_user_message(data)) + assert message is None + + data = {"messages": []} + message = asyncio.run(noma_guardrail._extract_user_message(data)) + assert message is None + + data = {} + message = asyncio.run(noma_guardrail._extract_user_message(data)) + assert message is None + + +class TestIntegration: + @pytest.mark.asyncio + async def test_full_guardrail_flow(self): + """Test full guardrail flow with multiple hooks""" + with patch.dict( + os.environ, + { + "NOMA_API_KEY": "test-api-key", + "NOMA_API_BASE": "https://api.test.noma.security/", + }, + ): + init_guardrails_v2( + all_guardrails=[ + { + "guardrail_name": "noma-pre-guard", + "litellm_params": { + "guardrail": "noma", + "mode": "pre_call", + "application_id": "test-app", + }, + }, + { + "guardrail_name": "noma-post-guard", + "litellm_params": { + "guardrail": "noma", + "mode": "post_call", + "application_id": "test-app", + }, + }, + ], + config_file_path="", + ) + + custom_loggers = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=litellm.integrations.custom_guardrail.CustomGuardrail + ) + ) + assert len(custom_loggers) >= 2 diff --git a/tests/test_litellm/test_logging_behavior.py b/tests/test_litellm/test_logging_behavior.py new file mode 100644 index 0000000000..24f92838ac --- /dev/null +++ b/tests/test_litellm/test_logging_behavior.py @@ -0,0 +1,638 @@ +import os +import tempfile +import re +import json +from pathlib import Path +from datetime import datetime + +import pytest + +# Import the loggers from litellm._logging +from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger + + +class TestLoggingBehavior: + """Test suite to verify logging behavior for all LiteLLM loggers.""" + + def read_log_file_contents(self, log_file_path): + """Helper method to read and return contents of log file.""" + if not os.path.exists(log_file_path): + return "" + + with open(log_file_path, 'r') as f: + return f.read() + + @pytest.fixture(autouse=True) + def setup_log_file(self, temp_log_file): + """Use the temp_log_file fixture to ensure proper isolation.""" + self.temp_log_path = temp_log_file + + # Set environment variable before importing/reloading + original_log_file = os.environ.get("LITELLM_LOG_FILE") + os.environ["LITELLM_LOG_FILE"] = temp_log_file + + # Force reload of the logging module to pick up new environment variable + import importlib + import litellm._logging + importlib.reload(litellm._logging) + + yield + + # Cleanup: Restore original environment variable + if original_log_file is not None: + os.environ["LITELLM_LOG_FILE"] = original_log_file + else: + os.environ.pop("LITELLM_LOG_FILE", None) + + # Reload again to restore original state + importlib.reload(litellm._logging) + + def test_verbose_logger_info_level(self): + """Test that verbose_logger writes to file with INFO level.""" + test_message = "INFO level test message from verbose_logger" + + # Log at INFO level + verbose_logger.info(test_message) + + # Force flush all handlers to ensure they write to disk + for handler in verbose_logger.handlers: + if hasattr(handler, 'flush'): + handler.flush() + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert test_message in log_contents, f"Message '{test_message}' should be found in log file" + + def test_verbose_logger_debug_level(self): + """Test that verbose_logger writes to file with DEBUG level.""" + test_message = "DEBUG level test message from verbose_logger" + + # Log at DEBUG level + verbose_logger.debug(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert test_message in log_contents, f"Message '{test_message}' should be found in log file" + + def test_verbose_proxy_logger_info_level(self): + """Test that verbose_proxy_logger writes to file with INFO level.""" + test_message = "INFO level test message from verbose_proxy_logger" + + # Log at INFO level + verbose_proxy_logger.info(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert test_message in log_contents, f"Message '{test_message}' should be found in log file" + + def test_verbose_proxy_logger_debug_level(self): + """Test that verbose_proxy_logger writes to file with DEBUG level.""" + test_message = "DEBUG level test message from verbose_proxy_logger" + + # Log at DEBUG level + verbose_proxy_logger.debug(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert test_message in log_contents, f"Message '{test_message}' should be found in log file" + + def test_verbose_router_logger_info_level(self): + """Test that verbose_router_logger writes to file with INFO level.""" + test_message = "INFO level test message from verbose_router_logger" + + # Log at INFO level + verbose_router_logger.info(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert test_message in log_contents, f"Message '{test_message}' should be found in log file" + + def test_verbose_router_logger_debug_level(self): + """Test that verbose_router_logger writes to file with DEBUG level.""" + test_message = "DEBUG level test message from verbose_router_logger" + + # Log at DEBUG level + verbose_router_logger.debug(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert test_message in log_contents, f"Message '{test_message}' should be found in log file" + + def test_log_format_includes_timestamp_and_level(self): + """Test that log entries include timestamp and level information.""" + test_message = "Format test message" + + # Log at INFO level + verbose_logger.info(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + + # Check for timestamp format (should be in HH:MM:SS format based on _logging.py) + assert re.search(r'\d{2}:\d{2}:\d{2}', log_contents), "Log should contain timestamp in HH:MM:SS format" + + # Check for level information + assert 'INFO' in log_contents, "Log should contain INFO level indicator" + + # Check for logger name + assert 'LiteLLM' in log_contents, "Log should contain LiteLLM logger name" + + def test_multiple_loggers_write_to_same_file(self): + """Test that all loggers write to the same file.""" + messages = { + 'verbose_logger': "Message from verbose_logger", + 'verbose_proxy_logger': "Message from verbose_proxy_logger", + 'verbose_router_logger': "Message from verbose_router_logger" + } + + # Log messages from different loggers + verbose_logger.info(messages['verbose_logger']) + verbose_proxy_logger.info(messages['verbose_proxy_logger']) + verbose_router_logger.info(messages['verbose_router_logger']) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + + # Verify all messages are in the same file + for message in messages.values(): + assert message in log_contents, f"Message '{message}' should be found in log file" + + def test_log_file_is_not_empty(self): + """Test that the log file is not empty after logging.""" + # Log a message + verbose_logger.info("Test message to ensure file is not empty") + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + + # Verify file is not empty + assert len(log_contents.strip()) > 0, "Log file should not be empty after logging" + + +class TestJSONLoggingBehavior: + """Test suite to verify JSON logging behavior for all LiteLLM loggers.""" + + def read_log_file_contents(self, log_file_path): + """Helper method to read and return contents of log file.""" + if not os.path.exists(log_file_path): + return "" + + with open(log_file_path, 'r') as f: + return f.read() + + @pytest.fixture(autouse=True) + def setup_json_logging(self, temp_log_file): + """Set up JSON logging environment and ensure proper isolation.""" + self.temp_log_path = temp_log_file + + # Store original environment variables + original_log_file = os.environ.get("LITELLM_LOG_FILE") + original_json_logs = os.environ.get("JSON_LOGS") + + # Set environment variables for JSON logging + os.environ["LITELLM_LOG_FILE"] = temp_log_file + os.environ["JSON_LOGS"] = "True" + + # Force reload of the logging module to pick up new environment variables + import importlib + import litellm._logging + importlib.reload(litellm._logging) + + yield + + # Cleanup: Restore original environment variables + if original_log_file is not None: + os.environ["LITELLM_LOG_FILE"] = original_log_file + else: + os.environ.pop("LITELLM_LOG_FILE", None) + + if original_json_logs is not None: + os.environ["JSON_LOGS"] = original_json_logs + else: + os.environ.pop("JSON_LOGS", None) + + # Reload again to restore original state + importlib.reload(litellm._logging) + + def test_verbose_logger_json_info_level(self): + """Test that verbose_logger writes JSON formatted logs at INFO level.""" + test_message = "JSON INFO level test message from verbose_logger" + + # Log at INFO level + verbose_logger.info(test_message) + + # Force flush all handlers to ensure they write to disk + for handler in verbose_logger.handlers: + if hasattr(handler, 'flush'): + handler.flush() + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify structure + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + assert len(log_lines) > 0, "Should have at least one log line" + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + + # Verify JSON structure + assert "message" in target_log, "JSON log should contain 'message' field" + assert "level" in target_log, "JSON log should contain 'level' field" + assert "timestamp" in target_log, "JSON log should contain 'timestamp' field" + + # Verify content + assert target_log["message"] == test_message + assert target_log["level"] == "INFO" + + # Verify timestamp is in ISO 8601 format + timestamp_str = target_log["timestamp"] + try: + datetime.fromisoformat(timestamp_str) + except ValueError: + pytest.fail(f"Timestamp '{timestamp_str}' is not in valid ISO 8601 format") + + def test_verbose_logger_json_debug_level(self): + """Test that verbose_logger writes JSON formatted logs at DEBUG level.""" + test_message = "JSON DEBUG level test message from verbose_logger" + + # Log at DEBUG level + verbose_logger.debug(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify structure + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + assert target_log["level"] == "DEBUG" + + def test_verbose_proxy_logger_json_info_level(self): + """Test that verbose_proxy_logger writes JSON formatted logs at INFO level.""" + test_message = "JSON INFO level test message from verbose_proxy_logger" + + # Log at INFO level + verbose_proxy_logger.info(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify structure + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + + # Verify JSON structure and content + assert target_log["message"] == test_message + assert target_log["level"] == "INFO" + + # Verify timestamp is in ISO 8601 format + timestamp_str = target_log["timestamp"] + try: + datetime.fromisoformat(timestamp_str) + except ValueError: + pytest.fail(f"Timestamp '{timestamp_str}' is not in valid ISO 8601 format") + + def test_verbose_proxy_logger_json_debug_level(self): + """Test that verbose_proxy_logger writes JSON formatted logs at DEBUG level.""" + test_message = "JSON DEBUG level test message from verbose_proxy_logger" + + # Log at DEBUG level + verbose_proxy_logger.debug(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify structure + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + assert target_log["level"] == "DEBUG" + + def test_verbose_router_logger_json_info_level(self): + """Test that verbose_router_logger writes JSON formatted logs at INFO level.""" + test_message = "JSON INFO level test message from verbose_router_logger" + + # Log at INFO level + verbose_router_logger.info(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify structure + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + + # Verify JSON structure and content + assert target_log["message"] == test_message + assert target_log["level"] == "INFO" + + # Verify timestamp is in ISO 8601 format + timestamp_str = target_log["timestamp"] + try: + datetime.fromisoformat(timestamp_str) + except ValueError: + pytest.fail(f"Timestamp '{timestamp_str}' is not in valid ISO 8601 format") + + def test_verbose_router_logger_json_debug_level(self): + """Test that verbose_router_logger writes JSON formatted logs at DEBUG level.""" + test_message = "JSON DEBUG level test message from verbose_router_logger" + + # Log at DEBUG level + verbose_router_logger.debug(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify structure + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + assert target_log["level"] == "DEBUG" + + def test_json_output_is_valid_json(self): + """Test that all JSON log output can be parsed as valid JSON.""" + test_messages = [ + "JSON test message 1", + "JSON test message 2", + "JSON test message 3" + ] + + # Log messages from all loggers + verbose_logger.info(test_messages[0]) + verbose_proxy_logger.info(test_messages[1]) + verbose_router_logger.info(test_messages[2]) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse each line as JSON + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + parsed_logs = [] + + for line in log_lines: + try: + parsed = json.loads(line) + parsed_logs.append(parsed) + except json.JSONDecodeError as e: + pytest.fail(f"Failed to parse JSON log line: {line}. Error: {e}") + + assert len(parsed_logs) >= len(test_messages), f"Should have at least {len(test_messages)} parsed log entries" + + # Verify each parsed log has required fields + for parsed_log in parsed_logs: + assert isinstance(parsed_log, dict), "Parsed log should be a dictionary" + assert "message" in parsed_log, "Each log should have a 'message' field" + assert "level" in parsed_log, "Each log should have a 'level' field" + assert "timestamp" in parsed_log, "Each log should have a 'timestamp' field" + + def test_json_timestamp_iso8601_format(self): + """Test that JSON log timestamps are in ISO 8601 format.""" + test_message = "Timestamp format test message" + + # Log a message + verbose_logger.info(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify timestamp format + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + + timestamp_str = target_log["timestamp"] + + # Verify timestamp can be parsed as ISO 8601 + try: + parsed_timestamp = datetime.fromisoformat(timestamp_str) + assert isinstance(parsed_timestamp, datetime), "Parsed timestamp should be a datetime object" + except ValueError as e: + pytest.fail(f"Timestamp '{timestamp_str}' is not in valid ISO 8601 format. Error: {e}") + + # Verify timestamp format matches expected pattern (YYYY-MM-DDTHH:MM:SS.ffffff) + import re + iso8601_pattern = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?$' + assert re.match(iso8601_pattern, timestamp_str), f"Timestamp '{timestamp_str}' does not match ISO 8601 pattern" + + def test_json_logs_contain_expected_fields(self): + """Test that JSON logs contain all expected fields with correct types.""" + test_message = "Field validation test message" + + # Log a message + verbose_logger.info(test_message) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse JSON and verify fields + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + + # Find the line containing our test message + target_log = None + for line in log_lines: + try: + parsed = json.loads(line) + if parsed.get("message") == test_message: + target_log = parsed + break + except json.JSONDecodeError: + continue + + assert target_log is not None, f"Could not find JSON log entry with message: {test_message}" + + # Verify required fields exist and have correct types + assert "message" in target_log, "JSON log should contain 'message' field" + assert "level" in target_log, "JSON log should contain 'level' field" + assert "timestamp" in target_log, "JSON log should contain 'timestamp' field" + + assert isinstance(target_log["message"], str), "'message' field should be a string" + assert isinstance(target_log["level"], str), "'level' field should be a string" + assert isinstance(target_log["timestamp"], str), "'timestamp' field should be a string" + + # Verify field values + assert target_log["message"] == test_message + assert target_log["level"] in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], "Level should be a valid log level" + + def test_multiple_json_loggers_write_to_same_file(self): + """Test that all loggers write JSON formatted logs to the same file.""" + messages = { + 'verbose_logger': "JSON message from verbose_logger", + 'verbose_proxy_logger': "JSON message from verbose_proxy_logger", + 'verbose_router_logger': "JSON message from verbose_router_logger" + } + + # Log messages from different loggers + verbose_logger.info(messages['verbose_logger']) + verbose_proxy_logger.info(messages['verbose_proxy_logger']) + verbose_router_logger.info(messages['verbose_router_logger']) + + # Read log file contents + log_file_path = os.environ.get("LITELLM_LOG_FILE") + assert log_file_path is not None, "LITELLM_LOG_FILE environment variable should be set" + + log_contents = self.read_log_file_contents(log_file_path) + assert log_contents.strip(), "Log file should not be empty" + + # Parse all JSON logs + log_lines = [line.strip() for line in log_contents.strip().split('\n') if line.strip()] + parsed_logs = [] + + for line in log_lines: + try: + parsed = json.loads(line) + parsed_logs.append(parsed) + except json.JSONDecodeError: + continue + + # Find logs for each message + found_messages = set() + for parsed_log in parsed_logs: + message = parsed_log.get("message", "") + if message in messages.values(): + found_messages.add(message) + + # Verify all messages are found in JSON format + for message in messages.values(): + assert message in found_messages, f"Message '{message}' should be found in JSON logs" \ No newline at end of file diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index bd39fbfc9c..dba40e2214 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -957,7 +957,12 @@ def test_get_model_info_shows_supports_computer_use(): def test_pre_process_non_default_params(model, custom_llm_provider): from pydantic import BaseModel - from litellm.utils import pre_process_non_default_params + from litellm.utils import ProviderConfigManager, pre_process_non_default_params + + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders(custom_llm_provider) + ) class ResponseFormat(BaseModel): x: str @@ -974,6 +979,7 @@ def test_pre_process_non_default_params(model, custom_llm_provider): special_params=special_params, custom_llm_provider=custom_llm_provider, additional_drop_params=None, + provider_config=provider_config, ) print(processed_non_default_params) assert processed_non_default_params == {