Merge branch 'main' of https://github.com/BerriAI/litellm into litellm_fix_CVE

# Please enter a commit message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.
This commit is contained in:
Harshit28j
2026-02-24 21:09:16 +05:30
34 changed files with 2590 additions and 286 deletions
+99 -22
View File
@@ -120,7 +120,7 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
## Agentic Research API (Responses API)
## Agent API (Responses API)
Requires v1.72.6+
@@ -196,7 +196,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="Explain quantum computing in simple terms",
custom_llm_provider="perplexity",
max_output_tokens=500,
@@ -215,7 +215,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
model="perplexity/anthropic/claude-sonnet-4-5",
input="Write a short story about a robot learning to paint",
custom_llm_provider="perplexity",
max_output_tokens=500,
@@ -234,7 +234,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/google/gemini-2.0-flash-exp",
model="perplexity/google/gemini-2.5-flash",
input="Explain the concept of neural networks",
custom_llm_provider="perplexity",
max_output_tokens=500,
@@ -253,7 +253,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/xai/grok-2-1212",
model="perplexity/xai/grok-4-1-fast-non-reasoning",
input="What makes a good AI assistant?",
custom_llm_provider="perplexity",
max_output_tokens=500,
@@ -276,7 +276,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="What's the weather in San Francisco today?",
custom_llm_provider="perplexity",
tools=[{"type": "web_search"}],
@@ -286,6 +286,78 @@ response = responses(
print(response.output)
```
### Function Calling
The Agent API supports custom function tools. Pass function tools through unchanged:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-5.2",
input="What's the weather in San Francisco?",
custom_llm_provider="perplexity",
tools=[
{"type": "web_search"},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
},
},
},
],
instructions="Use tools when appropriate.",
)
print(response.output)
```
### Structured Outputs
Request JSON schema structured outputs via the `text` parameter:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/preset/pro-search",
input="Extract key facts about the Eiffel Tower",
custom_llm_provider="perplexity",
text={
"format": {
"type": "json_schema",
"name": "facts",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"height_meters": {"type": "number"},
"year_built": {"type": "integer"},
},
"required": ["name", "height_meters", "year_built"],
},
"strict": True,
}
},
)
print(response.output)
```
### Reasoning Effort (Responses API)
@@ -319,7 +391,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
model="perplexity/anthropic/claude-sonnet-4-5",
input=[
{"type": "message", "role": "system", "content": "You are a helpful assistant."},
{"type": "message", "role": "user", "content": "What are the latest AI developments?"},
@@ -343,7 +415,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="Tell me a story about space exploration",
custom_llm_provider="perplexity",
stream=True,
@@ -360,23 +432,28 @@ for chunk in response:
| Provider | Model Name | Function Call |
|----------|------------|---------------|
| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` |
| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` |
| OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` |
| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` |
| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` |
| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` |
| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` |
| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` |
| OpenAI | gpt-5.1 | `responses(model="perplexity/openai/gpt-5.1", ...)` |
| OpenAI | gpt-5-mini | `responses(model="perplexity/openai/gpt-5-mini", ...)` |
| Anthropic | claude-opus-4-6 | `responses(model="perplexity/anthropic/claude-opus-4-6", ...)` |
| Anthropic | claude-opus-4-5 | `responses(model="perplexity/anthropic/claude-opus-4-5", ...)` |
| Anthropic | claude-sonnet-4-5 | `responses(model="perplexity/anthropic/claude-sonnet-4-5", ...)` |
| Anthropic | claude-haiku-4-5 | `responses(model="perplexity/anthropic/claude-haiku-4-5", ...)` |
| Google | gemini-3-pro-preview | `responses(model="perplexity/google/gemini-3-pro-preview", ...)` |
| Google | gemini-3-flash-preview | `responses(model="perplexity/google/gemini-3-flash-preview", ...)` |
| Google | gemini-2.5-pro | `responses(model="perplexity/google/gemini-2.5-pro", ...)` |
| Google | gemini-2.5-flash | `responses(model="perplexity/google/gemini-2.5-flash", ...)` |
| xAI | grok-4-1-fast-non-reasoning | `responses(model="perplexity/xai/grok-4-1-fast-non-reasoning", ...)` |
| Perplexity | sonar | `responses(model="perplexity/perplexity/sonar", ...)` |
### Available Presets
| Preset Name | Function Call |
|----------------|--------------------------------------------------------|
| fast-search | `responses(model="perplexity/preset/fast-search", ...)`|
| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
| deep-research | `responses(model="perplexity/preset/deep-research", ...)`|
| Preset Name | Function Call |
|-------------|---------------|
| fast-search | `responses(model="perplexity/preset/fast-search", ...)` |
| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
| deep-research | `responses(model="perplexity/preset/deep-research", ...)` |
| advanced-deep-research | `responses(model="perplexity/preset/advanced-deep-research", ...)` |
### Complete Example
@@ -388,7 +465,7 @@ os.environ['PERPLEXITY_API_KEY'] = ""
# Comprehensive example with multiple features
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="Research the latest developments in quantum computing and provide sources",
custom_llm_provider="perplexity",
tools=[
+1 -1
View File
@@ -85,7 +85,7 @@ const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio";
// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
const ws = new WebSocket(url, {
headers: {
"api-key": `f28ab7b695af4154bc53498e5bdccb07`,
"api-key": `sk-1234`,
"OpenAI-Beta": "realtime=v1",
},
});
+53 -32
View File
@@ -1242,6 +1242,16 @@ def completion_cost( # noqa: PLR0915
)
elif call_type in _VIDEO_CALL_TYPES:
### VIDEO GENERATION COST CALCULATION ###
# Extract custom model_info for deployment-specific pricing
_video_model_info: Optional[ModelInfo] = None
if custom_pricing and litellm_logging_obj is not None:
_litellm_params = getattr(
litellm_logging_obj, "litellm_params", None
)
if _litellm_params is not None:
_metadata = _litellm_params.get("metadata", {}) or {}
_video_model_info = _metadata.get("model_info", None)
usage_obj = getattr(completion_response, "usage", None)
if completion_response is not None and usage_obj:
# Handle both dict and Pydantic Usage object
@@ -1262,12 +1272,14 @@ def completion_cost( # noqa: PLR0915
model=model,
duration_seconds=duration_seconds,
custom_llm_provider=custom_llm_provider,
model_info=_video_model_info,
)
# Fallback to default video cost calculation if no duration available
return default_video_cost_calculator(
model=model,
duration_seconds=0.0, # Default to 0 if no duration available
custom_llm_provider=custom_llm_provider,
model_info=_video_model_info,
)
elif call_type in _SPEECH_CALL_TYPES:
prompt_characters = litellm.utils._count_characters(text=prompt)
@@ -1892,6 +1904,7 @@ def default_video_cost_calculator(
model: str,
duration_seconds: float,
custom_llm_provider: Optional[str] = None,
model_info: Optional[ModelInfo] = None,
) -> float:
"""
Default video cost calculator for video generation
@@ -1900,6 +1913,9 @@ def default_video_cost_calculator(
model (str): Model name
duration_seconds (float): Duration of the generated video in seconds
custom_llm_provider (Optional[str]): Custom LLM provider
model_info (Optional[ModelInfo]): Deployment-level model info containing
custom video pricing. When provided, used before falling back to
the global litellm.model_cost lookup.
Returns:
float: Cost in USD for the video generation
@@ -1907,42 +1923,47 @@ def default_video_cost_calculator(
Raises:
Exception: If model pricing not found in cost map
"""
# Build model names for cost lookup
base_model_name = model
model_name_without_custom_llm_provider: Optional[str] = None
if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"):
model_name_without_custom_llm_provider = model.replace(
f"{custom_llm_provider}/", ""
)
base_model_name = (
f"{custom_llm_provider}/{model_name_without_custom_llm_provider}"
)
verbose_logger.debug(f"Looking up cost for video model: {base_model_name}")
model_without_provider = model.split("/")[-1]
# Try model with provider first, fall back to base model name
# Use custom model_info pricing if provided (deployment-specific pricing)
cost_info: Optional[dict] = None
models_to_check: List[Optional[str]] = [
base_model_name,
model,
model_without_provider,
model_name_without_custom_llm_provider,
]
for _model in models_to_check:
if _model is not None and _model in litellm.model_cost:
cost_info = litellm.model_cost[_model]
break
if model_info is not None:
cost_info = dict(model_info)
else:
# Build model names for cost lookup
base_model_name = model
model_name_without_custom_llm_provider: Optional[str] = None
if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"):
model_name_without_custom_llm_provider = model.replace(
f"{custom_llm_provider}/", ""
)
base_model_name = (
f"{custom_llm_provider}/{model_name_without_custom_llm_provider}"
)
verbose_logger.debug(f"Looking up cost for video model: {base_model_name}")
model_without_provider = model.split("/")[-1]
# Try model with provider first, fall back to base model name
models_to_check: List[Optional[str]] = [
base_model_name,
model,
model_without_provider,
model_name_without_custom_llm_provider,
]
for _model in models_to_check:
if _model is not None and _model in litellm.model_cost:
cost_info = litellm.model_cost[_model]
break
# If still not found, try with custom_llm_provider prefix
if cost_info is None and custom_llm_provider:
prefixed_model = f"{custom_llm_provider}/{model}"
if prefixed_model in litellm.model_cost:
cost_info = litellm.model_cost[prefixed_model]
# If still not found, try with custom_llm_provider prefix
if cost_info is None and custom_llm_provider:
prefixed_model = f"{custom_llm_provider}/{model}"
if prefixed_model in litellm.model_cost:
cost_info = litellm.model_cost[prefixed_model]
if cost_info is None:
raise Exception(
f"Model not found in cost map. Tried checking {models_to_check}"
f"Model not found in cost map for model={model}"
)
# Check for video-specific cost per second first
@@ -4709,6 +4709,7 @@ class StandardLoggingPayloadSetup:
custom_pricing: Optional[bool],
custom_llm_provider: Optional[str],
init_response_obj: Union[Any, BaseModel, dict],
api_base: Optional[str] = None,
) -> StandardLoggingModelInformation:
model_cost_name = _select_model_name_for_cost_calc(
model=None,
@@ -4723,7 +4724,9 @@ class StandardLoggingPayloadSetup:
else:
try:
_model_cost_information = litellm.get_model_info(
model=model_cost_name, custom_llm_provider=custom_llm_provider
model=model_cost_name,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
model_cost_information = StandardLoggingModelInformation(
model_map_key=model_cost_name,
@@ -5236,6 +5239,7 @@ def get_standard_logging_object_payload(
custom_pricing=custom_pricing,
custom_llm_provider=kwargs.get("custom_llm_provider"),
init_response_obj=init_response_obj,
api_base=litellm_params.get("api_base"),
)
response_cost: float = kwargs.get("response_cost", 0) or 0.0
@@ -1848,9 +1848,10 @@ def convert_to_anthropic_tool_invoke(
break
else:
# Regular tool_use
sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id)
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
type="tool_use",
id=tool_id,
id=sanitized_tool_id,
name=tool_name,
input=tool_input,
)
+3 -1
View File
@@ -28,6 +28,7 @@ from litellm.constants import (
AIOHTTP_CONNECTOR_LIMIT,
AIOHTTP_CONNECTOR_LIMIT_PER_HOST,
AIOHTTP_KEEPALIVE_TIMEOUT,
AIOHTTP_NEEDS_CLEANUP_CLOSED,
AIOHTTP_TTL_DNS_CACHE,
DEFAULT_SSL_CIPHERS,
)
@@ -876,9 +877,10 @@ class AsyncHTTPHandler:
transport_connector_kwargs = {
"keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT,
"ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE,
"enable_cleanup_closed": True,
**connector_kwargs,
}
if AIOHTTP_NEEDS_CLEANUP_CLOSED:
transport_connector_kwargs["enable_cleanup_closed"] = True
if AIOHTTP_CONNECTOR_LIMIT > 0:
transport_connector_kwargs["limit"] = AIOHTTP_CONNECTOR_LIMIT
if AIOHTTP_CONNECTOR_LIMIT_PER_HOST > 0:
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional,
from httpx._models import Headers, Response
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@@ -223,7 +223,9 @@ class OllamaConfig(BaseConfig):
or get_secret_str("OLLAMA_API_KEY")
)
def get_model_info(self, model: str) -> ModelInfoBase:
def get_model_info(
self, model: str, api_base: Optional[str] = None
) -> ModelInfoBase:
"""
curl http://localhost:11434/api/show -d '{
"name": "mistral"
@@ -231,7 +233,11 @@ class OllamaConfig(BaseConfig):
"""
if model.startswith("ollama/") or model.startswith("ollama_chat/"):
model = model.split("/", 1)[1]
api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
api_base = (
api_base
or get_secret_str("OLLAMA_API_BASE")
or "http://localhost:11434"
)
api_key = self.get_api_key()
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
@@ -242,8 +248,21 @@ class OllamaConfig(BaseConfig):
headers=headers,
)
except Exception as e:
raise Exception(
f"OllamaError: Error getting model info for {model}. Set Ollama API Base via `OLLAMA_API_BASE` environment variable. Error: {e}"
verbose_logger.debug(
"OllamaError: Could not get model info for %s from %s. Error: %s",
model,
api_base,
e,
)
return ModelInfoBase(
key=model,
litellm_provider="ollama",
mode="chat",
input_cost_per_token=0.0,
output_cost_per_token=0.0,
max_tokens=None,
max_input_tokens=None,
max_output_tokens=None,
)
model_info = response.json()
+12 -5
View File
@@ -7,7 +7,7 @@ from typing import Literal, Optional, Tuple
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import CallTypes, Usage
from litellm.types.utils import CallTypes, ModelInfo, Usage
from litellm.utils import get_model_info
@@ -129,7 +129,10 @@ def cost_per_second(
def video_generation_cost(
model: str, duration_seconds: float, custom_llm_provider: Optional[str] = None
model: str,
duration_seconds: float,
custom_llm_provider: Optional[str] = None,
model_info: Optional[ModelInfo] = None,
) -> float:
"""
Calculates the cost for video generation based on duration in seconds.
@@ -138,14 +141,18 @@ def video_generation_cost(
- model: str, the model name without provider prefix
- duration_seconds: float, the duration of the generated video in seconds
- custom_llm_provider: str, the custom llm provider
- model_info: Optional[dict], deployment-level model info containing
custom video pricing. When provided, skips the global
get_model_info() lookup so that deployment-specific pricing is used.
Returns:
float - total_cost_in_usd
"""
## GET MODEL INFO
model_info = get_model_info(
model=model, custom_llm_provider=custom_llm_provider or "openai"
)
if model_info is None:
model_info = get_model_info(
model=model, custom_llm_provider=custom_llm_provider or "openai"
)
# Check for video-specific cost per second
video_cost_per_second = model_info.get("output_cost_per_video_per_second")
@@ -1,5 +1,5 @@
"""
Perplexity Agentic Research API (Responses API) module
Perplexity Agent API (Responses API) module
"""
from .transformation import PerplexityResponsesConfig
@@ -1,5 +1,5 @@
"""
Transformation logic for Perplexity Agentic Research API (Responses API)
Transformation logic for Perplexity Agent API (Responses API)
This module handles the translation between OpenAI's Responses API format
and Perplexity's Responses API format, which supports:
@@ -32,10 +32,10 @@ from litellm.types.utils import LlmProviders
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
"""
Configuration for Perplexity Agentic Research API (Responses API)
Configuration for Perplexity Agent API (Responses API)
Reference: https://docs.perplexity.ai/agentic-research/quickstart
Reference: https://docs.perplexity.ai/docs/agent-api/overview
"""
@property
@@ -45,8 +45,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
def get_supported_openai_params(self, model: str) -> list:
"""
Perplexity Responses API supports a different set of parameters
Ref: https://docs.perplexity.ai/api-reference/responses-post
Params aligned with response-echo fields and Open Responses spec.
"""
return [
"max_output_tokens",
@@ -58,6 +59,23 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
"preset",
"instructions",
"models", # Model fallback support
"tool_choice",
"parallel_tool_calls",
"max_tool_calls",
"text",
"previous_response_id",
"store",
"background",
"truncation",
"metadata",
"safety_identifier",
"user",
"stream_options",
"top_logprobs",
"prompt_cache_key",
"frequency_penalty",
"presence_penalty",
"service_tier",
]
def validate_environment(
@@ -65,16 +83,15 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
) -> dict:
"""Validate environment and set up headers"""
# Get API key from environment
api_key = (
get_secret_str("PERPLEXITYAI_API_KEY")
or get_secret_str("PERPLEXITY_API_KEY")
api_key = get_secret_str("PERPLEXITYAI_API_KEY") or get_secret_str(
"PERPLEXITY_API_KEY"
)
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
headers["Content-Type"] = "application/json"
return headers
def get_complete_url(
@@ -84,15 +101,17 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
) -> str:
"""Get the complete URL for the Perplexity Responses API"""
if api_base is None:
api_base = get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai"
api_base = (
get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai"
)
# Ensure api_base doesn't end with a slash
api_base = api_base.rstrip("/")
# Add the responses endpoint
return f"{api_base}/v1/responses"
def map_openai_params(
def map_openai_params( # noqa: PLR0915
self,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
model: str,
@@ -100,78 +119,136 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
) -> Dict:
"""
Map OpenAI Responses API parameters to Perplexity format
Key differences:
- Supports 'preset' parameter for predefined configurations
- Supports 'instructions' parameter for system-level guidance
- Tools are specified differently (web_search, fetch_url)
"""
mapped_params: Dict[str, Any] = {}
# Map standard parameters
if response_api_optional_params.get("max_output_tokens"):
mapped_params["max_output_tokens"] = response_api_optional_params["max_output_tokens"]
mapped_params["max_output_tokens"] = response_api_optional_params[
"max_output_tokens"
]
if response_api_optional_params.get("temperature"):
mapped_params["temperature"] = response_api_optional_params["temperature"]
if response_api_optional_params.get("top_p"):
mapped_params["top_p"] = response_api_optional_params["top_p"]
if response_api_optional_params.get("stream"):
mapped_params["stream"] = response_api_optional_params["stream"]
if response_api_optional_params.get("stream_options"):
mapped_params["stream_options"] = response_api_optional_params["stream_options"]
mapped_params["stream_options"] = response_api_optional_params[
"stream_options"
]
# Map Perplexity-specific parameters (using .get() with Any dict access)
preset = response_api_optional_params.get("preset") # type: ignore
if preset:
mapped_params["preset"] = preset
instructions = response_api_optional_params.get("instructions") # type: ignore
if instructions:
mapped_params["instructions"] = instructions
if response_api_optional_params.get("reasoning"):
mapped_params["reasoning"] = response_api_optional_params["reasoning"]
tools = response_api_optional_params.get("tools")
if tools:
# Convert tools to list of dicts for transformation
tools_list = [dict(tool) if hasattr(tool, '__dict__') else tool for tool in tools] # type: ignore
tools_list = [dict(tool) if hasattr(tool, "__dict__") else tool for tool in tools] # type: ignore
mapped_params["tools"] = self._transform_tools(tools_list) # type: ignore
# Tool control
if response_api_optional_params.get("tool_choice"):
mapped_params["tool_choice"] = response_api_optional_params["tool_choice"]
if response_api_optional_params.get("parallel_tool_calls") is not None:
mapped_params["parallel_tool_calls"] = response_api_optional_params[
"parallel_tool_calls"
]
if response_api_optional_params.get("max_tool_calls"):
mapped_params["max_tool_calls"] = response_api_optional_params[
"max_tool_calls"
]
# Structured outputs
text_param = response_api_optional_params.get("text")
if text_param:
mapped_params["text"] = text_param
# Conversation continuity
if response_api_optional_params.get("previous_response_id"):
mapped_params["previous_response_id"] = response_api_optional_params[
"previous_response_id"
]
# Storage and lifecycle
if response_api_optional_params.get("store") is not None:
mapped_params["store"] = response_api_optional_params["store"]
if response_api_optional_params.get("background") is not None:
mapped_params["background"] = response_api_optional_params["background"]
if response_api_optional_params.get("truncation"):
mapped_params["truncation"] = response_api_optional_params["truncation"]
# Metadata
if response_api_optional_params.get("metadata"):
mapped_params["metadata"] = response_api_optional_params["metadata"]
if response_api_optional_params.get("safety_identifier"):
mapped_params["safety_identifier"] = response_api_optional_params[
"safety_identifier"
]
if response_api_optional_params.get("user"):
mapped_params["user"] = response_api_optional_params["user"]
# Additional
if response_api_optional_params.get("top_logprobs") is not None:
mapped_params["top_logprobs"] = response_api_optional_params["top_logprobs"]
if response_api_optional_params.get("prompt_cache_key"):
mapped_params["prompt_cache_key"] = response_api_optional_params[
"prompt_cache_key"
]
if response_api_optional_params.get("frequency_penalty") is not None:
mapped_params["frequency_penalty"] = response_api_optional_params[
"frequency_penalty" # type: ignore[typeddict-item]
]
if response_api_optional_params.get("presence_penalty") is not None:
mapped_params["presence_penalty"] = response_api_optional_params[
"presence_penalty" # type: ignore[typeddict-item]
]
if response_api_optional_params.get("service_tier"):
mapped_params["service_tier"] = response_api_optional_params["service_tier"]
return mapped_params
def _transform_tools(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Transform tools to Perplexity format
Perplexity supports:
Transform tools to Perplexity format.
Perplexity supports (per public OpenAPI spec):
- web_search: Performs web searches
- fetch_url: Fetches content from URLs
- function: Function Calling
"""
perplexity_tools = []
for tool in tools:
if isinstance(tool, dict):
tool_type = tool.get("type")
tool_type = tool.get("type", "")
# Direct Perplexity tool format
if tool_type in ["web_search", "fetch_url"]:
perplexity_tools.append(tool)
# OpenAI function format - try to map to Perplexity tools
# Function tools: Perplexity supports them natively
elif tool_type == "function":
function = tool.get("function", {})
function_name = function.get("name", "")
if function_name == "web_search" or "search" in function_name.lower():
perplexity_tools.append({"type": "web_search"})
elif function_name == "fetch_url" or "fetch" in function_name.lower():
perplexity_tools.append({"type": "fetch_url"})
perplexity_tools.append(tool)
return perplexity_tools
def transform_responses_api_request(
@@ -204,24 +281,26 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
"model": model,
"input": self._format_input(input),
}
# Add all optional parameters
for key, value in response_api_optional_request_params.items():
data[key] = value
return data
def _format_input(self, input: Union[str, ResponseInputParam]) -> Union[str, List[Dict[str, Any]]]:
def _format_input(
self, input: Union[str, ResponseInputParam]
) -> Union[str, List[Dict[str, Any]]]:
"""
Format input for Perplexity Responses API
The API accepts either:
- A simple string for single-turn queries
- An array of message objects for multi-turn conversations
"""
if isinstance(input, str):
return input
# Handle ResponseInputParam format
if isinstance(input, list):
formatted_messages = []
@@ -234,7 +313,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
}
formatted_messages.append(formatted_message)
return formatted_messages
return str(input)
def transform_response_api_response(
@@ -267,10 +346,14 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
# Transform usage to handle Perplexity's cost structure
usage_data = raw_response_json.get("usage", {})
transformed_usage_dict = self._transform_usage(usage_data)
# Convert usage dict to ResponseAPIUsage object
usage_obj = ResponseAPIUsage(**transformed_usage_dict) if transformed_usage_dict else None
usage_obj = (
ResponseAPIUsage(**transformed_usage_dict)
if transformed_usage_dict
else None
)
# Map Perplexity response to OpenAI Responses API format
response = ResponsesAPIResponse(
id=raw_response_json.get("id", ""),
@@ -283,11 +366,11 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
)
return response
def _transform_usage(self, usage_data: Dict[str, Any]) -> Dict[str, Any]:
"""
Transform Perplexity usage data to OpenAI format
Perplexity returns:
{
"input_tokens": 100,
@@ -300,7 +383,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
"total_cost": 0.0003
}
}
OpenAI expects:
{
"input_tokens": 100,
@@ -314,7 +397,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
"output_tokens": usage_data.get("output_tokens", 0),
"total_tokens": usage_data.get("total_tokens", 0),
}
# Transform cost from Perplexity format (dict) to OpenAI format (float)
cost_obj = usage_data.get("cost")
if isinstance(cost_obj, dict) and "total_cost" in cost_obj:
@@ -322,20 +405,20 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
verbose_logger.debug(
"Transformed Perplexity cost object to float: %s -> %s",
cost_obj,
cost_obj["total_cost"]
cost_obj["total_cost"],
)
elif cost_obj is not None:
# If cost is already a float/number, use it as-is
transformed["cost"] = cost_obj
# Add input_tokens_details if present
if "input_tokens_details" in usage_data:
transformed["input_tokens_details"] = usage_data["input_tokens_details"]
# Add output_tokens_details if present
if "output_tokens_details" in usage_data:
transformed["output_tokens_details"] = usage_data["output_tokens_details"]
return transformed
def transform_streaming_response(
@@ -353,10 +436,10 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
event_pydantic_model = PerplexityResponsesConfig.get_event_model_class(
event_type=event_type
)
# Transform Perplexity-specific fields to OpenAI format
parsed_chunk = self._transform_perplexity_chunk(parsed_chunk)
# Defensive: Handle error.code being null (similar to OpenAI implementation)
try:
error_obj = parsed_chunk.get("error")
@@ -375,13 +458,13 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
def _transform_perplexity_chunk(self, chunk: dict) -> dict:
"""
Transform Perplexity-specific fields in a streaming chunk to OpenAI format.
This handles:
- Converting Perplexity's cost object to a simple float
"""
# Make a copy to avoid modifying the original
chunk = dict(chunk)
# Transform usage.cost from Perplexity format to OpenAI format
# Perplexity: {"currency": "USD", "input_cost": 0.0001, "output_cost": 0.0002, "total_cost": 0.0003}
# OpenAI: 0.0003 (just the total_cost as a float)
@@ -400,10 +483,10 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
verbose_logger.debug(
"Transformed Perplexity cost object to float: %s -> %s",
cost_obj,
cost_obj["total_cost"]
cost_obj["total_cost"],
)
except Exception as e:
# If transformation fails, log and continue with original chunk
verbose_logger.debug("Failed to transform Perplexity cost object: %s", e)
return chunk
@@ -2463,8 +2463,8 @@ class MCPServerManager:
# Check if we should skip health check based on auth configuration
should_skip_health_check = False
# Skip if auth_type is oauth2
if server.needs_user_oauth_token:
# Skip if server requires per-user authentication (OAuth2 or passthrough auth)
if server.requires_per_user_auth:
should_skip_health_check = True
# Skip if auth_type is not none and authentication_token is missing
elif (
@@ -2604,6 +2604,7 @@ class MCPServerManager:
server.mcp_info.get("description") if server.mcp_info else None
),
url=server.url,
spec_path=server.spec_path,
transport=server.transport,
auth_type=server.auth_type,
created_at=datetime.now(),
+2 -1
View File
@@ -714,8 +714,9 @@ async def _initialize_shared_aiohttp_session():
connector_kwargs = {
"keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT,
"ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE,
"enable_cleanup_closed": True,
}
if AIOHTTP_NEEDS_CLEANUP_CLOSED:
connector_kwargs["enable_cleanup_closed"] = True
if AIOHTTP_CONNECTOR_LIMIT > 0:
connector_kwargs["limit"] = AIOHTTP_CONNECTOR_LIMIT
if AIOHTTP_CONNECTOR_LIMIT_PER_HOST > 0:
+368 -77
View File
@@ -13,8 +13,6 @@ from email.mime.text import MIMEText
from typing import (
TYPE_CHECKING,
Any,
Callable,
Coroutine,
Dict,
List,
Literal,
@@ -2277,6 +2275,11 @@ class PrismaClient:
0.0,
float(os.getenv("PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS", "0.1")),
)
self._engine_pidfd: int = -1
self._engine_pid: int = 0
self._watching_engine: bool = False
self._engine_confirmed_dead: bool = False
self._engine_wait_thread: Optional[threading.Thread] = None
verbose_proxy_logger.debug("Success - Created Prisma Client")
def get_request_status(
@@ -3544,31 +3547,368 @@ class PrismaClient:
)
raise e
def _get_engine_pid(self) -> int:
try:
engine = self.db._original_prisma._engine # type: ignore[attr-defined]
if engine is not None and engine.process is not None:
return engine.process.pid
except (AttributeError, TypeError):
pass
return 0
def _is_engine_alive(self) -> bool:
if self._engine_pid <= 0:
return True
try:
os.kill(self._engine_pid, 0)
return True
except ProcessLookupError:
return False
except (PermissionError, OSError):
return True
@staticmethod
def _reap_all_zombies() -> set:
"""Reap ALL zombie child processes via waitpid(-1, WNOHANG).
Returns a set of reaped PIDs. As PID 1 in Docker (or any
process that spawns children), we must reap ALL terminated
children to prevent zombie accumulation.
"""
reaped: set = set()
while True:
try:
pid, _ = os.waitpid(-1, os.WNOHANG)
if pid == 0:
break
reaped.add(pid)
except ChildProcessError:
break
return reaped
def _try_waitpid_watch(self, pid: int) -> bool:
"""Watch engine PID via os.waitpid() in a dedicated thread.
The thread blocks on os.waitpid(pid, 0) which is a kernel-level
wait and with zero CPU overhead, instant detection when the process exits.
When the process dies, the thread notifies the asyncio event loop
via call_soon_threadsafe.
Returns True if the thread was started, False on failure.
"""
try:
probe_pid, _ = os.waitpid(pid, os.WNOHANG)
except ChildProcessError:
verbose_proxy_logger.debug(
"PID %s is not a child process; skipping waitpid watch.", pid,
)
return False
if probe_pid == pid:
verbose_proxy_logger.warning(
"prisma-query-engine PID %s already dead at watch start.", pid,
)
self._engine_confirmed_dead = True
self._reap_all_zombies()
self._cleanup_engine_watcher()
asyncio.create_task(
self.attempt_db_reconnect(
reason="engine_process_death",
force=True,
)
)
return True
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return False
thread = threading.Thread(
target=self._waitpid_thread_func,
args=(pid, loop),
daemon=True,
name=f"prisma-engine-waitpid-{pid}",
)
thread.start()
self._engine_wait_thread = thread
return True
def _waitpid_thread_func(self, pid: int, loop: asyncio.AbstractEventLoop) -> None:
"""Thread function: block until engine PID exits, then notify event loop.
Note: uvloop/libuv may reap the child first via waitpid(-1, WNOHANG)
in its SIGCHLD handler. In that case our waitpid raises ChildProcessError.
we still notify the event loop because the engine is dead either way.
"""
try:
os.waitpid(pid, 0)
except ChildProcessError:
pass
except OSError:
pass
try:
loop.call_soon_threadsafe(self._on_engine_death_from_thread, pid)
except RuntimeError:
pass
def _on_engine_death_from_thread(self, dead_pid: int) -> None:
"""Called on the event loop thread when the waitpid thread detects engine death."""
if self._engine_confirmed_dead:
return
if dead_pid != self._engine_pid:
return
verbose_proxy_logger.error(
"prisma-query-engine PID %s exited (waitpid thread); triggering reconnect.",
dead_pid,
)
self._engine_confirmed_dead = True
self._reap_all_zombies()
self._cleanup_engine_watcher()
asyncio.create_task(
self.attempt_db_reconnect(
reason="engine_process_death",
force=True,
)
)
def _try_pidfd_watch(self, pid: int) -> bool:
"""
Watch engine PID via pidfd_open + asyncio event loop reader.
Returns True if pidfd watch was set up, False if unavailable or failed.
Broad OSError catch handles both ENOSYS and SECCOMP-blocked syscalls.
"""
if not hasattr(os, "pidfd_open"):
return False
fd = -1
try:
fd = os.pidfd_open(pid, 0) # type: ignore[attr-defined]
asyncio.get_running_loop().add_reader(fd, self._on_pidfd_readable)
self._engine_pidfd = fd
return True
except OSError:
if fd >= 0:
os.close(fd)
return False
def _on_pidfd_readable(self) -> None:
"""pidfd became readable: engine process exited or became zombie.
Sets _engine_confirmed_dead BEFORE cleanup so _run_reconnect_cycle
takes the heavy path (recreate Prisma client + re-arm watcher).
"""
if self._engine_confirmed_dead:
# Already handled -- just clean up pidfd resources.
if self._engine_pidfd >= 0:
try:
asyncio.get_running_loop().remove_reader(self._engine_pidfd)
except Exception:
pass
try:
os.close(self._engine_pidfd)
except OSError:
pass
self._engine_pidfd = -1
return
dead_pid = self._engine_pid
verbose_proxy_logger.error(
"prisma-query-engine PID %s exited (pidfd event); triggering reconnect.",
dead_pid,
)
self._engine_confirmed_dead = True
self._reap_all_zombies()
self._cleanup_engine_watcher()
asyncio.create_task(
self.attempt_db_reconnect(
reason="engine_process_death",
force=True,
)
)
async def _poll_engine_proc(self) -> None:
"""poll via os.kill(pid, 0) every 1s.
Only used when BOTH waitpid thread and pidfd are unavailable
(e.g., PID is not our child process and pidfd_open fails)
"""
while self._watching_engine and self._engine_pid > 0:
try:
os.kill(self._engine_pid, 0)
except ProcessLookupError:
verbose_proxy_logger.error(
"prisma-query-engine PID %s gone; triggering reconnect.",
self._engine_pid,
)
self._engine_confirmed_dead = True
self._reap_all_zombies()
self._cleanup_engine_watcher()
await self.attempt_db_reconnect(
reason="engine_process_death",
force=True,
)
return
except (PermissionError, OSError):
verbose_proxy_logger.debug(
"Cannot signal PID %s; stopping engine poll.",
self._engine_pid,
)
self._cleanup_engine_watcher()
return
await asyncio.sleep(1)
def _cleanup_engine_watcher(self) -> None:
"""Clean up pidfd reader, waitpid thread ref, or stop polling and reset state."""
self._watching_engine = False
if self._engine_pidfd >= 0:
try:
asyncio.get_running_loop().remove_reader(self._engine_pidfd)
except Exception:
pass
try:
os.close(self._engine_pidfd)
except OSError:
pass
self._engine_pidfd = -1
self._engine_wait_thread = None
self._engine_pid = 0
async def _start_engine_watcher(self) -> None:
"""
Start watching the Prisma query engine process for death.
Detection priority:
1. os.waitpid() in a dedicated thread, works with all event loops.
2. pidfd_open kernel fd registered with asyncio.
3. os.kill(pid, 0) polling (1s), last-resort fallback when neither
waitpid thread nor pidfd are available.
"""
if self._watching_engine or self._engine_pidfd >= 0 or self._engine_wait_thread is not None:
return
pid = self._get_engine_pid()
if pid == 0:
verbose_proxy_logger.debug("Could not find prisma-query-engine PID; engine death detection unavailable.")
return
self._engine_pid = pid
self._engine_confirmed_dead = False
verbose_proxy_logger.info("Found prisma-query-engine at PID %s.", pid)
waitpid_ok = self._try_waitpid_watch(pid)
pidfd_ok = False if waitpid_ok else self._try_pidfd_watch(pid)
if waitpid_ok:
verbose_proxy_logger.info(
"Watching engine PID %s via waitpid thread.", pid,
)
elif pidfd_ok:
verbose_proxy_logger.info(
"Watching engine PID %s via pidfd.", pid,
)
else:
verbose_proxy_logger.info(
"Watching engine PID %s via os.kill polling.", pid,
)
self._watching_engine = True
asyncio.create_task(self._poll_engine_proc())
def _stop_engine_watcher(self) -> None:
"""Stop watching the engine process and clean up all resources."""
self._cleanup_engine_watcher()
self._engine_confirmed_dead = False
verbose_proxy_logger.debug("Stopped engine process watcher.")
async def _run_reconnect_cycle(
self, timeout_seconds: Optional[float] = None
) -> None:
"""
Run a reconnect cycle with direct db operations and a single overall timeout
budget to avoid long retries on hot paths (e.g. auth).
Run a reconnect cycle with a single overall timeout budget.
Uses the _engine_confirmed_dead flag (set by waitpid thread / pidfd / poll
handlers) to choose between heavy reconnect (engine dead -- recreate
Prisma client, re-arm watcher) and lightweight reconnect (network
blip -- disconnect, connect, SELECT 1).
"""
async def _do_direct_reconnect() -> None:
try:
await self.db.disconnect()
except Exception as disconnect_err:
verbose_proxy_logger.debug(
"Prisma DB disconnect before reconnect failed (ignored): %s",
disconnect_err,
)
await self.db.connect()
await self.db.query_raw("SELECT 1")
effective_timeout = (
timeout_seconds
if timeout_seconds is not None
else self._db_watchdog_reconnect_timeout_seconds
timeout_seconds if timeout_seconds is not None else self._db_watchdog_reconnect_timeout_seconds
)
await asyncio.wait_for(_do_direct_reconnect(), timeout=effective_timeout)
engine_is_dead = self._engine_confirmed_dead or (
self._engine_pid > 0 and not self._is_engine_alive()
)
if engine_is_dead:
dead_pid = self._engine_pid
verbose_proxy_logger.warning(
"prisma-query-engine PID %s is dead; reconnecting.",
dead_pid,
)
self._reap_all_zombies()
self._cleanup_engine_watcher()
self._engine_confirmed_dead = False
async def _do_heavy_reconnect() -> None:
db_url = os.getenv("DATABASE_URL", "")
if not db_url:
verbose_proxy_logger.error("DATABASE_URL not set; cannot recreate Prisma client.")
raise RuntimeError("DATABASE_URL not set")
await self.db.recreate_prisma_client(db_url)
await self._start_engine_watcher()
await asyncio.wait_for(_do_heavy_reconnect(), timeout=effective_timeout)
else:
verbose_proxy_logger.debug("Performing Prisma DB reconnect (engine alive or unknown).")
async def _do_direct_reconnect() -> None:
try:
await self.db.disconnect()
except Exception as disconnect_err:
verbose_proxy_logger.debug(
"Prisma DB disconnect before reconnect failed (ignored): %s",
disconnect_err,
)
await self.db.connect()
await self.db.query_raw("SELECT 1")
await asyncio.wait_for(_do_direct_reconnect(), timeout=effective_timeout)
async def _attempt_reconnect_inside_lock(
self,
force: bool,
reason: str,
timeout_seconds: Optional[float],
) -> bool:
now = time.time()
if (
force is False
and now - self._db_last_reconnect_attempt_ts
< self._db_reconnect_cooldown_seconds
):
verbose_proxy_logger.debug(
"Skipping DB reconnect attempt inside lock due to cooldown. reason=%s",
reason,
)
return False
verbose_proxy_logger.warning(
"Attempting Prisma DB reconnect. reason=%s", reason
)
reconnect_succeeded = False
try:
await self._run_reconnect_cycle(timeout_seconds=timeout_seconds)
reconnect_succeeded = True
verbose_proxy_logger.info(
"Prisma DB reconnect succeeded. reason=%s", reason
)
except Exception as reconnect_err:
verbose_proxy_logger.error(
"Prisma DB reconnect failed. reason=%s error=%s",
reason,
reconnect_err,
)
finally:
self._db_last_reconnect_attempt_ts = time.time()
return reconnect_succeeded
async def attempt_db_reconnect(
self,
@@ -3595,59 +3935,10 @@ class PrismaClient:
)
return False
async def _attempt_reconnect_inside_lock() -> bool:
now = time.time()
if (
force is False
and now - self._db_last_reconnect_attempt_ts
< self._db_reconnect_cooldown_seconds
):
verbose_proxy_logger.debug(
"Skipping DB reconnect attempt inside lock due to cooldown. reason=%s",
reason,
)
return False
verbose_proxy_logger.warning(
"Attempting Prisma DB reconnect. reason=%s", reason
)
reconnect_succeeded = False
try:
await self._run_reconnect_cycle(timeout_seconds=timeout_seconds)
reconnect_succeeded = True
verbose_proxy_logger.info(
"Prisma DB reconnect succeeded. reason=%s", reason
)
except Exception as reconnect_err:
verbose_proxy_logger.error(
"Prisma DB reconnect failed. reason=%s error=%s",
reason,
reconnect_err,
)
finally:
# Start cooldown after reconnect attempt has completed.
self._db_last_reconnect_attempt_ts = time.time()
return reconnect_succeeded
if lock_timeout_seconds is None:
async with self._db_reconnect_lock:
return await _attempt_reconnect_inside_lock()
return await self._attempt_reconnect_inside_lock(force, reason, timeout_seconds)
return await self._attempt_reconnect_with_lock_timeout(
_attempt_reconnect_inside_lock,
reason=reason,
lock_timeout_seconds=lock_timeout_seconds,
)
async def _attempt_reconnect_with_lock_timeout(
self,
reconnect_fn: Callable[[], Coroutine[Any, Any, bool]],
reason: str,
lock_timeout_seconds: float,
) -> bool:
"""Acquire the reconnect lock with a timeout, then run reconnect_fn."""
lock_acquired_by_timeout_task = False
async def _acquire_reconnect_lock() -> bool:
@@ -3695,14 +3986,14 @@ class PrismaClient:
return False
try:
return await reconnect_fn()
return await self._attempt_reconnect_inside_lock(force, reason, timeout_seconds)
finally:
self._db_reconnect_lock.release()
async def start_db_health_watchdog_task(self) -> None:
"""
Start a background task that probes DB health and attempts reconnect on failure.
"""
"""Start background tasks that monitor DB health:
- A periodic SELECT 1 probe that triggers reconnect on network/connection failure.
- A process-level watcher that detects engine death via waitpid thread, pidfd, or os.kill polling."""
if self._db_health_watchdog_enabled is not True:
verbose_proxy_logger.debug(
"Prisma DB health watchdog disabled via PRISMA_HEALTH_WATCHDOG_ENABLED"
@@ -3720,11 +4011,11 @@ class PrismaClient:
self._db_health_watchdog_probe_timeout_seconds,
self._db_watchdog_reconnect_timeout_seconds,
)
await self._start_engine_watcher()
async def stop_db_health_watchdog_task(self) -> None:
"""
Stop DB health watchdog task gracefully.
"""
"""Stop DB health watchdog task and engine watcher gracefully."""
self._stop_engine_watcher()
if self._db_health_watchdog_task is None:
return
self._db_health_watchdog_task.cancel()
@@ -65,3 +65,25 @@ class MCPServer(BaseModel):
def needs_user_oauth_token(self) -> bool:
"""True if this is an OAuth2 server that relies on per-user tokens (no client_credentials)."""
return self.auth_type == MCPAuth.oauth2 and not self.has_client_credentials
@property
def requires_per_user_auth(self) -> bool:
"""
True if this server requires per-user/per-request authentication.
This includes:
- OAuth2 servers without client credentials
- Servers with auth_type=none but extra_headers configured for auth passthrough
Health checks should be skipped for these servers since they cannot
authenticate without user-provided credentials.
"""
# OAuth2 without client credentials
if self.needs_user_oauth_token:
return True
# PAT passthrough: auth_type is none but extra_headers includes auth headers
if self.auth_type == MCPAuth.none and self.extra_headers:
auth_header_names = {"authorization", "x-api-key", "api-key", "apikey"}
return any(h.lower() in auth_header_names for h in self.extra_headers)
return False
+18 -9
View File
@@ -5385,14 +5385,18 @@ def _get_max_position_embeddings(model_name: str) -> Optional[int]:
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _cached_get_model_info_helper(
model: str, custom_llm_provider: Optional[str]
model: str,
custom_llm_provider: Optional[str],
api_base: Optional[str] = None,
) -> ModelInfoBase:
"""
_get_model_info_helper wrapped with lru_cache
Speed Optimization to hit high RPS
"""
return _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
return _get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
)
def get_provider_info(
@@ -5428,7 +5432,9 @@ def _is_potential_model_name_in_model_cost(
def _get_model_info_helper( # noqa: PLR0915
model: str, custom_llm_provider: Optional[str] = None
model: str,
custom_llm_provider: Optional[str] = None,
api_base: Optional[str] = None,
) -> ModelInfoBase:
"""
Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
@@ -5486,7 +5492,7 @@ def _get_model_info_helper( # noqa: PLR0915
elif (
custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat"
) and not _is_potential_model_name_in_model_cost(potential_model_names):
return litellm.OllamaConfig().get_model_info(model)
return litellm.OllamaConfig().get_model_info(model, api_base=api_base)
else:
"""
Check if: (in order of specificity)
@@ -5725,8 +5731,6 @@ def _get_model_info_helper( # noqa: PLR0915
)
except Exception as e:
verbose_logger.debug(f"Error getting model info: {e}")
if "OllamaError" in str(e):
raise e
raise Exception(
"This model isn't mapped yet. model={}, custom_llm_provider={}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json.".format(
model, custom_llm_provider
@@ -5735,7 +5739,11 @@ def _get_model_info_helper( # noqa: PLR0915
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> ModelInfo:
def get_model_info(
model: str,
custom_llm_provider: Optional[str] = None,
api_base: Optional[str] = None,
) -> ModelInfo:
"""
Get a dict for the maximum tokens (context window), input_cost_per_token, output_cost_per_token for a given model.
@@ -5813,6 +5821,7 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
_model_info = _get_model_info_helper(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
provider_info = get_provider_info(
@@ -5823,8 +5832,8 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
if value is not None:
_model_info[key] = value # type: ignore
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(f"model_info: {_model_info}")
# if verbose_logger.isEnabledFor(logging.DEBUG):
# verbose_logger.debug(f"model_info: {_model_info}")
returned_model_info = ModelInfo(
**_model_info, supported_openai_params=supported_openai_params
+77 -18
View File
@@ -26555,65 +26555,124 @@
"supports_function_calling": true,
"supports_tool_choice": true
},
"perplexity/preset/fast-search": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_preset": true,
"supports_function_calling": true
},
"perplexity/preset/pro-search": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_preset": true
"supports_preset": true,
"supports_function_calling": true
},
"perplexity/openai/gpt-4o": {
"perplexity/preset/deep-research": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
"supports_preset": true,
"supports_function_calling": true
},
"perplexity/openai/gpt-4o-mini": {
"perplexity/preset/advanced-deep-research": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
"supports_preset": true,
"supports_function_calling": true
},
"perplexity/openai/gpt-5.2": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_function_calling": true
},
"perplexity/anthropic/claude-3-5-sonnet-20241022": {
"perplexity/openai/gpt-5.1": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/anthropic/claude-3-5-haiku-20241022": {
"perplexity/openai/gpt-5-mini": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/google/gemini-2.0-flash-exp": {
"perplexity/anthropic/claude-opus-4-6": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/google/gemini-2.0-flash-thinking-exp": {
"perplexity/anthropic/claude-opus-4-5": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": true
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/xai/grok-2-1212": {
"perplexity/anthropic/claude-sonnet-4-5": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/xai/grok-2-vision-1212": {
"perplexity/anthropic/claude-haiku-4-5": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/google/gemini-3-pro-preview": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/google/gemini-3-flash-preview": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/google/gemini-2.5-pro": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/google/gemini-2.5-flash": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/xai/grok-4-1-fast-non-reasoning": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false,
"supports_function_calling": true
},
"perplexity/perplexity/sonar": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false,
"supports_function_calling": true
},
"publicai/aisingapore/Qwen-SEA-LION-v4-32B-IT": {
"input_cost_per_token": 0.0,
@@ -0,0 +1,446 @@
"""
Tests for PrismaClient engine watchdog: death detection and automatic reconnect.
Covers:
- Engine PID discovery and liveness check
- Engine process gone (os.kill raises ProcessLookupError) reconnect triggered
- PermissionError from os.kill treated as alive (process exists but not ours)
- pidfd handler schedules attempt_db_reconnect even when lock is held
- waitpid thread instant cross-platform detection, triggers reconnect
- _run_reconnect_cycle branches: heavy path (engine dead) vs lightweight path (engine alive)
- _engine_confirmed_dead flag ensures heavy reconnect even after _engine_pid reset
- Successful heavy reconnect watcher re-armed for new process
- Missing DATABASE_URL graceful RuntimeError in reconnect cycle
- Shutdown polling loop exits cleanly
"""
import asyncio
import os
import threading
import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.proxy.utils import PrismaClient, ProxyLogging
@pytest.fixture(autouse=True)
def mock_prisma_binary():
"""Mock prisma.Prisma to avoid requiring generated Prisma binaries for unit tests."""
import sys
mock_module = MagicMock()
with patch.dict(sys.modules, {"prisma": mock_module}):
yield
@pytest.fixture
def mock_proxy_logging():
proxy_logging = AsyncMock(spec=ProxyLogging)
proxy_logging.failure_handler = AsyncMock()
return proxy_logging
@pytest.fixture
def engine_client(mock_proxy_logging) -> PrismaClient:
"""
Minimal PrismaClient fixture for engine watchdog tests.
Uses the real constructor pattern from PR #21706 (database_url).
"""
client = PrismaClient(database_url="mock://test", proxy_logging_obj=mock_proxy_logging)
client.db = MagicMock()
client.db.recreate_prisma_client = AsyncMock()
client.db.disconnect = AsyncMock(return_value=None)
client.db.connect = AsyncMock(return_value=None)
client.db.query_raw = AsyncMock(return_value=[{"result": 1}])
return client
# ---------------------------------------------------------------------------
# _is_engine_alive
# ---------------------------------------------------------------------------
def test_is_engine_alive_returns_true_when_pid_unknown(engine_client):
"""_is_engine_alive returns True when no engine PID is tracked."""
engine_client._engine_pid = 0
assert engine_client._is_engine_alive() is True
def test_is_engine_alive_returns_false_when_process_gone(engine_client):
"""_is_engine_alive returns False when os.kill raises ProcessLookupError."""
engine_client._engine_pid = 9999
with patch("os.kill", side_effect=ProcessLookupError):
assert engine_client._is_engine_alive() is False
def test_is_engine_alive_returns_true_on_permission_error(engine_client):
"""_is_engine_alive returns True when os.kill raises PermissionError (process exists but not ours)."""
engine_client._engine_pid = 1234
with patch("os.kill", side_effect=PermissionError):
assert engine_client._is_engine_alive() is True
def test_is_engine_alive_returns_true_for_running_process(engine_client):
"""_is_engine_alive returns True when os.kill succeeds (process running)."""
engine_client._engine_pid = 1234
with patch("os.kill"):
assert engine_client._is_engine_alive() is True
# ---------------------------------------------------------------------------
# _poll_engine_proc — calls attempt_db_reconnect on death
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_poll_missing_process_triggers_reconnect(engine_client) -> None:
"""Polling loop triggers attempt_db_reconnect when os.kill raises ProcessLookupError."""
engine_client._engine_pid = 1234
engine_client._watching_engine = True
engine_client.attempt_db_reconnect = AsyncMock(return_value=True)
with patch("os.kill", side_effect=ProcessLookupError):
await engine_client._poll_engine_proc()
engine_client.attempt_db_reconnect.assert_awaited_once_with(
reason="engine_process_death",
force=True,
)
@pytest.mark.asyncio
async def test_poll_permission_error_stops_polling(engine_client) -> None:
"""Polling loop stops cleanly when os.kill raises PermissionError (process not ours)."""
engine_client._engine_pid = 1234
engine_client._watching_engine = True
engine_client.attempt_db_reconnect = AsyncMock(return_value=True)
with patch("os.kill", side_effect=PermissionError):
await engine_client._poll_engine_proc()
# PermissionError means process exists but isn't ours — no reconnect, just stop polling
engine_client.attempt_db_reconnect.assert_not_awaited()
assert engine_client._watching_engine is False
assert engine_client._engine_pid == 0
@pytest.mark.asyncio
async def test_stop_loop_halts_polling(engine_client) -> None:
"""Polling loop exits cleanly when _stop_engine_watcher is called."""
engine_client._engine_pid = 1234
engine_client._watching_engine = True
async def stop_during_sleep(_duration: float) -> None:
engine_client._stop_engine_watcher()
with (
patch("os.kill"),
patch("asyncio.sleep", side_effect=stop_during_sleep),
):
await engine_client._poll_engine_proc()
assert engine_client._watching_engine is False
assert engine_client._engine_pid == 0
# ---------------------------------------------------------------------------
# _on_pidfd_readable — calls attempt_db_reconnect
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_pidfd_readable_schedules_reconnect(engine_client) -> None:
"""pidfd handler schedules attempt_db_reconnect via asyncio.create_task."""
engine_client._engine_pid = 1234
engine_client.attempt_db_reconnect = AsyncMock(return_value=True)
created_coros = []
def capture_task(coro):
created_coros.append(coro)
return MagicMock()
with patch("asyncio.create_task", side_effect=capture_task):
engine_client._on_pidfd_readable()
# Run the captured coroutine to completion
assert len(created_coros) == 1
await created_coros[0]
engine_client.attempt_db_reconnect.assert_awaited_once_with(
reason="engine_process_death",
force=True,
)
@pytest.mark.asyncio
async def test_pidfd_schedules_reconnect_task_when_lock_held(engine_client) -> None:
"""pidfd handler schedules reconnect task even when _db_reconnect_lock is held."""
engine_client._engine_pid = 1234
created_coros = []
def capture_task(coro):
created_coros.append(coro)
return MagicMock()
async with engine_client._db_reconnect_lock:
with patch("asyncio.create_task", side_effect=capture_task):
engine_client._on_pidfd_readable()
for coro in created_coros:
coro.close()
assert len(created_coros) == 1
# ---------------------------------------------------------------------------
# _run_reconnect_cycle — engine liveness branching
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_run_reconnect_cycle_uses_heavy_path_when_engine_dead(
engine_client,
) -> None:
"""_run_reconnect_cycle calls recreate_prisma_client when engine is dead."""
engine_client._engine_pid = 1234
engine_client._start_engine_watcher = AsyncMock()
with (
patch.object(engine_client, "_is_engine_alive", return_value=False),
patch.dict(os.environ, {"DATABASE_URL": "postgresql://test"}),
patch("os.waitpid", side_effect=ChildProcessError),
):
await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
engine_client.db.recreate_prisma_client.assert_awaited_once_with("postgresql://test")
engine_client._start_engine_watcher.assert_awaited_once()
engine_client.db.connect.assert_not_awaited()
@pytest.mark.asyncio
async def test_run_reconnect_cycle_uses_heavy_path_when_confirmed_dead(
engine_client,
) -> None:
"""_run_reconnect_cycle takes heavy path when _engine_confirmed_dead is set.
This is the critical race-condition fix: SIGCHLD/pidfd handlers set
_engine_confirmed_dead BEFORE _cleanup_engine_watcher resets _engine_pid
to 0, so the heavy path executes even after cleanup.
"""
engine_client._engine_pid = 0 # Already reset by cleanup!
engine_client._engine_confirmed_dead = True # But flag survives cleanup
engine_client._start_engine_watcher = AsyncMock()
with (
patch.dict(os.environ, {"DATABASE_URL": "postgresql://test"}),
patch("os.waitpid", side_effect=ChildProcessError),
):
await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
engine_client.db.recreate_prisma_client.assert_awaited_once_with("postgresql://test")
engine_client._start_engine_watcher.assert_awaited_once()
engine_client.db.connect.assert_not_awaited()
assert engine_client._engine_confirmed_dead is False # Reset after use
@pytest.mark.asyncio
async def test_run_reconnect_cycle_uses_lightweight_path_when_engine_alive(
engine_client,
) -> None:
"""_run_reconnect_cycle uses disconnect/connect when engine is alive."""
engine_client._engine_pid = 1234
with patch.object(engine_client, "_is_engine_alive", return_value=True):
await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
engine_client.db.connect.assert_awaited_once()
engine_client.db.query_raw.assert_awaited_once_with("SELECT 1")
engine_client.db.recreate_prisma_client.assert_not_awaited()
@pytest.mark.asyncio
async def test_run_reconnect_cycle_uses_lightweight_path_when_pid_unknown(
engine_client,
) -> None:
"""_run_reconnect_cycle uses lightweight path when engine PID is not tracked."""
engine_client._engine_pid = 0
await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
engine_client.db.connect.assert_awaited_once()
engine_client.db.query_raw.assert_awaited_once_with("SELECT 1")
engine_client.db.recreate_prisma_client.assert_not_awaited()
@pytest.mark.asyncio
async def test_run_reconnect_cycle_heavy_path_raises_without_database_url(
engine_client,
) -> None:
"""Heavy reconnect raises RuntimeError when DATABASE_URL is not set."""
engine_client._engine_pid = 1234
with (
patch.object(engine_client, "_is_engine_alive", return_value=False),
patch.dict(os.environ, {}, clear=True),
patch("os.waitpid", side_effect=ChildProcessError),
):
with pytest.raises(RuntimeError, match="DATABASE_URL not set"):
await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
engine_client.db.recreate_prisma_client.assert_not_awaited()
# ---------------------------------------------------------------------------
# start/stop lifecycle integration
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_start_watchdog_task_also_starts_engine_watcher(
engine_client,
) -> None:
"""start_db_health_watchdog_task() also starts engine watcher."""
engine_client._start_engine_watcher = AsyncMock()
loop = asyncio.get_running_loop()
dummy_task = loop.create_task(asyncio.sleep(3600))
def fake_create_task(coro):
coro.close()
return dummy_task
with patch("asyncio.create_task", side_effect=fake_create_task):
await engine_client.start_db_health_watchdog_task()
engine_client._start_engine_watcher.assert_awaited_once()
dummy_task.cancel()
try:
await dummy_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_stop_watchdog_task_also_stops_engine_watcher(
engine_client,
) -> None:
"""stop_db_health_watchdog_task() also stops engine watcher."""
engine_client._stop_engine_watcher = MagicMock()
loop = asyncio.get_running_loop()
dummy_task = loop.create_task(asyncio.sleep(3600))
engine_client._db_health_watchdog_task = dummy_task
await engine_client.stop_db_health_watchdog_task()
engine_client._stop_engine_watcher.assert_called_once()
assert engine_client._db_health_watchdog_task is None
# ---------------------------------------------------------------------------
# waitpid thread (cross-platform)
# ---------------------------------------------------------------------------
def test_try_waitpid_watch_returns_false_when_not_child(engine_client):
"""_try_waitpid_watch returns False when PID is not our child process."""
engine_client._engine_pid = 9999
with patch("os.waitpid", side_effect=ChildProcessError):
assert engine_client._try_waitpid_watch(9999) is False
assert engine_client._engine_wait_thread is None
def test_try_waitpid_watch_starts_thread_for_child(engine_client):
"""_try_waitpid_watch starts a daemon thread when PID is our child."""
engine_client._engine_pid = 1234
mock_thread = MagicMock()
mock_loop = MagicMock()
with (
patch("os.waitpid", return_value=(0, 0)),
patch("asyncio.get_running_loop", return_value=mock_loop),
patch("threading.Thread", return_value=mock_thread) as mock_thread_cls,
):
result = engine_client._try_waitpid_watch(1234)
assert result is True
mock_thread.start.assert_called_once()
assert engine_client._engine_wait_thread is mock_thread
@pytest.mark.asyncio
async def test_try_waitpid_watch_handles_already_dead_engine(engine_client) -> None:
"""_try_waitpid_watch detects engine already dead at watch start."""
engine_client._engine_pid = 1234
engine_client.attempt_db_reconnect = AsyncMock(return_value=True)
created_coros = []
def capture_task(coro):
created_coros.append(coro)
return MagicMock()
waitpid_calls = iter([(1234, 0)])
def mock_waitpid(pid, flags):
if pid == -1:
raise ChildProcessError
return next(waitpid_calls)
with (
patch("os.waitpid", side_effect=mock_waitpid),
patch("asyncio.create_task", side_effect=capture_task),
):
result = engine_client._try_waitpid_watch(1234)
assert result is True
assert engine_client._engine_confirmed_dead is True
assert len(created_coros) == 1
created_coros[0].close()
@pytest.mark.asyncio
async def test_on_engine_death_from_thread_triggers_reconnect(engine_client) -> None:
"""waitpid thread callback schedules attempt_db_reconnect."""
engine_client._engine_pid = 1234
engine_client.attempt_db_reconnect = AsyncMock(return_value=True)
created_coros = []
def capture_task(coro):
created_coros.append(coro)
return MagicMock()
with patch("asyncio.create_task", side_effect=capture_task):
engine_client._on_engine_death_from_thread(1234)
assert len(created_coros) == 1
await created_coros[0]
engine_client.attempt_db_reconnect.assert_awaited_once_with(
reason="engine_process_death",
force=True,
)
def test_on_engine_death_from_thread_no_double_trigger(engine_client):
"""waitpid thread callback does not trigger reconnect if already confirmed dead."""
engine_client._engine_pid = 1234
engine_client._engine_confirmed_dead = True
with patch("asyncio.create_task") as mock_create_task:
engine_client._on_engine_death_from_thread(1234)
mock_create_task.assert_not_called()
def test_on_engine_death_from_thread_ignores_stale_pid(engine_client):
"""waitpid thread callback ignores death notification for a stale PID."""
engine_client._engine_pid = 5678
with patch("asyncio.create_task") as mock_create_task:
engine_client._on_engine_death_from_thread(1234)
mock_create_task.assert_not_called()
@@ -973,6 +973,54 @@ def test_convert_to_anthropic_tool_invoke_regular_tool():
assert result[0]["input"] == {"location": "San Francisco"}
def test_convert_to_anthropic_tool_invoke_sanitizes_invalid_ids():
"""Test that tool_use IDs with invalid characters are sanitized.
Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.
IDs from external frameworks (e.g. MiniMax) may contain characters
like colons that violate this pattern.
"""
tool_calls = [
{
"id": "sessions_history:183",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"location": "Boston"}',
},
},
{
"id": "composio.NOTION_SEARCH",
"type": "function",
"function": {
"name": "search_notes",
"arguments": '{"query": "test"}',
},
},
]
result = convert_to_anthropic_tool_invoke(tool_calls)
assert len(result) == 2
# Colons replaced with underscores
assert result[0]["id"] == "sessions_history_183"
# Dots replaced with underscores
assert result[1]["id"] == "composio_NOTION_SEARCH"
# Valid IDs should pass through unchanged
valid_tool_calls = [
{
"id": "toolu_01ABC-xyz_123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"location": "NYC"}',
},
}
]
valid_result = convert_to_anthropic_tool_invoke(valid_tool_calls)
assert valid_result[0]["id"] == "toolu_01ABC-xyz_123"
def test_convert_to_anthropic_tool_invoke_server_tool():
"""
Test that server_tool_use (srvtoolu_) is reconstructed as server_tool_use.
@@ -0,0 +1,31 @@
from unittest.mock import MagicMock, patch
def test_create_aiohttp_transport_sets_enable_cleanup_closed_when_needed(monkeypatch):
from litellm.llms.custom_httpx import http_handler as http_handler_module
connector_mock = MagicMock(name="connector")
session_mock = MagicMock(name="session")
monkeypatch.setattr(http_handler_module, "AIOHTTP_NEEDS_CLEANUP_CLOSED", True)
with patch.object(http_handler_module, "TCPConnector", return_value=connector_mock) as mock_tcp_connector:
with patch.object(http_handler_module, "ClientSession", return_value=session_mock):
transport = http_handler_module.AsyncHTTPHandler._create_aiohttp_transport(shared_session=None)
transport._get_valid_client_session()
assert mock_tcp_connector.call_args.kwargs["enable_cleanup_closed"] is True
def test_create_aiohttp_transport_omits_enable_cleanup_closed_when_not_needed(monkeypatch):
from litellm.llms.custom_httpx import http_handler as http_handler_module
connector_mock = MagicMock(name="connector")
session_mock = MagicMock(name="session")
monkeypatch.setattr(http_handler_module, "AIOHTTP_NEEDS_CLEANUP_CLOSED", False)
with patch.object(http_handler_module, "TCPConnector", return_value=connector_mock) as mock_tcp_connector:
with patch.object(http_handler_module, "ClientSession", return_value=session_mock):
transport = http_handler_module.AsyncHTTPHandler._create_aiohttp_transport(shared_session=None)
transport._get_valid_client_session()
assert "enable_cleanup_closed" not in mock_tcp_connector.call_args.kwargs
@@ -138,6 +138,88 @@ class TestOllamaModelInfo:
assert models == ["ollama/llama2"]
class TestOllamaGetModelInfo:
"""Tests for OllamaConfig.get_model_info() api_base threading and graceful fallback."""
def test_get_model_info_uses_provided_api_base(self, monkeypatch):
"""When api_base is passed, get_model_info should use it instead of env var or default."""
from litellm.llms.ollama.completion.transformation import OllamaConfig
captured_urls = []
def mock_post(url, json, headers=None):
captured_urls.append(url)
resp = DummyResponse(
{"template": "{{ .System }} tools {{ .Prompt }}", "model_info": {"context_length": 4096}},
status_code=200,
)
return resp
monkeypatch.setattr("litellm.module_level_client.post", mock_post)
config = OllamaConfig()
result = config.get_model_info("llama3", api_base="http://my-remote-server:11434")
assert captured_urls[0] == "http://my-remote-server:11434/api/show"
assert result["max_tokens"] == 4096
def test_get_model_info_falls_back_to_env_var(self, monkeypatch):
"""When no api_base is passed, should fall back to OLLAMA_API_BASE env var."""
from litellm.llms.ollama.completion.transformation import OllamaConfig
captured_urls = []
def mock_post(url, json, headers=None):
captured_urls.append(url)
return DummyResponse({"template": "", "model_info": {}}, status_code=200)
monkeypatch.setattr("litellm.module_level_client.post", mock_post)
monkeypatch.setenv("OLLAMA_API_BASE", "http://env-server:11434")
config = OllamaConfig()
config.get_model_info("llama3")
assert captured_urls[0] == "http://env-server:11434/api/show"
def test_get_model_info_graceful_fallback_on_connection_error(self, monkeypatch):
"""When the Ollama server is unreachable, should return defaults instead of raising."""
from litellm.llms.ollama.completion.transformation import OllamaConfig
def mock_post(url, json, headers=None):
raise ConnectionError("Connection refused")
monkeypatch.setattr("litellm.module_level_client.post", mock_post)
monkeypatch.delenv("OLLAMA_API_BASE", raising=False)
config = OllamaConfig()
result = config.get_model_info("llama3", api_base="http://unreachable:11434")
assert result["key"] == "llama3"
assert result["litellm_provider"] == "ollama"
assert result["input_cost_per_token"] == 0.0
assert result["output_cost_per_token"] == 0.0
assert result["max_tokens"] is None
def test_get_model_info_strips_ollama_prefix(self, monkeypatch):
"""Should strip 'ollama/' or 'ollama_chat/' prefix from model name."""
from litellm.llms.ollama.completion.transformation import OllamaConfig
captured_json = []
def mock_post(url, json, headers=None):
captured_json.append(json)
return DummyResponse({"template": "", "model_info": {}}, status_code=200)
monkeypatch.setattr("litellm.module_level_client.post", mock_post)
config = OllamaConfig()
config.get_model_info("ollama/llama3", api_base="http://localhost:11434")
assert captured_json[0]["name"] == "llama3"
config.get_model_info("ollama_chat/llama3", api_base="http://localhost:11434")
assert captured_json[1]["name"] == "llama3"
class TestOllamaAuthHeaders:
"""Tests for Ollama authentication header handling in completion calls."""
@@ -0,0 +1,381 @@
"""
Tests for Perplexity Responses API transformation
Tests the PerplexityResponsesConfig class that handles Perplexity-specific
transformations for the Agent API (Responses API).
Source: litellm/llms/perplexity/responses/transformation.py
"""
import os
import sys
sys.path.insert(0, os.path.abspath("../../../../.."))
from litellm.llms.perplexity.responses.transformation import PerplexityResponsesConfig
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
class TestPerplexityResponsesTransformation:
"""Test Perplexity Responses API configuration and transformations"""
def test_function_tool_passthrough(self):
"""Function tools with name/description/parameters are preserved"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
},
},
},
},
}
]
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert "tools" in result
assert len(result["tools"]) == 1
assert result["tools"][0]["type"] == "function"
assert result["tools"][0]["function"]["name"] == "get_weather"
assert (
result["tools"][0]["function"]["description"] == "Get the current weather"
)
assert "parameters" in result["tools"][0]["function"]
def test_web_search_tool_passthrough(self):
"""web_search tools are passed through unchanged"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(tools=[{"type": "web_search"}])
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert "tools" in result
assert len(result["tools"]) == 1
assert result["tools"][0]["type"] == "web_search"
def test_fetch_url_tool_passthrough(self):
"""fetch_url tools are passed through"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(tools=[{"type": "fetch_url"}])
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert "tools" in result
assert len(result["tools"]) == 1
assert result["tools"][0]["type"] == "fetch_url"
def test_mixed_tools_function_and_web_search(self):
"""Mixed function and web_search tools are transformed correctly"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(
tools=[
{"type": "web_search"},
{
"type": "function",
"function": {
"name": "custom_tool",
"description": "A custom tool",
"parameters": {"type": "object"},
},
},
]
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert len(result["tools"]) == 2
assert result["tools"][0]["type"] == "web_search"
assert result["tools"][1]["type"] == "function"
assert result["tools"][1]["function"]["name"] == "custom_tool"
def test_tool_choice_mapping(self):
"""tool_choice passes through"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(
tool_choice="required", temperature=0.7
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert result.get("tool_choice") == "required"
def test_parallel_tool_calls(self):
"""parallel_tool_calls passes through"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(
parallel_tool_calls=True, temperature=0.7
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert result.get("parallel_tool_calls") is True
def test_max_tool_calls_mapping(self):
"""max_tool_calls passes through"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(max_tool_calls=5, temperature=0.7)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert result.get("max_tool_calls") == 5
def test_text_passthrough(self):
"""text param passes through as-is (Perplexity accepts Open Responses format directly)"""
config = PerplexityResponsesConfig()
text_value = {
"format": {
"type": "json_schema",
"name": "weather_response",
"schema": {
"type": "object",
"properties": {"temp": {"type": "number"}},
},
"strict": True,
}
}
params = ResponsesAPIOptionalRequestParams(
text=text_value,
temperature=0.7,
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert "text" in result
assert result["text"] == text_value
assert "response_format" not in result
def test_previous_response_id(self):
"""previous_response_id passes through"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(
previous_response_id="resp_abc123",
temperature=0.7,
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert result.get("previous_response_id") == "resp_abc123"
def test_store_background_truncation(self):
"""Lifecycle params pass through"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(
store=True,
background=False,
truncation="auto",
temperature=0.7,
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert result.get("store") is True
assert result.get("background") is False
assert result.get("truncation") == "auto"
def test_metadata_safety_identifier_user(self):
"""Metadata params pass through"""
config = PerplexityResponsesConfig()
params = ResponsesAPIOptionalRequestParams(
metadata={"request_id": "req_123"},
safety_identifier="safety_123",
user="user_456",
temperature=0.7,
)
result = config.map_openai_params(
response_api_optional_params=params,
model="perplexity/openai/gpt-5.2",
drop_params=False,
)
assert result.get("metadata") == {"request_id": "req_123"}
assert result.get("safety_identifier") == "safety_123"
assert result.get("user") == "user_456"
def test_all_supported_params_declared(self):
"""get_supported_openai_params returns complete list"""
config = PerplexityResponsesConfig()
supported = config.get_supported_openai_params("perplexity/openai/gpt-5.2")
expected = [
"max_output_tokens",
"stream",
"temperature",
"top_p",
"tools",
"reasoning",
"preset",
"instructions",
"models",
"tool_choice",
"parallel_tool_calls",
"max_tool_calls",
"text",
"previous_response_id",
"store",
"background",
"truncation",
"metadata",
"safety_identifier",
"user",
"stream_options",
"top_logprobs",
"prompt_cache_key",
"frequency_penalty",
"presence_penalty",
"service_tier",
]
for param in expected:
assert param in supported, f"Missing supported param: {param}"
def test_cost_transformation(self):
"""Perplexity cost dict to OpenAI float"""
config = PerplexityResponsesConfig()
usage_data = {
"input_tokens": 100,
"output_tokens": 200,
"total_tokens": 300,
"cost": {
"currency": "USD",
"input_cost": 0.0001,
"output_cost": 0.0002,
"total_cost": 0.0003,
},
}
result = config._transform_usage(usage_data)
assert result["input_tokens"] == 100
assert result["output_tokens"] == 200
assert result["total_tokens"] == 300
assert result["cost"] == 0.0003
def test_cost_transformation_float_passthrough(self):
"""Cost already float passes through"""
config = PerplexityResponsesConfig()
usage_data = {
"input_tokens": 100,
"output_tokens": 200,
"total_tokens": 300,
"cost": 0.0005,
}
result = config._transform_usage(usage_data)
assert result["cost"] == 0.0005
def test_preset_handling(self):
"""Preset model names work"""
config = PerplexityResponsesConfig()
data = config.transform_responses_api_request(
model="preset/pro-search",
input="What is AI?",
response_api_optional_request_params={"temperature": 0.7},
litellm_params={},
headers={},
)
assert data["preset"] == "pro-search"
assert data["input"] == "What is AI?"
assert "temperature" in data
def test_get_complete_url(self):
"""Correct endpoint URL"""
config = PerplexityResponsesConfig()
url = config.get_complete_url(api_base=None, litellm_params={})
assert url == "https://api.perplexity.ai/v1/responses"
custom_url = config.get_complete_url(
api_base="https://custom.perplexity.ai",
litellm_params={},
)
assert custom_url == "https://custom.perplexity.ai/v1/responses"
url_with_slash = config.get_complete_url(
api_base="https://api.perplexity.ai/",
litellm_params={},
)
assert url_with_slash == "https://api.perplexity.ai/v1/responses"
def test_perplexity_provider_config_registration(self):
"""Test that Perplexity provider returns PerplexityResponsesConfig"""
config = ProviderConfigManager.get_provider_responses_api_config(
model="perplexity/openai/gpt-5.2",
provider=LlmProviders.PERPLEXITY,
)
assert config is not None
assert isinstance(config, PerplexityResponsesConfig)
assert config.custom_llm_provider == LlmProviders.PERPLEXITY
@@ -1487,3 +1487,182 @@ async def test_discovery_root_includes_server_name_prefix():
assert response["scopes_supported"] == ["read", "write"]
finally:
global_mcp_server_manager.registry.clear()
@pytest.mark.asyncio
async def test_oauth_callback_redirects_with_state():
"""Test OAuth callback endpoint properly decodes state and redirects to client callback URL."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
callback,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
# Mock the state decoding
mock_state_data = {
"base_url": "http://localhost:3000/ui/mcp/oauth/callback",
"original_state": "test-uuid-state-123",
"code_challenge": "test_challenge",
"code_challenge_method": "S256",
"client_redirect_uri": "http://localhost:3000/ui/mcp/oauth/callback",
}
with patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash"
) as mock_decode:
mock_decode.return_value = mock_state_data
# Call callback endpoint with code and state
response = await callback(
code="test_authorization_code_12345",
state="encrypted_state_value",
)
# Should redirect to the client callback URL with code and original state
assert response.status_code == 302
assert "http://localhost:3000/ui/mcp/oauth/callback" in response.headers["location"]
assert "code=test_authorization_code_12345" in response.headers["location"]
assert "state=test-uuid-state-123" in response.headers["location"]
# Verify state was decoded
mock_decode.assert_called_once_with("encrypted_state_value")
@pytest.mark.asyncio
async def test_oauth_callback_handles_invalid_state():
"""Test OAuth callback returns error page when state decryption fails."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
callback,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
# Mock state decoding to raise an exception
with patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash"
) as mock_decode:
mock_decode.side_effect = Exception("Failed to decrypt state")
# Call callback endpoint with invalid state
response = await callback(
code="test_code",
state="invalid_encrypted_state",
)
# Should return HTML error page
assert response.status_code == 200
assert "Authentication incomplete" in response.body.decode()
@pytest.mark.asyncio
async def test_oauth_authorize_includes_scopes_from_server_config():
"""Test that authorize endpoint includes scopes from server configuration."""
try:
from fastapi import Request
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
authorize_with_server,
)
from litellm.proxy._types import MCPTransport
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
# Create server with specific scopes (e.g., GitLab requires 'ai_workflows')
oauth_server = MCPServer(
server_id="gitlab_server",
name="gitlab",
server_name="gitlab",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
authorization_url="https://gitlab.com/oauth/authorize",
token_url="https://gitlab.com/oauth/token",
client_id="test_client",
scopes=["api", "read_user", "ai_workflows"], # GitLab-specific scopes
)
mock_request = MagicMock(spec=Request)
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
with patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper"
) as mock_encrypt:
mock_encrypt.return_value = "encrypted_state"
# Call authorize without explicit scope parameter
response = await authorize_with_server(
request=mock_request,
mcp_server=oauth_server,
client_id="test_client",
redirect_uri="http://localhost:3000/callback",
state="test_state",
code_challenge="test_challenge",
code_challenge_method="S256",
response_type="code",
scope=None, # No scope in request, should use server's scopes
)
# Should redirect with scopes from server config
assert response.status_code in (307, 302)
redirect_url = response.headers["location"]
assert "scope=api+read_user+ai_workflows" in redirect_url or "scope=api%20read_user%20ai_workflows" in redirect_url
@pytest.mark.asyncio
async def test_oauth_authorize_prefers_request_scope_over_server_config():
"""Test that explicit scope parameter takes precedence over server configuration."""
try:
from fastapi import Request
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
authorize_with_server,
)
from litellm.proxy._types import MCPTransport
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
oauth_server = MCPServer(
server_id="test_server",
name="test",
server_name="test",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
authorization_url="https://provider.com/oauth/authorize",
token_url="https://provider.com/oauth/token",
client_id="test_client",
scopes=["default_scope1", "default_scope2"],
)
mock_request = MagicMock(spec=Request)
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
with patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper"
) as mock_encrypt:
mock_encrypt.return_value = "encrypted_state"
# Call authorize WITH explicit scope parameter
response = await authorize_with_server(
request=mock_request,
mcp_server=oauth_server,
client_id="test_client",
redirect_uri="http://localhost:3000/callback",
state="test_state",
code_challenge="test_challenge",
code_challenge_method="S256",
response_type="code",
scope="custom_scope1 custom_scope2", # Explicit scope should take precedence
)
# Should use the explicit scope, not server config
assert response.status_code in (307, 302)
redirect_url = response.headers["location"]
assert "scope=custom_scope1+custom_scope2" in redirect_url or "scope=custom_scope1%20custom_scope2" in redirect_url
assert "default_scope" not in redirect_url
@@ -1036,6 +1036,240 @@ class TestMCPServerManager:
assert result.status == "healthy"
assert result.health_check_error is None
@pytest.mark.asyncio
async def test_health_check_skips_passthrough_auth_with_authorization_header(self):
"""Test that health check is skipped for servers with passthrough Authorization header"""
manager = MCPServerManager()
# Mock server with auth_type=none and Authorization in extra_headers (passthrough auth)
server = MCPServer(
server_id="github-server",
name="github-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
authentication_token=None,
url="http://github-server.com",
extra_headers=["Authorization"], # Passthrough auth configured
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# _create_mcp_client should not be called (health check should be skipped)
manager._create_mcp_client = AsyncMock()
# Perform health check
result = await manager.health_check_server("github-server")
# Verify that client was not created (health check was skipped)
manager._create_mcp_client.assert_not_called()
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "github-server"
assert result.status == "unknown"
assert result.health_check_error is None
assert result.last_health_check is not None
@pytest.mark.asyncio
async def test_health_check_skips_passthrough_auth_with_api_key_header(self):
"""Test that health check is skipped for servers with passthrough x-api-key header"""
manager = MCPServerManager()
# Mock server with auth_type=none and x-api-key in extra_headers
server = MCPServer(
server_id="sourcegraph-server",
name="sourcegraph-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
authentication_token=None,
url="http://sourcegraph-server.com",
extra_headers=["x-api-key"], # Passthrough auth configured
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# _create_mcp_client should not be called
manager._create_mcp_client = AsyncMock()
# Perform health check
result = await manager.health_check_server("sourcegraph-server")
# Verify that client was not created (health check was skipped)
manager._create_mcp_client.assert_not_called()
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "sourcegraph-server"
assert result.status == "unknown"
assert result.health_check_error is None
assert result.last_health_check is not None
@pytest.mark.asyncio
async def test_health_check_runs_when_no_passthrough_auth(self):
"""Test that health check runs normally for servers with auth_type=none but no passthrough headers"""
manager = MCPServerManager()
# Mock server with auth_type=none but no extra_headers (no passthrough auth)
server = MCPServer(
server_id="public-server",
name="public-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
authentication_token=None,
url="http://public-server.com",
extra_headers=None, # No passthrough auth
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# Mock successful client
mock_client = AsyncMock()
mock_client.run_with_session = AsyncMock(return_value="ok")
manager._create_mcp_client = AsyncMock(return_value=mock_client)
# Perform health check
result = await manager.health_check_server("public-server")
# Verify that client WAS created (health check should run)
manager._create_mcp_client.assert_called_once()
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "public-server"
assert result.status == "healthy"
assert result.health_check_error is None
assert result.last_health_check is not None
@pytest.mark.asyncio
async def test_health_check_runs_when_extra_headers_no_auth(self):
"""Test that health check runs when extra_headers exist but don't include auth headers"""
manager = MCPServerManager()
# Mock server with extra_headers but no auth-related headers
server = MCPServer(
server_id="custom-server",
name="custom-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
authentication_token=None,
url="http://custom-server.com",
extra_headers=["X-Custom-Header", "X-Request-ID"], # Non-auth headers
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# Mock successful client
mock_client = AsyncMock()
mock_client.run_with_session = AsyncMock(return_value="ok")
manager._create_mcp_client = AsyncMock(return_value=mock_client)
# Perform health check
result = await manager.health_check_server("custom-server")
# Verify that client WAS created (health check should run)
manager._create_mcp_client.assert_called_once()
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "custom-server"
assert result.status == "healthy"
assert result.health_check_error is None
@pytest.mark.asyncio
async def test_requires_per_user_auth_property_oauth2(self):
"""Test that requires_per_user_auth returns True for OAuth2 without client credentials"""
# OAuth2 without client credentials
server = MCPServer(
server_id="oauth-server",
name="oauth-server",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
url="http://oauth-server.com",
client_id=None,
client_secret=None,
token_url=None,
)
assert server.requires_per_user_auth is True
assert server.needs_user_oauth_token is True
@pytest.mark.asyncio
async def test_requires_per_user_auth_property_oauth2_with_client_creds(self):
"""Test that requires_per_user_auth returns False for OAuth2 with client credentials"""
# OAuth2 with client credentials
server = MCPServer(
server_id="oauth-server",
name="oauth-server",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
url="http://oauth-server.com",
client_id="client-id",
client_secret="client-secret",
token_url="http://oauth-server.com/token",
)
assert server.requires_per_user_auth is False
assert server.has_client_credentials is True
@pytest.mark.asyncio
async def test_requires_per_user_auth_property_passthrough_auth(self):
"""Test that requires_per_user_auth returns True for passthrough auth (auth_type=none + Authorization header)"""
# Passthrough auth with Authorization header
server = MCPServer(
server_id="github-server",
name="github-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
url="http://github-server.com",
extra_headers=["Authorization"],
)
assert server.requires_per_user_auth is True
# Passthrough auth with x-api-key header
server2 = MCPServer(
server_id="sourcegraph-server",
name="sourcegraph-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
url="http://sourcegraph-server.com",
extra_headers=["x-api-key"],
)
assert server2.requires_per_user_auth is True
# Passthrough auth with api-key header (case insensitive)
server3 = MCPServer(
server_id="api-server",
name="api-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
url="http://api-server.com",
extra_headers=["API-Key"],
)
assert server3.requires_per_user_auth is True
@pytest.mark.asyncio
async def test_requires_per_user_auth_property_no_passthrough(self):
"""Test that requires_per_user_auth returns False when no passthrough auth is configured"""
# auth_type=none but no extra_headers
server = MCPServer(
server_id="public-server",
name="public-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
url="http://public-server.com",
extra_headers=None,
)
assert server.requires_per_user_auth is False
# auth_type=none with non-auth extra_headers
server2 = MCPServer(
server_id="custom-server",
name="custom-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
url="http://custom-server.com",
extra_headers=["X-Custom-Header", "X-Request-ID"],
)
assert server2.requires_per_user_auth is False
@pytest.mark.asyncio
async def test_register_openapi_tools_includes_static_headers(self, tmp_path):
"""Ensure OpenAPI-to-MCP tool calls include server.static_headers (Issue #19341)."""
@@ -0,0 +1,34 @@
import asyncio
from unittest.mock import MagicMock, patch
def test_initialize_shared_aiohttp_session_sets_enable_cleanup_closed_when_needed(
monkeypatch,
):
from litellm.proxy import proxy_server as proxy_server_module
connector_mock = MagicMock(name="connector")
session_mock = MagicMock(name="session")
monkeypatch.setattr(proxy_server_module, "AIOHTTP_NEEDS_CLEANUP_CLOSED", True)
with patch("aiohttp.TCPConnector", return_value=connector_mock) as mock_tcp_connector:
with patch("aiohttp.ClientSession", return_value=session_mock):
asyncio.run(proxy_server_module._initialize_shared_aiohttp_session())
assert mock_tcp_connector.call_args.kwargs["enable_cleanup_closed"] is True
def test_initialize_shared_aiohttp_session_omits_enable_cleanup_closed_when_not_needed(
monkeypatch,
):
from litellm.proxy import proxy_server as proxy_server_module
connector_mock = MagicMock(name="connector")
session_mock = MagicMock(name="session")
monkeypatch.setattr(proxy_server_module, "AIOHTTP_NEEDS_CLEANUP_CLOSED", False)
with patch("aiohttp.TCPConnector", return_value=connector_mock) as mock_tcp_connector:
with patch("aiohttp.ClientSession", return_value=session_mock):
asyncio.run(proxy_server_module._initialize_shared_aiohttp_session())
assert "enable_cleanup_closed" not in mock_tcp_connector.call_args.kwargs
@@ -243,6 +243,77 @@ class TestVideoGeneration:
custom_llm_provider="openai"
)
def test_video_generation_cost_with_custom_model_info(self):
"""Test that custom model_info pricing is applied for video generation.
When a deployment has custom pricing via model_info, it should be used
instead of looking up the global litellm.model_cost map.
Related: https://github.com/BerriAI/litellm/issues/21907
"""
model_info = {
"output_cost_per_video_per_second": 0.05,
}
cost = default_video_cost_calculator(
model="my-custom-video-model",
duration_seconds=10.0,
model_info=model_info,
)
assert cost == 0.5
def test_video_generation_cost_custom_model_info_fallback_to_per_second(self):
"""Test that output_cost_per_second is used as fallback when
output_cost_per_video_per_second is not set in custom model_info.
Related: https://github.com/BerriAI/litellm/issues/21907
"""
model_info = {
"output_cost_per_second": 0.10,
}
cost = default_video_cost_calculator(
model="my-custom-video-model",
duration_seconds=5.0,
model_info=model_info,
)
assert cost == 0.5
def test_video_generation_cost_custom_pricing_through_completion_cost(self):
"""Test that custom video pricing flows through completion_cost via litellm_logging_obj.
This tests the full cost calculation path: completion_cost extracts model_info
from litellm_logging_obj.litellm_params.metadata.model_info and passes it to
the video cost calculator.
Related: https://github.com/BerriAI/litellm/issues/21907
"""
from litellm.cost_calculator import completion_cost
# Create mock response with usage containing duration_seconds
mock_response = MagicMock()
mock_response.usage = MagicMock()
mock_response.usage.duration_seconds = 10.0
type(mock_response)._hidden_params = {}
# Create mock litellm_logging_obj with custom pricing
mock_logging_obj = MagicMock()
mock_logging_obj.litellm_params = {
"metadata": {
"model_info": {
"output_cost_per_video_per_second": 0.05,
}
}
}
cost = completion_cost(
completion_response=mock_response,
model="openai/hunyuanvideo",
call_type="create_video",
custom_llm_provider="openai",
custom_pricing=True,
litellm_logging_obj=mock_logging_obj,
)
assert cost == 0.5
def test_video_generation_with_files(self):
"""Test video generation with file uploads."""
config = OpenAIVideoConfig()
@@ -41,13 +41,16 @@ const McpOAuthCallbackContent = () => {
}
try {
// Store in both sessionStorage and localStorage for redundancy
window.sessionStorage.setItem(RESULT_STORAGE_KEY, JSON.stringify(payload));
window.localStorage.setItem(RESULT_STORAGE_KEY, JSON.stringify(payload));
} catch (err) {
console.error("Failed to persist OAuth callback payload", err);
// Silently ignore storage errors
}
const returnUrl = window.sessionStorage.getItem(RETURN_URL_STORAGE_KEY);
console.info("[MCP OAuth callback] returnUrl", returnUrl);
// Check both sessionStorage and localStorage for return URL
const returnUrl = window.sessionStorage.getItem(RETURN_URL_STORAGE_KEY) ||
window.localStorage.getItem(RETURN_URL_STORAGE_KEY);
const destination = returnUrl || resolveDefaultRedirect();
window.location.replace(destination);
}, [payload]);
@@ -129,6 +129,21 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
},
onTokenReceived: (token) => {
setOauthAccessToken(token?.access_token ?? null);
if (token?.access_token) {
const credentials = {
access_token: token.access_token,
...(token.refresh_token && { refresh_token: token.refresh_token }),
...(token.expires_in && { expires_in: token.expires_in }),
...(token.scope && { scope: token.scope }),
};
form.setFieldsValue({ credentials });
NotificationsManager.success(
"OAuth authorization successful! Please click 'Create MCP Server' to save the configuration."
);
}
},
onBeforeRedirect: persistCreateUiState,
});
@@ -47,7 +47,13 @@ export const mcpServerColumns = (
{
accessorKey: "transport",
header: "Transport",
cell: ({ getValue }) => <span>{((getValue() as string) || "http").toUpperCase()}</span>,
cell: ({ row }) => {
const transport = row.original.transport || "http";
const specPath = row.original.spec_path;
// If server has spec_path, display as "OPENAPI" instead of the raw transport type
const displayTransport = specPath && transport !== "stdio" ? "OPENAPI" : transport;
return <span>{displayTransport.toUpperCase()}</span>;
},
},
{
accessorKey: "auth_type",
@@ -117,6 +117,21 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
},
onTokenReceived: (token) => {
setOauthAccessToken(token?.access_token ?? null);
if (token?.access_token) {
const credentials = {
access_token: token.access_token,
...(token.refresh_token && { refresh_token: token.refresh_token }),
...(token.expires_in && { expires_in: token.expires_in }),
...(token.scope && { scope: token.scope }),
};
form.setFieldsValue({ credentials });
NotificationsManager.success(
"OAuth authorization successful! Please click 'Update MCP Server' to save the credentials."
);
}
},
onBeforeRedirect: persistEditUiState,
});
@@ -144,9 +159,9 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
}, [mcpServer.env]);
// If server has spec_path and no url, show it as "openapi" transport in the UI
// If server has spec_path, show it as "openapi" transport in the UI
const effectiveTransport = React.useMemo(() => {
if (mcpServer.spec_path && !mcpServer.url && mcpServer.transport !== "stdio") {
if (mcpServer.spec_path && mcpServer.transport !== "stdio") {
return TRANSPORT.OPENAPI;
}
return mcpServer.transport;
@@ -234,8 +249,13 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
}
}, [mcpServer]);
// Fetch tools when component mounts
// Fetch tools when component mounts or when OAuth token is received
// But only if the server has been properly saved (has a permanent server_id)
useEffect(() => {
// Don't fetch if server hasn't been saved yet (no permanent server_id)
if (!mcpServer.server_id || mcpServer.server_id.trim() === "") {
return;
}
fetchTools();
}, [mcpServer, accessToken, oauthAccessToken]);
@@ -131,7 +131,7 @@ export const MCPServerView: React.FC<MCPServerViewProps> = ({
<Card>
<Text>Transport</Text>
<div className="mt-2">
<Title>{handleTransport(mcpServer.transport ?? undefined)}</Title>
<Title>{handleTransport(mcpServer.transport ?? undefined, mcpServer.spec_path ?? undefined).toUpperCase()}</Title>
</div>
</Card>
@@ -171,6 +171,7 @@ export const MCPServerView: React.FC<MCPServerViewProps> = ({
userRole={userRole}
userID={userID}
serverAlias={mcpServer.alias}
extraHeaders={mcpServer.extra_headers}
/>
</TabPanel>
@@ -220,7 +221,7 @@ export const MCPServerView: React.FC<MCPServerViewProps> = ({
</div>
<div>
<Text className="font-medium">Transport</Text>
<div>{handleTransport(mcpServer.transport)}</div>
<div>{handleTransport(mcpServer.transport, mcpServer.spec_path).toUpperCase()}</div>
</div>
<div>
<Text className="font-medium">Extra Headers</Text>
@@ -1,12 +1,12 @@
import React, { useState } from "react";
import { useQuery, useMutation } from "@tanstack/react-query";
import { ToolTestPanel } from "./ToolTestPanel";
import { MCPTool, MCPToolsViewerProps, MCPContent, CallMCPToolResponse } from "./types";
import { MCPTool, MCPToolsViewerProps, MCPContent, CallMCPToolResponse, AUTH_TYPE } from "./types";
import { listMCPTools, callMCPTool } from "../networking";
import { Card, Title, Text } from "@tremor/react";
import { RobotOutlined, ToolOutlined, SearchOutlined } from "@ant-design/icons";
import { Input } from "antd";
import { RobotOutlined, ToolOutlined, SearchOutlined, LockOutlined, KeyOutlined } from "@ant-design/icons";
import { Input, Alert, Button as AntdButton } from "antd";
const MCPToolsViewer = ({
serverId,
@@ -14,23 +14,50 @@ const MCPToolsViewer = ({
auth_type,
userRole,
userID,
serverAlias, // Add serverAlias prop
serverAlias,
extraHeaders,
}: MCPToolsViewerProps) => {
const [selectedTool, setSelectedTool] = useState<MCPTool | null>(null);
const [toolResult, setToolResult] = useState<MCPContent[] | null>(null);
const [toolError, setToolError] = useState<Error | null>(null);
const [toolSearchTerm, setToolSearchTerm] = useState("");
// State for passthrough headers
const [passthroughHeaders, setPassthroughHeaders] = useState<Record<string, string>>({});
const [showHeaderInput, setShowHeaderInput] = useState(false);
// Check if this server has extra headers configured
const hasExtraHeaders = extraHeaders && extraHeaders.length > 0;
// Build custom headers for MCP server requests
const buildCustomHeaders = () => {
if (!serverAlias || !hasExtraHeaders) return undefined;
const customHeaders: Record<string, string> = {};
// Add passthrough headers with server-specific prefix
Object.entries(passthroughHeaders).forEach(([headerName, headerValue]) => {
if (headerValue && headerValue.trim()) {
// Format: x-mcp-{alias}-{header_name}
const mcpHeaderName = `x-mcp-${serverAlias}-${headerName.toLowerCase()}`;
customHeaders[mcpHeaderName] = headerValue;
}
});
return Object.keys(customHeaders).length > 0 ? customHeaders : undefined;
};
// Query to fetch MCP tools
const {
data: mcpToolsResponse,
isLoading: isLoadingTools,
error: mcpToolsError,
refetch: refetchTools,
} = useQuery({
queryKey: ["mcpTools", serverId],
queryKey: ["mcpTools", serverId, passthroughHeaders],
queryFn: () => {
if (!accessToken) throw new Error("Access Token required");
return listMCPTools(accessToken, serverId);
return listMCPTools(accessToken, serverId, buildCustomHeaders());
},
enabled: !!accessToken,
staleTime: 30000, // Consider data fresh for 30 seconds
@@ -42,7 +69,13 @@ const MCPToolsViewer = ({
if (!accessToken) throw new Error("Access Token required");
try {
const result: CallMCPToolResponse = await callMCPTool(accessToken, serverId, args.tool.name, args.arguments);
const result: CallMCPToolResponse = await callMCPTool(
accessToken,
serverId,
args.tool.name,
args.arguments,
{ customHeaders: buildCustomHeaders() }
);
return result;
} catch (error) {
throw error;
@@ -79,6 +112,80 @@ const MCPToolsViewer = ({
<Title className="text-xl font-semibold mb-6 mt-2">MCP Tools</Title>
<div className="flex flex-col flex-1">
{/* Extra Headers Input Section */}
{hasExtraHeaders && (
<div className="mb-4 p-3 bg-blue-50 border border-blue-200 rounded-lg">
<div className="flex items-center justify-between mb-2">
<div className="flex items-center">
<KeyOutlined className="text-blue-600 mr-2" />
<Text className="text-sm font-medium text-blue-800">
Additional Headers
</Text>
</div>
<AntdButton
size="small"
type="link"
onClick={() => setShowHeaderInput(!showHeaderInput)}
className="text-blue-700 p-0 h-auto"
>
{showHeaderInput ? "Hide" : "Configure"}
</AntdButton>
</div>
{!showHeaderInput && Object.keys(passthroughHeaders).length === 0 && (
<Text className="text-xs text-blue-700">
This server requires additional headers. Click "Configure" to provide values.
</Text>
)}
{showHeaderInput && (
<div className="mt-3 space-y-2">
{extraHeaders?.map((headerName) => (
<div key={headerName}>
<label className="block text-xs font-medium text-gray-700 mb-1">
{headerName}
</label>
<Input
size="small"
placeholder={`Enter ${headerName}`}
value={passthroughHeaders[headerName] || ""}
onChange={(e) => {
setPassthroughHeaders({
...passthroughHeaders,
[headerName]: e.target.value,
});
}}
prefix={<KeyOutlined className="text-gray-400" />}
className="rounded"
/>
</div>
))}
<AntdButton
size="small"
type="primary"
onClick={() => {
refetchTools();
setShowHeaderInput(false);
}}
disabled={Object.values(passthroughHeaders).every(v => !v || !v.trim())}
className="w-full mt-2"
>
Load Tools
</AntdButton>
</div>
)}
{!showHeaderInput && Object.keys(passthroughHeaders).length > 0 && (
<div className="mt-2">
<Text className="text-xs text-green-700 flex items-center">
<span className="inline-block w-2 h-2 bg-green-500 rounded-full mr-2"></span>
{Object.keys(passthroughHeaders).length} header(s) configured
</Text>
</div>
)}
</div>
)}
{/* Tool Selection - Show tools first */}
<div className="flex flex-col flex-1 min-h-0">
<Text className="font-medium block mb-3 text-gray-700 flex items-center">
@@ -25,11 +25,16 @@ export const TRANSPORT = {
OPENAPI: "openapi",
};
export const handleTransport = (transport?: string | null): string => {
export const handleTransport = (transport?: string | null, specPath?: string | null): string => {
if (transport === null || transport === undefined) {
return TRANSPORT.SSE;
}
// If server has spec_path, display as "openapi" instead of the raw transport type
if (specPath && transport !== TRANSPORT.STDIO) {
return TRANSPORT.OPENAPI;
}
return transport;
};
@@ -132,6 +137,7 @@ export interface MCPToolsViewerProps {
userRole: string | null;
userID: string | null;
serverAlias?: string | null;
extraHeaders?: string[] | null;
}
export interface MCPServer {
@@ -7147,7 +7147,11 @@ export const testSearchToolConnection = async (accessToken: string, litellmParam
}
};
export const listMCPTools = async (accessToken: string, serverId: string) => {
export const listMCPTools = async (
accessToken: string,
serverId: string,
customHeaders?: Record<string, string>
) => {
try {
// Construct base URL
let url = proxyBaseUrl
@@ -7159,6 +7163,7 @@ export const listMCPTools = async (accessToken: string, serverId: string) => {
const headers: Record<string, string> = {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
...customHeaders, // Merge custom headers for passthrough auth
};
const response = await fetch(url, {
@@ -7194,6 +7199,7 @@ export const listMCPTools = async (accessToken: string, serverId: string) => {
export interface CallMCPToolOptions {
guardrails?: string[];
customHeaders?: Record<string, string>;
}
export const callMCPTool = async (
@@ -7212,6 +7218,7 @@ export const callMCPTool = async (
const headers: Record<string, string> = {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
...(options?.customHeaders || {}), // Merge custom headers for passthrough auth
};
const body: Record<string, any> = {
@@ -1,6 +1,6 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import NotificationsManager from "@/components/molecules/notifications_manager";
import {
buildMcpOAuthAuthorizeUrl,
@@ -61,6 +61,7 @@ export const useMcpOAuthFlow = ({
const [status, setStatus] = useState<McpOAuthStatus>("idle");
const [error, setError] = useState<string | null>(null);
const [tokenResponse, setTokenResponse] = useState<Record<string, any> | null>(null);
const processingRef = useRef(false);
const FLOW_STATE_KEY = "litellm-mcp-oauth-flow-state";
const RESULT_KEY = "litellm-mcp-oauth-result";
@@ -75,6 +76,28 @@ export const useMcpOAuthFlow = ({
redirectUri: string;
};
const setStorageItem = (key: string, value: string) => {
if (typeof window === "undefined") return;
try {
// Store in both sessionStorage and localStorage for redundancy
window.sessionStorage.setItem(key, value);
window.localStorage.setItem(key, value);
} catch (err) {
console.warn(`Failed to set storage item ${key}`, err);
}
};
const getStorageItem = (key: string): string | null => {
if (typeof window === "undefined") return null;
try {
// Try sessionStorage first, fall back to localStorage
return window.sessionStorage.getItem(key) || window.localStorage.getItem(key);
} catch (err) {
console.warn(`Failed to get storage item ${key}`, err);
return null;
}
};
const clearStoredFlow = () => {
if (typeof window === "undefined") {
return;
@@ -83,6 +106,9 @@ export const useMcpOAuthFlow = ({
window.sessionStorage.removeItem(FLOW_STATE_KEY);
window.sessionStorage.removeItem(RESULT_KEY);
window.sessionStorage.removeItem(RETURN_URL_KEY);
window.localStorage.removeItem(FLOW_STATE_KEY);
window.localStorage.removeItem(RESULT_KEY);
window.localStorage.removeItem(RETURN_URL_KEY);
} catch (err) {
console.warn("Failed to clear OAuth storage", err);
}
@@ -187,10 +213,9 @@ export const useMcpOAuthFlow = ({
}
try {
window.sessionStorage.setItem(FLOW_STATE_KEY, JSON.stringify(flowState));
window.sessionStorage.setItem(RETURN_URL_KEY, window.location.href);
setStorageItem(FLOW_STATE_KEY, JSON.stringify(flowState));
setStorageItem(RETURN_URL_KEY, window.location.href);
} catch (storageErr) {
console.error("Unable to persist OAuth state", storageErr);
throw new Error("Unable to access browser storage for OAuth. Please enable storage and retry.");
}
@@ -209,19 +234,28 @@ export const useMcpOAuthFlow = ({
return;
}
// Prevent duplicate processing
if (processingRef.current) {
return;
}
let payload: Record<string, any> | null = null;
let flowState: StoredFlowState | null = null;
try {
const storedPayload = window.sessionStorage.getItem(RESULT_KEY);
const storedPayload = getStorageItem(RESULT_KEY);
if (!storedPayload) {
return;
}
// Mark as processing
processingRef.current = true;
payload = JSON.parse(storedPayload);
flowState = JSON.parse(window.sessionStorage.getItem(FLOW_STATE_KEY) || "null");
const storedFlowState = getStorageItem(FLOW_STATE_KEY);
flowState = storedFlowState ? JSON.parse(storedFlowState) : null;
} catch (err) {
console.error("Failed to read OAuth session state", err);
clearStoredFlow();
processingRef.current = false;
setError("Failed to resume OAuth flow. Please retry.");
setStatus("error");
NotificationsManager.error("Failed to resume OAuth flow. Please retry.");
@@ -229,14 +263,26 @@ export const useMcpOAuthFlow = ({
}
if (!payload) {
processingRef.current = false;
return;
}
window.sessionStorage.removeItem(RESULT_KEY);
// Clear the result key after reading it
if (typeof window !== "undefined") {
try {
window.sessionStorage.removeItem(RESULT_KEY);
window.localStorage.removeItem(RESULT_KEY);
} catch (err) {
// Silently ignore storage errors
}
}
try {
if (!flowState || !flowState.state || !flowState.codeVerifier || !flowState.serverId) {
throw new Error("Missing OAuth session state. Please retry.");
throw new Error(
"OAuth session state was lost. This can happen if you have strict browser privacy settings. " +
"Please try again and ensure cookies/storage is enabled."
);
}
if (!payload.state || payload.state !== flowState.state) {
throw new Error("OAuth state mismatch. Please retry.");
@@ -264,31 +310,21 @@ export const useMcpOAuthFlow = ({
setError(null);
NotificationsManager.success("OAuth token retrieved successfully");
} catch (err) {
console.error("OAuth flow failed", err);
const message = err instanceof Error ? err.message : String(err);
setError(message);
setStatus("error");
NotificationsManager.error(message);
} finally {
clearStoredFlow();
// Reset processing flag after a delay to allow UI updates
setTimeout(() => {
processingRef.current = false;
}, 1000);
}
}, [onTokenReceived]);
useEffect(() => {
let cancelled = false;
const maybeResume = async () => {
if (cancelled) {
return;
}
await resumeOAuthFlow();
};
maybeResume();
return () => {
cancelled = true;
};
resumeOAuthFlow();
}, [resumeOAuthFlow]);
return {