diff --git a/docs/my-website/docs/provider_registration/add_model_pricing.md b/docs/my-website/docs/provider_registration/add_model_pricing.md index b3df1865cd..ebf35c42e3 100644 --- a/docs/my-website/docs/provider_registration/add_model_pricing.md +++ b/docs/my-website/docs/provider_registration/add_model_pricing.md @@ -13,7 +13,6 @@ Here's the full specification with all available fields: ```json { "sample_spec": { - "aliases": ["optional list of alternate names for this model, e.g. dated versions like sample_spec-20250101"], "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, "computer_use_output_cost_per_1k_tokens": 0.0, @@ -122,28 +121,4 @@ Here's the full specification with all available fields: } ``` -### Using Aliases - -Many providers release the same model under multiple names — for example, a `latest` tag and a dated version like `claude-sonnet-4-5-20250929`. Instead of duplicating the entire entry, you can use the `aliases` field: - -```json -{ - "claude-sonnet-4-5": { - "aliases": ["claude-sonnet-4-5-20250929"], - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": true - } -} -``` - -At load time, each alias is expanded into a top-level entry sharing the same data as the canonical entry. The example above makes both `claude-sonnet-4-5` and `claude-sonnet-4-5-20250929` resolve with the same pricing and capabilities. - -:::info -This is different from [`model_alias_map`](../completion/model_alias.md), which is a runtime SDK/proxy feature for mapping user-facing model names to LiteLLM model identifiers. The `aliases` field here is for the model cost JSON only — it avoids duplicate entries for models that share identical pricing and capabilities. -::: +That's it! Your PR will be reviewed and merged. diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 48f4d8b8d3..6df570c72b 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -346,8 +346,6 @@ class DualCache(BaseCache): ) try: if self.in_memory_cache is not None: - if "ttl" not in kwargs and self.default_in_memory_ttl is not None: - kwargs["ttl"] = self.default_in_memory_ttl await self.in_memory_cache.async_set_cache(key, value, **kwargs) if self.redis_cache is not None and local_only is False: @@ -369,8 +367,6 @@ class DualCache(BaseCache): ) try: if self.in_memory_cache is not None: - if "ttl" not in kwargs and self.default_in_memory_ttl is not None: - kwargs["ttl"] = self.default_in_memory_ttl await self.in_memory_cache.async_set_cache_pipeline( cache_list=cache_list, **kwargs ) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index a841870f25..babb575ee3 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -390,7 +390,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ResponseOutputMessage, ResponseReasoningItem, ) - from openai.types.responses.response_output_item import ResponseApplyPatchToolCall from litellm.types.utils import Choices, Message @@ -449,18 +448,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): accumulated_tool_calls.append(tool_call_dict) tool_call_index += 1 - elif isinstance(item, ResponseApplyPatchToolCall): - from litellm.responses.litellm_completion_transformation.transformation import ( - LiteLLMCompletionResponsesConfig, - ) - - tool_call_dict = LiteLLMCompletionResponsesConfig.convert_apply_patch_tool_call_to_chat_completion_tool_call( - tool_call_item=item, - index=tool_call_index, - ) - accumulated_tool_calls.append(tool_call_dict) - tool_call_index += 1 - elif isinstance(item, dict) and handle_raw_dict_callback is not None: # Handle raw dict responses (e.g., from GPT-5 Codex) choice, index = handle_raw_dict_callback(item=item, index=index) @@ -1108,12 +1095,6 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): finish_reason = "tool_calls" if has_function_calls else "stop" - usage = None - if response_data.get("usage"): - from litellm.responses.utils import ResponseAPILoggingUtils - usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - response_data.get("usage") - ) return ModelResponseStream( choices=[ StreamingChoices( @@ -1121,8 +1102,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): delta=Delta(content=""), finish_reason=finish_reason, ) - ], - usage=usage + ] ) else: pass diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 6d2b4226ff..70c28c4e06 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -64,10 +64,12 @@ def duration_in_seconds(duration: str) -> int: now = time.time() current_time = datetime.fromtimestamp(now) - # Calculate target month and year, handling overflow past December - total_months = current_time.month - 1 + value # 0-indexed months - target_year = current_time.year + total_months // 12 - target_month = total_months % 12 + 1 # back to 1-indexed + if current_time.month == 12: + target_year = current_time.year + 1 + target_month = 1 + else: + target_year = current_time.year + target_month = current_time.month + value # Determine the day to set for next month target_day = current_time.day diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 406702503e..f9398979f9 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -11,7 +11,7 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True import json import os from importlib.resources import files -from typing import Dict, List, Optional +from typing import Optional import httpx @@ -183,54 +183,6 @@ def get_model_cost_map_source_info() -> dict: } -def _expand_model_aliases(model_cost: dict) -> dict: - """ - Expand ``aliases`` lists in model cost entries into top-level entries. - - Each alias gets a reference to the **same** dict object as the canonical - entry (zero memory overhead). The ``aliases`` key is removed from the - entry so downstream code never sees it. - - If an alias collides with an existing canonical entry the alias is - silently skipped and a warning is logged. - """ - aliases_to_add: Dict[str, dict] = {} - keys_with_aliases: List[str] = [] - - for model_name, model_info in model_cost.items(): - aliases: Optional[list] = model_info.get("aliases") - if aliases is None: - continue - keys_with_aliases.append(model_name) - if not aliases: - continue - for alias in aliases: - if alias in model_cost: - verbose_logger.warning( - "LiteLLM model alias conflict: alias '%s' (from '%s') " - "already exists as a canonical entry — skipping.", - alias, - model_name, - ) - continue - if alias in aliases_to_add: - verbose_logger.warning( - "LiteLLM model alias conflict: alias '%s' (from '%s') " - "was already claimed by another entry — skipping.", - alias, - model_name, - ) - continue - aliases_to_add[alias] = model_info # same dict reference - - # Remove the ``aliases`` key from entries so it doesn't pollute model info - for key in keys_with_aliases: - model_cost[key].pop("aliases", None) - - model_cost.update(aliases_to_add) - return model_cost - - def get_model_cost_map(url: str) -> dict: """ Public entry point — returns the model cost map dict. @@ -250,7 +202,7 @@ def get_model_cost_map(url: str) -> dict: _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None - return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map()) + return GetModelCostMap.load_local_model_cost_map() _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False @@ -266,7 +218,7 @@ def get_model_cost_map(url: str) -> dict: ) _cost_map_source_info.source = "local" _cost_map_source_info.fallback_reason = f"Remote fetch failed: {str(e)}" - return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map()) + return GetModelCostMap.load_local_model_cost_map() # Validate using cached count (cheap int comparison, no file I/O) if not GetModelCostMap.validate_model_cost_map( @@ -280,8 +232,8 @@ def get_model_cost_map(url: str) -> dict: ) _cost_map_source_info.source = "local" _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" - return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map()) + return GetModelCostMap.load_local_model_cost_map() _cost_map_source_info.source = "remote" _cost_map_source_info.fallback_reason = None - return _expand_model_aliases(content) + return content diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 41cc200141..ad68f3851a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -73,53 +73,6 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" -def _redact_standard_logging_object(model_call_details: dict): - """Redact messages and response inside standard_logging_object if present.""" - standard_logging_object = model_call_details.get("standard_logging_object") - if standard_logging_object is None: - return - - redacted_str = "redacted-by-litellm" - - if standard_logging_object.get("messages") is not None: - standard_logging_object["messages"] = [ - {"role": "user", "content": redacted_str} - ] - - response = standard_logging_object.get("response") - if response is not None: - if isinstance(response, dict) and "output" in response: - # ResponsesAPIResponse format - redact content in output items - if isinstance(response.get("output"), list): - for output_item in response["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - for content_item in output_item["content"]: - if ( - isinstance(content_item, dict) - and "text" in content_item - ): - content_item["text"] = redacted_str - elif isinstance(response, dict) and "choices" in response: - # ModelResponse dict format - redact content in choices - if isinstance(response.get("choices"), list): - for choice in response["choices"]: - if isinstance(choice, dict): - if "message" in choice and isinstance(choice["message"], dict): - choice["message"]["content"] = redacted_str - if "audio" in choice["message"]: - choice["message"]["audio"] = None - elif "delta" in choice and isinstance(choice["delta"], dict): - choice["delta"]["content"] = redacted_str - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - elif isinstance(response, str): - standard_logging_object["response"] = redacted_str - else: - # For other formats (empty dict, None, etc.), use simple text format - standard_logging_object["response"] = {"text": redacted_str} - - def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -161,29 +114,6 @@ def perform_redaction(model_call_details: dict, result): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: _redact_choice_content(choice) - elif isinstance(_result, dict) and "choices" in _result: - # Handle dict representation of ModelResponse (e.g., from model_dump()) - if _result.get("choices") is not None: - for choice in _result["choices"]: - if isinstance(choice, dict): - if "message" in choice and isinstance(choice["message"], dict): - choice["message"]["content"] = "redacted-by-litellm" - if "reasoning_content" in choice["message"]: - choice["message"]["reasoning_content"] = "redacted-by-litellm" - if "thinking_blocks" in choice["message"]: - choice["message"]["thinking_blocks"] = None - if "audio" in choice["message"]: - choice["message"]["audio"] = None - elif "delta" in choice and isinstance(choice["delta"], dict): - choice["delta"]["content"] = "redacted-by-litellm" - if "reasoning_content" in choice["delta"]: - choice["delta"]["reasoning_content"] = "redacted-by-litellm" - if "thinking_blocks" in choice["delta"]: - choice["delta"]["thinking_blocks"] = None - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - else: - _redact_choice_content(choice) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): _redact_responses_api_output(_result.output) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index 2967d5e394..78d6372d02 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -4,10 +4,7 @@ from typing import List import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.llms.openai.chat.gpt_5_transformation import ( - OpenAIGPT5Config, - _get_effort_level, -) +from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.types.llms.openai import AllMessageValues from .gpt_transformation import AzureOpenAIConfig @@ -84,21 +81,20 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) - effective_effort = _get_effort_level(reasoning_effort_value) # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning supports_none = self._supports_reasoning_effort_level(model, "none") - if effective_effort == "none" and not supports_none: + if reasoning_effort_value == "none" and not supports_none: if litellm.drop_params is True or ( drop_params is not None and drop_params is True ): non_default_params = non_default_params.copy() optional_params = optional_params.copy() - if _get_effort_level(non_default_params.get("reasoning_effort")) == "none": + if non_default_params.get("reasoning_effort") == "none": non_default_params.pop("reasoning_effort") - if _get_effort_level(optional_params.get("reasoning_effort")) == "none": + if optional_params.get("reasoning_effort") == "none": optional_params.pop("reasoning_effort") else: raise UnsupportedParamsError( @@ -121,19 +117,9 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): ) # Only drop reasoning_effort='none' for models that don't support it - result_effort = _get_effort_level(result.get("reasoning_effort")) - if result_effort == "none" and not supports_none: + if result.get("reasoning_effort") == "none" and not supports_none: result.pop("reasoning_effort") - # Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together. - # Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not). - if self.is_model_gpt_5_4_plus_model(model): - has_tools = bool( - non_default_params.get("tools") or optional_params.get("tools") - ) - if has_tools and result_effort not in (None, "none"): - result.pop("reasoning_effort", None) - return result def transform_request( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index ebb8ca759e..d210f294c6 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -51,7 +51,6 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ( ChatCompletionMessageToolCall, - CompletionTokensDetailsWrapper, Function, Message, ModelResponse, @@ -64,7 +63,6 @@ from litellm.utils import ( has_tool_call_blocks, last_assistant_with_tool_calls_has_no_thinking_blocks, supports_reasoning, - token_counter, ) from ..common_utils import ( @@ -1208,7 +1206,6 @@ class AmazonConverseConfig(BaseConfig): self._validate_request_metadata(request_metadata) output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None) - inference_params.pop("output_config", None) # Bedrock Converse doesn't support it # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { @@ -1623,11 +1620,7 @@ class AmazonConverseConfig(BaseConfig): thinking_blocks_list.append(_redacted_block) return thinking_blocks_list - def _transform_usage( - self, - usage: ConverseTokenUsageBlock, - reasoning_content: Optional[str] = None, - ) -> Usage: + def _transform_usage(self, usage: ConverseTokenUsageBlock) -> Usage: input_tokens = usage["inputTokens"] output_tokens = usage["outputTokens"] total_tokens = usage["totalTokens"] @@ -1644,19 +1637,6 @@ class AmazonConverseConfig(BaseConfig): prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cache_read_input_tokens ) - reasoning_tokens = ( - token_counter(text=reasoning_content, count_response_tokens=True) - if reasoning_content - else 0 - ) - completion_tokens_details = CompletionTokensDetailsWrapper( - reasoning_tokens=reasoning_tokens, - text_tokens=( - output_tokens - reasoning_tokens - if reasoning_tokens > 0 - else output_tokens - ), - ) openai_usage = Usage( prompt_tokens=input_tokens, completion_tokens=output_tokens, @@ -1664,7 +1644,6 @@ class AmazonConverseConfig(BaseConfig): prompt_tokens_details=prompt_tokens_details, cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, - completion_tokens_details=completion_tokens_details, ) return openai_usage @@ -2001,10 +1980,7 @@ class AmazonConverseConfig(BaseConfig): chat_completion_message["tool_calls"] = filtered_tools ## CALCULATING USAGE - bedrock returns usage in the headers - usage = self._transform_usage( - completion_response["usage"], - reasoning_content=chat_completion_message.get("reasoning_content"), - ) + usage = self._transform_usage(completion_response["usage"]) ## HANDLE TOOL CALLS _message = Message(**chat_completion_message) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index c9b6f330ec..7ec32fecc4 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -426,11 +426,8 @@ class FireworksAIConfig(OpenAIGPTConfig): "FIREWORKS_ACCOUNT_ID is not set. Please set the environment variable, to query Fireworks AI's `/models` endpoint." ) - base = api_base.rstrip("/") - if base.endswith("/v1"): - base = base[: -len("/v1")] response = litellm.module_level_client.get( - url=f"{base}/v1/accounts/{account_id}/models", + url=f"{api_base}/v1/accounts/{account_id}/models", headers={"Authorization": f"Bearer {api_key}"}, ) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index f186bc6085..beb76f3d80 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -25,22 +25,6 @@ def _normalize_reasoning_effort_for_chat_completion( return None -def _get_effort_level(value: Union[str, dict, None]) -> Optional[str]: - """Extract the effective effort level from reasoning_effort (string or dict). - - Use this for guards that compare effort level (e.g. xhigh validation, "none" checks). - Ensures dict inputs like {"effort": "none", "summary": "detailed"} are correctly - treated as effort="none" for validation purposes. - """ - if value is None: - return None - if isinstance(value, str): - return value - if isinstance(value, dict) and "effort" in value: - return value["effort"] - return None - - class OpenAIGPT5Config(OpenAIGPTConfig): """Configuration for gpt-5 models including GPT-5-Codex variants. @@ -86,19 +70,6 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model_name = model.split("/")[-1] return model_name.startswith("gpt-5.4") - @classmethod - def is_model_gpt_5_4_plus_model(cls, model: str) -> bool: - """Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro).""" - model_name = model.split("/")[-1] - if not model_name.startswith("gpt-5."): - return False - try: - version_str = model_name.replace("gpt-5.", "").split("-")[0] - major = version_str.split(".")[0] - return int(major) >= 4 - except (ValueError, IndexError): - return False - @classmethod def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: """Check if the model supports a specific reasoning_effort level. @@ -179,32 +150,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # Get raw reasoning_effort and effective effort level for all guards. - # Use effective_effort (extracted string) for xhigh validation, "none" checks, and - # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} - # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors. + # Normalize reasoning_effort: chat completion API expects a string, not a dict + # (e.g. {'effort': 'high', 'summary': 'detailed'} -> 'high') raw_reasoning_effort = ( non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) - effective_effort = _get_effort_level(raw_reasoning_effort) + normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) + if raw_reasoning_effort is not None and normalized is not None: + if "reasoning_effort" in non_default_params: + non_default_params["reasoning_effort"] = normalized + if "reasoning_effort" in optional_params: + optional_params["reasoning_effort"] = normalized - # Normalize to string for Chat Completions API when dict has only "effort". - # Preserve full dict (e.g. {"effort": "high", "summary": "detailed"}) for Responses API. - if isinstance(raw_reasoning_effort, dict) and set(raw_reasoning_effort.keys()) <= {"effort"}: - normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) - if normalized is not None: - if "reasoning_effort" in non_default_params: - non_default_params["reasoning_effort"] = normalized - if "reasoning_effort" in optional_params: - optional_params["reasoning_effort"] = normalized - - reasoning_effort = ( - non_default_params.get("reasoning_effort") - or optional_params.get("reasoning_effort") - or raw_reasoning_effort - ) - if effective_effort is not None and effective_effort == "xhigh": + reasoning_effort = normalized or raw_reasoning_effort + if reasoning_effort is not None and reasoning_effort == "xhigh": if not self._supports_reasoning_effort_level(model, "xhigh"): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) @@ -231,20 +191,17 @@ class OpenAIGPT5Config(OpenAIGPTConfig): has_tools = bool( non_default_params.get("tools") or optional_params.get("tools") ) - if has_tools and effective_effort not in (None, "none"): - # Check if this will be routed to Responses API - # If so, keep reasoning_effort; otherwise drop it for chat completions API - if not self.is_model_gpt_5_4_plus_model(model): - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - reasoning_effort = None + if has_tools and reasoning_effort not in (None, "none"): + non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) + reasoning_effort = None # gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none" supports_none = self._supports_reasoning_effort_level(model, "none") if supports_none: sampling_params = ["logprobs", "top_logprobs", "top_p"] has_sampling = any(p in non_default_params for p in sampling_params) - if has_sampling and effective_effort not in (None, "none"): + if has_sampling and reasoning_effort not in (None, "none"): if litellm.drop_params or drop_params: for p in sampling_params: non_default_params.pop(p, None) @@ -254,7 +211,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): "gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when " "reasoning_effort='none'. Current reasoning_effort='{}'. " "To drop unsupported params set `litellm.drop_params = True`" - ).format(effective_effort), + ).format(reasoning_effort), status_code=400, ) @@ -262,7 +219,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: # models supporting reasoning_effort="none" also support flexible temperature - if supports_none and (effective_effort == "none" or effective_effort is None): + if supports_none and (reasoning_effort == "none" or reasoning_effort is None): optional_params["temperature"] = temperature_value elif temperature_value == 1: optional_params["temperature"] = temperature_value diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index efbb218f57..2a30dc5ef3 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -583,17 +583,35 @@ class SagemakerLLM(BaseAWSLLM): ### BOTO3 INIT import boto3 - # Use _load_credentials to support role assumption (aws_role_name, aws_session_name) - credentials, aws_region_name = self._load_credentials(optional_params) + # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) + aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_region_name = optional_params.pop("aws_region_name", None) - # Create boto3 session with the loaded credentials - session = boto3.Session( - aws_access_key_id=credentials.access_key, - aws_secret_access_key=credentials.secret_key, - aws_session_token=credentials.token, - region_name=aws_region_name, - ) - client = session.client(service_name="sagemaker-runtime") + if aws_access_key_id is not None: + # uses auth params passed to completion + # aws_access_key_id is not None, assume user is trying to auth using litellm.completion + client = boto3.client( + service_name="sagemaker-runtime", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=aws_region_name, + ) + else: + # aws_access_key_id is None, assume user is trying to auth using env variables + # boto3 automaticaly reads env variables + + # we need to read region name from env + # I assume majority of users use .env for auth + region_name = ( + get_secret("AWS_REGION_NAME") + or aws_region_name # get region from config file if specified + or "us-west-2" # default to us-west-2 if region not specified + ) + client = boto3.client( + service_name="sagemaker-runtime", + region_name=region_name, + ) # pop streaming if it's in the optional params as 'stream' raises an error with sagemaker inference_params = deepcopy(optional_params) @@ -610,9 +628,7 @@ class SagemakerLLM(BaseAWSLLM): #### EMBEDDING LOGIC # Transform request based on model type provider_config = SagemakerEmbeddingConfig.get_model_config(model) - request_data = provider_config.transform_embedding_request( - model, input, optional_params, {} - ) + request_data = provider_config.transform_embedding_request(model, input, optional_params, {}) data = json.dumps(request_data).encode("utf-8") ## LOGGING @@ -657,19 +673,19 @@ class SagemakerLLM(BaseAWSLLM): ) print_verbose(f"raw model_response: {response}") - + # Transform response based on model type from httpx import Response as HttpxResponse - + # Create a mock httpx Response object for the transformation mock_response = HttpxResponse( status_code=200, - content=json.dumps(response).encode("utf-8"), - headers={"content-type": "application/json"}, + content=json.dumps(response).encode('utf-8'), + headers={"content-type": "application/json"} ) - + model_response = EmbeddingResponse() - + # Use the request_data that was already transformed above return provider_config.transform_embedding_response( model=model, @@ -679,5 +695,5 @@ class SagemakerLLM(BaseAWSLLM): api_key=None, request_data=request_data, optional_params=optional_params, - litellm_params=litellm_params or {}, + litellm_params=litellm_params or {} ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 54148c9492..57889284a8 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -529,18 +529,12 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 raise e -# Keys that LiteLLM consumes internally and must never be forwarded to the -_LITELLM_INTERNAL_EXTRA_BODY_KEYS: frozenset = frozenset({"cache", "tags"}) - - def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: """Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values.""" extra_body: Optional[dict] = optional_params.pop("extra_body", None) if extra_body is not None: data_dict: dict = data # type: ignore[assignment] for k, v in extra_body.items(): - if k in _LITELLM_INTERNAL_EXTRA_BODY_KEYS: - continue if k in data_dict and isinstance(data_dict[k], dict) and isinstance(v, dict): data_dict[k].update(v) else: diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 5ad3cf444f..5f6cb87b26 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -92,24 +92,7 @@ def get_base_url(spec: Dict[str, Any], spec_path: Optional[str] = None) -> str: """Extract base URL from OpenAPI spec.""" # OpenAPI 3.x if "servers" in spec and spec["servers"]: - server_url = spec["servers"][0]["url"] - - # If the server URL is relative (starts with /), derive base from spec_path - if server_url.startswith("/") and spec_path: - if spec_path.startswith("http://") or spec_path.startswith("https://"): - # Extract base URL from spec_path (e.g., https://petstore3.swagger.io/api/v3/openapi.json) - # Combine domain with the relative server URL - from urllib.parse import urlparse - parsed = urlparse(spec_path) - base_domain = f"{parsed.scheme}://{parsed.netloc}" - full_base_url = base_domain + server_url - verbose_logger.info( - f"OpenAPI spec has relative server URL '{server_url}'. " - f"Deriving base from spec_path: {full_base_url}" - ) - return full_base_url - - return server_url + return spec["servers"][0]["url"] # OpenAPI 2.x (Swagger) elif "host" in spec: scheme = spec.get("schemes", ["https"])[0] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 7898f03e01..99f6a5234a 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -711,7 +711,6 @@ if MCP_AVAILABLE: Checks both the full tool name and unprefixed version (without server prefix). This allows users to configure simple tool names regardless of prefixing. - Comparison is case-insensitive to handle OpenAPI operationIds that may be in camelCase. Args: tool_name: The tool name to check (may be prefixed like "server-tool_name") @@ -724,15 +723,13 @@ if MCP_AVAILABLE: split_server_prefix_from_name, ) - # Normalize filter list to lowercase for case-insensitive comparison - filter_list_lower = [f.lower() for f in filter_list] - - if tool_name.lower() in filter_list_lower: + # Check if the full name is in the list + if tool_name in filter_list: return True - # Check if the unprefixed name is in the list (case-insensitive) + # Check if the unprefixed name is in the list unprefixed_name, _ = split_server_prefix_from_name(tool_name) - return unprefixed_name.lower() in filter_list_lower + return unprefixed_name in filter_list def filter_tools_by_allowed_tools( tools: List[MCPTool], diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 13b26eef43..32f209a763 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -108,23 +108,16 @@ def get_key_models( """ all_models: List[str] = [] if len(user_api_key_dict.models) > 0: - all_models = list(user_api_key_dict.models) # copy to avoid mutating cached objects + all_models = user_api_key_dict.models if SpecialModelNames.all_team_models.value in all_models: - all_models = list(user_api_key_dict.team_models) # copy to avoid mutating cached objects + all_models = user_api_key_dict.team_models if SpecialModelNames.all_proxy_models.value in all_models: - all_models = list(proxy_model_list) # copy to avoid mutating caller's list - if include_model_access_groups: - all_models.extend(model_access_groups.keys()) + all_models = proxy_model_list all_models = _get_models_from_access_groups( - model_access_groups=model_access_groups, - all_models=all_models, - include_model_access_groups=include_model_access_groups, + model_access_groups=model_access_groups, all_models=all_models ) - # deduplicate while preserving order - all_models = list(dict.fromkeys(all_models)) - verbose_proxy_logger.debug("ALL KEY MODELS - {}".format(len(all_models))) return all_models @@ -148,8 +141,8 @@ def get_team_models( all_models_set.update(team_models) if SpecialModelNames.all_proxy_models.value in all_models_set: all_models_set.update(proxy_model_list) - if include_model_access_groups: - all_models_set.update(model_access_groups.keys()) + + all_models = list(all_models_set) all_models = _get_models_from_access_groups( model_access_groups=model_access_groups, @@ -157,9 +150,6 @@ def get_team_models( include_model_access_groups=include_model_access_groups, ) - # deduplicate while preserving order - all_models = list(dict.fromkeys(all_models)) - verbose_proxy_logger.debug("ALL TEAM MODELS - {}".format(len(all_models))) return all_models diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 5fa9546e00..9f228bb118 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -142,47 +142,17 @@ async def get_credentials( tags=["credential management"], response_model=CredentialItem, ) -async def get_credential_by_name( - request: Request, - fastapi_response: Response, - credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - [BETA] endpoint. This might change unexpectedly. - """ - try: - for credential in litellm.credential_list: - if credential.credential_name == credential_name: - masked_credential = CredentialItem( - credential_name=credential.credential_name, - credential_values=_get_masked_values( - credential.credential_values, - unmasked_length=4, - number_of_asterisks=4, - ), - credential_info=credential.credential_info, - ) - return masked_credential - raise HTTPException( - status_code=404, - detail="Credential not found. Got credential name: " + credential_name, - ) - except Exception as e: - verbose_proxy_logger.exception(e) - raise handle_exception_on_proxy(e) - - @router.get( "/credentials/by_model/{model_id}", dependencies=[Depends(user_api_key_auth)], tags=["credential management"], response_model=CredentialItem, ) -async def get_credential_by_model( +async def get_credential( request: Request, fastapi_response: Response, - model_id: str = Path(..., description="The model ID to look up credentials for"), + credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), + model_id: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -191,25 +161,48 @@ async def get_credential_by_model( from litellm.proxy.proxy_server import llm_router try: - if llm_router is None: - raise HTTPException(status_code=500, detail="LLM router not found") - model = llm_router.get_deployment(model_id) - if model is None: - raise HTTPException(status_code=404, detail="Model not found") - credential_values = llm_router.get_deployment_credentials(model_id) - if credential_values is None: - raise HTTPException(status_code=404, detail="Model not found") - masked_credential_values = _get_masked_values( - credential_values, - unmasked_length=4, - number_of_asterisks=4, - ) - credential = CredentialItem( - credential_name="{}-credential-{}".format(model.model_name, model_id), - credential_values=masked_credential_values, - credential_info={}, - ) - return credential + if model_id: + if llm_router is None: + raise HTTPException(status_code=500, detail="LLM router not found") + model = llm_router.get_deployment(model_id) + if model is None: + raise HTTPException(status_code=404, detail="Model not found") + credential_values = llm_router.get_deployment_credentials(model_id) + if credential_values is None: + raise HTTPException(status_code=404, detail="Model not found") + masked_credential_values = _get_masked_values( + credential_values, + unmasked_length=4, + number_of_asterisks=4, + ) + credential = CredentialItem( + credential_name="{}-credential-{}".format(model.model_name, model_id), + credential_values=masked_credential_values, + credential_info={}, + ) + # return credential object + return credential + elif credential_name: + for credential in litellm.credential_list: + if credential.credential_name == credential_name: + masked_credential = CredentialItem( + credential_name=credential.credential_name, + credential_values=_get_masked_values( + credential.credential_values, + unmasked_length=4, + number_of_asterisks=4, + ), + credential_info=credential.credential_info, + ) + return masked_credential + raise HTTPException( + status_code=404, + detail="Credential not found. Got credential name: " + credential_name, + ) + else: + raise HTTPException( + status_code=404, detail="Credential name or model ID required" + ) except Exception as e: verbose_proxy_logger.exception(e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index ee1868fc74..633de86aa6 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2827,6 +2827,21 @@ async def validate_membership( ) +def _unfurl_all_proxy_models( + team_info: LiteLLM_TeamTable, llm_router: Router +) -> LiteLLM_TeamTable: + if ( + SpecialModelNames.all_proxy_models.value in team_info.models + and llm_router is not None + ): + team_models: set[str] = set() # make set to avoid duplicates + for model in team_info.models: + if model != SpecialModelNames.all_proxy_models.value: + team_models.add(model) + for model in llm_router.get_model_names(): + team_models.add(model) + team_info.models = list(team_models) + return team_info async def _add_team_member_budget_table( @@ -2957,6 +2972,9 @@ async def team_info( team_info_response_object=_team_info, ) + # ## UNFURL 'all-proxy-models' into the team_info.models list ## + # if llm_router is not None: + # _team_info = _unfurl_all_proxy_models(_team_info, llm_router) response_object = TeamInfoResponseObject( team_id=team_id, team_info=_team_info, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 4d95fda0a4..356807415d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2062,8 +2062,7 @@ class InitPassThroughEndpointHelpers: """ ## CHECK IF MAPPED PASS THROUGH ENDPOINT for mapped_route in LiteLLMRoutes.mapped_pass_through_routes.value: - full_mapped_route = InitPassThroughEndpointHelpers._build_full_path_with_root(mapped_route) - if route.startswith(full_mapped_route): + if route.startswith(mapped_route): return True # Fast path: check if any registered route key contains this path diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index e8704a6a33..5b58fbe70a 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1461,21 +1461,11 @@ async def _get_spend_report_for_time_range( dependencies=[Depends(user_api_key_auth)], responses={ 200: { - "description": "The calculated cost", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "cost": { - "type": "number", - "description": "The calculated cost", - "example": 0.0, - } - }, - } - } - }, + "cost": { + "description": "The calculated cost", + "example": 0.0, + "type": "float", + } } }, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a99c2a8ed5..19845d7c49 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -292,14 +292,14 @@ class LiteLLMCompletionResponsesConfig: ) _messages = litellm_completion_request.get("messages") or [] session_messages = chat_completion_session.get("messages") or [] - + # If session messages are empty (e.g., no database in test environment), # we still need to process the new input messages # Store original _messages before combining for safety check original_new_messages = _messages.copy() if _messages else [] - + combined_messages = session_messages + _messages - + # Fix: Ensure tool_results have corresponding tool_calls in previous assistant message # Pass tools parameter to help reconstruct tool_calls if not in cache tools = litellm_completion_request.get("tools") or [] @@ -307,7 +307,7 @@ class LiteLLMCompletionResponsesConfig: messages=combined_messages, tools=tools ) - + # Safety check: Ensure we don't end up with empty messages # This can happen when using previous_response_id without a database (e.g., in tests) # and session messages are empty but new input messages exist @@ -338,7 +338,7 @@ class LiteLLMCompletionResponsesConfig: model=litellm_completion_request.get("model", ""), llm_provider=litellm_completion_request.get("custom_llm_provider", ""), ) - + litellm_completion_request["messages"] = combined_messages litellm_completion_request["litellm_trace_id"] = chat_completion_session.get( "litellm_session_id" @@ -384,45 +384,10 @@ class LiteLLMCompletionResponsesConfig: if call_id_raw: existing_tool_call_ids.add(str(call_id_raw)) - ######################################################### - # Merge consecutive function_call items into a single assistant - # message. Anthropic requires that all tool_use blocks appear in - # ONE assistant message immediately followed by the tool_result - # blocks. Without this merging, each function_call creates its own - # assistant message, producing back-to-back assistant messages that - # Anthropic rejects with "tool_use ids were found without - # tool_result blocks immediately after". - ######################################################### - if messages: - last_msg = messages[-1] - last_role = ( - last_msg.get("role") - if isinstance(last_msg, dict) - else getattr(last_msg, "role", None) - ) - if last_role == "assistant": - for new_msg in chat_completion_messages: - new_role = ( - new_msg.get("role") - if isinstance(new_msg, dict) - else getattr(new_msg, "role", None) - ) - if new_role == "assistant": - new_tcs = ( - new_msg.get("tool_calls") - if isinstance(new_msg, dict) - else getattr(new_msg, "tool_calls", None) - ) or [] - for tc in new_tcs: - LiteLLMCompletionResponsesConfig._add_tool_call_to_assistant( - last_msg, tc - ) - continue - ######################################################### # If Input Item is a Tool Call Output, add it to the tool_call_output_messages list - # preserving the ordering of tool call outputs. Some models require the tool - # result to immediately follow the assistant tool call. + # preserving the ordering of tool call outputs. Some models require the tool + # result to immediately follow the assistant tool call. ######################################################### if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( input_item=_input @@ -795,14 +760,14 @@ class LiteLLMCompletionResponsesConfig: ]: """ Ensure that tool_result messages have corresponding tool_calls in the previous assistant message. - + This is critical for Anthropic API which requires that each tool_result block has a corresponding tool_use block in the previous assistant message. - + Args: messages: List of messages that may include tool_result messages tools: Optional list of tools that can be used to reconstruct tool_calls if not in cache - + Returns: List of messages with tool_calls added to assistant messages when needed """ @@ -821,29 +786,29 @@ class LiteLLMCompletionResponsesConfig: ] ] = list(copy.deepcopy(messages)) messages_to_remove = [] - + # Count non-tool messages to avoid removing all messages # This prevents empty messages list when using previous_response_id without a database non_tool_messages_count = sum( 1 for msg in fixed_messages if msg.get("role") != "tool" ) - + for i, message in enumerate(fixed_messages): # Only process tool messages - check role first to narrow the type if message.get("role") != "tool": continue - + # At this point, we know it's a tool message, so it should have tool_call_id # Use get() with default to safely access tool_call_id tool_call_id_raw = message.get("tool_call_id") if isinstance(message, dict) else getattr(message, "tool_call_id", None) tool_call_id: str = ( str(tool_call_id_raw) if tool_call_id_raw is not None else "" ) - + prev_assistant_idx = LiteLLMCompletionResponsesConfig._find_previous_assistant_idx( fixed_messages, i ) - + # Try to recover empty tool_call_id from previous assistant message if not tool_call_id and prev_assistant_idx is not None: prev_assistant = fixed_messages[prev_assistant_idx] @@ -858,7 +823,7 @@ class LiteLLMCompletionResponsesConfig: message_dict["tool_call_id"] = tool_call_id elif hasattr(message, "tool_call_id"): setattr(message, "tool_call_id", tool_call_id) - + # Only remove messages with empty tool_call_id if we have other non-tool messages # This prevents ending up with an empty messages list when using previous_response_id # without a database (e.g., in tests where session messages are empty) @@ -870,7 +835,7 @@ class LiteLLMCompletionResponsesConfig: # If no non-tool messages, keep the tool message even with empty call_id # The API will return a proper error message about the missing tool_use block continue - + # Check if the previous assistant message has the corresponding tool_call # This needs to run for ALL tool messages with a valid tool_call_id, # not just those that had an empty tool_call_id initially @@ -879,12 +844,12 @@ class LiteLLMCompletionResponsesConfig: tool_calls = LiteLLMCompletionResponsesConfig._get_tool_calls_list( prev_assistant ) - + if not LiteLLMCompletionResponsesConfig._check_tool_call_exists( tool_calls, tool_call_id ): _tool_use_definition = TOOL_CALLS_CACHE.get_cache(key=tool_call_id) - + if not _tool_use_definition and tools: _tool_use_definition = ( LiteLLMCompletionResponsesConfig._reconstruct_tool_call_from_tools( @@ -909,11 +874,11 @@ class LiteLLMCompletionResponsesConfig: LiteLLMCompletionResponsesConfig._add_tool_call_to_assistant( prev_assistant, tool_call_chunk ) - + # Remove messages with empty tool_call_id that couldn't be fixed for idx in reversed(messages_to_remove): fixed_messages.pop(idx) - + return fixed_messages @staticmethod @@ -1558,39 +1523,6 @@ class LiteLLMCompletionResponsesConfig: return tool_call_dict - @staticmethod - def convert_apply_patch_tool_call_to_chat_completion_tool_call( - tool_call_item: Any, - index: int = 0, - ) -> Dict[str, Any]: - """ - Convert ResponseApplyPatchToolCall to ChatCompletionToolCallChunk format. - - The operation (create_file / update_file / delete_file) is serialised - as JSON so it appears in function.arguments, just like any other - tool call. - - Args: - tool_call_item: ResponseApplyPatchToolCall object with call_id and operation - index: The index of this tool call - - Returns: - Dictionary in ChatCompletionToolCallChunk format - """ - import json - - operation_dict = tool_call_item.operation.model_dump() - tool_call_dict: Dict[str, Any] = { - "id": tool_call_item.call_id, - "function": { - "name": "apply_patch", - "arguments": json.dumps(operation_dict), - }, - "type": "function", - "index": index, - } - return tool_call_dict - @staticmethod def transform_chat_completion_response_to_responses_api_response( request_input: Union[str, ResponseInputParam], diff --git a/litellm/router.py b/litellm/router.py index 44e9667d5c..43b53d14d7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5484,10 +5484,6 @@ class Router: return response except Exception as e: - # Always track the latest error so we raise the most - # recent exception instead of the first one. - original_exception = e - ## LOGGING kwargs = self.log_retry(kwargs=kwargs, e=e) remaining_retries = num_retries - current_attempt - 1 @@ -5502,24 +5498,6 @@ class Router: ) else: _healthy_deployments = [] - - # Check if this error is non-retryable (e.g., 400 context - # window exceeded). If so, raise immediately instead of - # continuing the retry loop. Respect retry policy - # precedence - only check when no retry policy applies. - if not _retry_policy_applies: - try: - self.should_retry_this_error( - error=e, - healthy_deployments=_healthy_deployments, - all_deployments=_all_deployments, - context_window_fallbacks=context_window_fallbacks, - regular_fallbacks=fallbacks, - content_policy_fallbacks=content_policy_fallbacks, - ) - except Exception: - raise e - _timeout = self._time_to_sleep_before_retry( e=e, remaining_retries=remaining_retries, diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index e09b5c1456..0449a843bd 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -490,22 +490,20 @@ class LowestLatencyLoggingHandler(CustomLogger): # get average latency or average ttft (depending on streaming/non-streaming) total: float = 0.0 - use_ttft = ( + if ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 - ) - if use_ttft: + ): for _call_latency in item_ttft_latency: if isinstance(_call_latency, float): total += _call_latency - item_latency = total / len(item_ttft_latency) else: for _call_latency in item_latency: if isinstance(_call_latency, float): total += _call_latency - item_latency = total / len(item_latency) + item_latency = total / len(item_latency) # -------------- # # Debugging Logic diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0b3f87fbe0..b1d4d5a116 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,24 +458,6 @@ "interactions": true } }, - "charity_engine": { - "display_name": "Charity Engine (`charity_engine`)", - "url": "https://docs.litellm.ai/docs/providers/charity_engine", - "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "a2a": false, - "interactions": false - } - }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { diff --git a/tests/llm_translation/test_skills_api.py b/tests/llm_translation/test_skills_api.py index b340167133..7565ba7440 100644 --- a/tests/llm_translation/test_skills_api.py +++ b/tests/llm_translation/test_skills_api.py @@ -44,25 +44,19 @@ def create_skill_zip(skill_name: str, unique_suffix: Optional[str] = None): skill_dir = test_dir / skill_name # Create a zip file containing the skill directory - # When unique_suffix is set, folder name must match skill name in SKILL.md (Anthropic requirement) - zip_folder_name = f"{skill_name}-{unique_suffix}" if unique_suffix else skill_name zip_path = test_dir / f"{skill_name}.zip" with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.write(skill_dir, arcname=skill_name) + if unique_suffix is not None: - # Rewrite SKILL.md with a unique name and use matching folder name + # Rewrite SKILL.md with a unique name to avoid API conflicts skill_md = (skill_dir / "SKILL.md").read_text() skill_md = skill_md.replace( f"name: {skill_name}", - f"name: {zip_folder_name}", + f"name: {skill_name}-{unique_suffix}", ) - zf.writestr(f"{zip_folder_name}/SKILL.md", skill_md) - # Add any other files in the skill dir (e.g. subdirs) under the new folder name - for f in skill_dir.rglob("*"): - if f.is_file() and f.name != "SKILL.md": - rel = f.relative_to(skill_dir) - zf.write(f, arcname=f"{zip_folder_name}/{rel}") + zf.writestr(f"{skill_name}/SKILL.md", skill_md) else: - zf.write(skill_dir, arcname=skill_name) zf.write(skill_dir / "SKILL.md", arcname=f"{skill_name}/SKILL.md") try: diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index ead387599d..fcdfcfe6e7 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1300,11 +1300,9 @@ def test_logging_async_cache_hit_sync_call(turn_off_message_logging): "redacted-by-litellm" == standard_logging_object["messages"][0]["content"] ) - # response is a full ModelResponse dict (choices format) since d84e5e381acf - assert ( - standard_logging_object["response"]["choices"][0]["message"]["content"] - == "redacted-by-litellm" - ) + assert {"text": "redacted-by-litellm"} == standard_logging_object[ + "response" + ] def test_logging_standard_payload_failure_call(): diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 0391a5a895..0536ec7205 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -45,8 +45,7 @@ async def test_global_redaction_on(): await asyncio.sleep(1) standard_logging_payload = test_custom_logger.logged_standard_logging_payload assert standard_logging_payload is not None - response = standard_logging_payload["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload", @@ -76,8 +75,7 @@ async def test_global_redaction_with_dynamic_params(turn_off_message_logging): ) if turn_off_message_logging is True: - response = standard_logging_payload["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -110,8 +108,7 @@ async def test_global_redaction_off_with_dynamic_params(turn_off_message_logging json.dumps(standard_logging_payload, indent=2), ) if turn_off_message_logging is True: - response = standard_logging_payload["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -393,8 +390,7 @@ async def test_redaction_with_streaming_response(): assert standard_logging_payload is not None # Verify that redaction worked without pickle errors - response = standard_logging_payload["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload for streaming with coroutine handling", @@ -481,6 +477,5 @@ async def test_redaction_with_metadata_completion_api(): # Verify the helper function works correctly - with get_metadata_variable_name_from_kwargs, # the system checks the appropriate field for headers - response = standard_logging_payload["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 606f25ddf4..9974c23e4b 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -1,11 +1,9 @@ import asyncio -import time from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.caching.dual_cache import DualCache -from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache @@ -58,104 +56,3 @@ async def test_dual_cache_async_batch_get_cache_rolls_back_redis_reservation_on_ assert mock_async_batch_get_cache.call_count == 2 assert "shared_a" not in dual_cache.last_redis_batch_access_time assert "shared_b" not in dual_cache.last_redis_batch_access_time - - -@pytest.mark.asyncio -async def test_dual_cache_async_set_cache_injects_default_in_memory_ttl(): - """ - Test that async_set_cache injects default_in_memory_ttl into kwargs - when no explicit ttl is provided, matching the sync set_cache behavior. - - Regression test for: async_set_cache was missing the TTL injection that - sync set_cache has, causing InMemoryCache to use its own default_ttl (600s) - instead of DualCache's default_in_memory_ttl. - """ - in_memory_cache = InMemoryCache(default_ttl=600) - dual_cache = DualCache( - in_memory_cache=in_memory_cache, - default_in_memory_ttl=60, - ) - - before = time.time() - await dual_cache.async_set_cache(key="test_key", value="test_value") - after = time.time() - - # The TTL stored should reflect default_in_memory_ttl (60s), not - # InMemoryCache's default_ttl (600s) - expiry = in_memory_cache.ttl_dict["test_key"] - assert expiry >= before + 60 - assert expiry <= after + 60 - - -@pytest.mark.asyncio -async def test_dual_cache_async_set_cache_respects_explicit_ttl(): - """ - Test that async_set_cache does NOT override an explicitly provided ttl. - """ - in_memory_cache = InMemoryCache(default_ttl=600) - dual_cache = DualCache( - in_memory_cache=in_memory_cache, - default_in_memory_ttl=60, - ) - - before = time.time() - await dual_cache.async_set_cache(key="test_key", value="test_value", ttl=30) - after = time.time() - - # The explicit ttl=30 should be used, not default_in_memory_ttl (60) - expiry = in_memory_cache.ttl_dict["test_key"] - assert expiry >= before + 30 - assert expiry <= after + 30 - - -@pytest.mark.asyncio -async def test_dual_cache_async_set_cache_pipeline_injects_default_in_memory_ttl(): - """ - Test that async_set_cache_pipeline injects default_in_memory_ttl into kwargs - when no explicit ttl is provided. - """ - in_memory_cache = InMemoryCache(default_ttl=600) - dual_cache = DualCache( - in_memory_cache=in_memory_cache, - default_in_memory_ttl=60, - ) - - cache_list = [("key_a", "value_a"), ("key_b", "value_b")] - - before = time.time() - await dual_cache.async_set_cache_pipeline(cache_list=cache_list) - after = time.time() - - for key in ["key_a", "key_b"]: - expiry = in_memory_cache.ttl_dict[key] - assert expiry >= before + 60 - assert expiry <= after + 60 - - -@pytest.mark.asyncio -async def test_dual_cache_sync_and_async_set_cache_use_same_ttl(): - """ - Test that sync set_cache and async async_set_cache produce the same TTL - when no explicit ttl is provided, ensuring parity between the two paths. - """ - in_memory_sync = InMemoryCache(default_ttl=600) - dual_cache_sync = DualCache( - in_memory_cache=in_memory_sync, - default_in_memory_ttl=60, - ) - - in_memory_async = InMemoryCache(default_ttl=600) - dual_cache_async = DualCache( - in_memory_cache=in_memory_async, - default_in_memory_ttl=60, - ) - - dual_cache_sync.set_cache(key="test_key", value="test_value") - await dual_cache_async.async_set_cache(key="test_key", value="test_value") - - sync_expiry = in_memory_sync.ttl_dict["test_key"] - async_expiry = in_memory_async.ttl_dict["test_key"] - - # Both should use default_in_memory_ttl=60, so their expiry times - # should be within a small tolerance of each other - assert abs(sync_expiry - async_expiry) < 1.0 diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 8c72b7725a..ef3d7534d9 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -738,58 +738,7 @@ def test_response_completed_with_message_only_emits_stop_finish_reason(): ) - -def test_response_completed_preserves_usage_with_cached_tokens(): - """ - Test that response.completed correctly translates Responses API usage - (input_tokens_details) to chat completion usage (prompt_tokens_details). - - This is a regression test for an issue where streaming with models that - use the Responses API bridge (e.g. gpt-5.2-codex) would drop - prompt_tokens_details, causing cached_tokens to always be None. - """ - from litellm.completion_extras.litellm_responses_transformation.transformation import ( - OpenAiResponsesToChatCompletionStreamIterator, - ) - - iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) - - chunk = { - "type": "response.completed", - "response": { - "id": "resp_789", - "status": "completed", - "output": [ - { - "type": "message", - "id": "msg_abc", - "role": "assistant", - "content": [{"type": "output_text", "text": "Six"}], - "status": "completed", - } - ], - "usage": { - "input_tokens": 1226, - "output_tokens": 5, - "total_tokens": 1231, - "input_tokens_details": {"cached_tokens": 1024}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - } - - result = iterator.chunk_parser(chunk) - - assert result.usage is not None, "usage should be set on response.completed chunk" - assert result.usage.prompt_tokens == 1226, "prompt_tokens should map from input_tokens" - assert result.usage.completion_tokens == 5, "completion_tokens should map from output_tokens" - assert result.usage.prompt_tokens_details is not None, "prompt_tokens_details should be set" - assert result.usage.prompt_tokens_details.cached_tokens == 1024, ( - "cached_tokens should be preserved from input_tokens_details" - ) - - -def test_function_call_done_emits_is_finished(): +def test_function_call_done_does_not_emit_finish_reason(): """ Test that OUTPUT_ITEM_DONE for a function_call does NOT emit finish_reason. The response.completed event handles the terminal finish_reason correctly. @@ -1378,138 +1327,6 @@ def test_transform_response_preserves_annotations(): print("✓ Annotations from Responses API are correctly preserved in Chat Completions format") -def test_apply_patch_tool_call_converted_to_chat_completion_tool_call(): - """ - Test that ResponseApplyPatchToolCall items from the Responses API are - correctly converted to ChatCompletions-style tool calls by the bridge. - - This is a regression test for a bug where litellm.completion() with a - responses/ model prefix crashed when the model returned an - apply_patch_call, because _convert_response_output_to_choices did not - handle ResponseApplyPatchToolCall items. The model DID use the tool, - but the bridge silently dropped it (or raised an error), while the - native litellm.responses() path worked correctly. - """ - import json - from unittest.mock import Mock - - from openai.types.responses.response_apply_patch_tool_call import ( - OperationCreateFile, - ) - from openai.types.responses.response_output_item import ( - ResponseApplyPatchToolCall, - ) - - from litellm.completion_extras.litellm_responses_transformation.transformation import ( - LiteLLMResponsesTransformationHandler, - ) - from litellm.types.llms.openai import ( - InputTokensDetails, - OutputTokensDetails, - ResponseAPIUsage, - ResponsesAPIResponse, - ) - from litellm.types.utils import ModelResponse, Usage - - handler = LiteLLMResponsesTransformationHandler() - - # Build an apply_patch_call item like the model would return - operation = OperationCreateFile( - diff="--- /dev/null\n+++ b/hello.py\n@@ -0,0 +1 @@\n+print('hello world')\n", - path="hello.py", - type="create_file", - ) - apply_patch_item = ResponseApplyPatchToolCall( - id="apc_001", - call_id="call_patch_hello", - operation=operation, - status="completed", - type="apply_patch_call", - ) - - # Minimal usage - usage = ResponseAPIUsage( - input_tokens=30, - input_tokens_details=InputTokensDetails(cached_tokens=0), - output_tokens=40, - output_tokens_details=OutputTokensDetails(reasoning_tokens=0), - total_tokens=70, - ) - - raw_response = ResponsesAPIResponse( - id="resp_apply_patch_test", - created_at=1234567890, - error=None, - incomplete_details=None, - instructions=None, - metadata={}, - model="gpt-5.2-codex", - object="response", - output=[apply_patch_item], - parallel_tool_calls=True, - temperature=1.0, - tool_choice="auto", - tools=[], - top_p=1.0, - max_output_tokens=None, - previous_response_id=None, - reasoning=None, - status="completed", - text=None, - truncation="disabled", - usage=usage, - user=None, - store=True, - background=False, - ) - - model_response = ModelResponse( - id="chatcmpl-apply-patch", - created=1234567890, - model=None, - object="chat.completion", - choices=[], - usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), - ) - - logging_obj = Mock() - - result = handler.transform_response( - model="gpt-5.2-codex", - raw_response=raw_response, - model_response=model_response, - logging_obj=logging_obj, - request_data={"model": "gpt-5.2-codex"}, - messages=[ - {"role": "system", "content": "You are a coding assistant."}, - {"role": "user", "content": "Create hello.py"}, - ], - optional_params={}, - litellm_params={}, - encoding=Mock(), - ) - - # Should have exactly one choice with finish_reason="tool_calls" - assert len(result.choices) == 1, f"Expected 1 choice, got {len(result.choices)}" - - choice = result.choices[0] - assert choice.finish_reason == "tool_calls" - - # The choice should contain one tool call for apply_patch - tool_calls = choice.message.tool_calls - assert tool_calls is not None, "tool_calls should not be None" - assert len(tool_calls) == 1, f"Expected 1 tool_call, got {len(tool_calls)}" - - tc = tool_calls[0] - assert tc["id"] == "call_patch_hello" - assert tc["type"] == "function" - assert tc["function"]["name"] == "apply_patch" - - # The operation should be serialised as JSON in arguments - args = json.loads(tc["function"]["arguments"]) - assert args["type"] == "create_file" - assert args["path"] == "hello.py" - assert "print('hello world')" in args["diff"] def test_multi_tool_call_stream_no_premature_finish(): """ Regression test for multi-tool-call streaming bug. @@ -1961,35 +1778,3 @@ def test_parallel_tool_calls_comprehensive_streaming_integration(): ) print("✓ Parallel tool calls with split argument deltas stream correctly end-to-end") - - -def test_map_optional_params_preserves_reasoning_summary(): - """Test that reasoning_effort dict with summary field is preserved. - - Regression test for: User reported that summary field was being dropped - when routing to Responses API. The dict format should be fully preserved. - """ - from litellm.completion_extras.litellm_responses_transformation.transformation import ( - LiteLLMResponsesTransformationHandler, - ) - from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams - - handler = LiteLLMResponsesTransformationHandler() - - optional_params = { - "stream": False, - "tools": [{"type": "function", "function": {"name": "test_tool"}}], - "tool_choice": "auto", - "reasoning_effort": {"effort": "high", "summary": "detailed"}, - } - - responses_api_request = ResponsesAPIOptionalRequestParams() - handler._map_optional_params_to_responses_api_request( - optional_params, responses_api_request - ) - - # Verify reasoning_effort dict with summary was fully preserved - assert "reasoning" in responses_api_request - assert responses_api_request["reasoning"] == {"effort": "high", "summary": "detailed"} - assert responses_api_request["reasoning"]["effort"] == "high" - assert responses_api_request["reasoning"]["summary"] == "detailed" diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 635359563b..25f3d1364f 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -192,23 +192,6 @@ def test_azure_gpt5_1_series_temperature_handling(config: AzureOpenAIGPT5Config) assert params["temperature"] == 0.6 -def test_azure_gpt5_4_drops_reasoning_effort_when_tools_present(config: AzureOpenAIGPT5Config): - """Azure Chat Completions: gpt-5.4+ drops reasoning_effort when tools are present. - - OpenAI routes tools+reasoning to Responses API; Azure does not, so we drop reasoning_effort. - """ - tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] - params = config.map_openai_params( - non_default_params={"reasoning_effort": "high", "tools": tools}, - optional_params={}, - model="gpt5_series/gpt-5.4", - drop_params=False, - api_version="2024-05-01-preview", - ) - assert "reasoning_effort" not in params - assert params["tools"] == tools - - def test_azure_gpt5_reasoning_effort_none_error(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5 (non-5.1) raises error for reasoning_effort='none' when drop_params=False.""" with pytest.raises(litellm.utils.UnsupportedParamsError): diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 9892a0403b..345f3ae7c5 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -43,29 +43,6 @@ def test_transform_usage(): ) assert openai_usage._cache_creation_input_tokens == usage["cacheWriteInputTokens"] assert openai_usage._cache_read_input_tokens == usage["cacheReadInputTokens"] - # completion_tokens_details should always be populated - assert openai_usage.completion_tokens_details is not None - assert openai_usage.completion_tokens_details.reasoning_tokens == 0 - assert openai_usage.completion_tokens_details.text_tokens == usage["outputTokens"] - - -def test_transform_usage_with_reasoning_content(): - """Test that completion_tokens_details correctly tracks reasoning vs text tokens.""" - usage = ConverseTokenUsageBlock( - **{ - "inputTokens": 10, - "outputTokens": 100, - "totalTokens": 110, - } - ) - config = AmazonConverseConfig() - reasoning_text = "Let me think about this step by step." - openai_usage = config._transform_usage(usage, reasoning_content=reasoning_text) - assert openai_usage.completion_tokens_details is not None - assert openai_usage.completion_tokens_details.reasoning_tokens > 0 - assert openai_usage.completion_tokens_details.text_tokens == ( - usage["outputTokens"] - openai_usage.completion_tokens_details.reasoning_tokens - ) def test_transform_system_message(): @@ -3193,33 +3170,6 @@ def test_transform_request_with_output_config(): assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" -def test_output_config_snake_case_stripped_from_bedrock_converse_request(): - """Test that output_config (snake_case) is stripped from Bedrock Converse requests. - - Bedrock Converse API doesn't support the output_config parameter (Anthropic-only). - Nova and other Converse models reject requests with extraneous output_config. - """ - config = AmazonConverseConfig() - messages = [{"role": "user", "content": "test"}] - optional_params = { - "output_config": {"effort": "high"}, - } - - result = config._transform_request( - model="us.amazon.nova-pro-v1:0", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={}, - ) - - # output_config must not appear in additionalModelRequestFields - additional = result.get("additionalModelRequestFields", {}) - assert "output_config" not in additional, ( - f"output_config should be stripped for Bedrock Converse, got: {list(additional.keys())}" - ) - - def test_transform_response_native_structured_output(): """Test response handling when model returns JSON as text content (native structured output).""" response_json = { diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 5d5aaa64c8..8006ffdff1 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -110,60 +110,6 @@ def test_get_supported_openai_params_reasoning_effort(): assert "reasoning_effort" not in unsupported_params -@pytest.mark.parametrize( - "api_base, expected_url_prefix", - [ - ( - "https://api.fireworks.ai/inference/v1", - "https://api.fireworks.ai/inference/v1/accounts/", - ), - ( - "https://api.fireworks.ai/inference/v1/", - "https://api.fireworks.ai/inference/v1/accounts/", - ), - ( - "https://custom-host.example.com/v1", - "https://custom-host.example.com/v1/accounts/", - ), - ( - "https://custom-host.example.com/api", - "https://custom-host.example.com/api/v1/accounts/", - ), - ], - ids=["default", "trailing-slash", "custom-with-v1", "custom-without-v1"], -) -def test_get_models_url_no_double_v1(api_base, expected_url_prefix): - """Ensure get_models never produces a /v1/v1/ URL segment (fixes #23106).""" - config = FireworksAIConfig() - account_id = "fireworks" - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] - } - - with ( - patch("litellm.module_level_client.get", return_value=mock_response) as mock_get, - patch( - "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", - side_effect=lambda key: { - "FIREWORKS_API_KEY": "test-key", - "FIREWORKS_API_BASE": api_base, - "FIREWORKS_ACCOUNT_ID": account_id, - }.get(key), - ), - ): - result = config.get_models(api_key="test-key", api_base=api_base) - - called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "") - assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" - assert called_url.startswith(expected_url_prefix), ( - f"URL {called_url} does not start with {expected_url_prefix}" - ) - assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] - - def test_transform_messages_helper_removes_provider_specific_fields(): """ Test that _transform_messages_helper removes provider_specific_fields from messages. diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 90fdc2d20d..39ff0a4f4d 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -13,7 +13,6 @@ from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config class TestOpenAIGPTConfig: @@ -325,195 +324,3 @@ class TestPromptCacheParams: ) assert optional_params.get("prompt_cache_key") == "my-cache-key" assert optional_params.get("prompt_cache_retention") == "24h" - - -class TestGPT5ReasoningEffortPreservation: - """Tests for GPT-5 reasoning_effort dict preservation for Responses API.""" - - def setup_method(self): - self.config = OpenAIGPT5Config() - - def test_reasoning_effort_string_preserved(self): - """Test that reasoning_effort as string is preserved.""" - non_default_params = {"reasoning_effort": "high"} - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.4", - drop_params=False, - ) - - # String format should be preserved - assert non_default_params.get("reasoning_effort") == "high" - - def test_reasoning_effort_dict_with_only_effort_normalized(self): - """Test that reasoning_effort dict with only 'effort' key is normalized to string.""" - non_default_params = {"reasoning_effort": {"effort": "high"}} - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.4", - drop_params=False, - ) - - # Dict with only 'effort' should be normalized to string - assert non_default_params.get("reasoning_effort") == "high" - - def test_reasoning_effort_dict_with_summary_preserved(self): - """Test that reasoning_effort dict with 'summary' field is preserved for Responses API. - - Regression test for: User reported that summary field was being dropped when - routing to Responses API. The dict format with additional fields should be - preserved so it can be properly handled by the Responses API transformation. - """ - non_default_params = {"reasoning_effort": {"effort": "high", "summary": "detailed"}} - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.4", - drop_params=False, - ) - - # Dict with additional fields should be preserved - assert non_default_params.get("reasoning_effort") == {"effort": "high", "summary": "detailed"} - assert isinstance(non_default_params.get("reasoning_effort"), dict) - assert non_default_params["reasoning_effort"]["effort"] == "high" - assert non_default_params["reasoning_effort"]["summary"] == "detailed" - - def test_reasoning_effort_dict_with_generate_summary_preserved(self): - """Test that reasoning_effort dict with 'generate_summary' field is preserved.""" - non_default_params = {"reasoning_effort": {"effort": "medium", "generate_summary": "auto"}} - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.4", - drop_params=False, - ) - - # Dict with additional fields should be preserved - assert non_default_params.get("reasoning_effort") == {"effort": "medium", "generate_summary": "auto"} - assert isinstance(non_default_params.get("reasoning_effort"), dict) - - def test_reasoning_effort_dict_with_all_fields_preserved(self): - """Test that reasoning_effort dict with all fields is preserved.""" - non_default_params = { - "reasoning_effort": { - "effort": "high", - "summary": "detailed", - "generate_summary": "concise" - } - } - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.4", - drop_params=False, - ) - - # Dict with all fields should be preserved - reasoning = non_default_params.get("reasoning_effort") - assert isinstance(reasoning, dict) - assert reasoning["effort"] == "high" - assert reasoning["summary"] == "detailed" - assert reasoning["generate_summary"] == "concise" - - def test_reasoning_effort_dict_xhigh_triggers_validation(self): - """xhigh-dict: effective effort is extracted for model-support validation. - - When reasoning_effort={"effort": "xhigh", "summary": "detailed"} is passed to a model - that doesn't support xhigh (e.g. gpt-5.1), the xhigh guard must fire. - """ - import litellm - - non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} - optional_params = {} - - with pytest.raises(litellm.utils.UnsupportedParamsError): - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.1", - drop_params=False, - ) - - def test_reasoning_effort_dict_xhigh_dropped_when_requested(self): - """xhigh-dict with drop_params=True: reasoning_effort is dropped.""" - non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.1", - drop_params=True, - ) - - assert "reasoning_effort" not in non_default_params - - def test_reasoning_effort_dict_none_treated_as_none_for_tools(self): - """none-dict: {"effort": "none", "summary": "detailed"} is treated as effort=none. - - Tool-drop guard should NOT fire; reasoning_effort should be kept. - """ - tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] - non_default_params = {"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools} - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.4", - drop_params=False, - ) - - assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} - assert non_default_params.get("tools") == tools - - def test_reasoning_effort_dict_none_treated_as_none_for_sampling(self): - """none-dict: {"effort": "none", "summary": "detailed"} allows logprobs/top_p. - - Sampling-param guard should NOT fire; logprobs should be kept. - """ - non_default_params = { - "reasoning_effort": {"effort": "none", "summary": "detailed"}, - "logprobs": True, - } - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.1", - drop_params=False, - ) - - assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} - assert non_default_params.get("logprobs") is True - - def test_reasoning_effort_dict_none_allows_temperature(self): - """none-dict: {"effort": "none", "summary": "detailed"} allows non-default temperature.""" - non_default_params = { - "reasoning_effort": {"effort": "none", "summary": "detailed"}, - "temperature": 0.5, - } - optional_params = {} - - self.config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5.1", - drop_params=False, - ) - - assert optional_params.get("temperature") == 0.5 - assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 13d2ebab14..b136f8774b 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -324,11 +324,10 @@ def test_gpt5_4_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): assert params["reasoning_effort"] == "xhigh" -def test_gpt5_preserves_reasoning_effort_dict_with_summary(config: OpenAIConfig): - """Dict with summary/generate_summary is preserved for Responses API. +def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): + """Chat completion API expects reasoning_effort as a string, not a dict. Config/deployments may pass Responses API format: {'effort': 'high', 'summary': 'detailed'}. - We preserve the full dict so it reaches the Responses API transformation. """ params = config.map_openai_params( non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, @@ -336,82 +335,18 @@ def test_gpt5_preserves_reasoning_effort_dict_with_summary(config: OpenAIConfig) model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == {"effort": "high", "summary": "detailed"} + assert params["reasoning_effort"] == "high" -def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): - """Dict with effort='xhigh' triggers xhigh model-support validation. - - Regression: when reasoning_effort is a dict, effective_effort must be used for - the xhigh guard so validation is not silently skipped. - """ - with pytest.raises(litellm.utils.UnsupportedParamsError): - config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, - optional_params={}, - model="gpt-5.1", - drop_params=False, - ) - - -def test_gpt5_xhigh_dict_accepted_for_supported_model(config: OpenAIConfig): - """Dict with effort='xhigh' passes through for gpt-5.4+.""" - params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, - optional_params={}, - model="gpt-5.4", - drop_params=False, - ) - assert params["reasoning_effort"] == {"effort": "xhigh", "summary": "detailed"} - - -def test_gpt5_none_dict_with_tools_no_tool_drop(config: OpenAIConfig): - """Dict with effort='none' and tools: no tool-drop, reasoning_effort preserved. - - Regression: effective_effort='none' must be used for tool-drop guard so - {"effort": "none", "summary": "detailed"} is not incorrectly treated as non-none. - """ - tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] - params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools}, - optional_params={}, - model="gpt-5.4", - drop_params=False, - ) - assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} - assert params["tools"] == tools - - -def test_gpt5_none_dict_with_sampling_params_allowed(config: OpenAIConfig): - """Dict with effort='none' allows logprobs/top_p/top_logprobs. - - Regression: effective_effort='none' must be used for sampling guard so - {"effort": "none", "summary": "detailed"} does not incorrectly trigger sampling errors. - """ - params = config.map_openai_params( - non_default_params={ - "reasoning_effort": {"effort": "none", "summary": "detailed"}, - "logprobs": True, - "top_p": 0.9, - }, - optional_params={}, - model="gpt-5.1", - drop_params=False, - ) - assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} - assert params["logprobs"] is True - assert params["top_p"] == 0.9 - - -def test_gpt5_preserves_reasoning_effort_dict_with_summary_from_optional_params(config: OpenAIConfig): - """reasoning_effort dict with summary in optional_params is preserved.""" +def test_gpt5_normalizes_reasoning_effort_dict_from_optional_params(config: OpenAIConfig): + """reasoning_effort dict in optional_params (e.g. from model config) is normalized.""" params = config.map_openai_params( non_default_params={}, optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} + assert params["reasoning_effort"] == "medium" def test_gpt5_4_drops_reasoning_effort_when_tools_present(config: OpenAIConfig): diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py deleted file mode 100644 index 82c84af5e2..0000000000 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py +++ /dev/null @@ -1,243 +0,0 @@ -""" -Test cases for SageMaker embedding role assumption support - -This module tests that the SageMaker embedding handler properly supports -AWS IAM role assumption via aws_role_name and aws_session_name parameters, -matching the behavior of the completion handler. -""" - -import json -import os -import sys -from datetime import timezone -from unittest.mock import MagicMock, call, patch - -sys.path.insert(0, os.path.abspath("../../../../..")) - -from botocore.credentials import Credentials - -from litellm.llms.sagemaker.completion.handler import SagemakerLLM -from litellm.types.utils import EmbeddingResponse - - -class TestSagemakerEmbeddingRoleAssumption: - """Test that SageMaker embedding supports role assumption like completion does""" - - def setup_method(self): - self.sagemaker_llm = SagemakerLLM() - - def test_embedding_uses_load_credentials(self): - """ - Test that embedding() calls _load_credentials() to support role assumption. - This ensures aws_role_name and aws_session_name parameters are properly handled. - """ - # Mock credentials that would be returned after role assumption - mock_credentials = Credentials( - access_key="assumed-access-key", - secret_key="assumed-secret-key", - token="assumed-session-token", - ) - - # Mock the SageMaker client response - mock_sagemaker_client = MagicMock() - mock_sagemaker_client.invoke_endpoint.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) - ) - } - - # Mock boto3.Session to return our mock client - mock_session = MagicMock() - mock_session.client.return_value = mock_sagemaker_client - - with patch.object( - self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-east-1") - ) as mock_load_creds, patch("boto3.Session", return_value=mock_session): - - # Create mock logging object - mock_logging = MagicMock() - - optional_params = { - "aws_role_name": "arn:aws:iam::123456789012:role/TestRole", - "aws_session_name": "test-session", - } - - self.sagemaker_llm.embedding( - model="test-endpoint", - input=["hello world"], - model_response=EmbeddingResponse(), - print_verbose=print, - encoding=None, - logging_obj=mock_logging, - optional_params=optional_params, - ) - - # Verify _load_credentials was called with the optional_params - mock_load_creds.assert_called_once() - - # Verify boto3.Session was created with the assumed credentials - mock_session_calls = mock_session.client.call_args_list - assert len(mock_session_calls) == 1 - assert mock_session_calls[0] == call(service_name="sagemaker-runtime") - - def test_embedding_role_assumption_with_sts(self): - """ - Test the full role assumption flow for embeddings, similar to completion. - Verifies that STS assume_role is called when aws_role_name is provided. - """ - # Mock the STS client for role assumption - mock_sts_client = MagicMock() - - # Mock the STS response with proper expiration handling - mock_expiry = MagicMock() - mock_expiry.tzinfo = timezone.utc - time_diff = MagicMock() - time_diff.total_seconds.return_value = 3600 - mock_expiry.__sub__ = MagicMock(return_value=time_diff) - - mock_sts_response = { - "Credentials": { - "AccessKeyId": "assumed-access-key", - "SecretAccessKey": "assumed-secret-key", - "SessionToken": "assumed-session-token", - "Expiration": mock_expiry, - } - } - mock_sts_client.assume_role.return_value = mock_sts_response - - # Mock the SageMaker client response - mock_sagemaker_client = MagicMock() - mock_sagemaker_client.invoke_endpoint.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) - ) - } - - # Mock boto3.Session for SageMaker client creation - mock_session = MagicMock() - mock_session.client.return_value = mock_sagemaker_client - - def mock_boto3_client(service_name, **kwargs): - if service_name == "sts": - return mock_sts_client - return mock_sagemaker_client - - with patch("boto3.client", side_effect=mock_boto3_client), \ - patch("boto3.Session", return_value=mock_session): - - mock_logging = MagicMock() - - optional_params = { - "aws_role_name": "arn:aws:iam::123456789012:role/CrossAccountRole", - "aws_session_name": "litellm-embedding-session", - "aws_region_name": "us-east-1", - } - - self.sagemaker_llm.embedding( - model="test-endpoint", - input=["hello world"], - model_response=EmbeddingResponse(), - print_verbose=print, - encoding=None, - logging_obj=mock_logging, - optional_params=optional_params, - ) - - # Verify STS assume_role was called with correct parameters - mock_sts_client.assume_role.assert_called_once() - call_args = mock_sts_client.assume_role.call_args - assert call_args[1]["RoleArn"] == "arn:aws:iam::123456789012:role/CrossAccountRole" - assert call_args[1]["RoleSessionName"] == "litellm-embedding-session" - - def test_embedding_without_role_assumption(self): - """ - Test that embedding works without role assumption when aws_role_name is not provided. - Should use default credentials from environment/instance profile. - """ - # Mock the SageMaker client response - mock_sagemaker_client = MagicMock() - mock_sagemaker_client.invoke_endpoint.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) - ) - } - - mock_session = MagicMock() - mock_session.client.return_value = mock_sagemaker_client - - # Mock credentials returned from environment - mock_credentials = Credentials( - access_key="env-access-key", - secret_key="env-secret-key", - token=None, - ) - - with patch.object( - self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-west-2") - ), patch("boto3.Session", return_value=mock_session): - - mock_logging = MagicMock() - - # No aws_role_name provided - optional_params = { - "aws_region_name": "us-west-2", - } - - result = self.sagemaker_llm.embedding( - model="test-endpoint", - input=["hello world"], - model_response=EmbeddingResponse(), - print_verbose=print, - encoding=None, - logging_obj=mock_logging, - optional_params=optional_params, - ) - - # Should still work and return embeddings - assert result is not None - - def test_embedding_session_created_with_assumed_credentials(self): - """ - Test that boto3.Session is created with the credentials from role assumption. - This verifies the credentials flow from _load_credentials to the SageMaker client. - """ - mock_credentials = Credentials( - access_key="assumed-key", - secret_key="assumed-secret", - token="assumed-token", - ) - - mock_sagemaker_client = MagicMock() - mock_sagemaker_client.invoke_endpoint.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) - ) - } - - with patch.object( - self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-east-1") - ), patch("boto3.Session") as mock_session_class: - - mock_session = MagicMock() - mock_session.client.return_value = mock_sagemaker_client - mock_session_class.return_value = mock_session - - mock_logging = MagicMock() - - self.sagemaker_llm.embedding( - model="test-endpoint", - input=["hello world"], - model_response=EmbeddingResponse(), - print_verbose=print, - encoding=None, - logging_obj=mock_logging, - optional_params={}, - ) - - # Verify Session was created with the assumed credentials - mock_session_class.assert_called_once_with( - aws_access_key_id="assumed-key", - aws_secret_access_key="assumed-secret", - aws_session_token="assumed-token", - region_name="us-east-1", - ) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index f3c82e439c..b264964b14 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -126,75 +126,6 @@ def test_vertex_ai_includes_labels(): -def test_extra_body_cache_not_forwarded_to_vertex_ai(): - """ - 'cache' inside extra_body is a LiteLLM-internal proxy caching control. - It must NOT be forwarded to the Vertex AI request body. - - Regression test for: "Invalid JSON payload received. Unknown name \"cache\": Cannot find field." - Vertex AI enforces a strict JSON schema and rejects any unknown field. - """ - messages = [{"role": "user", "content": "test"}] - optional_params = { - "extra_body": { - "cache": {"use-cache": True, "ttl": 86400}, # LiteLLM-internal - "some_vertex_param": "value", # legitimate provider extra - }, - } - litellm_params = {} - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params=litellm_params, - cached_content=None, - ) - - # 'cache' must be stripped — Vertex AI has no such field - assert "cache" not in result, ( - "extra_body.cache must not be forwarded to Vertex AI. " - "Vertex AI rejects it with 400: Unknown name \"cache\": Cannot find field." - ) - - # Other legitimate extra_body keys should still pass through - assert "some_vertex_param" in result - assert result["some_vertex_param"] == "value" - - # Core request fields must be present - assert "contents" in result - - -def test_extra_body_tags_not_forwarded_to_vertex_ai(): - """ - 'tags' inside extra_body is a LiteLLM-internal param for logging/tracking. - It must NOT be forwarded to the Vertex AI request body. - Documented in litellm_proxy.md: "Send tags by including them in the extra_body parameter" - """ - messages = [{"role": "user", "content": "test"}] - optional_params = { - "extra_body": { - "tags": ["user:alice", "env:prod"], - "custom_param": "allowed", - }, - } - litellm_params = {} - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params=litellm_params, - cached_content=None, - ) - - assert "tags" not in result - assert "custom_param" in result - assert result["custom_param"] == "allowed" - - def test_metadata_to_labels_vertex_only(): """Test that metadata->labels conversion only happens for Vertex AI""" messages = [{"role": "user", "content": "test"}] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index a104ac2257..de2ec13b4a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2093,150 +2093,3 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab assert spend_meta["tool_count_total"] == 1 assert spend_meta["allowed_server_count"] == 1 assert spend_meta["per_server_tool_counts"]["server_a"] == 1 - - -def test_tool_name_matches_case_insensitive(): - """Test that _tool_name_matches performs case-insensitive comparison. - - This is critical for OpenAPI-based MCP servers where: - 1. operationIds are often in camelCase (e.g., 'addPet', 'updatePet') - 2. Tool names are lowercased during registration (e.g., 'addpet', 'updatepet') - 3. allowed_tools configuration may use the original camelCase names - - Without case-insensitive matching, all tools would be filtered out. - """ - try: - from litellm.proxy._experimental.mcp_server.server import _tool_name_matches - except ImportError: - pytest.skip("MCP server not available") - - # Test case 1: Unprefixed tool name with camelCase in filter list - assert _tool_name_matches("addpet", ["addPet", "updatePet"]) is True - assert _tool_name_matches("updatepet", ["addPet", "updatePet"]) is True - assert _tool_name_matches("deletepet", ["addPet", "updatePet"]) is False - - # Test case 2: Prefixed tool name with camelCase in filter list - assert _tool_name_matches("per_store-addpet", ["addPet", "updatePet"]) is True - assert _tool_name_matches("per_store-updatepet", ["addPet", "updatePet"]) is True - assert _tool_name_matches("per_store-deletepet", ["addPet", "updatePet"]) is False - - # Test case 3: Mixed case variations - assert _tool_name_matches("findPetsByStatus", ["findpetsbystatus"]) is True - assert _tool_name_matches("findpetsbystatus", ["findPetsByStatus"]) is True - assert _tool_name_matches("FINDPETSBYSTATUS", ["findPetsByStatus"]) is True - - # Test case 4: Full prefixed name in filter list (case-insensitive) - assert _tool_name_matches("server-addPet", ["server-addpet"]) is True - assert _tool_name_matches("server-addpet", ["server-addPet"]) is True - - # Test case 5: Ensure non-matching names still don't match - assert _tool_name_matches("addpet", ["deletePet", "updatePet"]) is False - assert _tool_name_matches("server-addpet", ["deletePet", "updatePet"]) is False - - -def test_filter_tools_by_allowed_tools_case_insensitive(): - """Test that filter_tools_by_allowed_tools handles case-insensitive matching. - - Ensures that OpenAPI tools with lowercase names can be filtered using - camelCase allowed_tools configuration from the OpenAPI spec. - """ - try: - from litellm.proxy._experimental.mcp_server.server import ( - filter_tools_by_allowed_tools, - ) - from litellm.types.mcp_server.tool_registry import MCPTool - except ImportError: - pytest.skip("MCP server not available") - - # Mock handler function - def mock_handler(**kwargs): - return kwargs - - # Create mock tools with lowercase names (as registered from OpenAPI) - tools = [ - MCPTool( - name="per_store-addpet", - description="Add a pet", - input_schema={"type": "object"}, - handler=mock_handler, - ), - MCPTool( - name="per_store-updatepet", - description="Update a pet", - input_schema={"type": "object"}, - handler=mock_handler, - ), - MCPTool( - name="per_store-deletepet", - description="Delete a pet", - input_schema={"type": "object"}, - handler=mock_handler, - ), - MCPTool( - name="per_store-findpetsbystatus", - description="Find pets by status", - input_schema={"type": "object"}, - handler=mock_handler, - ), - ] - - # Create mock server with camelCase allowed_tools (as from OpenAPI spec) - server = MCPServer( - server_id="test-server", - name="per_store", - transport=MCPTransport.http, - allowed_tools=["addPet", "updatePet", "findPetsByStatus"], - ) - - # Filter tools - filtered_tools = filter_tools_by_allowed_tools(tools, server) - - # Should return 3 tools (case-insensitive match) - assert len(filtered_tools) == 3 - assert any(t.name == "per_store-addpet" for t in filtered_tools) - assert any(t.name == "per_store-updatepet" for t in filtered_tools) - assert any(t.name == "per_store-findpetsbystatus" for t in filtered_tools) - assert not any(t.name == "per_store-deletepet" for t in filtered_tools) - - -def test_filter_tools_by_allowed_tools_no_filter(): - """Test that filter_tools_by_allowed_tools returns all tools when no filter is set.""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - filter_tools_by_allowed_tools, - ) - from litellm.types.mcp_server.tool_registry import MCPTool - except ImportError: - pytest.skip("MCP server not available") - - # Mock handler function - def mock_handler(**kwargs): - return kwargs - - tools = [ - MCPTool( - name="fusion_litellm_mcp-model_list", - description="List models", - input_schema={"type": "object"}, - handler=mock_handler, - ), - MCPTool( - name="fusion_litellm_mcp-chat_completion", - description="Chat completion", - input_schema={"type": "object"}, - handler=mock_handler, - ), - ] - - # Server with no allowed_tools filter - server = MCPServer( - server_id="test-server", - name="fusion_litellm_mcp", - transport=MCPTransport.http, - allowed_tools=None, - ) - - filtered_tools = filter_tools_by_allowed_tools(tools, server) - - # Should return all tools when no filter is configured - assert len(filtered_tools) == 2 diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index c43621d7f7..193b014f03 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -21,140 +21,6 @@ def test_get_team_models_for_all_models_and_team_only_models(): assert set(result) == set(combined_models) -def test_get_team_models_all_proxy_models_includes_access_groups(): - """ - When a team has 'all-proxy-models' and include_model_access_groups=True, - the result should include model access group names (e.g. 'claude-model-group') - in addition to individual model names. - """ - from litellm.proxy.auth.model_checks import get_team_models - - team_models = ["all-proxy-models"] - proxy_model_list = ["model1", "model2"] - model_access_groups = { - "group-a": ["model1"], - "group-b": ["model2"], - } - - result = get_team_models( - team_models, proxy_model_list, model_access_groups, include_model_access_groups=True - ) - assert "group-a" in result - assert "group-b" in result - assert "model1" in result - assert "model2" in result - assert len(result) == len(set(result)), "result should have no duplicates" - - -def test_get_team_models_all_proxy_models_without_include_flag(): - """ - When include_model_access_groups=False, access group names should NOT - appear in the result even with 'all-proxy-models'. - """ - from litellm.proxy.auth.model_checks import get_team_models - - team_models = ["all-proxy-models"] - proxy_model_list = ["model1", "model2"] - model_access_groups = { - "group-a": ["model1"], - "group-b": ["model2"], - } - - result = get_team_models( - team_models, proxy_model_list, model_access_groups, include_model_access_groups=False - ) - assert "group-a" not in result - assert "group-b" not in result - assert "model1" in result - assert "model2" in result - - -def test_get_key_models_all_proxy_models_includes_access_groups(): - """ - When a key has 'all-proxy-models' and include_model_access_groups=True, - the result should include model access group names. - """ - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.auth.model_checks import get_key_models - - user_api_key_dict = UserAPIKeyAuth( - models=["all-proxy-models"], - api_key="test-key", - ) - proxy_model_list = ["model1", "model2"] - model_access_groups = { - "group-a": ["model1"], - } - - result = get_key_models( - user_api_key_dict=user_api_key_dict, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - include_model_access_groups=True, - ) - assert "group-a" in result - assert "model1" in result - assert "model2" in result - assert len(result) == len(set(result)), "result should have no duplicates" - - -def test_get_key_models_passes_include_model_access_groups(): - """ - When a key explicitly has an access group name in its models list and - include_model_access_groups=True, the group name should be retained - (not stripped by _get_models_from_access_groups). - """ - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.auth.model_checks import get_key_models - - user_api_key_dict = UserAPIKeyAuth( - models=["group-a"], - api_key="test-key", - ) - proxy_model_list = ["model1", "model2"] - model_access_groups = { - "group-a": ["model1", "model2"], - } - - result = get_key_models( - user_api_key_dict=user_api_key_dict, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - include_model_access_groups=True, - ) - assert "group-a" in result - assert "model1" in result - assert "model2" in result - - -def test_get_key_models_does_not_mutate_input(): - """ - get_key_models must not mutate user_api_key_dict.models in-place. - _get_models_from_access_groups uses .pop()/.extend() which would corrupt - cached UserAPIKeyAuth objects if all_models were an alias instead of a copy. - """ - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.auth.model_checks import get_key_models - - original_models = ["group-a", "extra-model"] - user_api_key_dict = UserAPIKeyAuth( - models=list(original_models), # give it a list - api_key="test-key", - ) - model_access_groups = { - "group-a": ["model1", "model2"], - } - - _ = get_key_models( - user_api_key_dict=user_api_key_dict, - proxy_model_list=["model1", "model2"], - model_access_groups=model_access_groups, - include_model_access_groups=False, - ) - # The original models list on the auth object must be unchanged - assert user_api_key_dict.models == original_models - - @pytest.mark.parametrize( "key_models,team_models,proxy_model_list,model_list,expected", [ diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 71420c23ad..7ec97ddc18 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -2369,42 +2369,3 @@ def test_get_registered_pass_through_route_with_custom_root(): # Clean up _registered_pass_through_routes.clear() - - -def test_mapped_pass_through_routes_with_server_root_path(): - """ - Mapped passthrough routes (vertex_ai, bedrock, etc) should match - even when SERVER_ROOT_PATH is set and the incoming route is prefixed. - - Regression test for https://github.com/BerriAI/litellm/issues/22272 - """ - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( - InitPassThroughEndpointHelpers, - ) - - with patch( - "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path" - ) as mock_get_root: - mock_get_root.return_value = "/litellm" - - # prefixed route should match mapped routes like /vertex_ai - assert ( - InitPassThroughEndpointHelpers.is_registered_pass_through_route( - "/litellm/vertex_ai/v1/projects/foo" - ) - is True - ) - assert ( - InitPassThroughEndpointHelpers.is_registered_pass_through_route( - "/litellm/bedrock/model/invoke" - ) - is True - ) - - # bare route without prefix should not match when root is set - assert ( - InitPassThroughEndpointHelpers.is_registered_pass_through_route( - "/vertex_ai/v1/projects/foo" - ) - is False - ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 3249a7ec79..9a64e641b5 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1071,10 +1071,9 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs) # When redaction is enabled and response is a dict (not ModelResponse), - # perform_redaction redacts content in-place within the choices structure + # perform_redaction returns {"text": "redacted-by-litellm"} parsed_response = json.loads(response_result) - assert parsed_response["choices"][0]["message"]["content"] == "redacted-by-litellm" - assert parsed_response["choices"][0]["message"]["role"] == "assistant" + assert parsed_response == {"text": "redacted-by-litellm"} @patch("litellm.secret_managers.main.get_secret_bool") diff --git a/tests/test_litellm/proxy/test_openapi_schema_validation.py b/tests/test_litellm/proxy/test_openapi_schema_validation.py deleted file mode 100644 index aafe08f303..0000000000 --- a/tests/test_litellm/proxy/test_openapi_schema_validation.py +++ /dev/null @@ -1,142 +0,0 @@ -""" -Test that the OpenAPI schema generated by FastAPI is valid for specific endpoints. - -Validates fixes for: -- /spend/calculate response schema (must use proper OpenAPI 3.x content wrapper) -- /credentials/by_model/{model_id} path parameter (must not leak credential_name) - -Related issue: https://github.com/BerriAI/litellm/issues/21305 -""" - -import pytest - - -class TestSpendCalculateOpenAPISchema: - """Test /spend/calculate response schema is valid OpenAPI 3.x.""" - - def test_response_schema_has_description(self): - """The 200 response must have a 'description' field per OpenAPI 3.x spec.""" - from litellm.proxy.spend_tracking.spend_management_endpoints import router - - for route in router.routes: - if hasattr(route, "path") and route.path == "/spend/calculate": - responses = route.responses or {} - response_200 = responses.get(200, {}) - assert "description" in response_200, ( - "/spend/calculate 200 response must have a 'description' field" - ) - break - else: - pytest.fail("/spend/calculate route not found in router") - - def test_response_schema_has_content_wrapper(self): - """The 200 response must use 'content' wrapper, not bare properties.""" - from litellm.proxy.spend_tracking.spend_management_endpoints import router - - for route in router.routes: - if hasattr(route, "path") and route.path == "/spend/calculate": - responses = route.responses or {} - response_200 = responses.get(200, {}) - # Must NOT have 'cost' as a top-level key (invalid OpenAPI) - assert "cost" not in response_200, ( - "/spend/calculate 200 response must not have 'cost' as a " - "top-level property - use 'content' wrapper instead" - ) - # Must have 'content' wrapper - assert "content" in response_200, ( - "/spend/calculate 200 response must have a 'content' field" - ) - content = response_200["content"] - assert "application/json" in content - assert "schema" in content["application/json"] - break - else: - pytest.fail("/spend/calculate route not found in router") - - -class TestCredentialEndpointsOpenAPISchema: - """Test /credentials endpoints have correct path parameters.""" - - def test_by_name_and_by_model_are_separate_handlers(self): - """ - /credentials/by_name/{credential_name} and /credentials/by_model/{model_id} - must be separate handler functions so each only declares its own path params. - """ - from litellm.proxy.credential_endpoints.endpoints import router - - by_name_routes = [] - by_model_routes = [] - for route in router.routes: - if not hasattr(route, "path"): - continue - if "by_name" in route.path: - by_name_routes.append(route) - elif "by_model" in route.path: - by_model_routes.append(route) - - assert len(by_name_routes) == 1, "Expected exactly one by_name route" - assert len(by_model_routes) == 1, "Expected exactly one by_model route" - - # They must be different endpoint functions - by_name_endpoint = by_name_routes[0].endpoint - by_model_endpoint = by_model_routes[0].endpoint - assert by_name_endpoint is not by_model_endpoint, ( - "by_name and by_model must be separate handler functions " - "to avoid path parameter conflicts in OpenAPI spec" - ) - - def test_by_model_route_does_not_require_credential_name(self): - """ - The /credentials/by_model/{model_id} route must NOT have - credential_name as a parameter. - """ - import inspect - from litellm.proxy.credential_endpoints.endpoints import ( - get_credential_by_model, - ) - - sig = inspect.signature(get_credential_by_model) - param_names = list(sig.parameters.keys()) - assert "credential_name" not in param_names, ( - "get_credential_by_model must not have a credential_name parameter" - ) - - def test_by_name_route_does_not_require_model_id(self): - """ - The /credentials/by_name/{credential_name} route must NOT have - model_id as a parameter. - """ - import inspect - from litellm.proxy.credential_endpoints.endpoints import ( - get_credential_by_name, - ) - - sig = inspect.signature(get_credential_by_name) - param_names = list(sig.parameters.keys()) - assert "model_id" not in param_names, ( - "get_credential_by_name must not have a model_id parameter" - ) - - def test_by_model_has_model_id_path_param(self): - """The by_model handler must accept model_id as a path parameter.""" - import inspect - from litellm.proxy.credential_endpoints.endpoints import ( - get_credential_by_model, - ) - - sig = inspect.signature(get_credential_by_model) - assert "model_id" in sig.parameters, ( - "get_credential_by_model must have a model_id parameter" - ) - - def test_by_name_has_credential_name_path_param(self): - """The by_name handler must accept credential_name as a path parameter.""" - import inspect - from litellm.proxy.credential_endpoints.endpoints import ( - get_credential_by_name, - ) - - sig = inspect.signature(get_credential_by_name) - assert "credential_name" in sig.parameters, ( - "get_credential_by_name must have a credential_name parameter" - ) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index a931a9bc93..6d6162437c 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1774,128 +1774,3 @@ class TestStreamingIDConsistency: # Verify it matches the cached ID assert iterator._cached_item_id is not None assert iterator._cached_item_id == text_done_id - - def test_parallel_tool_calls_merged_into_single_assistant_message(self): - """ - Regression test: multi-turn parallel tool calls via the Responses API must - produce a single assistant message with all tool_calls, not one assistant - message per function_call item. - - When the model responds with two parallel tool calls (e.g. get_weather for - SF and NYC), the next Responses API request includes two consecutive - function_call items followed by two function_call_output items. - - Without the fix each function_call becomes its own assistant message, - producing back-to-back assistant messages that Anthropic/Vertex AI rejects: - "tool_use ids were found without tool_result blocks immediately after". - """ - input_items = [ - {"type": "message", "role": "user", "content": "Weather in SF and NYC?"}, - # Two parallel tool calls from the previous assistant response - { - "type": "function_call", - "call_id": "toolu_01", - "name": "get_weather", - "arguments": '{"city": "SF"}', - }, - { - "type": "function_call", - "call_id": "toolu_02", - "name": "get_weather", - "arguments": '{"city": "NYC"}', - }, - # Tool results - {"type": "function_call_output", "call_id": "toolu_01", "output": "72°F"}, - {"type": "function_call_output", "call_id": "toolu_02", "output": "55°F"}, - ] - - messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=input_items - ) - - roles = [ - m.get("role") if isinstance(m, dict) else getattr(m, "role", None) - for m in messages - ] - - # Must not have two consecutive assistant messages - for i in range(len(roles) - 1): - assert not ( - roles[i] == "assistant" and roles[i + 1] == "assistant" - ), f"Consecutive assistant messages at indices {i} and {i+1}: {roles}" - - # The single assistant message must contain BOTH tool_calls - assistant_messages = [ - m for m in messages - if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) - == "assistant" - ] - assert len(assistant_messages) == 1, ( - f"Expected 1 assistant message, got {len(assistant_messages)}" - ) - - assistant_msg = assistant_messages[0] - tool_calls = ( - assistant_msg.get("tool_calls") - if isinstance(assistant_msg, dict) - else getattr(assistant_msg, "tool_calls", None) - ) - assert tool_calls is not None and len(tool_calls) == 2, ( - f"Expected 2 tool_calls in the merged assistant message, got: {tool_calls}" - ) - - call_ids = [ - (tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)) - for tc in tool_calls - ] - assert "toolu_01" in call_ids, f"toolu_01 missing from tool_calls: {call_ids}" - assert "toolu_02" in call_ids, f"toolu_02 missing from tool_calls: {call_ids}" - - # Both tool messages must be present - tool_messages = [ - m for m in messages - if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) - == "tool" - ] - assert len(tool_messages) == 2, ( - f"Expected 2 tool messages, got {len(tool_messages)}" - ) - - def test_single_tool_call_still_works_after_merge_fix(self): - """ - Ensure the parallel-tool-call merging fix does not break the existing - single-tool-call path. - """ - input_items = [ - {"type": "message", "role": "user", "content": "Weather in SF?"}, - { - "type": "function_call", - "call_id": "toolu_01", - "name": "get_weather", - "arguments": '{"city": "SF"}', - }, - {"type": "function_call_output", "call_id": "toolu_01", "output": "72°F"}, - ] - - messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=input_items - ) - - roles = [ - m.get("role") if isinstance(m, dict) else getattr(m, "role", None) - for m in messages - ] - - assert "user" in roles - assert "assistant" in roles - assert "tool" in roles - - assistant_messages = [m for m in messages if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) == "assistant"] - assert len(assistant_messages) == 1 - - tool_calls = ( - assistant_messages[0].get("tool_calls") - if isinstance(assistant_messages[0], dict) - else getattr(assistant_messages[0], "tool_calls", None) - ) - assert tool_calls is not None and len(tool_calls) == 1 diff --git a/tests/test_litellm/test_model_cost_aliases.py b/tests/test_litellm/test_model_cost_aliases.py deleted file mode 100644 index c6702ab13e..0000000000 --- a/tests/test_litellm/test_model_cost_aliases.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -Tests for the ``aliases`` feature in the model cost map. - -The ``_expand_model_aliases`` function processes ``aliases`` lists from model -entries, creating shared dict references for alias entries at load time. -""" - -import logging - -import pytest - -from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _make_model_cost(**entries) -> dict: - """Build a small model_cost dict from keyword args (model_name → info).""" - return dict(entries) - - -# --------------------------------------------------------------------------- -# Core expansion behaviour -# --------------------------------------------------------------------------- - - -class TestExpandModelAliases: - """Unit tests for _expand_model_aliases.""" - - def test_basic_expansion(self): - """Aliases are added as top-level entries in model_cost.""" - model_cost = { - "my-model-latest": { - "aliases": ["my-model-20250101"], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - assert "my-model-20250101" in result - assert result["my-model-20250101"]["input_cost_per_token"] == 1e-06 - assert result["my-model-20250101"]["litellm_provider"] == "test" - - def test_multiple_aliases(self): - """A single entry can declare multiple aliases.""" - model_cost = { - "provider/model-latest": { - "aliases": ["provider/model-v1", "provider/model-v2"], - "input_cost_per_token": 5e-06, - "litellm_provider": "provider", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - assert "provider/model-v1" in result - assert "provider/model-v2" in result - - def test_shared_dict_reference(self): - """Alias entries share the same dict object as the canonical entry (no copy).""" - model_cost = { - "canonical-model": { - "aliases": ["alias-model"], - "input_cost_per_token": 2e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - assert result["alias-model"] is result["canonical-model"] - - def test_aliases_key_removed(self): - """The ``aliases`` key is removed from the entry after expansion.""" - model_cost = { - "my-model": { - "aliases": ["my-model-alias"], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - assert "aliases" not in result["my-model"] - assert "aliases" not in result["my-model-alias"] - - def test_entries_without_aliases_unchanged(self): - """Entries with no ``aliases`` key are left untouched.""" - model_cost = { - "plain-model": { - "input_cost_per_token": 3e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - assert "plain-model" in result - assert result["plain-model"]["input_cost_per_token"] == 3e-06 - assert len(result) == 1 - - def test_empty_aliases_list(self): - """An empty ``aliases`` list is treated the same as no aliases.""" - model_cost = { - "model-a": { - "aliases": [], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - assert len(result) == 1 - assert "model-a" in result - - -# --------------------------------------------------------------------------- -# Conflict handling -# --------------------------------------------------------------------------- - - -class TestAliasConflicts: - """Tests for alias conflict detection and handling.""" - - def test_alias_conflicts_with_canonical_entry(self, caplog): - """Alias that matches an existing canonical entry is skipped with a warning.""" - model_cost = { - "model-latest": { - "aliases": ["model-dated"], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - "model-dated": { - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - with caplog.at_level(logging.WARNING, logger="LiteLLM"): - result = _expand_model_aliases(model_cost) - - # The canonical "model-dated" entry is preserved, not overwritten - assert "model-dated" in result - assert "alias conflict" in caplog.text.lower() - - def test_duplicate_alias_across_entries(self, caplog): - """Same alias claimed by two different entries: second one is skipped.""" - model_cost = { - "model-a": { - "aliases": ["shared-alias"], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - "model-b": { - "aliases": ["shared-alias"], - "input_cost_per_token": 2e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - with caplog.at_level(logging.WARNING, logger="LiteLLM"): - result = _expand_model_aliases(model_cost) - - # "shared-alias" should point to model-a (first one wins) - assert "shared-alias" in result - assert result["shared-alias"]["input_cost_per_token"] == 1e-06 - - def test_canonical_entry_not_overwritten_by_alias(self): - """An alias must never overwrite an existing canonical entry's data.""" - original_cost = 9.99e-06 - model_cost = { - "existing-model": { - "input_cost_per_token": original_cost, - "litellm_provider": "test", - "mode": "chat", - }, - "other-model": { - "aliases": ["existing-model"], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - # Original entry must be preserved - assert result["existing-model"]["input_cost_per_token"] == original_cost - - -# --------------------------------------------------------------------------- -# Integration with model_cost dict mutation -# --------------------------------------------------------------------------- - - -class TestAliasIntegration: - """Higher-level tests verifying aliases work with the model_cost dict.""" - - def test_mutation_through_alias_visible_on_canonical(self): - """Since alias is a shared reference, mutations are visible on both.""" - model_cost = { - "canonical": { - "aliases": ["alias"], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - # Mutate via alias - result["alias"]["input_cost_per_token"] = 999 - assert result["canonical"]["input_cost_per_token"] == 999 - - def test_mixed_entries_with_and_without_aliases(self): - """A model_cost dict with a mix of aliased and plain entries.""" - model_cost = { - "model-with-alias": { - "aliases": ["alias-1", "alias-2"], - "input_cost_per_token": 1e-06, - "litellm_provider": "test", - "mode": "chat", - }, - "plain-model": { - "input_cost_per_token": 2e-06, - "litellm_provider": "test", - "mode": "chat", - }, - } - result = _expand_model_aliases(model_cost) - - assert len(result) == 4 # 2 canonical + 2 aliases - assert "alias-1" in result - assert "alias-2" in result - assert "plain-model" in result - assert "model-with-alias" in result - - def test_expand_on_empty_dict(self): - """Expanding an empty dict returns an empty dict.""" - assert _expand_model_aliases({}) == {} diff --git a/tests/test_litellm/test_router_retry_non_retryable_errors.py b/tests/test_litellm/test_router_retry_non_retryable_errors.py deleted file mode 100644 index 20a1c979a0..0000000000 --- a/tests/test_litellm/test_router_retry_non_retryable_errors.py +++ /dev/null @@ -1,251 +0,0 @@ -""" -Test that the Router retry loop correctly handles non-retryable errors. - -Verifies that: -1. Non-retryable errors (e.g., 400 ContextWindowExceeded) inside the retry loop - break out immediately instead of being swallowed. -2. original_exception is updated to the latest error, not stuck on the first. -3. Retryable errors (e.g., 429 RateLimitError) still retry normally. - -Regression tests for https://github.com/BerriAI/litellm/issues/21343 -""" - -from unittest.mock import AsyncMock, patch - -import pytest - -import litellm -from litellm import Router - - -def _make_rate_limit_error(message="Rate limited"): - """Create a RateLimitError for testing.""" - return litellm.RateLimitError( - message=message, - llm_provider="bedrock", - model="anthropic.claude-v2", - ) - - -def _make_context_window_error(message="prompt is too long: 1205821 tokens > 200000"): - """Create a ContextWindowExceededError for testing.""" - return litellm.ContextWindowExceededError( - message=message, - llm_provider="vertex_ai", - model="claude-3-opus", - ) - - -def _make_bad_request_error(message="Invalid request"): - """Create a BadRequestError for testing.""" - return litellm.BadRequestError( - message=message, - llm_provider="openai", - model="gpt-4", - ) - - -def _make_not_found_error(message="Model not found"): - """Create a NotFoundError for testing.""" - return litellm.NotFoundError( - message=message, - llm_provider="openai", - model="gpt-99", - ) - - -def _create_router(num_retries=2): - """Create a Router with two deployments for testing.""" - return Router( - model_list=[ - { - "model_name": "test-model", - "litellm_params": { - "model": "openai/gpt-4", - "api_key": "fake-key-1", - }, - }, - { - "model_name": "test-model", - "litellm_params": { - "model": "openai/gpt-4", - "api_key": "fake-key-2", - }, - }, - ], - num_retries=num_retries, - ) - - -def _base_kwargs(): - """Return kwargs required by async_function_with_retries.""" - return { - "model": "test-model", - "messages": [{"role": "user", "content": "test"}], - "original_function": AsyncMock(), - "metadata": {}, - } - - -@pytest.mark.asyncio -async def test_non_retryable_error_in_retry_loop_raises_immediately(): - """ - When a non-retryable error (400 ContextWindowExceeded) occurs inside the - retry loop, the router should raise it immediately instead of swallowing it - and raising the original error. - - Scenario: First call -> 429, Retry -> 400 (non-retryable) - Expected: ContextWindowExceededError is raised, NOT RateLimitError - """ - router = _create_router(num_retries=2) - - rate_limit_error = _make_rate_limit_error() - context_window_error = _make_context_window_error() - - call_count = 0 - - async def mock_make_call(*args, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise rate_limit_error - else: - raise context_window_error - - with patch.object(router, "make_call", side_effect=mock_make_call), \ - patch.object(router, "_async_get_healthy_deployments", - return_value=(["d1", "d2"], ["d1", "d2"])), \ - patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ - patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): - with pytest.raises(litellm.ContextWindowExceededError): - await router.async_function_with_retries( - num_retries=2, - **_base_kwargs(), - ) - - -@pytest.mark.asyncio -async def test_bad_request_error_in_retry_loop_raises_immediately(): - """ - A generic 400 BadRequestError inside the retry loop should also break out - immediately since 400 is not retryable. - """ - router = _create_router(num_retries=2) - - rate_limit_error = _make_rate_limit_error() - bad_request_error = _make_bad_request_error() - - call_count = 0 - - async def mock_make_call(*args, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise rate_limit_error - else: - raise bad_request_error - - with patch.object(router, "make_call", side_effect=mock_make_call), \ - patch.object(router, "_async_get_healthy_deployments", - return_value=(["d1", "d2"], ["d1", "d2"])), \ - patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ - patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): - with pytest.raises(litellm.BadRequestError): - await router.async_function_with_retries( - num_retries=2, - **_base_kwargs(), - ) - - -@pytest.mark.asyncio -async def test_original_exception_updated_to_latest_error(): - """ - When all retries are exhausted with retryable errors, the LAST error - should be raised, not the first one. - """ - router = _create_router(num_retries=2) - - call_count = 0 - - async def mock_make_call(*args, **kwargs): - nonlocal call_count - call_count += 1 - raise _make_rate_limit_error(f"Rate limit attempt {call_count}") - - with patch.object(router, "make_call", side_effect=mock_make_call), \ - patch.object(router, "_async_get_healthy_deployments", - return_value=(["d1", "d2"], ["d1", "d2"])), \ - patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ - patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): - with pytest.raises(litellm.RateLimitError) as exc_info: - await router.async_function_with_retries( - num_retries=2, - **_base_kwargs(), - ) - # Should be the LAST error, not the first - assert "Rate limit attempt 3" in str(exc_info.value) - - -@pytest.mark.asyncio -async def test_retryable_errors_still_retry_normally(): - """ - Retryable errors (429 RateLimitError) should still be retried the - configured number of times before raising. - """ - router = _create_router(num_retries=3) - - call_count = 0 - - async def mock_make_call(*args, **kwargs): - nonlocal call_count - call_count += 1 - raise _make_rate_limit_error(f"Rate limit attempt {call_count}") - - with patch.object(router, "make_call", side_effect=mock_make_call), \ - patch.object(router, "_async_get_healthy_deployments", - return_value=(["d1", "d2"], ["d1", "d2"])), \ - patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ - patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): - with pytest.raises(litellm.RateLimitError): - await router.async_function_with_retries( - num_retries=3, - **_base_kwargs(), - ) - - # Initial call + 3 retries = 4 total calls - assert call_count == 4 - - -@pytest.mark.asyncio -async def test_not_found_error_in_retry_loop_raises_immediately(): - """ - A 404 NotFoundError inside the retry loop should break out immediately. - """ - router = _create_router(num_retries=2) - - rate_limit_error = _make_rate_limit_error() - not_found_error = _make_not_found_error() - - call_count = 0 - - async def mock_make_call(*args, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise rate_limit_error - else: - raise not_found_error - - with patch.object(router, "make_call", side_effect=mock_make_call), \ - patch.object(router, "_async_get_healthy_deployments", - return_value=(["d1", "d2"], ["d1", "d2"])), \ - patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ - patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): - with pytest.raises(litellm.NotFoundError): - await router.async_function_with_retries( - num_retries=2, - **_base_kwargs(), - ) - - # Only 2 calls: initial + first retry that hits non-retryable - assert call_count == 2 diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 35f87e8770..4fd513b0d2 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -262,8 +262,8 @@ it("should display user email correctly", async () => { }); }); -it("should show loading message only on initial load (isPending)", () => { - // Mock initial loading state +it("should show skeleton loaders when isLoading is true", () => { + // Mock loading state mockUseKeys.mockReturnValue({ data: null, isPending: true, @@ -283,7 +283,7 @@ it("should show loading message only on initial load (isPending)", () => { renderWithProviders(); - // Check that loading message is shown on initial load + // Check that loading message is shown expect(screen.getByText("🚅 Loading keys...")).toBeInTheDocument(); // Check that actual key data is not shown @@ -795,79 +795,3 @@ describe("pagination display – total count and page count", () => { }); }); }); - -describe("refetch button", () => { - it("should show Fetch button in normal state", () => { - renderWithProviders(); - - const fetchButton = screen.getByTitle("Fetch data"); - expect(fetchButton).toBeInTheDocument(); - expect(fetchButton).not.toBeDisabled(); - expect(screen.getByText("Fetch")).toBeInTheDocument(); - }); - - it("should show Fetching state and keep table data visible during refetch", () => { - mockUseKeys.mockReturnValue({ - data: { - keys: [mockKey], - total_count: 1, - current_page: 1, - total_pages: 1, - } as KeysResponse, - isPending: false, - isFetching: true, - refetch: vi.fn(), - } as any); - - renderWithProviders(); - - // Button should show "Fetching" and be disabled - expect(screen.getByText("Fetching")).toBeInTheDocument(); - const fetchButton = screen.getByTitle("Fetch data"); - expect(fetchButton).toBeDisabled(); - - // Table data should still be visible (stale data) - expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); - - // "Loading keys..." should NOT appear during refetch - expect(screen.queryByText("🚅 Loading keys...")).not.toBeInTheDocument(); - }); - - it("should call refetch when Fetch button is clicked", () => { - const mockRefetch = vi.fn(); - mockUseKeys.mockReturnValue({ - data: { - keys: [mockKey], - total_count: 1, - current_page: 1, - total_pages: 1, - } as KeysResponse, - isPending: false, - isFetching: false, - refetch: mockRefetch, - } as any); - - renderWithProviders(); - - const fetchButton = screen.getByTitle("Fetch data"); - fireEvent.click(fetchButton); - - expect(mockRefetch).toHaveBeenCalledTimes(1); - }); - - it("should show Fetch button enabled on error so user can retry", () => { - mockUseKeys.mockReturnValue({ - data: null, - isPending: false, - isFetching: false, - isError: true, - refetch: vi.fn(), - } as any); - - renderWithProviders(); - - const fetchButton = screen.getByTitle("Fetch data"); - expect(fetchButton).not.toBeDisabled(); - expect(screen.getByText("Fetch")).toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index d9c590938f..fe9d58b979 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -24,9 +24,9 @@ import { TableRow, Text, } from "@tremor/react"; -import { InfoCircleOutlined, SyncOutlined } from "@ant-design/icons"; -import { Button as AntButton, Popover, Skeleton, Tooltip } from "antd"; -import React, { useEffect, useDeferredValue, useMemo, useState } from "react"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { Popover, Skeleton, Tooltip } from "antd"; +import React, { useEffect, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { useFilterLogic } from "../key_team_helpers/filter_logic"; import { PaginatedKeyAliasSelect } from "../KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect"; @@ -81,7 +81,6 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo data: keys, isPending: isLoading, isFetching, - isError, refetch, } = useKeys(tablePagination.pageIndex + 1, tablePagination.pageSize, { sortBy: sortBy || undefined, @@ -98,15 +97,6 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo organizations, }); - // Defer the transition so the button stays in loading state until the table - // has rendered with the new data (mirrors the spend-logs pattern) - const isFetchingDeferred = useDeferredValue(isFetching); - const isButtonLoading = (isFetching || isFetchingDeferred) && !isError; - - const handleRefresh = () => { - refetch(); - }; - const totalCount = filteredTotalCount ?? keys?.total_count ?? 0; // Add a useEffect to call refresh when a key is created @@ -616,28 +606,16 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo
-
- {isLoading ? ( - - ) : ( - - Showing {rangeLabel} of {totalCount} results - - )} - - } - onClick={handleRefresh} - disabled={isButtonLoading} - title="Fetch data" - > - {isButtonLoading ? "Fetching" : "Fetch"} - -
+ {isLoading || isFetching ? ( + + ) : ( + + Showing {rangeLabel} of {totalCount} results + + )}
- {isLoading ? ( + {isLoading || isFetching ? ( ) : ( @@ -645,24 +623,24 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo )} - {isLoading ? ( + {isLoading || isFetching ? ( ) : ( )} - {isLoading ? ( + {isLoading || isFetching ? ( ) : (