From 8d5db4f712cf94eeacee130eb3557b910155096d Mon Sep 17 00:00:00 2001 From: jtsaw <166962251+jtsaw@users.noreply.github.com> Date: Tue, 17 Feb 2026 21:10:50 -0800 Subject: [PATCH 01/40] fix handling of ResponseApplyPatchToolCall in completion bridge (#20913) * fix handling of ResponseApplyPatchToolCall in completion bridge * refactor * style: fix black formatting * fix: clean up lint errors in test file (unused imports, print statements, formatting) * refactor: extract _map_optional_params_to_responses_api to fix PLR0915 * what * this linter cannot be me * revert cause idk what's going on * weird * idk why this got removed * revert more stuff * revert pt 3 --- .../transformation.py | 19 +- .../transformation.py | 77 +++++--- ...responses_transformation_transformation.py | 174 ++++++++++++++++-- 3 files changed, 225 insertions(+), 45 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index e546a0dbb0..5de9a48985 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -401,6 +401,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ResponseOutputMessage, ResponseReasoningItem, ) + from openai.types.responses.response_output_item import ResponseApplyPatchToolCall from litellm.types.utils import Choices, Message @@ -457,6 +458,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): accumulated_tool_calls.append(tool_call_dict) tool_call_index += 1 + elif isinstance(item, ResponseApplyPatchToolCall): + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + tool_call_dict = LiteLLMCompletionResponsesConfig.convert_apply_patch_tool_call_to_chat_completion_tool_call( + tool_call_item=item, + index=tool_call_index, + ) + accumulated_tool_calls.append(tool_call_dict) + tool_call_index += 1 + elif isinstance(item, dict) and handle_raw_dict_callback is not None: # Handle raw dict responses (e.g., from GPT-5 Codex) choice, index = handle_raw_dict_callback(item=item, index=index) @@ -533,7 +546,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raw_response.usage ), ) - + # Preserve hidden params from the ResponsesAPIResponse, especially the headers # which contain important provider information like x-request-id raw_response_hidden_params = getattr(raw_response, "_hidden_params", {}) @@ -550,7 +563,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): model_response._hidden_params[key] = merged_headers else: model_response._hidden_params[key] = value - + return model_response def get_model_response_iterator( @@ -855,7 +868,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return {"format": {"type": "text"}} return None - + @staticmethod def _convert_annotations_to_chat_format( annotations: Optional[List[Any]], diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b8379b28c3..8daa8e49d1 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -291,14 +291,14 @@ class LiteLLMCompletionResponsesConfig: ) _messages = litellm_completion_request.get("messages") or [] session_messages = chat_completion_session.get("messages") or [] - + # If session messages are empty (e.g., no database in test environment), # we still need to process the new input messages # Store original _messages before combining for safety check original_new_messages = _messages.copy() if _messages else [] - + combined_messages = session_messages + _messages - + # Fix: Ensure tool_results have corresponding tool_calls in previous assistant message # Pass tools parameter to help reconstruct tool_calls if not in cache tools = litellm_completion_request.get("tools") or [] @@ -306,7 +306,7 @@ class LiteLLMCompletionResponsesConfig: messages=combined_messages, tools=tools ) - + # Safety check: Ensure we don't end up with empty messages # This can happen when using previous_response_id without a database (e.g., in tests) # and session messages are empty but new input messages exist @@ -337,7 +337,7 @@ class LiteLLMCompletionResponsesConfig: model=litellm_completion_request.get("model", ""), llm_provider=litellm_completion_request.get("custom_llm_provider", ""), ) - + litellm_completion_request["messages"] = combined_messages litellm_completion_request["litellm_trace_id"] = chat_completion_session.get( "litellm_session_id" @@ -385,8 +385,8 @@ class LiteLLMCompletionResponsesConfig: ######################################################### # If Input Item is a Tool Call Output, add it to the tool_call_output_messages list - # preserving the ordering of tool call outputs. Some models require the tool - # result to immediately follow the assistant tool call. + # preserving the ordering of tool call outputs. Some models require the tool + # result to immediately follow the assistant tool call. ######################################################### if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( input_item=_input @@ -743,47 +743,47 @@ class LiteLLMCompletionResponsesConfig: ) -> List[Union[AllMessageValues, GenericChatCompletionMessage, ChatCompletionResponseMessage]]: """ Ensure that tool_result messages have corresponding tool_calls in the previous assistant message. - + This is critical for Anthropic API which requires that each tool_result block has a corresponding tool_use block in the previous assistant message. - + Args: messages: List of messages that may include tool_result messages tools: Optional list of tools that can be used to reconstruct tool_calls if not in cache - + Returns: List of messages with tool_calls added to assistant messages when needed """ if not messages: return messages - + # Create a deep copy to avoid modifying the original import copy fixed_messages = copy.deepcopy(messages) messages_to_remove = [] - + # Count non-tool messages to avoid removing all messages # This prevents empty messages list when using previous_response_id without a database non_tool_messages_count = sum( 1 for msg in fixed_messages if msg.get("role") != "tool" ) - + for i, message in enumerate(fixed_messages): # Only process tool messages - check role first to narrow the type if message.get("role") != "tool": continue - + # At this point, we know it's a tool message, so it should have tool_call_id # Use get() with default to safely access tool_call_id tool_call_id_raw = message.get("tool_call_id") if isinstance(message, dict) else getattr(message, "tool_call_id", None) tool_call_id: str = ( str(tool_call_id_raw) if tool_call_id_raw is not None else "" ) - + prev_assistant_idx = LiteLLMCompletionResponsesConfig._find_previous_assistant_idx( fixed_messages, i ) - + # Try to recover empty tool_call_id from previous assistant message if not tool_call_id and prev_assistant_idx is not None: prev_assistant = fixed_messages[prev_assistant_idx] @@ -798,7 +798,7 @@ class LiteLLMCompletionResponsesConfig: message_dict["tool_call_id"] = tool_call_id elif hasattr(message, "tool_call_id"): setattr(message, "tool_call_id", tool_call_id) - + # Only remove messages with empty tool_call_id if we have other non-tool messages # This prevents ending up with an empty messages list when using previous_response_id # without a database (e.g., in tests where session messages are empty) @@ -810,7 +810,7 @@ class LiteLLMCompletionResponsesConfig: # If no non-tool messages, keep the tool message even with empty call_id # The API will return a proper error message about the missing tool_use block continue - + # Check if the previous assistant message has the corresponding tool_call # This needs to run for ALL tool messages with a valid tool_call_id, # not just those that had an empty tool_call_id initially @@ -819,12 +819,12 @@ class LiteLLMCompletionResponsesConfig: tool_calls = LiteLLMCompletionResponsesConfig._get_tool_calls_list( prev_assistant ) - + if not LiteLLMCompletionResponsesConfig._check_tool_call_exists( tool_calls, tool_call_id ): _tool_use_definition = TOOL_CALLS_CACHE.get_cache(key=tool_call_id) - + if not _tool_use_definition and tools: _tool_use_definition = ( LiteLLMCompletionResponsesConfig._reconstruct_tool_call_from_tools( @@ -849,11 +849,11 @@ class LiteLLMCompletionResponsesConfig: LiteLLMCompletionResponsesConfig._add_tool_call_to_assistant( prev_assistant, tool_call_chunk ) - + # Remove messages with empty tool_call_id that couldn't be fixed for idx in reversed(messages_to_remove): fixed_messages.pop(idx) - + return fixed_messages @staticmethod @@ -1454,6 +1454,39 @@ class LiteLLMCompletionResponsesConfig: return tool_call_dict + @staticmethod + def convert_apply_patch_tool_call_to_chat_completion_tool_call( + tool_call_item: Any, + index: int = 0, + ) -> Dict[str, Any]: + """ + Convert ResponseApplyPatchToolCall to ChatCompletionToolCallChunk format. + + The operation (create_file / update_file / delete_file) is serialised + as JSON so it appears in function.arguments, just like any other + tool call. + + Args: + tool_call_item: ResponseApplyPatchToolCall object with call_id and operation + index: The index of this tool call + + Returns: + Dictionary in ChatCompletionToolCallChunk format + """ + import json + + operation_dict = tool_call_item.operation.model_dump() + tool_call_dict: Dict[str, Any] = { + "id": tool_call_item.call_id, + "function": { + "name": "apply_patch", + "arguments": json.dumps(operation_dict), + }, + "type": "function", + "index": index, + } + return tool_call_dict + @staticmethod def transform_chat_completion_response_to_responses_api_response( request_input: Union[str, ResponseInputParam], diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index f8a082ee30..25e8a1f330 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1012,11 +1012,11 @@ def test_multiple_tool_calls_in_single_choice(): def test_map_reasoning_effort_adds_summary_detailed(): """ Test that _map_reasoning_effort behavior with reasoning_auto_summary flag. - + By default (flag=False), summary should NOT be added to avoid: 1. Breaking for users without verified OpenAI orgs (400 errors) 2. Making requests more expensive by including summary reasoning tokens - + When flag is enabled (flag=True or env var), summary="detailed" is added. """ import os @@ -1030,64 +1030,64 @@ def test_map_reasoning_effort_adds_summary_detailed(): # Test all string effort levels - DEFAULT BEHAVIOR (no summary) effort_levels = ["none", "low", "medium", "high", "xhigh", "minimal"] - + # Save original flag value original_flag = litellm.reasoning_auto_summary original_env = os.environ.get("LITELLM_REASONING_AUTO_SUMMARY") - + try: # Test 1: Default behavior (flag=False, no env var) - NO summary litellm.reasoning_auto_summary = False if "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] - + for effort in effort_levels: result = handler._map_reasoning_effort(effort) - + assert result is not None, f"Result should not be None for effort={effort}" assert result["effort"] == effort, f"Effort should be {effort}" assert "summary" not in result, f"Summary should NOT be present by default for effort={effort}" - + print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}' (no summary by default)") - + # Test 2: With flag enabled - summary IS added litellm.reasoning_auto_summary = True - + for effort in effort_levels: result = handler._map_reasoning_effort(effort) - + assert result is not None, f"Result should not be None for effort={effort}" assert result["effort"] == effort, f"Effort should be {effort}" assert result["summary"] == "detailed", f"Summary should be 'detailed' when flag is enabled for effort={effort}" - + print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}', summary='detailed' (flag enabled)") - + # Test 3: With env var enabled (flag disabled) - summary IS added litellm.reasoning_auto_summary = False os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" - + result = handler._map_reasoning_effort("high") assert result["summary"] == "detailed", "Summary should be 'detailed' when env var is enabled" print("✓ LITELLM_REASONING_AUTO_SUMMARY env var works correctly") - + # Test 4: Dict input is passed through as-is (no modification) litellm.reasoning_auto_summary = False if "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] - + dict_input = {"effort": "high", "summary": "custom_summary"} result_dict = handler._map_reasoning_effort(dict_input) assert result_dict["effort"] == "high" assert result_dict["summary"] == "custom_summary" print("✓ Dict input is passed through without modification") - + # Test 5: None/unknown values return None result_unknown = handler._map_reasoning_effort("unknown_value") assert result_unknown is None print("✓ Unknown reasoning_effort values return None") - + print("✓ All reasoning_effort behaviors work correctly with flag/env var control") - + finally: # Restore original values litellm.reasoning_auto_summary = original_flag @@ -1100,10 +1100,10 @@ def test_map_reasoning_effort_adds_summary_detailed(): def test_transform_response_preserves_annotations(): """ Test that annotations from Responses API are preserved when transforming to Chat Completions format. - + This is a regression test for the bug where annotations (like url_citation) were being dropped during the transformation from ResponsesAPIResponse to ModelResponse. - + The fix ensures annotations are extracted from ResponseOutputText content items and passed through to the Message object in the Chat Completions response. """ @@ -1278,3 +1278,137 @@ def test_transform_response_preserves_annotations(): assert result.usage.total_tokens == 30 print("✓ Annotations from Responses API are correctly preserved in Chat Completions format") + + +def test_apply_patch_tool_call_converted_to_chat_completion_tool_call(): + """ + Test that ResponseApplyPatchToolCall items from the Responses API are + correctly converted to ChatCompletions-style tool calls by the bridge. + + This is a regression test for a bug where litellm.completion() with a + responses/ model prefix crashed when the model returned an + apply_patch_call, because _convert_response_output_to_choices did not + handle ResponseApplyPatchToolCall items. The model DID use the tool, + but the bridge silently dropped it (or raised an error), while the + native litellm.responses() path worked correctly. + """ + import json + from unittest.mock import Mock + + from openai.types.responses.response_apply_patch_tool_call import ( + OperationCreateFile, + ) + from openai.types.responses.response_output_item import ( + ResponseApplyPatchToolCall, + ) + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.llms.openai import ( + InputTokensDetails, + OutputTokensDetails, + ResponseAPIUsage, + ResponsesAPIResponse, + ) + from litellm.types.utils import ModelResponse, Usage + + handler = LiteLLMResponsesTransformationHandler() + + # Build an apply_patch_call item like the model would return + operation = OperationCreateFile( + diff="--- /dev/null\n+++ b/hello.py\n@@ -0,0 +1 @@\n+print('hello world')\n", + path="hello.py", + type="create_file", + ) + apply_patch_item = ResponseApplyPatchToolCall( + id="apc_001", + call_id="call_patch_hello", + operation=operation, + status="completed", + type="apply_patch_call", + ) + + # Minimal usage + usage = ResponseAPIUsage( + input_tokens=30, + input_tokens_details=InputTokensDetails(cached_tokens=0), + output_tokens=40, + output_tokens_details=OutputTokensDetails(reasoning_tokens=0), + total_tokens=70, + ) + + raw_response = ResponsesAPIResponse( + id="resp_apply_patch_test", + created_at=1234567890, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model="gpt-5.2-codex", + object="response", + output=[apply_patch_item], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + max_output_tokens=None, + previous_response_id=None, + reasoning=None, + status="completed", + text=None, + truncation="disabled", + usage=usage, + user=None, + store=True, + background=False, + ) + + model_response = ModelResponse( + id="chatcmpl-apply-patch", + created=1234567890, + model=None, + object="chat.completion", + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + + logging_obj = Mock() + + result = handler.transform_response( + model="gpt-5.2-codex", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.2-codex"}, + messages=[ + {"role": "system", "content": "You are a coding assistant."}, + {"role": "user", "content": "Create hello.py"}, + ], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + # Should have exactly one choice with finish_reason="tool_calls" + assert len(result.choices) == 1, f"Expected 1 choice, got {len(result.choices)}" + + choice = result.choices[0] + assert choice.finish_reason == "tool_calls" + + # The choice should contain one tool call for apply_patch + tool_calls = choice.message.tool_calls + assert tool_calls is not None, "tool_calls should not be None" + assert len(tool_calls) == 1, f"Expected 1 tool_call, got {len(tool_calls)}" + + tc = tool_calls[0] + assert tc["id"] == "call_patch_hello" + assert tc["type"] == "function" + assert tc["function"]["name"] == "apply_patch" + + # The operation should be serialised as JSON in arguments + args = json.loads(tc["function"]["arguments"]) + assert args["type"] == "create_file" + assert args["path"] == "hello.py" + assert "print('hello world')" in args["diff"] From ae613b2d36f92a700077884234b0076af67cfb85 Mon Sep 17 00:00:00 2001 From: Atharva Jaiswal <92455570+AtharvaJaiswal005@users.noreply.github.com> Date: Wed, 18 Feb 2026 12:28:08 +0530 Subject: [PATCH 02/40] fix(router): break retry loop on non-retryable errors (#21370) The retry loop in async_function_with_retries catches all exceptions blindly and continues retrying even for non-retryable errors like 400 ContextWindowExceeded or 404 NotFoundError. This causes the original retryable error to be raised instead of the actual non-retryable one. Changes: - Update original_exception to latest error on each retry attempt - Add should_retry_this_error() check inside the retry loop to break out immediately on non-retryable errors - Respect _retry_policy_applies precedence Fixes #21343 --- litellm/router.py | 22 ++ .../test_router_retry_non_retryable_errors.py | 251 ++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 tests/test_litellm/test_router_retry_non_retryable_errors.py diff --git a/litellm/router.py b/litellm/router.py index 888c97ca0b..3fac761ce6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5149,6 +5149,10 @@ class Router: return response except Exception as e: + # Always track the latest error so we raise the most + # recent exception instead of the first one. + original_exception = e + ## LOGGING kwargs = self.log_retry(kwargs=kwargs, e=e) remaining_retries = num_retries - current_attempt - 1 @@ -5163,6 +5167,24 @@ class Router: ) else: _healthy_deployments = [] + + # Check if this error is non-retryable (e.g., 400 context + # window exceeded). If so, raise immediately instead of + # continuing the retry loop. Respect retry policy + # precedence - only check when no retry policy applies. + if not _retry_policy_applies: + try: + self.should_retry_this_error( + error=e, + healthy_deployments=_healthy_deployments, + all_deployments=_all_deployments, + context_window_fallbacks=context_window_fallbacks, + regular_fallbacks=fallbacks, + content_policy_fallbacks=content_policy_fallbacks, + ) + except Exception: + raise e + _timeout = self._time_to_sleep_before_retry( e=e, remaining_retries=remaining_retries, diff --git a/tests/test_litellm/test_router_retry_non_retryable_errors.py b/tests/test_litellm/test_router_retry_non_retryable_errors.py new file mode 100644 index 0000000000..20a1c979a0 --- /dev/null +++ b/tests/test_litellm/test_router_retry_non_retryable_errors.py @@ -0,0 +1,251 @@ +""" +Test that the Router retry loop correctly handles non-retryable errors. + +Verifies that: +1. Non-retryable errors (e.g., 400 ContextWindowExceeded) inside the retry loop + break out immediately instead of being swallowed. +2. original_exception is updated to the latest error, not stuck on the first. +3. Retryable errors (e.g., 429 RateLimitError) still retry normally. + +Regression tests for https://github.com/BerriAI/litellm/issues/21343 +""" + +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm import Router + + +def _make_rate_limit_error(message="Rate limited"): + """Create a RateLimitError for testing.""" + return litellm.RateLimitError( + message=message, + llm_provider="bedrock", + model="anthropic.claude-v2", + ) + + +def _make_context_window_error(message="prompt is too long: 1205821 tokens > 200000"): + """Create a ContextWindowExceededError for testing.""" + return litellm.ContextWindowExceededError( + message=message, + llm_provider="vertex_ai", + model="claude-3-opus", + ) + + +def _make_bad_request_error(message="Invalid request"): + """Create a BadRequestError for testing.""" + return litellm.BadRequestError( + message=message, + llm_provider="openai", + model="gpt-4", + ) + + +def _make_not_found_error(message="Model not found"): + """Create a NotFoundError for testing.""" + return litellm.NotFoundError( + message=message, + llm_provider="openai", + model="gpt-99", + ) + + +def _create_router(num_retries=2): + """Create a Router with two deployments for testing.""" + return Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "fake-key-1", + }, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "fake-key-2", + }, + }, + ], + num_retries=num_retries, + ) + + +def _base_kwargs(): + """Return kwargs required by async_function_with_retries.""" + return { + "model": "test-model", + "messages": [{"role": "user", "content": "test"}], + "original_function": AsyncMock(), + "metadata": {}, + } + + +@pytest.mark.asyncio +async def test_non_retryable_error_in_retry_loop_raises_immediately(): + """ + When a non-retryable error (400 ContextWindowExceeded) occurs inside the + retry loop, the router should raise it immediately instead of swallowing it + and raising the original error. + + Scenario: First call -> 429, Retry -> 400 (non-retryable) + Expected: ContextWindowExceededError is raised, NOT RateLimitError + """ + router = _create_router(num_retries=2) + + rate_limit_error = _make_rate_limit_error() + context_window_error = _make_context_window_error() + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise rate_limit_error + else: + raise context_window_error + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.ContextWindowExceededError): + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + + +@pytest.mark.asyncio +async def test_bad_request_error_in_retry_loop_raises_immediately(): + """ + A generic 400 BadRequestError inside the retry loop should also break out + immediately since 400 is not retryable. + """ + router = _create_router(num_retries=2) + + rate_limit_error = _make_rate_limit_error() + bad_request_error = _make_bad_request_error() + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise rate_limit_error + else: + raise bad_request_error + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.BadRequestError): + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + + +@pytest.mark.asyncio +async def test_original_exception_updated_to_latest_error(): + """ + When all retries are exhausted with retryable errors, the LAST error + should be raised, not the first one. + """ + router = _create_router(num_retries=2) + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + raise _make_rate_limit_error(f"Rate limit attempt {call_count}") + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.RateLimitError) as exc_info: + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + # Should be the LAST error, not the first + assert "Rate limit attempt 3" in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_retryable_errors_still_retry_normally(): + """ + Retryable errors (429 RateLimitError) should still be retried the + configured number of times before raising. + """ + router = _create_router(num_retries=3) + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + raise _make_rate_limit_error(f"Rate limit attempt {call_count}") + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.RateLimitError): + await router.async_function_with_retries( + num_retries=3, + **_base_kwargs(), + ) + + # Initial call + 3 retries = 4 total calls + assert call_count == 4 + + +@pytest.mark.asyncio +async def test_not_found_error_in_retry_loop_raises_immediately(): + """ + A 404 NotFoundError inside the retry loop should break out immediately. + """ + router = _create_router(num_retries=2) + + rate_limit_error = _make_rate_limit_error() + not_found_error = _make_not_found_error() + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise rate_limit_error + else: + raise not_found_error + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.NotFoundError): + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + + # Only 2 calls: initial + first retry that hits non-retryable + assert call_count == 2 From 42afba9cdd3ec78270c84da0e6e915d158105434 Mon Sep 17 00:00:00 2001 From: Atharva Jaiswal <92455570+AtharvaJaiswal005@users.noreply.github.com> Date: Wed, 18 Feb 2026 12:29:01 +0530 Subject: [PATCH 03/40] Fix invalid OpenAPI schema for /spend/calculate and /credentials endpoints (#21369) - /spend/calculate: wrap response in proper OpenAPI 3.x content structure - /credentials: split stacked route decorators into separate handlers to eliminate path parameter conflict between by_name and by_model routes --- .../proxy/credential_endpoints/endpoints.py | 97 ++++++------ .../spend_management_endpoints.py | 20 ++- .../proxy/test_openapi_schema_validation.py | 142 ++++++++++++++++++ 3 files changed, 209 insertions(+), 50 deletions(-) create mode 100644 tests/test_litellm/proxy/test_openapi_schema_validation.py diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 9f228bb118..5fa9546e00 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -142,17 +142,47 @@ async def get_credentials( tags=["credential management"], response_model=CredentialItem, ) +async def get_credential_by_name( + request: Request, + fastapi_response: Response, + credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + [BETA] endpoint. This might change unexpectedly. + """ + try: + for credential in litellm.credential_list: + if credential.credential_name == credential_name: + masked_credential = CredentialItem( + credential_name=credential.credential_name, + credential_values=_get_masked_values( + credential.credential_values, + unmasked_length=4, + number_of_asterisks=4, + ), + credential_info=credential.credential_info, + ) + return masked_credential + raise HTTPException( + status_code=404, + detail="Credential not found. Got credential name: " + credential_name, + ) + except Exception as e: + verbose_proxy_logger.exception(e) + raise handle_exception_on_proxy(e) + + @router.get( "/credentials/by_model/{model_id}", dependencies=[Depends(user_api_key_auth)], tags=["credential management"], response_model=CredentialItem, ) -async def get_credential( +async def get_credential_by_model( request: Request, fastapi_response: Response, - credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), - model_id: Optional[str] = None, + model_id: str = Path(..., description="The model ID to look up credentials for"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -161,48 +191,25 @@ async def get_credential( from litellm.proxy.proxy_server import llm_router try: - if model_id: - if llm_router is None: - raise HTTPException(status_code=500, detail="LLM router not found") - model = llm_router.get_deployment(model_id) - if model is None: - raise HTTPException(status_code=404, detail="Model not found") - credential_values = llm_router.get_deployment_credentials(model_id) - if credential_values is None: - raise HTTPException(status_code=404, detail="Model not found") - masked_credential_values = _get_masked_values( - credential_values, - unmasked_length=4, - number_of_asterisks=4, - ) - credential = CredentialItem( - credential_name="{}-credential-{}".format(model.model_name, model_id), - credential_values=masked_credential_values, - credential_info={}, - ) - # return credential object - return credential - elif credential_name: - for credential in litellm.credential_list: - if credential.credential_name == credential_name: - masked_credential = CredentialItem( - credential_name=credential.credential_name, - credential_values=_get_masked_values( - credential.credential_values, - unmasked_length=4, - number_of_asterisks=4, - ), - credential_info=credential.credential_info, - ) - return masked_credential - raise HTTPException( - status_code=404, - detail="Credential not found. Got credential name: " + credential_name, - ) - else: - raise HTTPException( - status_code=404, detail="Credential name or model ID required" - ) + if llm_router is None: + raise HTTPException(status_code=500, detail="LLM router not found") + model = llm_router.get_deployment(model_id) + if model is None: + raise HTTPException(status_code=404, detail="Model not found") + credential_values = llm_router.get_deployment_credentials(model_id) + if credential_values is None: + raise HTTPException(status_code=404, detail="Model not found") + masked_credential_values = _get_masked_values( + credential_values, + unmasked_length=4, + number_of_asterisks=4, + ) + credential = CredentialItem( + credential_name="{}-credential-{}".format(model.model_name, model_id), + credential_values=masked_credential_values, + credential_info={}, + ) + return credential except Exception as e: verbose_proxy_logger.exception(e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 08aaa85169..92770a5c80 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1461,11 +1461,21 @@ async def _get_spend_report_for_time_range( dependencies=[Depends(user_api_key_auth)], responses={ 200: { - "cost": { - "description": "The calculated cost", - "example": 0.0, - "type": "float", - } + "description": "The calculated cost", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "cost": { + "type": "number", + "description": "The calculated cost", + "example": 0.0, + } + }, + } + } + }, } }, ) diff --git a/tests/test_litellm/proxy/test_openapi_schema_validation.py b/tests/test_litellm/proxy/test_openapi_schema_validation.py new file mode 100644 index 0000000000..aafe08f303 --- /dev/null +++ b/tests/test_litellm/proxy/test_openapi_schema_validation.py @@ -0,0 +1,142 @@ +""" +Test that the OpenAPI schema generated by FastAPI is valid for specific endpoints. + +Validates fixes for: +- /spend/calculate response schema (must use proper OpenAPI 3.x content wrapper) +- /credentials/by_model/{model_id} path parameter (must not leak credential_name) + +Related issue: https://github.com/BerriAI/litellm/issues/21305 +""" + +import pytest + + +class TestSpendCalculateOpenAPISchema: + """Test /spend/calculate response schema is valid OpenAPI 3.x.""" + + def test_response_schema_has_description(self): + """The 200 response must have a 'description' field per OpenAPI 3.x spec.""" + from litellm.proxy.spend_tracking.spend_management_endpoints import router + + for route in router.routes: + if hasattr(route, "path") and route.path == "/spend/calculate": + responses = route.responses or {} + response_200 = responses.get(200, {}) + assert "description" in response_200, ( + "/spend/calculate 200 response must have a 'description' field" + ) + break + else: + pytest.fail("/spend/calculate route not found in router") + + def test_response_schema_has_content_wrapper(self): + """The 200 response must use 'content' wrapper, not bare properties.""" + from litellm.proxy.spend_tracking.spend_management_endpoints import router + + for route in router.routes: + if hasattr(route, "path") and route.path == "/spend/calculate": + responses = route.responses or {} + response_200 = responses.get(200, {}) + # Must NOT have 'cost' as a top-level key (invalid OpenAPI) + assert "cost" not in response_200, ( + "/spend/calculate 200 response must not have 'cost' as a " + "top-level property - use 'content' wrapper instead" + ) + # Must have 'content' wrapper + assert "content" in response_200, ( + "/spend/calculate 200 response must have a 'content' field" + ) + content = response_200["content"] + assert "application/json" in content + assert "schema" in content["application/json"] + break + else: + pytest.fail("/spend/calculate route not found in router") + + +class TestCredentialEndpointsOpenAPISchema: + """Test /credentials endpoints have correct path parameters.""" + + def test_by_name_and_by_model_are_separate_handlers(self): + """ + /credentials/by_name/{credential_name} and /credentials/by_model/{model_id} + must be separate handler functions so each only declares its own path params. + """ + from litellm.proxy.credential_endpoints.endpoints import router + + by_name_routes = [] + by_model_routes = [] + for route in router.routes: + if not hasattr(route, "path"): + continue + if "by_name" in route.path: + by_name_routes.append(route) + elif "by_model" in route.path: + by_model_routes.append(route) + + assert len(by_name_routes) == 1, "Expected exactly one by_name route" + assert len(by_model_routes) == 1, "Expected exactly one by_model route" + + # They must be different endpoint functions + by_name_endpoint = by_name_routes[0].endpoint + by_model_endpoint = by_model_routes[0].endpoint + assert by_name_endpoint is not by_model_endpoint, ( + "by_name and by_model must be separate handler functions " + "to avoid path parameter conflicts in OpenAPI spec" + ) + + def test_by_model_route_does_not_require_credential_name(self): + """ + The /credentials/by_model/{model_id} route must NOT have + credential_name as a parameter. + """ + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_model, + ) + + sig = inspect.signature(get_credential_by_model) + param_names = list(sig.parameters.keys()) + assert "credential_name" not in param_names, ( + "get_credential_by_model must not have a credential_name parameter" + ) + + def test_by_name_route_does_not_require_model_id(self): + """ + The /credentials/by_name/{credential_name} route must NOT have + model_id as a parameter. + """ + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_name, + ) + + sig = inspect.signature(get_credential_by_name) + param_names = list(sig.parameters.keys()) + assert "model_id" not in param_names, ( + "get_credential_by_name must not have a model_id parameter" + ) + + def test_by_model_has_model_id_path_param(self): + """The by_model handler must accept model_id as a path parameter.""" + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_model, + ) + + sig = inspect.signature(get_credential_by_model) + assert "model_id" in sig.parameters, ( + "get_credential_by_model must have a model_id parameter" + ) + + def test_by_name_has_credential_name_path_param(self): + """The by_name handler must accept credential_name as a path parameter.""" + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_name, + ) + + sig = inspect.signature(get_credential_by_name) + assert "credential_name" in sig.parameters, ( + "get_credential_by_name must have a credential_name parameter" + ) From 8c8d1debee7f91078998f7eac0f15dae57db167c Mon Sep 17 00:00:00 2001 From: Kerem Turgutlu Date: Tue, 3 Mar 2026 08:51:06 +0300 Subject: [PATCH 04/40] fix: preserve usage/cached_tokens in Responses API streaming bridge (#22194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The response.completed handler in the completion→responses streaming bridge was discarding the usage object, causing prompt_tokens_details (and cached_tokens) to always be None when streaming with models that use the Responses API (e.g. gpt-5.2-codex, gpt-5.3-codex). Extract usage from the response.completed event and translate it via the existing _transform_response_api_usage_to_chat_usage helper. Fixes #22192 --- .../transformation.py | 9 +++- ...responses_transformation_transformation.py | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 1704861686..413c19bfc2 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1088,6 +1088,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): finish_reason = "tool_calls" if has_function_calls else "stop" + usage = None + if response_data.get("usage"): + from litellm.responses.utils import ResponseAPILoggingUtils + usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + response_data.get("usage") + ) return ModelResponseStream( choices=[ StreamingChoices( @@ -1095,7 +1101,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): delta=Delta(content=""), finish_reason=finish_reason, ) - ] + ], + usage=usage ) else: pass diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 3021fff9a2..cdafe24799 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -738,6 +738,57 @@ def test_response_completed_with_message_only_emits_stop_finish_reason(): ) + +def test_response_completed_preserves_usage_with_cached_tokens(): + """ + Test that response.completed correctly translates Responses API usage + (input_tokens_details) to chat completion usage (prompt_tokens_details). + + This is a regression test for an issue where streaming with models that + use the Responses API bridge (e.g. gpt-5.2-codex) would drop + prompt_tokens_details, causing cached_tokens to always be None. + """ + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + + chunk = { + "type": "response.completed", + "response": { + "id": "resp_789", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_abc", + "role": "assistant", + "content": [{"type": "output_text", "text": "Six"}], + "status": "completed", + } + ], + "usage": { + "input_tokens": 1226, + "output_tokens": 5, + "total_tokens": 1231, + "input_tokens_details": {"cached_tokens": 1024}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + } + + result = iterator.chunk_parser(chunk) + + assert result.usage is not None, "usage should be set on response.completed chunk" + assert result.usage.prompt_tokens == 1226, "prompt_tokens should map from input_tokens" + assert result.usage.completion_tokens == 5, "completion_tokens should map from output_tokens" + assert result.usage.prompt_tokens_details is not None, "prompt_tokens_details should be set" + assert result.usage.prompt_tokens_details.cached_tokens == 1024, ( + "cached_tokens should be preserved from input_tokens_details" + ) + + def test_function_call_done_emits_is_finished(): """ Test that OUTPUT_ITEM_DONE for a function_call still emits is_finished=True. From 239f044721a34901bfaa1216bf0079149adf0fb3 Mon Sep 17 00:00:00 2001 From: pnookala-godaddy <93624827+pnookala-godaddy@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:52:32 -0800 Subject: [PATCH 05/40] fix(caching): inject default_in_memory_ttl in DualCache async_set_cache and async_set_cache_pipeline (#22241) DualCache.async_set_cache and async_set_cache_pipeline were missing the default_in_memory_ttl injection that the sync set_cache method has. This caused InMemoryCache to fall back to its own default_ttl (600s) instead of using DualCache's configured default_in_memory_ttl (typically 60s). This is particularly impactful for end-user budget enforcement in the proxy, where cached spend values could remain stale for 10 minutes instead of 1 minute, allowing users to exceed their budgets. --- litellm/caching/dual_cache.py | 4 + tests/test_litellm/caching/test_dual_cache.py | 103 ++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 6df570c72b..48f4d8b8d3 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -346,6 +346,8 @@ class DualCache(BaseCache): ) try: if self.in_memory_cache is not None: + if "ttl" not in kwargs and self.default_in_memory_ttl is not None: + kwargs["ttl"] = self.default_in_memory_ttl await self.in_memory_cache.async_set_cache(key, value, **kwargs) if self.redis_cache is not None and local_only is False: @@ -367,6 +369,8 @@ class DualCache(BaseCache): ) try: if self.in_memory_cache is not None: + if "ttl" not in kwargs and self.default_in_memory_ttl is not None: + kwargs["ttl"] = self.default_in_memory_ttl await self.in_memory_cache.async_set_cache_pipeline( cache_list=cache_list, **kwargs ) diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 9974c23e4b..606f25ddf4 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -1,9 +1,11 @@ import asyncio +import time from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache @@ -56,3 +58,104 @@ async def test_dual_cache_async_batch_get_cache_rolls_back_redis_reservation_on_ assert mock_async_batch_get_cache.call_count == 2 assert "shared_a" not in dual_cache.last_redis_batch_access_time assert "shared_b" not in dual_cache.last_redis_batch_access_time + + +@pytest.mark.asyncio +async def test_dual_cache_async_set_cache_injects_default_in_memory_ttl(): + """ + Test that async_set_cache injects default_in_memory_ttl into kwargs + when no explicit ttl is provided, matching the sync set_cache behavior. + + Regression test for: async_set_cache was missing the TTL injection that + sync set_cache has, causing InMemoryCache to use its own default_ttl (600s) + instead of DualCache's default_in_memory_ttl. + """ + in_memory_cache = InMemoryCache(default_ttl=600) + dual_cache = DualCache( + in_memory_cache=in_memory_cache, + default_in_memory_ttl=60, + ) + + before = time.time() + await dual_cache.async_set_cache(key="test_key", value="test_value") + after = time.time() + + # The TTL stored should reflect default_in_memory_ttl (60s), not + # InMemoryCache's default_ttl (600s) + expiry = in_memory_cache.ttl_dict["test_key"] + assert expiry >= before + 60 + assert expiry <= after + 60 + + +@pytest.mark.asyncio +async def test_dual_cache_async_set_cache_respects_explicit_ttl(): + """ + Test that async_set_cache does NOT override an explicitly provided ttl. + """ + in_memory_cache = InMemoryCache(default_ttl=600) + dual_cache = DualCache( + in_memory_cache=in_memory_cache, + default_in_memory_ttl=60, + ) + + before = time.time() + await dual_cache.async_set_cache(key="test_key", value="test_value", ttl=30) + after = time.time() + + # The explicit ttl=30 should be used, not default_in_memory_ttl (60) + expiry = in_memory_cache.ttl_dict["test_key"] + assert expiry >= before + 30 + assert expiry <= after + 30 + + +@pytest.mark.asyncio +async def test_dual_cache_async_set_cache_pipeline_injects_default_in_memory_ttl(): + """ + Test that async_set_cache_pipeline injects default_in_memory_ttl into kwargs + when no explicit ttl is provided. + """ + in_memory_cache = InMemoryCache(default_ttl=600) + dual_cache = DualCache( + in_memory_cache=in_memory_cache, + default_in_memory_ttl=60, + ) + + cache_list = [("key_a", "value_a"), ("key_b", "value_b")] + + before = time.time() + await dual_cache.async_set_cache_pipeline(cache_list=cache_list) + after = time.time() + + for key in ["key_a", "key_b"]: + expiry = in_memory_cache.ttl_dict[key] + assert expiry >= before + 60 + assert expiry <= after + 60 + + +@pytest.mark.asyncio +async def test_dual_cache_sync_and_async_set_cache_use_same_ttl(): + """ + Test that sync set_cache and async async_set_cache produce the same TTL + when no explicit ttl is provided, ensuring parity between the two paths. + """ + in_memory_sync = InMemoryCache(default_ttl=600) + dual_cache_sync = DualCache( + in_memory_cache=in_memory_sync, + default_in_memory_ttl=60, + ) + + in_memory_async = InMemoryCache(default_ttl=600) + dual_cache_async = DualCache( + in_memory_cache=in_memory_async, + default_in_memory_ttl=60, + ) + + dual_cache_sync.set_cache(key="test_key", value="test_value") + await dual_cache_async.async_set_cache(key="test_key", value="test_value") + + sync_expiry = in_memory_sync.ttl_dict["test_key"] + async_expiry = in_memory_async.ttl_dict["test_key"] + + # Both should use default_in_memory_ttl=60, so their expiry times + # should be within a small tolerance of each other + assert abs(sync_expiry - async_expiry) < 1.0 From 52c5f2af6bb0649e9e3eee85935742fb47e9facc Mon Sep 17 00:00:00 2001 From: Umut Polat <52835619+umut-polat@users.noreply.github.com> Date: Tue, 3 Mar 2026 08:56:37 +0300 Subject: [PATCH 06/40] fix: apply server root path to mapped passthrough route matching (#22310) mapped passthrough routes (vertex_ai, bedrock, etc) were compared against the raw request path without prepending SERVER_ROOT_PATH. db-registered routes already used _build_full_path_with_root for this but the mapped routes branch was missed. fixes #22272 --- .../pass_through_endpoints.py | 3 +- .../test_pass_through_endpoints.py | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 356807415d..4d95fda0a4 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2062,7 +2062,8 @@ class InitPassThroughEndpointHelpers: """ ## CHECK IF MAPPED PASS THROUGH ENDPOINT for mapped_route in LiteLLMRoutes.mapped_pass_through_routes.value: - if route.startswith(mapped_route): + full_mapped_route = InitPassThroughEndpointHelpers._build_full_path_with_root(mapped_route) + if route.startswith(full_mapped_route): return True # Fast path: check if any registered route key contains this path diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 7ec97ddc18..71420c23ad 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -2369,3 +2369,42 @@ def test_get_registered_pass_through_route_with_custom_root(): # Clean up _registered_pass_through_routes.clear() + + +def test_mapped_pass_through_routes_with_server_root_path(): + """ + Mapped passthrough routes (vertex_ai, bedrock, etc) should match + even when SERVER_ROOT_PATH is set and the incoming route is prefixed. + + Regression test for https://github.com/BerriAI/litellm/issues/22272 + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path" + ) as mock_get_root: + mock_get_root.return_value = "/litellm" + + # prefixed route should match mapped routes like /vertex_ai + assert ( + InitPassThroughEndpointHelpers.is_registered_pass_through_route( + "/litellm/vertex_ai/v1/projects/foo" + ) + is True + ) + assert ( + InitPassThroughEndpointHelpers.is_registered_pass_through_route( + "/litellm/bedrock/model/invoke" + ) + is True + ) + + # bare route without prefix should not match when root is set + assert ( + InitPassThroughEndpointHelpers.is_registered_pass_through_route( + "/vertex_ai/v1/projects/foo" + ) + is False + ) From 4dc277e427a14c437a7a23c4db1f59853af6c287 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Mar 2026 10:21:29 +0530 Subject: [PATCH 07/40] fix(vertex_ai): strip LiteLLM-internal keys from extra_body before merging to Gemini request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #20950 added extra_body forwarding to Vertex AI Gemini. LiteLLM-internal keys (cache, tags) were being merged into the request body, causing Vertex AI to reject with 400: 'Unknown name "cache": Cannot find field.' - Add _LITELLM_INTERNAL_EXTRA_BODY_KEYS frozenset (cache, tags) - Skip these keys in _pop_and_merge_extra_body before merging - Add regression tests for cache and tags stripping Fixes regression from 1.79.3 → 1.81.12 when using proxy cache with extra_body={"cache": {"use-cache": True, "ttl": 86400}} Made-with: Cursor --- .../llms/vertex_ai/gemini/transformation.py | 6 ++ ...odel_prices_and_context_window_backup.json | 36 ++++++++++ .../test_vertex_ai_gemini_transformation.py | 69 +++++++++++++++++++ 3 files changed, 111 insertions(+) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 57889284a8..54148c9492 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -529,12 +529,18 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 raise e +# Keys that LiteLLM consumes internally and must never be forwarded to the +_LITELLM_INTERNAL_EXTRA_BODY_KEYS: frozenset = frozenset({"cache", "tags"}) + + def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: """Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values.""" extra_body: Optional[dict] = optional_params.pop("extra_body", None) if extra_body is not None: data_dict: dict = data # type: ignore[assignment] for k, v in extra_body.items(): + if k in _LITELLM_INTERNAL_EXTRA_BODY_KEYS: + continue if k in data_dict and isinstance(data_dict[k], dict) and isinstance(v, dict): data_dict[k].update(v) else: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 900894f74d..ae256ed078 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16799,6 +16799,42 @@ "supports_vision": true, "supports_web_search": true }, + "gemini/gemini-3.1-flash-image-preview": { + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.045, + "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "rpm": 1000, + "tpm": 4000000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "gemini/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index b264964b14..f3c82e439c 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -126,6 +126,75 @@ def test_vertex_ai_includes_labels(): +def test_extra_body_cache_not_forwarded_to_vertex_ai(): + """ + 'cache' inside extra_body is a LiteLLM-internal proxy caching control. + It must NOT be forwarded to the Vertex AI request body. + + Regression test for: "Invalid JSON payload received. Unknown name \"cache\": Cannot find field." + Vertex AI enforces a strict JSON schema and rejects any unknown field. + """ + messages = [{"role": "user", "content": "test"}] + optional_params = { + "extra_body": { + "cache": {"use-cache": True, "ttl": 86400}, # LiteLLM-internal + "some_vertex_param": "value", # legitimate provider extra + }, + } + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + # 'cache' must be stripped — Vertex AI has no such field + assert "cache" not in result, ( + "extra_body.cache must not be forwarded to Vertex AI. " + "Vertex AI rejects it with 400: Unknown name \"cache\": Cannot find field." + ) + + # Other legitimate extra_body keys should still pass through + assert "some_vertex_param" in result + assert result["some_vertex_param"] == "value" + + # Core request fields must be present + assert "contents" in result + + +def test_extra_body_tags_not_forwarded_to_vertex_ai(): + """ + 'tags' inside extra_body is a LiteLLM-internal param for logging/tracking. + It must NOT be forwarded to the Vertex AI request body. + Documented in litellm_proxy.md: "Send tags by including them in the extra_body parameter" + """ + messages = [{"role": "user", "content": "test"}] + optional_params = { + "extra_body": { + "tags": ["user:alice", "env:prod"], + "custom_param": "allowed", + }, + } + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + assert "tags" not in result + assert "custom_param" in result + assert result["custom_param"] == "allowed" + + def test_metadata_to_labels_vertex_only(): """Test that metadata->labels conversion only happens for Vertex AI""" messages = [{"role": "user", "content": "test"}] From 8cf80a14d95f871485130587f5ffffcb6d58d181 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Mar 2026 18:28:11 +0530 Subject: [PATCH 08/40] fix(openai): preserve reasoning_effort summary field for Responses API When reasoning_effort is passed as a dict with additional fields like 'summary' or 'generate_summary', preserve the full dict format instead of normalizing it to a string. This ensures that when requests are routed to the OpenAI Responses API, all reasoning parameters are correctly included. The normalization to string format now only happens for simple dicts with just the 'effort' key, which is appropriate for the Chat Completions API. Fixes issue where summary field was being dropped when routing gpt-5.4+ requests with tools + reasoning to Responses API. Made-with: Cursor --- .../llms/openai/chat/gpt_5_transformation.py | 31 ++++-- ...responses_transformation_transformation.py | 32 ++++++ .../chat/test_openai_gpt_transformation.py | 102 ++++++++++++++++++ 3 files changed, 158 insertions(+), 7 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index beb76f3d80..6b3c741191 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -151,17 +151,34 @@ class OpenAIGPT5Config(OpenAIGPTConfig): ) # Normalize reasoning_effort: chat completion API expects a string, not a dict - # (e.g. {'effort': 'high', 'summary': 'detailed'} -> 'high') + # (e.g. {'effort': 'high'} -> 'high') + # BUT: preserve dict format if it has additional fields like 'summary' for Responses API raw_reasoning_effort = ( non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) - normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) - if raw_reasoning_effort is not None and normalized is not None: - if "reasoning_effort" in non_default_params: - non_default_params["reasoning_effort"] = normalized - if "reasoning_effort" in optional_params: - optional_params["reasoning_effort"] = normalized + + # Only normalize if it's a simple dict with just 'effort' key + # Preserve dict format if it has additional fields (e.g., 'summary') for Responses API + should_normalize = False + if isinstance(raw_reasoning_effort, dict): + # Only normalize if dict has only 'effort' key (or is empty) + if set(raw_reasoning_effort.keys()) == {"effort"} or len(raw_reasoning_effort) == 0: + should_normalize = True + elif isinstance(raw_reasoning_effort, str): + # String format is already normalized + should_normalize = False + + if should_normalize: + normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) + if raw_reasoning_effort is not None and normalized is not None: + if "reasoning_effort" in non_default_params: + non_default_params["reasoning_effort"] = normalized + if "reasoning_effort" in optional_params: + optional_params["reasoning_effort"] = normalized + else: + # Keep the original format (string or dict with additional fields) + normalized = raw_reasoning_effort reasoning_effort = normalized or raw_reasoning_effort if reasoning_effort is not None and reasoning_effort == "xhigh": diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index ef3d7534d9..c5dc0409fa 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1778,3 +1778,35 @@ def test_parallel_tool_calls_comprehensive_streaming_integration(): ) print("✓ Parallel tool calls with split argument deltas stream correctly end-to-end") + + +def test_map_optional_params_preserves_reasoning_summary(): + """Test that reasoning_effort dict with summary field is preserved. + + Regression test for: User reported that summary field was being dropped + when routing to Responses API. The dict format should be fully preserved. + """ + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + handler = LiteLLMResponsesTransformationHandler() + + optional_params = { + "stream": False, + "tools": [{"type": "function", "function": {"name": "test_tool"}}], + "tool_choice": "auto", + "reasoning_effort": {"effort": "high", "summary": "detailed"}, + } + + responses_api_request = ResponsesAPIOptionalRequestParams() + handler._map_optional_params_to_responses_api_request( + optional_params, responses_api_request + ) + + # Verify reasoning_effort dict with summary was fully preserved + assert "reasoning" in responses_api_request + assert responses_api_request["reasoning"] == {"effort": "high", "summary": "detailed"} + assert responses_api_request["reasoning"]["effort"] == "high" + assert responses_api_request["reasoning"]["summary"] == "detailed" diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 39ff0a4f4d..a78abbced2 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -13,6 +13,7 @@ from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config class TestOpenAIGPTConfig: @@ -324,3 +325,104 @@ class TestPromptCacheParams: ) assert optional_params.get("prompt_cache_key") == "my-cache-key" assert optional_params.get("prompt_cache_retention") == "24h" + + +class TestGPT5ReasoningEffortPreservation: + """Tests for GPT-5 reasoning_effort dict preservation for Responses API.""" + + def setup_method(self): + self.config = OpenAIGPT5Config() + + def test_reasoning_effort_string_preserved(self): + """Test that reasoning_effort as string is preserved.""" + non_default_params = {"reasoning_effort": "high"} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # String format should be preserved + assert non_default_params.get("reasoning_effort") == "high" + + def test_reasoning_effort_dict_with_only_effort_normalized(self): + """Test that reasoning_effort dict with only 'effort' key is normalized to string.""" + non_default_params = {"reasoning_effort": {"effort": "high"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with only 'effort' should be normalized to string + assert non_default_params.get("reasoning_effort") == "high" + + def test_reasoning_effort_dict_with_summary_preserved(self): + """Test that reasoning_effort dict with 'summary' field is preserved for Responses API. + + Regression test for: User reported that summary field was being dropped when + routing to Responses API. The dict format with additional fields should be + preserved so it can be properly handled by the Responses API transformation. + """ + non_default_params = {"reasoning_effort": {"effort": "high", "summary": "detailed"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with additional fields should be preserved + assert non_default_params.get("reasoning_effort") == {"effort": "high", "summary": "detailed"} + assert isinstance(non_default_params.get("reasoning_effort"), dict) + assert non_default_params["reasoning_effort"]["effort"] == "high" + assert non_default_params["reasoning_effort"]["summary"] == "detailed" + + def test_reasoning_effort_dict_with_generate_summary_preserved(self): + """Test that reasoning_effort dict with 'generate_summary' field is preserved.""" + non_default_params = {"reasoning_effort": {"effort": "medium", "generate_summary": "auto"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with additional fields should be preserved + assert non_default_params.get("reasoning_effort") == {"effort": "medium", "generate_summary": "auto"} + assert isinstance(non_default_params.get("reasoning_effort"), dict) + + def test_reasoning_effort_dict_with_all_fields_preserved(self): + """Test that reasoning_effort dict with all fields is preserved.""" + non_default_params = { + "reasoning_effort": { + "effort": "high", + "summary": "detailed", + "generate_summary": "concise" + } + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with all fields should be preserved + reasoning = non_default_params.get("reasoning_effort") + assert isinstance(reasoning, dict) + assert reasoning["effort"] == "high" + assert reasoning["summary"] == "detailed" + assert reasoning["generate_summary"] == "concise" From ee3ecb59941211a36c2adf14850b5add8c8c81f7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Mar 2026 18:41:07 +0530 Subject: [PATCH 09/40] fix(openai): preserve reasoning_effort summary + fix xhigh/none guards for dict inputs - Add _get_effort_level() to extract effective effort from string or dict - Use effective_effort for xhigh validation, tool-drop, sampling, temperature guards - Preserve dict format when it has summary/generate_summary for Responses API - Add tests: xhigh-dict validation, none-dict for tools/sampling/temperature - Update tests: dict-with-summary now preserved (not normalized) Made-with: Cursor --- .../llms/azure/chat/gpt_5_transformation.py | 24 ++++- .../llms/openai/chat/gpt_5_transformation.py | 84 +++++++++++------ .../chat/test_azure_gpt5_transformation.py | 17 ++++ .../chat/test_openai_gpt_transformation.py | 91 +++++++++++++++++++ .../llms/openai/test_gpt5_transformation.py | 77 ++++++++++++++-- 5 files changed, 253 insertions(+), 40 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index 78d6372d02..2967d5e394 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -4,7 +4,10 @@ from typing import List import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_5_transformation import ( + OpenAIGPT5Config, + _get_effort_level, +) from litellm.types.llms.openai import AllMessageValues from .gpt_transformation import AzureOpenAIConfig @@ -81,20 +84,21 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) + effective_effort = _get_effort_level(reasoning_effort_value) # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning supports_none = self._supports_reasoning_effort_level(model, "none") - if reasoning_effort_value == "none" and not supports_none: + if effective_effort == "none" and not supports_none: if litellm.drop_params is True or ( drop_params is not None and drop_params is True ): non_default_params = non_default_params.copy() optional_params = optional_params.copy() - if non_default_params.get("reasoning_effort") == "none": + if _get_effort_level(non_default_params.get("reasoning_effort")) == "none": non_default_params.pop("reasoning_effort") - if optional_params.get("reasoning_effort") == "none": + if _get_effort_level(optional_params.get("reasoning_effort")) == "none": optional_params.pop("reasoning_effort") else: raise UnsupportedParamsError( @@ -117,9 +121,19 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): ) # Only drop reasoning_effort='none' for models that don't support it - if result.get("reasoning_effort") == "none" and not supports_none: + result_effort = _get_effort_level(result.get("reasoning_effort")) + if result_effort == "none" and not supports_none: result.pop("reasoning_effort") + # Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together. + # Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not). + if self.is_model_gpt_5_4_plus_model(model): + has_tools = bool( + non_default_params.get("tools") or optional_params.get("tools") + ) + if has_tools and result_effort not in (None, "none"): + result.pop("reasoning_effort", None) + return result def transform_request( diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 6b3c741191..f186bc6085 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -25,6 +25,22 @@ def _normalize_reasoning_effort_for_chat_completion( return None +def _get_effort_level(value: Union[str, dict, None]) -> Optional[str]: + """Extract the effective effort level from reasoning_effort (string or dict). + + Use this for guards that compare effort level (e.g. xhigh validation, "none" checks). + Ensures dict inputs like {"effort": "none", "summary": "detailed"} are correctly + treated as effort="none" for validation purposes. + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, dict) and "effort" in value: + return value["effort"] + return None + + class OpenAIGPT5Config(OpenAIGPTConfig): """Configuration for gpt-5 models including GPT-5-Codex variants. @@ -70,6 +86,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model_name = model.split("/")[-1] return model_name.startswith("gpt-5.4") + @classmethod + def is_model_gpt_5_4_plus_model(cls, model: str) -> bool: + """Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro).""" + model_name = model.split("/")[-1] + if not model_name.startswith("gpt-5."): + return False + try: + version_str = model_name.replace("gpt-5.", "").split("-")[0] + major = version_str.split(".")[0] + return int(major) >= 4 + except (ValueError, IndexError): + return False + @classmethod def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: """Check if the model supports a specific reasoning_effort level. @@ -150,38 +179,32 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # Normalize reasoning_effort: chat completion API expects a string, not a dict - # (e.g. {'effort': 'high'} -> 'high') - # BUT: preserve dict format if it has additional fields like 'summary' for Responses API + # Get raw reasoning_effort and effective effort level for all guards. + # Use effective_effort (extracted string) for xhigh validation, "none" checks, and + # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} + # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors. raw_reasoning_effort = ( non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) - - # Only normalize if it's a simple dict with just 'effort' key - # Preserve dict format if it has additional fields (e.g., 'summary') for Responses API - should_normalize = False - if isinstance(raw_reasoning_effort, dict): - # Only normalize if dict has only 'effort' key (or is empty) - if set(raw_reasoning_effort.keys()) == {"effort"} or len(raw_reasoning_effort) == 0: - should_normalize = True - elif isinstance(raw_reasoning_effort, str): - # String format is already normalized - should_normalize = False - - if should_normalize: + effective_effort = _get_effort_level(raw_reasoning_effort) + + # Normalize to string for Chat Completions API when dict has only "effort". + # Preserve full dict (e.g. {"effort": "high", "summary": "detailed"}) for Responses API. + if isinstance(raw_reasoning_effort, dict) and set(raw_reasoning_effort.keys()) <= {"effort"}: normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) - if raw_reasoning_effort is not None and normalized is not None: + if normalized is not None: if "reasoning_effort" in non_default_params: non_default_params["reasoning_effort"] = normalized if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - else: - # Keep the original format (string or dict with additional fields) - normalized = raw_reasoning_effort - reasoning_effort = normalized or raw_reasoning_effort - if reasoning_effort is not None and reasoning_effort == "xhigh": + reasoning_effort = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + or raw_reasoning_effort + ) + if effective_effort is not None and effective_effort == "xhigh": if not self._supports_reasoning_effort_level(model, "xhigh"): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) @@ -208,17 +231,20 @@ class OpenAIGPT5Config(OpenAIGPTConfig): has_tools = bool( non_default_params.get("tools") or optional_params.get("tools") ) - if has_tools and reasoning_effort not in (None, "none"): - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - reasoning_effort = None + if has_tools and effective_effort not in (None, "none"): + # Check if this will be routed to Responses API + # If so, keep reasoning_effort; otherwise drop it for chat completions API + if not self.is_model_gpt_5_4_plus_model(model): + non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) + reasoning_effort = None # gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none" supports_none = self._supports_reasoning_effort_level(model, "none") if supports_none: sampling_params = ["logprobs", "top_logprobs", "top_p"] has_sampling = any(p in non_default_params for p in sampling_params) - if has_sampling and reasoning_effort not in (None, "none"): + if has_sampling and effective_effort not in (None, "none"): if litellm.drop_params or drop_params: for p in sampling_params: non_default_params.pop(p, None) @@ -228,7 +254,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): "gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when " "reasoning_effort='none'. Current reasoning_effort='{}'. " "To drop unsupported params set `litellm.drop_params = True`" - ).format(reasoning_effort), + ).format(effective_effort), status_code=400, ) @@ -236,7 +262,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: # models supporting reasoning_effort="none" also support flexible temperature - if supports_none and (reasoning_effort == "none" or reasoning_effort is None): + if supports_none and (effective_effort == "none" or effective_effort is None): optional_params["temperature"] = temperature_value elif temperature_value == 1: optional_params["temperature"] = temperature_value diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 25f3d1364f..635359563b 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -192,6 +192,23 @@ def test_azure_gpt5_1_series_temperature_handling(config: AzureOpenAIGPT5Config) assert params["temperature"] == 0.6 +def test_azure_gpt5_4_drops_reasoning_effort_when_tools_present(config: AzureOpenAIGPT5Config): + """Azure Chat Completions: gpt-5.4+ drops reasoning_effort when tools are present. + + OpenAI routes tools+reasoning to Responses API; Azure does not, so we drop reasoning_effort. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + params = config.map_openai_params( + non_default_params={"reasoning_effort": "high", "tools": tools}, + optional_params={}, + model="gpt5_series/gpt-5.4", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert "reasoning_effort" not in params + assert params["tools"] == tools + + def test_azure_gpt5_reasoning_effort_none_error(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5 (non-5.1) raises error for reasoning_effort='none' when drop_params=False.""" with pytest.raises(litellm.utils.UnsupportedParamsError): diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index a78abbced2..90fdc2d20d 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -426,3 +426,94 @@ class TestGPT5ReasoningEffortPreservation: assert reasoning["effort"] == "high" assert reasoning["summary"] == "detailed" assert reasoning["generate_summary"] == "concise" + + def test_reasoning_effort_dict_xhigh_triggers_validation(self): + """xhigh-dict: effective effort is extracted for model-support validation. + + When reasoning_effort={"effort": "xhigh", "summary": "detailed"} is passed to a model + that doesn't support xhigh (e.g. gpt-5.1), the xhigh guard must fire. + """ + import litellm + + non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} + optional_params = {} + + with pytest.raises(litellm.utils.UnsupportedParamsError): + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + def test_reasoning_effort_dict_xhigh_dropped_when_requested(self): + """xhigh-dict with drop_params=True: reasoning_effort is dropped.""" + non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=True, + ) + + assert "reasoning_effort" not in non_default_params + + def test_reasoning_effort_dict_none_treated_as_none_for_tools(self): + """none-dict: {"effort": "none", "summary": "detailed"} is treated as effort=none. + + Tool-drop guard should NOT fire; reasoning_effort should be kept. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + non_default_params = {"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} + assert non_default_params.get("tools") == tools + + def test_reasoning_effort_dict_none_treated_as_none_for_sampling(self): + """none-dict: {"effort": "none", "summary": "detailed"} allows logprobs/top_p. + + Sampling-param guard should NOT fire; logprobs should be kept. + """ + non_default_params = { + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "logprobs": True, + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} + assert non_default_params.get("logprobs") is True + + def test_reasoning_effort_dict_none_allows_temperature(self): + """none-dict: {"effort": "none", "summary": "detailed"} allows non-default temperature.""" + non_default_params = { + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "temperature": 0.5, + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + assert optional_params.get("temperature") == 0.5 + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index b136f8774b..13d2ebab14 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -324,10 +324,11 @@ def test_gpt5_4_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): assert params["reasoning_effort"] == "xhigh" -def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): - """Chat completion API expects reasoning_effort as a string, not a dict. +def test_gpt5_preserves_reasoning_effort_dict_with_summary(config: OpenAIConfig): + """Dict with summary/generate_summary is preserved for Responses API. Config/deployments may pass Responses API format: {'effort': 'high', 'summary': 'detailed'}. + We preserve the full dict so it reaches the Responses API transformation. """ params = config.map_openai_params( non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, @@ -335,18 +336,82 @@ def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == "high" + assert params["reasoning_effort"] == {"effort": "high", "summary": "detailed"} -def test_gpt5_normalizes_reasoning_effort_dict_from_optional_params(config: OpenAIConfig): - """reasoning_effort dict in optional_params (e.g. from model config) is normalized.""" +def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): + """Dict with effort='xhigh' triggers xhigh model-support validation. + + Regression: when reasoning_effort is a dict, effective_effort must be used for + the xhigh guard so validation is not silently skipped. + """ + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + + +def test_gpt5_xhigh_dict_accepted_for_supported_model(config: OpenAIConfig): + """Dict with effort='xhigh' passes through for gpt-5.4+.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "xhigh", "summary": "detailed"} + + +def test_gpt5_none_dict_with_tools_no_tool_drop(config: OpenAIConfig): + """Dict with effort='none' and tools: no tool-drop, reasoning_effort preserved. + + Regression: effective_effort='none' must be used for tool-drop guard so + {"effort": "none", "summary": "detailed"} is not incorrectly treated as non-none. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} + assert params["tools"] == tools + + +def test_gpt5_none_dict_with_sampling_params_allowed(config: OpenAIConfig): + """Dict with effort='none' allows logprobs/top_p/top_logprobs. + + Regression: effective_effort='none' must be used for sampling guard so + {"effort": "none", "summary": "detailed"} does not incorrectly trigger sampling errors. + """ + params = config.map_openai_params( + non_default_params={ + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "logprobs": True, + "top_p": 0.9, + }, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} + assert params["logprobs"] is True + assert params["top_p"] == 0.9 + + +def test_gpt5_preserves_reasoning_effort_dict_with_summary_from_optional_params(config: OpenAIConfig): + """reasoning_effort dict with summary in optional_params is preserved.""" params = config.map_openai_params( non_default_params={}, optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == "medium" + assert params["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} def test_gpt5_4_drops_reasoning_effort_when_tools_present(config: OpenAIConfig): From 9b15f639a4a6edc37a6096d258b8e605f94dae4b Mon Sep 17 00:00:00 2001 From: Varad Khonde <72742264+Varad2001@users.noreply.github.com> Date: Mon, 9 Mar 2026 21:31:31 +0530 Subject: [PATCH 10/40] fix(responses): merge parallel function_call items into single assistant message (#23116) --- .../transformation.py | 35 +++++ .../test_litellm_completion_responses.py | 125 ++++++++++++++++++ 2 files changed, 160 insertions(+) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 19845d7c49..d901bbc210 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -384,6 +384,41 @@ class LiteLLMCompletionResponsesConfig: if call_id_raw: existing_tool_call_ids.add(str(call_id_raw)) + ######################################################### + # Merge consecutive function_call items into a single assistant + # message. Anthropic requires that all tool_use blocks appear in + # ONE assistant message immediately followed by the tool_result + # blocks. Without this merging, each function_call creates its own + # assistant message, producing back-to-back assistant messages that + # Anthropic rejects with "tool_use ids were found without + # tool_result blocks immediately after". + ######################################################### + if messages: + last_msg = messages[-1] + last_role = ( + last_msg.get("role") + if isinstance(last_msg, dict) + else getattr(last_msg, "role", None) + ) + if last_role == "assistant": + for new_msg in chat_completion_messages: + new_role = ( + new_msg.get("role") + if isinstance(new_msg, dict) + else getattr(new_msg, "role", None) + ) + if new_role == "assistant": + new_tcs = ( + new_msg.get("tool_calls") + if isinstance(new_msg, dict) + else getattr(new_msg, "tool_calls", None) + ) or [] + for tc in new_tcs: + LiteLLMCompletionResponsesConfig._add_tool_call_to_assistant( + last_msg, tc + ) + continue + ######################################################### # If Input Item is a Tool Call Output, add it to the tool_call_output_messages list # preserving the ordering of tool call outputs. Some models require the tool diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 6d6162437c..a931a9bc93 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1774,3 +1774,128 @@ class TestStreamingIDConsistency: # Verify it matches the cached ID assert iterator._cached_item_id is not None assert iterator._cached_item_id == text_done_id + + def test_parallel_tool_calls_merged_into_single_assistant_message(self): + """ + Regression test: multi-turn parallel tool calls via the Responses API must + produce a single assistant message with all tool_calls, not one assistant + message per function_call item. + + When the model responds with two parallel tool calls (e.g. get_weather for + SF and NYC), the next Responses API request includes two consecutive + function_call items followed by two function_call_output items. + + Without the fix each function_call becomes its own assistant message, + producing back-to-back assistant messages that Anthropic/Vertex AI rejects: + "tool_use ids were found without tool_result blocks immediately after". + """ + input_items = [ + {"type": "message", "role": "user", "content": "Weather in SF and NYC?"}, + # Two parallel tool calls from the previous assistant response + { + "type": "function_call", + "call_id": "toolu_01", + "name": "get_weather", + "arguments": '{"city": "SF"}', + }, + { + "type": "function_call", + "call_id": "toolu_02", + "name": "get_weather", + "arguments": '{"city": "NYC"}', + }, + # Tool results + {"type": "function_call_output", "call_id": "toolu_01", "output": "72°F"}, + {"type": "function_call_output", "call_id": "toolu_02", "output": "55°F"}, + ] + + messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( + input=input_items + ) + + roles = [ + m.get("role") if isinstance(m, dict) else getattr(m, "role", None) + for m in messages + ] + + # Must not have two consecutive assistant messages + for i in range(len(roles) - 1): + assert not ( + roles[i] == "assistant" and roles[i + 1] == "assistant" + ), f"Consecutive assistant messages at indices {i} and {i+1}: {roles}" + + # The single assistant message must contain BOTH tool_calls + assistant_messages = [ + m for m in messages + if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) + == "assistant" + ] + assert len(assistant_messages) == 1, ( + f"Expected 1 assistant message, got {len(assistant_messages)}" + ) + + assistant_msg = assistant_messages[0] + tool_calls = ( + assistant_msg.get("tool_calls") + if isinstance(assistant_msg, dict) + else getattr(assistant_msg, "tool_calls", None) + ) + assert tool_calls is not None and len(tool_calls) == 2, ( + f"Expected 2 tool_calls in the merged assistant message, got: {tool_calls}" + ) + + call_ids = [ + (tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)) + for tc in tool_calls + ] + assert "toolu_01" in call_ids, f"toolu_01 missing from tool_calls: {call_ids}" + assert "toolu_02" in call_ids, f"toolu_02 missing from tool_calls: {call_ids}" + + # Both tool messages must be present + tool_messages = [ + m for m in messages + if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) + == "tool" + ] + assert len(tool_messages) == 2, ( + f"Expected 2 tool messages, got {len(tool_messages)}" + ) + + def test_single_tool_call_still_works_after_merge_fix(self): + """ + Ensure the parallel-tool-call merging fix does not break the existing + single-tool-call path. + """ + input_items = [ + {"type": "message", "role": "user", "content": "Weather in SF?"}, + { + "type": "function_call", + "call_id": "toolu_01", + "name": "get_weather", + "arguments": '{"city": "SF"}', + }, + {"type": "function_call_output", "call_id": "toolu_01", "output": "72°F"}, + ] + + messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( + input=input_items + ) + + roles = [ + m.get("role") if isinstance(m, dict) else getattr(m, "role", None) + for m in messages + ] + + assert "user" in roles + assert "assistant" in roles + assert "tool" in roles + + assistant_messages = [m for m in messages if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) == "assistant"] + assert len(assistant_messages) == 1 + + tool_calls = ( + assistant_messages[0].get("tool_calls") + if isinstance(assistant_messages[0], dict) + else getattr(assistant_messages[0], "tool_calls", None) + ) + assert tool_calls is not None and len(tool_calls) == 1 From 9500fc18d189ca2335a9dfdc693833e31a168f1b Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 9 Mar 2026 19:33:52 -0700 Subject: [PATCH 11/40] Fix TypeError: LiteLLM_Params.__init__() got multiple values for argument 'self' (#23220) The bug occurred when user data inadvertently contained reserved Python keywords like 'self', 'params', or '__class__' as keys. When such a dict was unpacked via **kwargs to LiteLLM_Params() or GenericLiteLLMParams(), Python raised TypeError because 'self' was passed both implicitly and as a keyword argument. The fix: - Add a Pydantic model_validator(mode='before') to GenericLiteLLMParams that filters out reserved keys ('self', 'params', '__class__') before validation - Move the max_retries str-to-int conversion into the same validator - Remove the custom __init__ methods from both GenericLiteLLMParams and LiteLLM_Params, since the validator now handles the preprocessing - Clean up unused VERTEX_CREDENTIALS_TYPES import This fix applies to all classes that inherit from GenericLiteLLMParams, including LiteLLM_Params and updateLiteLLMParams. Added comprehensive tests in tests/test_litellm/test_litellm_params_reserved_keys.py Co-authored-by: Cursor Agent --- litellm/types/router.py | 131 +++--------------- .../test_litellm_params_reserved_keys.py | 92 ++++++++++++ 2 files changed, 111 insertions(+), 112 deletions(-) create mode 100644 tests/test_litellm/test_litellm_params_reserved_keys.py diff --git a/litellm/types/router.py b/litellm/types/router.py index d917d845ad..f0c1ea5e32 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing import Any, Dict, List, Literal, Optional, Tuple, Union, get_type_hints import httpx -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from typing_extensions import Required, TypedDict from litellm._uuid import uuid @@ -16,7 +16,6 @@ from litellm._uuid import uuid from .completion import CompletionRequest from .embedding import EmbeddingRequest from .llms.openai import OpenAIFileObject -from .llms.vertex_ai import VERTEX_CREDENTIALS_TYPES from .search import SearchProvider from .utils import CustomPricingLiteLLMParams, ModelResponse @@ -162,6 +161,9 @@ class CredentialLiteLLMParams(BaseModel): watsonx_region_name: Optional[str] = None +_RESERVED_INIT_KEYS = frozenset({"self", "params", "__class__"}) + + class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ LiteLLM Params without 'model' arg (used across completion / assistants api) @@ -215,76 +217,21 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): vector_store_id: Optional[str] = None milvus_text_field: Optional[str] = None - def __init__( - self, - custom_llm_provider: Optional[str] = None, - max_retries: Optional[Union[int, str]] = None, - tpm: Optional[int] = None, - rpm: Optional[int] = None, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - api_version: Optional[str] = None, - timeout: Optional[Union[float, str]] = None, # if str, pass in as os.environ/ - stream_timeout: Optional[Union[float, str]] = ( - None # timeout when making stream=True calls, if str, pass in as os.environ/ - ), - organization: Optional[str] = None, # for openai orgs - ## LOGGING PARAMS ## - litellm_trace_id: Optional[str] = None, - ## UNIFIED PROJECT/REGION ## - region_name: Optional[str] = None, - ## VERTEX AI ## - vertex_project: Optional[str] = None, - vertex_location: Optional[str] = None, - vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES] = None, - ## AWS BEDROCK / SAGEMAKER ## - aws_access_key_id: Optional[str] = None, - aws_secret_access_key: Optional[str] = None, - aws_region_name: Optional[str] = None, - ## IBM WATSONX ## - watsonx_region_name: Optional[str] = None, - input_cost_per_token: Optional[float] = None, - output_cost_per_token: Optional[float] = None, - input_cost_per_second: Optional[float] = None, - output_cost_per_second: Optional[float] = None, - max_file_size_mb: Optional[float] = None, - # Deployment budgets - max_budget: Optional[float] = None, - budget_duration: Optional[str] = None, - # Pass through params - use_in_pass_through: Optional[bool] = False, - # Dynamic param to force using litellm proxy - use_litellm_proxy: Optional[bool] = False, - # This will merge the reasoning content in the choices - merge_reasoning_content_in_choices: Optional[bool] = False, - model_info: Optional[Dict] = None, - mock_response: Optional[Union[str, ModelResponse, Exception, Any]] = None, - # auto-router params - auto_router_config_path: Optional[str] = None, - auto_router_config: Optional[str] = None, - auto_router_default_model: Optional[str] = None, - auto_router_embedding_model: Optional[str] = None, - # complexity-router params - complexity_router_config: Optional[Dict] = None, - complexity_router_default_model: Optional[str] = None, - # Batch/File API Params - s3_bucket_name: Optional[str] = None, - s3_encryption_key_id: Optional[str] = None, - gcs_bucket_name: Optional[str] = None, - **params, - ): - args = locals() - args.pop("max_retries", None) - args.pop("self", None) - args.pop("params", None) - args.pop("__class__", None) - if max_retries is not None and isinstance(max_retries, str): - max_retries = int(max_retries) # cast to int - # We need to keep max_retries in args since it's a parameter of GenericLiteLLMParams - args[ - "max_retries" - ] = max_retries # Put max_retries back in args after popping it - super().__init__(**args, **params) + @model_validator(mode="before") + @classmethod + def preprocess_input_data(cls, data: Any) -> Any: + """ + Pre-process input data before validation: + 1. Filter out reserved Python keywords ('self', 'params', '__class__') to prevent + 'got multiple values for argument' errors when user data contains these keys. + 2. Convert max_retries from string to int if needed. + """ + if isinstance(data, dict): + filtered = {k: v for k, v in data.items() if k not in _RESERVED_INIT_KEYS} + if "max_retries" in filtered and isinstance(filtered["max_retries"], str): + filtered["max_retries"] = int(filtered["max_retries"]) + return filtered + return data def __contains__(self, key): # Define custom behavior for the 'in' operator @@ -311,46 +258,6 @@ class LiteLLM_Params(GenericLiteLLMParams): model: str model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) - def __init__( - self, - model: str, - custom_llm_provider: Optional[str] = None, - max_retries: Optional[Union[int, str]] = None, - tpm: Optional[int] = None, - rpm: Optional[int] = None, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - api_version: Optional[str] = None, - timeout: Optional[Union[float, str]] = None, # if str, pass in as os.environ/ - stream_timeout: Optional[Union[float, str]] = ( - None # timeout when making stream=True calls, if str, pass in as os.environ/ - ), - organization: Optional[str] = None, # for openai orgs - ## VERTEX AI ## - vertex_project: Optional[str] = None, - vertex_location: Optional[str] = None, - ## AWS BEDROCK / SAGEMAKER ## - aws_access_key_id: Optional[str] = None, - aws_secret_access_key: Optional[str] = None, - aws_region_name: Optional[str] = None, - # OpenAI / Azure Whisper - # set a max-size of file that can be passed to litellm proxy - max_file_size_mb: Optional[float] = None, - # will use deployment on pass-through endpoints if True - use_in_pass_through: Optional[bool] = False, - use_litellm_proxy: Optional[bool] = False, - **params, - ): - args = locals() - args.pop("max_retries", None) - args.pop("self", None) - args.pop("params", None) - args.pop("__class__", None) - if max_retries is not None and isinstance(max_retries, str): - max_retries = int(max_retries) # cast to int - args["max_retries"] = max_retries - super().__init__(**{**args, **params}) - def __contains__(self, key): # Define custom behavior for the 'in' operator return hasattr(self, key) diff --git a/tests/test_litellm/test_litellm_params_reserved_keys.py b/tests/test_litellm/test_litellm_params_reserved_keys.py new file mode 100644 index 0000000000..f49651bd81 --- /dev/null +++ b/tests/test_litellm/test_litellm_params_reserved_keys.py @@ -0,0 +1,92 @@ +""" +Test that LiteLLM_Params and GenericLiteLLMParams handle reserved keys gracefully. + +This test verifies the fix for the bug where passing a dict containing 'self', +'params', or '__class__' keys to LiteLLM_Params() would cause: + TypeError: LiteLLM_Params.__init__() got multiple values for argument 'self' +""" + +import pytest + +from litellm.types.router import GenericLiteLLMParams, LiteLLM_Params + + +class TestLiteLLMParamsReservedKeys: + """Test that reserved keys in input data are filtered out gracefully.""" + + def test_litellm_params_with_self_key(self): + """Test LiteLLM_Params handles 'self' key in input dict.""" + params_dict = {"model": "gpt-4", "self": "some_value", "api_key": "test-key"} + params = LiteLLM_Params(**params_dict) + assert params.model == "gpt-4" + assert params.api_key == "test-key" + assert not hasattr(params, "self") or params.get("self") is None + + def test_litellm_params_with_params_key(self): + """Test LiteLLM_Params handles 'params' key in input dict.""" + params_dict = {"model": "gpt-4", "params": "bad_value"} + params = LiteLLM_Params(**params_dict) + assert params.model == "gpt-4" + + def test_litellm_params_with_class_key(self): + """Test LiteLLM_Params handles '__class__' key in input dict.""" + params_dict = {"model": "gpt-4", "__class__": "bad_value"} + params = LiteLLM_Params(**params_dict) + assert params.model == "gpt-4" + + def test_generic_litellm_params_with_self_key(self): + """Test GenericLiteLLMParams handles 'self' key in input dict.""" + params_dict = {"self": "some_value", "api_key": "test-key"} + params = GenericLiteLLMParams(**params_dict) + assert params.api_key == "test-key" + + def test_generic_litellm_params_with_params_key(self): + """Test GenericLiteLLMParams handles 'params' key in input dict.""" + params_dict = {"params": "bad_value", "api_key": "test-key"} + params = GenericLiteLLMParams(**params_dict) + assert params.api_key == "test-key" + + def test_generic_litellm_params_with_class_key(self): + """Test GenericLiteLLMParams handles '__class__' key in input dict.""" + params_dict = {"__class__": "bad_value", "api_key": "test-key"} + params = GenericLiteLLMParams(**params_dict) + assert params.api_key == "test-key" + + def test_max_retries_string_conversion(self): + """Test that max_retries is converted from string to int.""" + params = LiteLLM_Params(model="gpt-4", max_retries="5") + assert params.max_retries == 5 + assert isinstance(params.max_retries, int) + + def test_extra_fields_preserved(self): + """Test that extra fields are preserved when reserved keys are filtered.""" + params_dict = { + "model": "gpt-4", + "self": "ignored", + "custom_field": "custom_value", + } + params = LiteLLM_Params(**params_dict) + assert params.model == "gpt-4" + assert params.custom_field == "custom_value" + + def test_normal_instantiation_still_works(self): + """Test that normal instantiation without reserved keys works.""" + params = LiteLLM_Params( + model="gpt-4", api_key="test-key", custom_llm_provider="openai" + ) + assert params.model == "gpt-4" + assert params.api_key == "test-key" + assert params.custom_llm_provider == "openai" + + def test_multiple_reserved_keys(self): + """Test filtering multiple reserved keys at once.""" + params_dict = { + "model": "gpt-4", + "self": "value1", + "params": "value2", + "__class__": "value3", + "api_key": "test-key", + } + params = LiteLLM_Params(**params_dict) + assert params.model == "gpt-4" + assert params.api_key == "test-key" From f44e67b0f199d4dc9aaf8c8f134b1ee3afb5a1ee Mon Sep 17 00:00:00 2001 From: Marty Sullivan Date: Mon, 9 Mar 2026 22:43:53 -0400 Subject: [PATCH 12/40] 2026-03-09-azure-updates (#23159) * add new azure gpt models * add versionless azure/gpt-5.4 models * Undated azure/gpt-5.4 alias missing supports_service_tier * indicate service tier support for azure/gpt-5.3-chat * fix priority tier pricing for new azure/gpt models --- ...odel_prices_and_context_window_backup.json | 189 ++++++++++++++++++ model_prices_and_context_window.json | 189 ++++++++++++++++++ 2 files changed, 378 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 177a2bf52e..194af4895f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4207,6 +4207,41 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-5.3-chat": { + "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, @@ -4299,6 +4334,160 @@ "supports_vision": true, "supports_web_search": true }, + "azure/gpt-5.4": { + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_priority": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_272k_tokens": 5e-06, + "input_cost_per_token_priority": 5e-06, + "input_cost_per_token_above_272k_tokens_priority": 1e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_above_272k_tokens": 2.25e-05, + "output_cost_per_token_priority": 3e-05, + "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "azure/gpt-5.4-2026-03-05": { + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_priority": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_272k_tokens": 5e-06, + "input_cost_per_token_priority": 5e-06, + "input_cost_per_token_above_272k_tokens_priority": 1e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_above_272k_tokens": 2.25e-05, + "output_cost_per_token_priority": 3e-05, + "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "azure/gpt-5.4-pro": { + "cache_read_input_token_cost": 3e-06, + "cache_read_input_token_cost_above_272k_tokens": 6e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 6e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 0.00018, + "output_cost_per_token_above_272k_tokens": 0.00027, + "supported_endpoints": [ + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure/gpt-5.4-pro-2026-03-05": { + "cache_read_input_token_cost": 3e-06, + "cache_read_input_token_cost_above_272k_tokens": 6e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 6e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 0.00018, + "output_cost_per_token_above_272k_tokens": 0.00027, + "supported_endpoints": [ + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure/gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, "cache_read_input_token_cost": 1.25e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 177a2bf52e..194af4895f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4207,6 +4207,41 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/gpt-5.3-chat": { + "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, @@ -4299,6 +4334,160 @@ "supports_vision": true, "supports_web_search": true }, + "azure/gpt-5.4": { + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_priority": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_272k_tokens": 5e-06, + "input_cost_per_token_priority": 5e-06, + "input_cost_per_token_above_272k_tokens_priority": 1e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_above_272k_tokens": 2.25e-05, + "output_cost_per_token_priority": 3e-05, + "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "azure/gpt-5.4-2026-03-05": { + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_priority": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_272k_tokens": 5e-06, + "input_cost_per_token_priority": 5e-06, + "input_cost_per_token_above_272k_tokens_priority": 1e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_above_272k_tokens": 2.25e-05, + "output_cost_per_token_priority": 3e-05, + "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "azure/gpt-5.4-pro": { + "cache_read_input_token_cost": 3e-06, + "cache_read_input_token_cost_above_272k_tokens": 6e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 6e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 0.00018, + "output_cost_per_token_above_272k_tokens": 0.00027, + "supported_endpoints": [ + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure/gpt-5.4-pro-2026-03-05": { + "cache_read_input_token_cost": 3e-06, + "cache_read_input_token_cost_above_272k_tokens": 6e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 6e-05, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 0.00018, + "output_cost_per_token_above_272k_tokens": 0.00027, + "supported_endpoints": [ + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure/gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, "cache_read_input_token_cost": 1.25e-06, From 0d3735f9c0ee8102d43dcba9b39f4028c22d59c2 Mon Sep 17 00:00:00 2001 From: JiangNan <1394485448@qq.com> Date: Tue, 10 Mar 2026 10:46:26 +0800 Subject: [PATCH 13/40] fix: handle month overflow in duration_in_seconds for multi-month durations (#23099) When value > 1 (e.g., "2mo") and current_month + value > 12, target_month exceeds valid range (1-12), causing ValueError in datetime constructor. For example, calling duration_in_seconds("2mo") in November produces target_month=13. Use modular arithmetic to correctly wrap months and increment year. Signed-off-by: JiangNan <1394485448@qq.com> --- litellm/litellm_core_utils/duration_parser.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 70c28c4e06..6d2b4226ff 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -64,12 +64,10 @@ def duration_in_seconds(duration: str) -> int: now = time.time() current_time = datetime.fromtimestamp(now) - if current_time.month == 12: - target_year = current_time.year + 1 - target_month = 1 - else: - target_year = current_time.year - target_month = current_time.month + value + # Calculate target month and year, handling overflow past December + total_months = current_time.month - 1 + value # 0-indexed months + target_year = current_time.year + total_months // 12 + target_month = total_months % 12 + 1 # back to 1-indexed # Determine the day to set for next month target_day = current_time.day From 3ce37e6c35f41d9e337e9db6c6e58b3616e4699b Mon Sep 17 00:00:00 2001 From: JiangNan <1394485448@qq.com> Date: Tue, 10 Mar 2026 10:49:31 +0800 Subject: [PATCH 14/40] fix: use correct list length when averaging TTFT latency for streaming requests (#23100) In _get_available_deployments, when streaming mode is active the code sums time-to-first-token values from item_ttft_latency but divides by len(item_latency) instead of len(item_ttft_latency). These lists can have different lengths, producing an incorrect average that skews lowest-latency routing decisions for streaming requests. Signed-off-by: JiangNan <1394485448@qq.com> --- litellm/router_strategy/lowest_latency.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 0449a843bd..e09b5c1456 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -490,20 +490,22 @@ class LowestLatencyLoggingHandler(CustomLogger): # get average latency or average ttft (depending on streaming/non-streaming) total: float = 0.0 - if ( + use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 - ): + ) + if use_ttft: for _call_latency in item_ttft_latency: if isinstance(_call_latency, float): total += _call_latency + item_latency = total / len(item_ttft_latency) else: for _call_latency in item_latency: if isinstance(_call_latency, float): total += _call_latency - item_latency = total / len(item_latency) + item_latency = total / len(item_latency) # -------------- # # Debugging Logic From 9314963697120c289c489f4decb49bea2e8d55e9 Mon Sep 17 00:00:00 2001 From: zxshen Date: Tue, 10 Mar 2026 10:49:58 +0800 Subject: [PATCH 15/40] fix(fireworks): strip duplicate /v1 from models endpoint URL (#23113) _get_openai_compatible_provider_info already returns an api_base ending in /v1, but get_models prepended another /v1, producing .../inference/v1/v1/accounts/... which 404s. Strip the trailing /v1 from api_base before re-adding it so that both the default and any user-supplied base work correctly. Add parametrized tests covering the default URL, trailing-slash, custom base with /v1, and custom base without /v1. Fixes #23106 --- .../llms/fireworks_ai/chat/transformation.py | 5 +- .../test_fireworks_ai_chat_transformation.py | 54 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 7ec32fecc4..c9b6f330ec 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -426,8 +426,11 @@ class FireworksAIConfig(OpenAIGPTConfig): "FIREWORKS_ACCOUNT_ID is not set. Please set the environment variable, to query Fireworks AI's `/models` endpoint." ) + base = api_base.rstrip("/") + if base.endswith("/v1"): + base = base[: -len("/v1")] response = litellm.module_level_client.get( - url=f"{api_base}/v1/accounts/{account_id}/models", + url=f"{base}/v1/accounts/{account_id}/models", headers={"Authorization": f"Bearer {api_key}"}, ) diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 8006ffdff1..5d5aaa64c8 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -110,6 +110,60 @@ def test_get_supported_openai_params_reasoning_effort(): assert "reasoning_effort" not in unsupported_params +@pytest.mark.parametrize( + "api_base, expected_url_prefix", + [ + ( + "https://api.fireworks.ai/inference/v1", + "https://api.fireworks.ai/inference/v1/accounts/", + ), + ( + "https://api.fireworks.ai/inference/v1/", + "https://api.fireworks.ai/inference/v1/accounts/", + ), + ( + "https://custom-host.example.com/v1", + "https://custom-host.example.com/v1/accounts/", + ), + ( + "https://custom-host.example.com/api", + "https://custom-host.example.com/api/v1/accounts/", + ), + ], + ids=["default", "trailing-slash", "custom-with-v1", "custom-without-v1"], +) +def test_get_models_url_no_double_v1(api_base, expected_url_prefix): + """Ensure get_models never produces a /v1/v1/ URL segment (fixes #23106).""" + config = FireworksAIConfig() + account_id = "fireworks" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] + } + + with ( + patch("litellm.module_level_client.get", return_value=mock_response) as mock_get, + patch( + "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", + side_effect=lambda key: { + "FIREWORKS_API_KEY": "test-key", + "FIREWORKS_API_BASE": api_base, + "FIREWORKS_ACCOUNT_ID": account_id, + }.get(key), + ), + ): + result = config.get_models(api_key="test-key", api_base=api_base) + + called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "") + assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" + assert called_url.startswith(expected_url_prefix), ( + f"URL {called_url} does not start with {expected_url_prefix}" + ) + assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] + + def test_transform_messages_helper_removes_provider_specific_fields(): """ Test that _transform_messages_helper removes provider_specific_fields from messages. From 2c738cc939c408cd0e85772bd503297c7d363197 Mon Sep 17 00:00:00 2001 From: Maxwell Calkin <101308415+MaxwellCalkin@users.noreply.github.com> Date: Mon, 9 Mar 2026 22:51:25 -0400 Subject: [PATCH 16/40] fix: strip empty text content blocks in /v1/messages endpoint (#23097) Claude's API returns assistant messages with empty text blocks ({"type": "text", "text": ""}) alongside tool_use blocks during multi-turn tool-use conversations. These blocks are rejected when sent back to the API with "text content blocks must be non-empty". Sanitization already exists for other code paths (/v1/chat/completions for both Anthropic and Bedrock), but NOT for the /v1/messages native path. This adds the same treatment by stripping empty text blocks from messages in async_anthropic_messages_handler before they are forwarded to the provider. Fixes #22930 --- litellm/llms/custom_httpx/llm_http_handler.py | 60 +++++ ...est_v1_messages_empty_text_sanitization.py | 247 ++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1cef3e9ce1..6a5d669cad 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -152,6 +152,59 @@ else: LiteLLMLoggingObj = Any +def _sanitize_anthropic_messages_empty_text_blocks( + messages: List[Dict], +) -> List[Dict]: + """ + Strip empty text content blocks from Anthropic-format messages. + + Claude's API returns assistant messages with ``{"type": "text", "text": ""}`` + alongside ``tool_use`` blocks, but rejects them when sent back in subsequent + requests. This helper removes those empty text blocks so the /v1/messages + native path doesn't forward them as-is. + + - If a content list contains a mix of empty text blocks and other blocks + (e.g. tool_use), the empty text blocks are removed. + - If *all* blocks in a content list are empty text, the content is replaced + with a single non-empty placeholder to avoid sending an empty array. + + Ref: https://github.com/BerriAI/litellm/issues/22930 + """ + sanitized: List[Dict] = [] + for message in messages: + content = message.get("content") + if not isinstance(content, list): + sanitized.append(message) + continue + + filtered = [ + block + for block in content + if not ( + isinstance(block, dict) + and block.get("type") == "text" + and not block.get("text", "").strip() + ) + ] + + if filtered == content: + # Nothing was removed — keep original message as-is. + sanitized.append(message) + elif filtered: + # Some empty text blocks removed, but other content remains. + new_message = message.copy() + new_message["content"] = filtered + sanitized.append(new_message) + else: + # All blocks were empty text blocks. Replace with a placeholder + # so we don't send an empty content array. + new_message = message.copy() + new_message["content"] = [{"type": "text", "text": "..."}] + sanitized.append(new_message) + + return sanitized + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -1905,6 +1958,13 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) + # Sanitize empty text content blocks from messages before forwarding. + # Claude's API returns assistant messages with empty text blocks + # ({"type": "text", "text": ""}) alongside tool_use blocks, but rejects + # them when sent back. Strip these to prevent 400 errors. + # Ref: https://github.com/BerriAI/litellm/issues/22930 + messages = _sanitize_anthropic_messages_empty_text_blocks(messages) + # Prepare request body request_body = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, diff --git a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py new file mode 100644 index 0000000000..b397b5a484 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py @@ -0,0 +1,247 @@ +""" +Test empty text content block sanitization for the /v1/messages native path. + +The Anthropic API returns assistant messages with empty text blocks +({"type": "text", "text": ""}) alongside tool_use blocks, but rejects +them when sent back. The /v1/messages endpoint must strip these before +forwarding to providers. + +Ref: https://github.com/BerriAI/litellm/issues/22930 +""" + +import pytest + +from litellm.llms.custom_httpx.llm_http_handler import ( + _sanitize_anthropic_messages_empty_text_blocks, +) + + +class TestSanitizeAnthropicMessagesEmptyTextBlocks: + """Unit tests for _sanitize_anthropic_messages_empty_text_blocks.""" + + def test_strips_empty_text_alongside_tool_use(self): + """ + The most common case from the bug report: an assistant message + containing an empty text block next to a tool_use block. + """ + messages = [ + {"role": "user", "content": "Run the command."}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "toolu_xxx", + "name": "Bash", + "input": {"command": "ls"}, + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result) == 2 + assert result[0] == messages[0] # user message unchanged + # assistant content should only have the tool_use block + assert len(result[1]["content"]) == 1 + assert result[1]["content"][0]["type"] == "tool_use" + + def test_preserves_nonempty_text_blocks(self): + """Non-empty text blocks must not be removed.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Let me check that."}, + { + "type": "tool_use", + "id": "toolu_yyy", + "name": "Bash", + "input": {"command": "pwd"}, + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 2 + assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} + + def test_whitespace_only_text_block_stripped(self): + """Whitespace-only text blocks should also be stripped.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": " \n\t "}, + { + "type": "tool_use", + "id": "toolu_zzz", + "name": "Bash", + "input": {}, + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 1 + assert result[0]["content"][0]["type"] == "tool_use" + + def test_all_empty_text_blocks_replaced_with_placeholder(self): + """ + If all content blocks are empty text, replace with a placeholder + to avoid sending an empty content array. + """ + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 1 + assert result[0]["content"][0]["type"] == "text" + assert result[0]["content"][0]["text"].strip() # must be non-empty + + def test_string_content_untouched(self): + """Messages with string content should pass through unchanged.""" + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"}, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert result == messages + + def test_no_content_key_untouched(self): + """Messages without a content key should pass through.""" + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant"}, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert result == messages + + def test_user_message_content_list_also_sanitized(self): + """ + Empty text blocks should be stripped from user messages too, + not just assistant messages. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": ""}, + {"type": "text", "text": "actual question"}, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 1 + assert result[0]["content"][0]["text"] == "actual question" + + def test_tool_result_content_blocks_untouched(self): + """ + tool_result content blocks should not be affected — only + {"type": "text", "text": ""} blocks are stripped. + """ + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_xxx", + "content": "", + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert result == messages + + def test_multiple_messages_mixed(self): + """End-to-end scenario with multiple messages, some needing sanitization.""" + messages = [ + {"role": "user", "content": "Run ls"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "toolu_1", + "name": "Bash", + "input": {"command": "ls"}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": "file1.txt\nfile2.txt", + }, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here are the files:"}, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + # First message: string content, unchanged + assert result[0] == messages[0] + # Second message: empty text stripped, only tool_use remains + assert len(result[1]["content"]) == 1 + assert result[1]["content"][0]["type"] == "tool_use" + # Third message: tool_result, unchanged + assert result[2] == messages[2] + # Fourth message: non-empty text, unchanged + assert result[3] == messages[3] + + def test_does_not_mutate_original_messages(self): + """The function should not modify the input list or its dicts.""" + original_content = [ + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "toolu_1", + "name": "Bash", + "input": {}, + }, + ] + messages = [ + { + "role": "assistant", + "content": original_content, + }, + ] + + _sanitize_anthropic_messages_empty_text_blocks(messages) + + # Original message content should be unchanged + assert len(messages[0]["content"]) == 2 + assert messages[0]["content"][0] == {"type": "text", "text": ""} From 30b82c3a0cef9ffa3abcdbd1768ee9cc026f3825 Mon Sep 17 00:00:00 2001 From: tristanolive Date: Tue, 10 Mar 2026 03:46:43 +0000 Subject: [PATCH 17/40] feat(charity_engine): add Charity Engine provider (#23223) * feat(charity_engine): add Charity Engine provider Charity Engine is a crowdsourced distributed computing platform that donates processing power to charitable causes. Its inference API provides OpenAI-compatible chat, completions, and embeddings endpoints. * test(charity_engine): add provider config and resolution tests Verify JSONProviderRegistry config, provider list membership, model routing for charity_engine/, and Router compatibility. * feat(charity_engine): add Charity Engine to LlmProviders enum Enables provider_list membership and LlmProviders.CHARITY_ENGINE resolution required by the provider and test suite. * fix(charity_engine): remove api_base_env to fix non-deterministic test The CHARITY_ENGINE_API_BASE env var could override the base_url in CI, causing test_charity_engine_provider_resolution to fail intermittently. * fix(charity_engine): remove trailing slash from base_url --- litellm/llms/openai_like/providers.json | 7 ++ litellm/types/utils.py | 1 + .../llms/openai_like/test_charity_engine.py | 101 ++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 tests/test_litellm/llms/openai_like/test_charity_engine.py diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index b3125d4ad3..275c352b39 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -94,5 +94,12 @@ "assemblyai": { "base_url": "https://llm-gateway.assemblyai.com/v1", "api_key_env": "ASSEMBLYAI_API_KEY" + }, + "charity_engine": { + "base_url": "https://api.charityengine.services/remotejobs/v2/inference", + "api_key_env": "CHARITY_ENGINE_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } } diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8ae0cf2892..b5d5c06924 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3177,6 +3177,7 @@ class LlmProviders(str, Enum): TOPAZ = "topaz" SAP_GENERATIVE_AI_HUB = "sap" ASSEMBLYAI = "assemblyai" + CHARITY_ENGINE = "charity_engine" GITHUB_COPILOT = "github_copilot" SNOWFLAKE = "snowflake" GRADIENT_AI = "gradient_ai" diff --git a/tests/test_litellm/llms/openai_like/test_charity_engine.py b/tests/test_litellm/llms/openai_like/test_charity_engine.py new file mode 100644 index 0000000000..5d6a751b62 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_charity_engine.py @@ -0,0 +1,101 @@ +""" +Tests for Charity Engine provider configuration and integration. +""" + +import os +import sys + +try: + import pytest +except ImportError: + pytest = None + +# Add workspace to path +workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +sys.path.insert(0, workspace_path) + +import litellm + + +class TestCharityEngineProviderConfig: + """Test Charity Engine provider configuration""" + + def test_charity_engine_in_provider_list(self): + """Test that charity_engine is in the provider list""" + from litellm import LlmProviders + + assert hasattr(LlmProviders, "CHARITY_ENGINE") + assert LlmProviders.CHARITY_ENGINE.value == "charity_engine" + assert "charity_engine" in litellm.provider_list + + def test_charity_engine_json_config_exists(self): + """Test that charity_engine is configured in providers.json""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("charity_engine") + + charity_engine = JSONProviderRegistry.get("charity_engine") + assert charity_engine is not None + assert charity_engine.base_url == "https://api.charityengine.services/remotejobs/v2/inference" + assert charity_engine.api_key_env == "CHARITY_ENGINE_API_KEY" + assert charity_engine.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_charity_engine_provider_resolution(self): + """Test that provider resolution finds charity_engine""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="charity_engine/gemma3:270m", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "gemma3:270m" + assert provider == "charity_engine" + assert api_base == "https://api.charityengine.services/remotejobs/v2/inference" + + def test_charity_engine_router_config(self): + """Test that charity_engine can be used in Router configuration""" + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "gemma3-270m", + "litellm_params": { + "model": "charity_engine/gemma3:270m", + "api_key": "test-key", + }, + } + ] + ) + + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "gemma3-270m" + + +if __name__ == "__main__": + print("Testing Charity Engine Provider...") + + test_config = TestCharityEngineProviderConfig() + + print("\n1. Testing provider in list...") + test_config.test_charity_engine_in_provider_list() + print(" ✓ charity_engine in provider list") + + print("\n2. Testing JSON config...") + test_config.test_charity_engine_json_config_exists() + print(" ✓ charity_engine JSON config loaded") + + print("\n3. Testing provider resolution...") + test_config.test_charity_engine_provider_resolution() + print(" ✓ Provider resolution works") + + print("\n4. Testing router configuration...") + test_config.test_charity_engine_router_config() + print(" ✓ Router configuration works") + + print("\n" + "=" * 50) + print("✓ All configuration tests passed!") + print("=" * 50) From dd6f0d6c55179634b5a5b8d1f670d97c56891b6c Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 9 Mar 2026 20:56:27 -0700 Subject: [PATCH 18/40] fix: forward recognized OpenAI params from kwargs in completion() (#23224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that arrives via completion(**kwargs) is now automatically forwarded to get_optional_params(), even if it's not a named parameter of completion(). Previously, get_non_default_completion_params() excluded params in OPENAI_CHAT_COMPLETION_PARAMS (assuming they'd be forwarded via the named-param path), while optional_param_args only contained explicitly named params. Params like 'store' that were in the known-params list but not named params fell through both paths and were silently dropped. The fix adds a 7-line loop after building optional_param_args that forwards any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES. This means new OpenAI params only need to be added to the constants dict — no boilerplate changes to 3+ function signatures required. Fixes #23087 Co-authored-by: Cursor Agent --- litellm/main.py | 8 + .../llms/openai/chat/test_store_param.py | 188 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 tests/test_litellm/llms/openai/chat/test_store_param.py diff --git a/litellm/main.py b/litellm/main.py index 364519e1fe..e23baadb79 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -65,6 +65,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.constants import ( + DEFAULT_CHAT_COMPLETION_PARAM_VALUES, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) @@ -1487,6 +1488,13 @@ def completion( # type: ignore # noqa: PLR0915 "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v optional_params = get_optional_params( **optional_param_args, **non_default_params ) diff --git a/tests/test_litellm/llms/openai/chat/test_store_param.py b/tests/test_litellm/llms/openai/chat/test_store_param.py new file mode 100644 index 0000000000..0fd4799dae --- /dev/null +++ b/tests/test_litellm/llms/openai/chat/test_store_param.py @@ -0,0 +1,188 @@ +""" +Tests for the `store` parameter being correctly forwarded to OpenAI. + +Related issue: https://github.com/BerriAI/litellm/issues/23087 + +The `store` parameter was listed in OPENAI_CHAT_COMPLETION_PARAMS and +DEFAULT_CHAT_COMPLETION_PARAM_VALUES but was silently dropped because +get_non_default_completion_params() excluded it (as a "known" param) +while optional_param_args didn't include it (not a named param of +completion()). The fix adds a safety net in completion() that forwards +any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that aren't +already in optional_param_args. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES +from litellm.utils import get_non_default_completion_params, get_optional_params + + +class TestStoreParamForwarding: + """Tests that `store` flows through the parameter processing pipeline.""" + + def test_store_true_forwarded_for_openai(self): + """should forward store=True for OpenAI models via kwargs""" + result = get_optional_params( + model="gpt-5.1", + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True + + def test_store_false_forwarded_for_openai(self): + """should forward store=False for OpenAI models""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + store=False, + ) + assert result.get("store") is False + + def test_store_none_not_forwarded(self): + """should not include store when it is None (default)""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + ) + assert "store" not in result + + def test_store_with_gpt5_models(self): + """should forward store=True for GPT-5 family models""" + for model in ["gpt-5.1", "gpt-5.2"]: + result = get_optional_params( + model=model, + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True, f"store not forwarded for {model}" + + def test_store_in_supported_params(self): + """should list store as a supported OpenAI param""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + for model in ["gpt-4o", "gpt-5.1", "gpt-5.2"]: + supported = config.get_supported_openai_params(model) + assert "store" in supported, f"store not in supported params for {model}" + + def test_store_in_transform_request(self): + """should include store in the final transformed request body""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + + def test_store_true_with_metadata(self): + """should forward both store and metadata when both are set""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True, "metadata": {"key": "value"}} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + assert result.get("metadata") == {"key": "value"} + + +class TestDefaultParamValuesSafetyNet: + """Tests that any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES flows + through completion() even without being a named parameter.""" + + def test_known_openai_param_excluded_from_non_default(self): + """should confirm get_non_default_completion_params excludes known OpenAI params""" + kwargs = {"store": True, "temperature": 0.5} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert "store" not in non_default + assert "temperature" not in non_default + + def test_unknown_param_included_in_non_default(self): + """should pass through unknown provider-specific params""" + kwargs = {"my_custom_provider_param": "foo"} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert non_default.get("my_custom_provider_param") == "foo" + + def test_safety_net_forwards_recognized_kwargs(self): + """should forward kwargs in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + that are not already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"store": True, "metadata": {"key": "value"}} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["store"] is True + assert optional_param_args["metadata"] == {"key": "value"} + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_does_not_override_existing(self): + """should not override a param that's already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"temperature": 0.9} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_skips_none_values(self): + """should not forward params with None value (the default)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": None} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert "store" not in optional_param_args + + def test_safety_net_forwards_falsy_non_none(self): + """should forward store=False (falsy but not None)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": False} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args.get("store") is False From 33d4b93bee36ed1419f3043ada679e6011fed022 Mon Sep 17 00:00:00 2001 From: jymmi Date: Mon, 9 Mar 2026 20:58:14 -0700 Subject: [PATCH 19/40] fix(sagemaker): Add role assumption support for embedding endpoint (#20435) The SageMaker embedding handler was not using _load_credentials(), which meant aws_role_name and aws_session_name parameters were ignored. This prevented cross-account role assumption for embeddings while it worked for completions. Changes: - Replace direct boto3 client creation with _load_credentials() call - Create boto3.Session with assumed credentials - Add comprehensive unit tests for role assumption This aligns the embedding handler behavior with the completion handler, which already supports role assumption via the BaseAWSLLM.get_credentials() method. Fixes cross-account SageMaker embedding access where users need to assume a role in another account to invoke endpoints. --- litellm/llms/sagemaker/completion/handler.py | 56 ++-- ...est_sagemaker_embedding_role_assumption.py | 243 ++++++++++++++++++ 2 files changed, 263 insertions(+), 36 deletions(-) create mode 100644 tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index 2a30dc5ef3..efbb218f57 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -583,35 +583,17 @@ class SagemakerLLM(BaseAWSLLM): ### BOTO3 INIT import boto3 - # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them - aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) - aws_access_key_id = optional_params.pop("aws_access_key_id", None) - aws_region_name = optional_params.pop("aws_region_name", None) + # Use _load_credentials to support role assumption (aws_role_name, aws_session_name) + credentials, aws_region_name = self._load_credentials(optional_params) - if aws_access_key_id is not None: - # uses auth params passed to completion - # aws_access_key_id is not None, assume user is trying to auth using litellm.completion - client = boto3.client( - service_name="sagemaker-runtime", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - region_name=aws_region_name, - ) - else: - # aws_access_key_id is None, assume user is trying to auth using env variables - # boto3 automaticaly reads env variables - - # we need to read region name from env - # I assume majority of users use .env for auth - region_name = ( - get_secret("AWS_REGION_NAME") - or aws_region_name # get region from config file if specified - or "us-west-2" # default to us-west-2 if region not specified - ) - client = boto3.client( - service_name="sagemaker-runtime", - region_name=region_name, - ) + # Create boto3 session with the loaded credentials + session = boto3.Session( + aws_access_key_id=credentials.access_key, + aws_secret_access_key=credentials.secret_key, + aws_session_token=credentials.token, + region_name=aws_region_name, + ) + client = session.client(service_name="sagemaker-runtime") # pop streaming if it's in the optional params as 'stream' raises an error with sagemaker inference_params = deepcopy(optional_params) @@ -628,7 +610,9 @@ class SagemakerLLM(BaseAWSLLM): #### EMBEDDING LOGIC # Transform request based on model type provider_config = SagemakerEmbeddingConfig.get_model_config(model) - request_data = provider_config.transform_embedding_request(model, input, optional_params, {}) + request_data = provider_config.transform_embedding_request( + model, input, optional_params, {} + ) data = json.dumps(request_data).encode("utf-8") ## LOGGING @@ -673,19 +657,19 @@ class SagemakerLLM(BaseAWSLLM): ) print_verbose(f"raw model_response: {response}") - + # Transform response based on model type from httpx import Response as HttpxResponse - + # Create a mock httpx Response object for the transformation mock_response = HttpxResponse( status_code=200, - content=json.dumps(response).encode('utf-8'), - headers={"content-type": "application/json"} + content=json.dumps(response).encode("utf-8"), + headers={"content-type": "application/json"}, ) - + model_response = EmbeddingResponse() - + # Use the request_data that was already transformed above return provider_config.transform_embedding_response( model=model, @@ -695,5 +679,5 @@ class SagemakerLLM(BaseAWSLLM): api_key=None, request_data=request_data, optional_params=optional_params, - litellm_params=litellm_params or {} + litellm_params=litellm_params or {}, ) diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py new file mode 100644 index 0000000000..82c84af5e2 --- /dev/null +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py @@ -0,0 +1,243 @@ +""" +Test cases for SageMaker embedding role assumption support + +This module tests that the SageMaker embedding handler properly supports +AWS IAM role assumption via aws_role_name and aws_session_name parameters, +matching the behavior of the completion handler. +""" + +import json +import os +import sys +from datetime import timezone +from unittest.mock import MagicMock, call, patch + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from botocore.credentials import Credentials + +from litellm.llms.sagemaker.completion.handler import SagemakerLLM +from litellm.types.utils import EmbeddingResponse + + +class TestSagemakerEmbeddingRoleAssumption: + """Test that SageMaker embedding supports role assumption like completion does""" + + def setup_method(self): + self.sagemaker_llm = SagemakerLLM() + + def test_embedding_uses_load_credentials(self): + """ + Test that embedding() calls _load_credentials() to support role assumption. + This ensures aws_role_name and aws_session_name parameters are properly handled. + """ + # Mock credentials that would be returned after role assumption + mock_credentials = Credentials( + access_key="assumed-access-key", + secret_key="assumed-secret-key", + token="assumed-session-token", + ) + + # Mock the SageMaker client response + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + # Mock boto3.Session to return our mock client + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + + with patch.object( + self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ) as mock_load_creds, patch("boto3.Session", return_value=mock_session): + + # Create mock logging object + mock_logging = MagicMock() + + optional_params = { + "aws_role_name": "arn:aws:iam::123456789012:role/TestRole", + "aws_session_name": "test-session", + } + + self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params=optional_params, + ) + + # Verify _load_credentials was called with the optional_params + mock_load_creds.assert_called_once() + + # Verify boto3.Session was created with the assumed credentials + mock_session_calls = mock_session.client.call_args_list + assert len(mock_session_calls) == 1 + assert mock_session_calls[0] == call(service_name="sagemaker-runtime") + + def test_embedding_role_assumption_with_sts(self): + """ + Test the full role assumption flow for embeddings, similar to completion. + Verifies that STS assume_role is called when aws_role_name is provided. + """ + # Mock the STS client for role assumption + mock_sts_client = MagicMock() + + # Mock the STS response with proper expiration handling + mock_expiry = MagicMock() + mock_expiry.tzinfo = timezone.utc + time_diff = MagicMock() + time_diff.total_seconds.return_value = 3600 + mock_expiry.__sub__ = MagicMock(return_value=time_diff) + + mock_sts_response = { + "Credentials": { + "AccessKeyId": "assumed-access-key", + "SecretAccessKey": "assumed-secret-key", + "SessionToken": "assumed-session-token", + "Expiration": mock_expiry, + } + } + mock_sts_client.assume_role.return_value = mock_sts_response + + # Mock the SageMaker client response + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + # Mock boto3.Session for SageMaker client creation + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + + def mock_boto3_client(service_name, **kwargs): + if service_name == "sts": + return mock_sts_client + return mock_sagemaker_client + + with patch("boto3.client", side_effect=mock_boto3_client), \ + patch("boto3.Session", return_value=mock_session): + + mock_logging = MagicMock() + + optional_params = { + "aws_role_name": "arn:aws:iam::123456789012:role/CrossAccountRole", + "aws_session_name": "litellm-embedding-session", + "aws_region_name": "us-east-1", + } + + self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params=optional_params, + ) + + # Verify STS assume_role was called with correct parameters + mock_sts_client.assume_role.assert_called_once() + call_args = mock_sts_client.assume_role.call_args + assert call_args[1]["RoleArn"] == "arn:aws:iam::123456789012:role/CrossAccountRole" + assert call_args[1]["RoleSessionName"] == "litellm-embedding-session" + + def test_embedding_without_role_assumption(self): + """ + Test that embedding works without role assumption when aws_role_name is not provided. + Should use default credentials from environment/instance profile. + """ + # Mock the SageMaker client response + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + + # Mock credentials returned from environment + mock_credentials = Credentials( + access_key="env-access-key", + secret_key="env-secret-key", + token=None, + ) + + with patch.object( + self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-west-2") + ), patch("boto3.Session", return_value=mock_session): + + mock_logging = MagicMock() + + # No aws_role_name provided + optional_params = { + "aws_region_name": "us-west-2", + } + + result = self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params=optional_params, + ) + + # Should still work and return embeddings + assert result is not None + + def test_embedding_session_created_with_assumed_credentials(self): + """ + Test that boto3.Session is created with the credentials from role assumption. + This verifies the credentials flow from _load_credentials to the SageMaker client. + """ + mock_credentials = Credentials( + access_key="assumed-key", + secret_key="assumed-secret", + token="assumed-token", + ) + + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + with patch.object( + self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), patch("boto3.Session") as mock_session_class: + + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + mock_session_class.return_value = mock_session + + mock_logging = MagicMock() + + self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params={}, + ) + + # Verify Session was created with the assumed credentials + mock_session_class.assert_called_once_with( + aws_access_key_id="assumed-key", + aws_secret_access_key="assumed-secret", + aws_session_token="assumed-token", + region_name="us-east-1", + ) From 325df8d62aaaaa8d079ed2269b781dcdfcd0202a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:45:28 +0530 Subject: [PATCH 20/40] Fix logging tests --- litellm/litellm_core_utils/redact_messages.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ad68f3851a..ddeb24d04a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -123,6 +123,16 @@ def perform_redaction(model_call_details: dict, result): elif isinstance(_result, litellm.EmbeddingResponse): if hasattr(_result, "data") and _result.data is not None: _result.data = [] + elif isinstance(_result, dict) and "choices" in _result: + # ModelResponse.model_dump() returns dict - redact choices in place + if isinstance(_result.get("choices"), list) and len(_result["choices"]) > 0: + choice = _result["choices"][0] + if isinstance(choice, dict) and "message" in choice: + msg = choice["message"] + if isinstance(msg, dict) and "content" in msg: + msg["content"] = "redacted-by-litellm" + if isinstance(msg, dict) and "audio" in msg: + msg["audio"] = None else: return {"text": "redacted-by-litellm"} return _result From 56be0a651f2d355be8ea9a0a1f2f71b2dc56cf80 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:50:37 +0530 Subject: [PATCH 21/40] fix: add charity_engine to provider_endpoints_support.json Made-with: Cursor --- provider_endpoints_support.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b1d4d5a116..0b3f87fbe0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,6 +458,24 @@ "interactions": true } }, + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { From c1b860b3c1f98c94892eaf30ddf7b32a46e20194 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:53:19 +0530 Subject: [PATCH 22/40] Revert "fix: strip empty text content blocks in /v1/messages endpoint (#23097)" This reverts commit 2c738cc939c408cd0e85772bd503297c7d363197. --- litellm/llms/custom_httpx/llm_http_handler.py | 60 ----- ...est_v1_messages_empty_text_sanitization.py | 247 ------------------ 2 files changed, 307 deletions(-) delete mode 100644 tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6a5d669cad..1cef3e9ce1 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -152,59 +152,6 @@ else: LiteLLMLoggingObj = Any -def _sanitize_anthropic_messages_empty_text_blocks( - messages: List[Dict], -) -> List[Dict]: - """ - Strip empty text content blocks from Anthropic-format messages. - - Claude's API returns assistant messages with ``{"type": "text", "text": ""}`` - alongside ``tool_use`` blocks, but rejects them when sent back in subsequent - requests. This helper removes those empty text blocks so the /v1/messages - native path doesn't forward them as-is. - - - If a content list contains a mix of empty text blocks and other blocks - (e.g. tool_use), the empty text blocks are removed. - - If *all* blocks in a content list are empty text, the content is replaced - with a single non-empty placeholder to avoid sending an empty array. - - Ref: https://github.com/BerriAI/litellm/issues/22930 - """ - sanitized: List[Dict] = [] - for message in messages: - content = message.get("content") - if not isinstance(content, list): - sanitized.append(message) - continue - - filtered = [ - block - for block in content - if not ( - isinstance(block, dict) - and block.get("type") == "text" - and not block.get("text", "").strip() - ) - ] - - if filtered == content: - # Nothing was removed — keep original message as-is. - sanitized.append(message) - elif filtered: - # Some empty text blocks removed, but other content remains. - new_message = message.copy() - new_message["content"] = filtered - sanitized.append(new_message) - else: - # All blocks were empty text blocks. Replace with a placeholder - # so we don't send an empty content array. - new_message = message.copy() - new_message["content"] = [{"type": "text", "text": "..."}] - sanitized.append(new_message) - - return sanitized - - class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -1958,13 +1905,6 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) - # Sanitize empty text content blocks from messages before forwarding. - # Claude's API returns assistant messages with empty text blocks - # ({"type": "text", "text": ""}) alongside tool_use blocks, but rejects - # them when sent back. Strip these to prevent 400 errors. - # Ref: https://github.com/BerriAI/litellm/issues/22930 - messages = _sanitize_anthropic_messages_empty_text_blocks(messages) - # Prepare request body request_body = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, diff --git a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py deleted file mode 100644 index b397b5a484..0000000000 --- a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -Test empty text content block sanitization for the /v1/messages native path. - -The Anthropic API returns assistant messages with empty text blocks -({"type": "text", "text": ""}) alongside tool_use blocks, but rejects -them when sent back. The /v1/messages endpoint must strip these before -forwarding to providers. - -Ref: https://github.com/BerriAI/litellm/issues/22930 -""" - -import pytest - -from litellm.llms.custom_httpx.llm_http_handler import ( - _sanitize_anthropic_messages_empty_text_blocks, -) - - -class TestSanitizeAnthropicMessagesEmptyTextBlocks: - """Unit tests for _sanitize_anthropic_messages_empty_text_blocks.""" - - def test_strips_empty_text_alongside_tool_use(self): - """ - The most common case from the bug report: an assistant message - containing an empty text block next to a tool_use block. - """ - messages = [ - {"role": "user", "content": "Run the command."}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_xxx", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result) == 2 - assert result[0] == messages[0] # user message unchanged - # assistant content should only have the tool_use block - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - - def test_preserves_nonempty_text_blocks(self): - """Non-empty text blocks must not be removed.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Let me check that."}, - { - "type": "tool_use", - "id": "toolu_yyy", - "name": "Bash", - "input": {"command": "pwd"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 2 - assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} - - def test_whitespace_only_text_block_stripped(self): - """Whitespace-only text blocks should also be stripped.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": " \n\t "}, - { - "type": "tool_use", - "id": "toolu_zzz", - "name": "Bash", - "input": {}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "tool_use" - - def test_all_empty_text_blocks_replaced_with_placeholder(self): - """ - If all content blocks are empty text, replace with a placeholder - to avoid sending an empty content array. - """ - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "text" - assert result[0]["content"][0]["text"].strip() # must be non-empty - - def test_string_content_untouched(self): - """Messages with string content should pass through unchanged.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_no_content_key_untouched(self): - """Messages without a content key should pass through.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_user_message_content_list_also_sanitized(self): - """ - Empty text blocks should be stripped from user messages too, - not just assistant messages. - """ - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": ""}, - {"type": "text", "text": "actual question"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["text"] == "actual question" - - def test_tool_result_content_blocks_untouched(self): - """ - tool_result content blocks should not be affected — only - {"type": "text", "text": ""} blocks are stripped. - """ - messages = [ - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_xxx", - "content": "", - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_multiple_messages_mixed(self): - """End-to-end scenario with multiple messages, some needing sanitization.""" - messages = [ - {"role": "user", "content": "Run ls"}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": "file1.txt\nfile2.txt", - }, - ], - }, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Here are the files:"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - # First message: string content, unchanged - assert result[0] == messages[0] - # Second message: empty text stripped, only tool_use remains - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - # Third message: tool_result, unchanged - assert result[2] == messages[2] - # Fourth message: non-empty text, unchanged - assert result[3] == messages[3] - - def test_does_not_mutate_original_messages(self): - """The function should not modify the input list or its dicts.""" - original_content = [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {}, - }, - ] - messages = [ - { - "role": "assistant", - "content": original_content, - }, - ] - - _sanitize_anthropic_messages_empty_text_blocks(messages) - - # Original message content should be unchanged - assert len(messages[0]["content"]) == 2 - assert messages[0]["content"][0] == {"type": "text", "text": ""} From 2cb47727b62d63417d41032aeefb7728df1a3442 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 9 Mar 2026 20:56:27 -0700 Subject: [PATCH 23/40] fix: forward recognized OpenAI params from kwargs in completion() (#23224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that arrives via completion(**kwargs) is now automatically forwarded to get_optional_params(), even if it's not a named parameter of completion(). Previously, get_non_default_completion_params() excluded params in OPENAI_CHAT_COMPLETION_PARAMS (assuming they'd be forwarded via the named-param path), while optional_param_args only contained explicitly named params. Params like 'store' that were in the known-params list but not named params fell through both paths and were silently dropped. The fix adds a 7-line loop after building optional_param_args that forwards any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES. This means new OpenAI params only need to be added to the constants dict — no boilerplate changes to 3+ function signatures required. Fixes #23087 Co-authored-by: Cursor Agent --- litellm/main.py | 8 + .../llms/openai/chat/test_store_param.py | 188 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 tests/test_litellm/llms/openai/chat/test_store_param.py diff --git a/litellm/main.py b/litellm/main.py index 364519e1fe..e23baadb79 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -65,6 +65,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.constants import ( + DEFAULT_CHAT_COMPLETION_PARAM_VALUES, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) @@ -1487,6 +1488,13 @@ def completion( # type: ignore # noqa: PLR0915 "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v optional_params = get_optional_params( **optional_param_args, **non_default_params ) diff --git a/tests/test_litellm/llms/openai/chat/test_store_param.py b/tests/test_litellm/llms/openai/chat/test_store_param.py new file mode 100644 index 0000000000..0fd4799dae --- /dev/null +++ b/tests/test_litellm/llms/openai/chat/test_store_param.py @@ -0,0 +1,188 @@ +""" +Tests for the `store` parameter being correctly forwarded to OpenAI. + +Related issue: https://github.com/BerriAI/litellm/issues/23087 + +The `store` parameter was listed in OPENAI_CHAT_COMPLETION_PARAMS and +DEFAULT_CHAT_COMPLETION_PARAM_VALUES but was silently dropped because +get_non_default_completion_params() excluded it (as a "known" param) +while optional_param_args didn't include it (not a named param of +completion()). The fix adds a safety net in completion() that forwards +any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that aren't +already in optional_param_args. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES +from litellm.utils import get_non_default_completion_params, get_optional_params + + +class TestStoreParamForwarding: + """Tests that `store` flows through the parameter processing pipeline.""" + + def test_store_true_forwarded_for_openai(self): + """should forward store=True for OpenAI models via kwargs""" + result = get_optional_params( + model="gpt-5.1", + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True + + def test_store_false_forwarded_for_openai(self): + """should forward store=False for OpenAI models""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + store=False, + ) + assert result.get("store") is False + + def test_store_none_not_forwarded(self): + """should not include store when it is None (default)""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + ) + assert "store" not in result + + def test_store_with_gpt5_models(self): + """should forward store=True for GPT-5 family models""" + for model in ["gpt-5.1", "gpt-5.2"]: + result = get_optional_params( + model=model, + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True, f"store not forwarded for {model}" + + def test_store_in_supported_params(self): + """should list store as a supported OpenAI param""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + for model in ["gpt-4o", "gpt-5.1", "gpt-5.2"]: + supported = config.get_supported_openai_params(model) + assert "store" in supported, f"store not in supported params for {model}" + + def test_store_in_transform_request(self): + """should include store in the final transformed request body""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + + def test_store_true_with_metadata(self): + """should forward both store and metadata when both are set""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True, "metadata": {"key": "value"}} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + assert result.get("metadata") == {"key": "value"} + + +class TestDefaultParamValuesSafetyNet: + """Tests that any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES flows + through completion() even without being a named parameter.""" + + def test_known_openai_param_excluded_from_non_default(self): + """should confirm get_non_default_completion_params excludes known OpenAI params""" + kwargs = {"store": True, "temperature": 0.5} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert "store" not in non_default + assert "temperature" not in non_default + + def test_unknown_param_included_in_non_default(self): + """should pass through unknown provider-specific params""" + kwargs = {"my_custom_provider_param": "foo"} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert non_default.get("my_custom_provider_param") == "foo" + + def test_safety_net_forwards_recognized_kwargs(self): + """should forward kwargs in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + that are not already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"store": True, "metadata": {"key": "value"}} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["store"] is True + assert optional_param_args["metadata"] == {"key": "value"} + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_does_not_override_existing(self): + """should not override a param that's already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"temperature": 0.9} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_skips_none_values(self): + """should not forward params with None value (the default)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": None} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert "store" not in optional_param_args + + def test_safety_net_forwards_falsy_non_none(self): + """should forward store=False (falsy but not None)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": False} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args.get("store") is False From 7542845e8db7cceffbbe3d0ada8f5360ec469800 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:53:19 +0530 Subject: [PATCH 24/40] Revert "fix: strip empty text content blocks in /v1/messages endpoint (#23097)" This reverts commit 2c738cc939c408cd0e85772bd503297c7d363197. --- litellm/llms/custom_httpx/llm_http_handler.py | 60 ----- ...est_v1_messages_empty_text_sanitization.py | 247 ------------------ 2 files changed, 307 deletions(-) delete mode 100644 tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6a5d669cad..1cef3e9ce1 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -152,59 +152,6 @@ else: LiteLLMLoggingObj = Any -def _sanitize_anthropic_messages_empty_text_blocks( - messages: List[Dict], -) -> List[Dict]: - """ - Strip empty text content blocks from Anthropic-format messages. - - Claude's API returns assistant messages with ``{"type": "text", "text": ""}`` - alongside ``tool_use`` blocks, but rejects them when sent back in subsequent - requests. This helper removes those empty text blocks so the /v1/messages - native path doesn't forward them as-is. - - - If a content list contains a mix of empty text blocks and other blocks - (e.g. tool_use), the empty text blocks are removed. - - If *all* blocks in a content list are empty text, the content is replaced - with a single non-empty placeholder to avoid sending an empty array. - - Ref: https://github.com/BerriAI/litellm/issues/22930 - """ - sanitized: List[Dict] = [] - for message in messages: - content = message.get("content") - if not isinstance(content, list): - sanitized.append(message) - continue - - filtered = [ - block - for block in content - if not ( - isinstance(block, dict) - and block.get("type") == "text" - and not block.get("text", "").strip() - ) - ] - - if filtered == content: - # Nothing was removed — keep original message as-is. - sanitized.append(message) - elif filtered: - # Some empty text blocks removed, but other content remains. - new_message = message.copy() - new_message["content"] = filtered - sanitized.append(new_message) - else: - # All blocks were empty text blocks. Replace with a placeholder - # so we don't send an empty content array. - new_message = message.copy() - new_message["content"] = [{"type": "text", "text": "..."}] - sanitized.append(new_message) - - return sanitized - - class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -1958,13 +1905,6 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) - # Sanitize empty text content blocks from messages before forwarding. - # Claude's API returns assistant messages with empty text blocks - # ({"type": "text", "text": ""}) alongside tool_use blocks, but rejects - # them when sent back. Strip these to prevent 400 errors. - # Ref: https://github.com/BerriAI/litellm/issues/22930 - messages = _sanitize_anthropic_messages_empty_text_blocks(messages) - # Prepare request body request_body = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, diff --git a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py deleted file mode 100644 index b397b5a484..0000000000 --- a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -Test empty text content block sanitization for the /v1/messages native path. - -The Anthropic API returns assistant messages with empty text blocks -({"type": "text", "text": ""}) alongside tool_use blocks, but rejects -them when sent back. The /v1/messages endpoint must strip these before -forwarding to providers. - -Ref: https://github.com/BerriAI/litellm/issues/22930 -""" - -import pytest - -from litellm.llms.custom_httpx.llm_http_handler import ( - _sanitize_anthropic_messages_empty_text_blocks, -) - - -class TestSanitizeAnthropicMessagesEmptyTextBlocks: - """Unit tests for _sanitize_anthropic_messages_empty_text_blocks.""" - - def test_strips_empty_text_alongside_tool_use(self): - """ - The most common case from the bug report: an assistant message - containing an empty text block next to a tool_use block. - """ - messages = [ - {"role": "user", "content": "Run the command."}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_xxx", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result) == 2 - assert result[0] == messages[0] # user message unchanged - # assistant content should only have the tool_use block - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - - def test_preserves_nonempty_text_blocks(self): - """Non-empty text blocks must not be removed.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Let me check that."}, - { - "type": "tool_use", - "id": "toolu_yyy", - "name": "Bash", - "input": {"command": "pwd"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 2 - assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} - - def test_whitespace_only_text_block_stripped(self): - """Whitespace-only text blocks should also be stripped.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": " \n\t "}, - { - "type": "tool_use", - "id": "toolu_zzz", - "name": "Bash", - "input": {}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "tool_use" - - def test_all_empty_text_blocks_replaced_with_placeholder(self): - """ - If all content blocks are empty text, replace with a placeholder - to avoid sending an empty content array. - """ - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "text" - assert result[0]["content"][0]["text"].strip() # must be non-empty - - def test_string_content_untouched(self): - """Messages with string content should pass through unchanged.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_no_content_key_untouched(self): - """Messages without a content key should pass through.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_user_message_content_list_also_sanitized(self): - """ - Empty text blocks should be stripped from user messages too, - not just assistant messages. - """ - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": ""}, - {"type": "text", "text": "actual question"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["text"] == "actual question" - - def test_tool_result_content_blocks_untouched(self): - """ - tool_result content blocks should not be affected — only - {"type": "text", "text": ""} blocks are stripped. - """ - messages = [ - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_xxx", - "content": "", - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_multiple_messages_mixed(self): - """End-to-end scenario with multiple messages, some needing sanitization.""" - messages = [ - {"role": "user", "content": "Run ls"}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": "file1.txt\nfile2.txt", - }, - ], - }, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Here are the files:"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - # First message: string content, unchanged - assert result[0] == messages[0] - # Second message: empty text stripped, only tool_use remains - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - # Third message: tool_result, unchanged - assert result[2] == messages[2] - # Fourth message: non-empty text, unchanged - assert result[3] == messages[3] - - def test_does_not_mutate_original_messages(self): - """The function should not modify the input list or its dicts.""" - original_content = [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {}, - }, - ] - messages = [ - { - "role": "assistant", - "content": original_content, - }, - ] - - _sanitize_anthropic_messages_empty_text_blocks(messages) - - # Original message content should be unchanged - assert len(messages[0]["content"]) == 2 - assert messages[0]["content"][0] == {"type": "text", "text": ""} From 3f30f6a49c7456a7dc19b3539646f7fe3b97cc39 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:59:53 +0530 Subject: [PATCH 25/40] Revert "Fix logging tests" --- litellm/litellm_core_utils/redact_messages.py | 10 ---------- provider_endpoints_support.json | 18 ------------------ 2 files changed, 28 deletions(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ddeb24d04a..ad68f3851a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -123,16 +123,6 @@ def perform_redaction(model_call_details: dict, result): elif isinstance(_result, litellm.EmbeddingResponse): if hasattr(_result, "data") and _result.data is not None: _result.data = [] - elif isinstance(_result, dict) and "choices" in _result: - # ModelResponse.model_dump() returns dict - redact choices in place - if isinstance(_result.get("choices"), list) and len(_result["choices"]) > 0: - choice = _result["choices"][0] - if isinstance(choice, dict) and "message" in choice: - msg = choice["message"] - if isinstance(msg, dict) and "content" in msg: - msg["content"] = "redacted-by-litellm" - if isinstance(msg, dict) and "audio" in msg: - msg["audio"] = None else: return {"text": "redacted-by-litellm"} return _result diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0b3f87fbe0..b1d4d5a116 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,24 +458,6 @@ "interactions": true } }, - "charity_engine": { - "display_name": "Charity Engine (`charity_engine`)", - "url": "https://docs.litellm.ai/docs/providers/charity_engine", - "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "a2a": false, - "interactions": false - } - }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { From 504e66ccd4b9a1052be23472a56cd630d1df4bdc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 10:22:00 +0530 Subject: [PATCH 26/40] =?UTF-8?q?Revert=20"fix:=20forward=20recognized=20O?= =?UTF-8?q?penAI=20params=20from=20kwargs=20in=20completion()=20(#2?= =?UTF-8?q?=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit dd6f0d6c55179634b5a5b8d1f670d97c56891b6c. --- litellm/main.py | 8 - .../llms/openai/chat/test_store_param.py | 188 ------------------ 2 files changed, 196 deletions(-) delete mode 100644 tests/test_litellm/llms/openai/chat/test_store_param.py diff --git a/litellm/main.py b/litellm/main.py index e23baadb79..364519e1fe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -65,7 +65,6 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.constants import ( - DEFAULT_CHAT_COMPLETION_PARAM_VALUES, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) @@ -1488,13 +1487,6 @@ def completion( # type: ignore # noqa: PLR0915 "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v optional_params = get_optional_params( **optional_param_args, **non_default_params ) diff --git a/tests/test_litellm/llms/openai/chat/test_store_param.py b/tests/test_litellm/llms/openai/chat/test_store_param.py deleted file mode 100644 index 0fd4799dae..0000000000 --- a/tests/test_litellm/llms/openai/chat/test_store_param.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -Tests for the `store` parameter being correctly forwarded to OpenAI. - -Related issue: https://github.com/BerriAI/litellm/issues/23087 - -The `store` parameter was listed in OPENAI_CHAT_COMPLETION_PARAMS and -DEFAULT_CHAT_COMPLETION_PARAM_VALUES but was silently dropped because -get_non_default_completion_params() excluded it (as a "known" param) -while optional_param_args didn't include it (not a named param of -completion()). The fix adds a safety net in completion() that forwards -any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that aren't -already in optional_param_args. -""" - -import os -import sys - -sys.path.insert(0, os.path.abspath("../../../../..")) - -from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES -from litellm.utils import get_non_default_completion_params, get_optional_params - - -class TestStoreParamForwarding: - """Tests that `store` flows through the parameter processing pipeline.""" - - def test_store_true_forwarded_for_openai(self): - """should forward store=True for OpenAI models via kwargs""" - result = get_optional_params( - model="gpt-5.1", - custom_llm_provider="openai", - store=True, - ) - assert result.get("store") is True - - def test_store_false_forwarded_for_openai(self): - """should forward store=False for OpenAI models""" - result = get_optional_params( - model="gpt-4o", - custom_llm_provider="openai", - store=False, - ) - assert result.get("store") is False - - def test_store_none_not_forwarded(self): - """should not include store when it is None (default)""" - result = get_optional_params( - model="gpt-4o", - custom_llm_provider="openai", - ) - assert "store" not in result - - def test_store_with_gpt5_models(self): - """should forward store=True for GPT-5 family models""" - for model in ["gpt-5.1", "gpt-5.2"]: - result = get_optional_params( - model=model, - custom_llm_provider="openai", - store=True, - ) - assert result.get("store") is True, f"store not forwarded for {model}" - - def test_store_in_supported_params(self): - """should list store as a supported OpenAI param""" - from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig - - config = OpenAIGPTConfig() - for model in ["gpt-4o", "gpt-5.1", "gpt-5.2"]: - supported = config.get_supported_openai_params(model) - assert "store" in supported, f"store not in supported params for {model}" - - def test_store_in_transform_request(self): - """should include store in the final transformed request body""" - from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig - - config = OpenAIGPTConfig() - messages = [{"role": "user", "content": "Hello"}] - optional_params = {"store": True} - result = config.transform_request( - model="gpt-5.1", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={}, - ) - assert result.get("store") is True - - def test_store_true_with_metadata(self): - """should forward both store and metadata when both are set""" - from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig - - config = OpenAIGPTConfig() - messages = [{"role": "user", "content": "Hello"}] - optional_params = {"store": True, "metadata": {"key": "value"}} - result = config.transform_request( - model="gpt-5.1", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={}, - ) - assert result.get("store") is True - assert result.get("metadata") == {"key": "value"} - - -class TestDefaultParamValuesSafetyNet: - """Tests that any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES flows - through completion() even without being a named parameter.""" - - def test_known_openai_param_excluded_from_non_default(self): - """should confirm get_non_default_completion_params excludes known OpenAI params""" - kwargs = {"store": True, "temperature": 0.5} - non_default = get_non_default_completion_params(kwargs=kwargs) - assert "store" not in non_default - assert "temperature" not in non_default - - def test_unknown_param_included_in_non_default(self): - """should pass through unknown provider-specific params""" - kwargs = {"my_custom_provider_param": "foo"} - non_default = get_non_default_completion_params(kwargs=kwargs) - assert non_default.get("my_custom_provider_param") == "foo" - - def test_safety_net_forwards_recognized_kwargs(self): - """should forward kwargs in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - that are not already in optional_param_args""" - optional_param_args = { - "model": "gpt-5.1", - "custom_llm_provider": "openai", - "temperature": 0.7, - } - kwargs = {"store": True, "metadata": {"key": "value"}} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert optional_param_args["store"] is True - assert optional_param_args["metadata"] == {"key": "value"} - assert optional_param_args["temperature"] == 0.7 - - def test_safety_net_does_not_override_existing(self): - """should not override a param that's already in optional_param_args""" - optional_param_args = { - "model": "gpt-5.1", - "custom_llm_provider": "openai", - "temperature": 0.7, - } - kwargs = {"temperature": 0.9} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert optional_param_args["temperature"] == 0.7 - - def test_safety_net_skips_none_values(self): - """should not forward params with None value (the default)""" - optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} - kwargs = {"store": None} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert "store" not in optional_param_args - - def test_safety_net_forwards_falsy_non_none(self): - """should forward store=False (falsy but not None)""" - optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} - kwargs = {"store": False} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert optional_param_args.get("store") is False From b08445837bd7fef2f2adf996dfd33db031b0aa3a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 10:21:49 +0530 Subject: [PATCH 27/40] fix(logging): preserve ModelResponse choices format in redacted standard_logging_object + add Charity Engine provider endpoint - Fix perform_redaction to handle dict representation of ModelResponse (from model_dump()) - Preserve full choices structure when redacting, redact content/audio in place - Add _redact_standard_logging_object helper for standard_logging_object field - Update test_logging_redaction_e2e_test assertions to expect choices format - Add charity_engine to provider_endpoints_support.json Fixes: test_standard_logging_payload, test_standard_logging_payload_audio Made-with: Cursor --- litellm/litellm_core_utils/redact_messages.py | 70 +++++++++++++++++++ provider_endpoints_support.json | 18 +++++ .../test_logging_redaction_e2e_test.py | 15 ++-- 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ad68f3851a..41cc200141 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -73,6 +73,53 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_standard_logging_object(model_call_details: dict): + """Redact messages and response inside standard_logging_object if present.""" + standard_logging_object = model_call_details.get("standard_logging_object") + if standard_logging_object is None: + return + + redacted_str = "redacted-by-litellm" + + if standard_logging_object.get("messages") is not None: + standard_logging_object["messages"] = [ + {"role": "user", "content": redacted_str} + ] + + response = standard_logging_object.get("response") + if response is not None: + if isinstance(response, dict) and "output" in response: + # ResponsesAPIResponse format - redact content in output items + if isinstance(response.get("output"), list): + for output_item in response["output"]: + if isinstance(output_item, dict) and "content" in output_item: + if isinstance(output_item["content"], list): + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + elif isinstance(response, dict) and "choices" in response: + # ModelResponse dict format - redact content in choices + if isinstance(response.get("choices"), list): + for choice in response["choices"]: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = redacted_str + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = redacted_str + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + elif isinstance(response, str): + standard_logging_object["response"] = redacted_str + else: + # For other formats (empty dict, None, etc.), use simple text format + standard_logging_object["response"] = {"text": redacted_str} + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -114,6 +161,29 @@ def perform_redaction(model_call_details: dict, result): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: _redact_choice_content(choice) + elif isinstance(_result, dict) and "choices" in _result: + # Handle dict representation of ModelResponse (e.g., from model_dump()) + if _result.get("choices") is not None: + for choice in _result["choices"]: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = "redacted-by-litellm" + if "reasoning_content" in choice["message"]: + choice["message"]["reasoning_content"] = "redacted-by-litellm" + if "thinking_blocks" in choice["message"]: + choice["message"]["thinking_blocks"] = None + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = "redacted-by-litellm" + if "reasoning_content" in choice["delta"]: + choice["delta"]["reasoning_content"] = "redacted-by-litellm" + if "thinking_blocks" in choice["delta"]: + choice["delta"]["thinking_blocks"] = None + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + else: + _redact_choice_content(choice) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): _redact_responses_api_output(_result.output) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b1d4d5a116..0b3f87fbe0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,6 +458,24 @@ "interactions": true } }, + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 0536ec7205..0391a5a895 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -45,7 +45,8 @@ async def test_global_redaction_on(): await asyncio.sleep(1) standard_logging_payload = test_custom_logger.logged_standard_logging_payload assert standard_logging_payload is not None - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload", @@ -75,7 +76,8 @@ async def test_global_redaction_with_dynamic_params(turn_off_message_logging): ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -108,7 +110,8 @@ async def test_global_redaction_off_with_dynamic_params(turn_off_message_logging json.dumps(standard_logging_payload, indent=2), ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -390,7 +393,8 @@ async def test_redaction_with_streaming_response(): assert standard_logging_payload is not None # Verify that redaction worked without pickle errors - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload for streaming with coroutine handling", @@ -477,5 +481,6 @@ async def test_redaction_with_metadata_completion_api(): # Verify the helper function works correctly - with get_metadata_variable_name_from_kwargs, # the system checks the appropriate field for headers - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" From c8297332009e3c95a41960659593c67bd2607408 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 22:30:07 -0700 Subject: [PATCH 28/40] [Fix] Include model access groups when expanding All Proxy Models When a team has "all-proxy-models", the model list expansion now includes model access group names so they appear in the UI key creation form. Also fixes get_key_models not forwarding include_model_access_groups to _get_models_from_access_groups, and removes unused _unfurl_all_proxy_models. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/auth/model_checks.py | 10 +- .../management_endpoints/team_endpoints.py | 18 --- .../proxy/auth/test_model_checks.py | 104 ++++++++++++++++++ 3 files changed, 112 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 32f209a763..4ca1449208 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -112,10 +112,14 @@ def get_key_models( if SpecialModelNames.all_team_models.value in all_models: all_models = user_api_key_dict.team_models if SpecialModelNames.all_proxy_models.value in all_models: - all_models = proxy_model_list + all_models = list(proxy_model_list) # copy to avoid mutating caller's list + if include_model_access_groups: + all_models.extend(model_access_groups.keys()) all_models = _get_models_from_access_groups( - model_access_groups=model_access_groups, all_models=all_models + model_access_groups=model_access_groups, + all_models=all_models, + include_model_access_groups=include_model_access_groups, ) verbose_proxy_logger.debug("ALL KEY MODELS - {}".format(len(all_models))) @@ -141,6 +145,8 @@ def get_team_models( all_models_set.update(team_models) if SpecialModelNames.all_proxy_models.value in all_models_set: all_models_set.update(proxy_model_list) + if include_model_access_groups: + all_models_set.update(model_access_groups.keys()) all_models = list(all_models_set) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 633de86aa6..ee1868fc74 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2827,21 +2827,6 @@ async def validate_membership( ) -def _unfurl_all_proxy_models( - team_info: LiteLLM_TeamTable, llm_router: Router -) -> LiteLLM_TeamTable: - if ( - SpecialModelNames.all_proxy_models.value in team_info.models - and llm_router is not None - ): - team_models: set[str] = set() # make set to avoid duplicates - for model in team_info.models: - if model != SpecialModelNames.all_proxy_models.value: - team_models.add(model) - for model in llm_router.get_model_names(): - team_models.add(model) - team_info.models = list(team_models) - return team_info async def _add_team_member_budget_table( @@ -2972,9 +2957,6 @@ async def team_info( team_info_response_object=_team_info, ) - # ## UNFURL 'all-proxy-models' into the team_info.models list ## - # if llm_router is not None: - # _team_info = _unfurl_all_proxy_models(_team_info, llm_router) response_object = TeamInfoResponseObject( team_id=team_id, team_info=_team_info, diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 193b014f03..739ff25b7d 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -21,6 +21,110 @@ def test_get_team_models_for_all_models_and_team_only_models(): assert set(result) == set(combined_models) +def test_get_team_models_all_proxy_models_includes_access_groups(): + """ + When a team has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names (e.g. 'claude-model-group') + in addition to individual model names. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=True + ) + assert "group-a" in result + assert "group-b" in result + assert "model1" in result + assert "model2" in result + + +def test_get_team_models_all_proxy_models_without_include_flag(): + """ + When include_model_access_groups=False, access group names should NOT + appear in the result even with 'all-proxy-models'. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=False + ) + assert "group-a" not in result + assert "group-b" not in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_all_proxy_models_includes_access_groups(): + """ + When a key has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["all-proxy-models"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_passes_include_model_access_groups(): + """ + When a key explicitly has an access group name in its models list and + include_model_access_groups=True, the group name should be retained + (not stripped by _get_models_from_access_groups). + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["group-a"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1", "model2"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + + @pytest.mark.parametrize( "key_models,team_models,proxy_model_list,model_list,expected", [ From 1cf191d9ad3a0732126d67b33813625419adafad Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 22:44:45 -0700 Subject: [PATCH 29/40] [Fix] Deduplicate model lists and remove dead assignment Adds dedup to get_key_models and get_team_models to prevent duplicate entries when access group member models overlap with proxy_model_list. Removes dead assignment of all_models in get_team_models. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/auth/model_checks.py | 8 ++++++-- tests/test_litellm/proxy/auth/test_model_checks.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 4ca1449208..ccbc2f0194 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -122,6 +122,9 @@ def get_key_models( include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL KEY MODELS - {}".format(len(all_models))) return all_models @@ -148,14 +151,15 @@ def get_team_models( if include_model_access_groups: all_models_set.update(model_access_groups.keys()) - all_models = list(all_models_set) - all_models = _get_models_from_access_groups( model_access_groups=model_access_groups, all_models=list(all_models_set), include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL TEAM MODELS - {}".format(len(all_models))) return all_models diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 739ff25b7d..2b484bf975 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -43,6 +43,7 @@ def test_get_team_models_all_proxy_models_includes_access_groups(): assert "group-b" in result assert "model1" in result assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" def test_get_team_models_all_proxy_models_without_include_flag(): @@ -94,6 +95,7 @@ def test_get_key_models_all_proxy_models_includes_access_groups(): assert "group-a" in result assert "model1" in result assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" def test_get_key_models_passes_include_model_access_groups(): From 1755a281bd619eadbdad5501254694724ab78ae9 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 22:55:53 -0700 Subject: [PATCH 30/40] Fix mutation bug: copy lists in get_key_models to prevent corrupting cached UserAPIKeyAuth `all_models = user_api_key_dict.models` was creating an alias, so `_get_models_from_access_groups` (which uses `.pop()`/`.extend()`) would mutate the cached object in-place. Now both `.models` and `.team_models` assignments create copies via `list()`. Added test to verify the input is not mutated. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/auth/model_checks.py | 4 +-- .../proxy/auth/test_model_checks.py | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index ccbc2f0194..13b26eef43 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -108,9 +108,9 @@ def get_key_models( """ all_models: List[str] = [] if len(user_api_key_dict.models) > 0: - all_models = user_api_key_dict.models + all_models = list(user_api_key_dict.models) # copy to avoid mutating cached objects if SpecialModelNames.all_team_models.value in all_models: - all_models = user_api_key_dict.team_models + all_models = list(user_api_key_dict.team_models) # copy to avoid mutating cached objects if SpecialModelNames.all_proxy_models.value in all_models: all_models = list(proxy_model_list) # copy to avoid mutating caller's list if include_model_access_groups: diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 2b484bf975..c43621d7f7 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -127,6 +127,34 @@ def test_get_key_models_passes_include_model_access_groups(): assert "model2" in result +def test_get_key_models_does_not_mutate_input(): + """ + get_key_models must not mutate user_api_key_dict.models in-place. + _get_models_from_access_groups uses .pop()/.extend() which would corrupt + cached UserAPIKeyAuth objects if all_models were an alias instead of a copy. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + original_models = ["group-a", "extra-model"] + user_api_key_dict = UserAPIKeyAuth( + models=list(original_models), # give it a list + api_key="test-key", + ) + model_access_groups = { + "group-a": ["model1", "model2"], + } + + _ = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=["model1", "model2"], + model_access_groups=model_access_groups, + include_model_access_groups=False, + ) + # The original models list on the auth object must be unchanged + assert user_api_key_dict.models == original_models + + @pytest.mark.parametrize( "key_models,team_models,proxy_model_list,model_list,expected", [ From db99fdeff3ab664b4292aa3c8a8c19d147c7162e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 11:34:23 +0530 Subject: [PATCH 31/40] fix(mcp): OpenAPI tool listing and execution for relative URLs and camelCase - Fix case-insensitive tool name matching in _tool_name_matches() so that OpenAPI operationIds (camelCase) match lowercase registered tool names when filtering by allowed_tools - Fix get_base_url() to resolve relative server URLs (e.g. /api/v3) by deriving full base URL from spec_path when OpenAPI spec has relative URLs - Add tests for case-insensitive matching and filter_tools_by_allowed_tools Made-with: Cursor --- .../mcp_server/openapi_to_mcp_generator.py | 19 ++- .../proxy/_experimental/mcp_server/server.py | 11 +- .../mcp_server/test_mcp_server.py | 147 ++++++++++++++++++ 3 files changed, 172 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 5f6cb87b26..5ad3cf444f 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -92,7 +92,24 @@ def get_base_url(spec: Dict[str, Any], spec_path: Optional[str] = None) -> str: """Extract base URL from OpenAPI spec.""" # OpenAPI 3.x if "servers" in spec and spec["servers"]: - return spec["servers"][0]["url"] + server_url = spec["servers"][0]["url"] + + # If the server URL is relative (starts with /), derive base from spec_path + if server_url.startswith("/") and spec_path: + if spec_path.startswith("http://") or spec_path.startswith("https://"): + # Extract base URL from spec_path (e.g., https://petstore3.swagger.io/api/v3/openapi.json) + # Combine domain with the relative server URL + from urllib.parse import urlparse + parsed = urlparse(spec_path) + base_domain = f"{parsed.scheme}://{parsed.netloc}" + full_base_url = base_domain + server_url + verbose_logger.info( + f"OpenAPI spec has relative server URL '{server_url}'. " + f"Deriving base from spec_path: {full_base_url}" + ) + return full_base_url + + return server_url # OpenAPI 2.x (Swagger) elif "host" in spec: scheme = spec.get("schemes", ["https"])[0] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 99f6a5234a..7898f03e01 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -711,6 +711,7 @@ if MCP_AVAILABLE: Checks both the full tool name and unprefixed version (without server prefix). This allows users to configure simple tool names regardless of prefixing. + Comparison is case-insensitive to handle OpenAPI operationIds that may be in camelCase. Args: tool_name: The tool name to check (may be prefixed like "server-tool_name") @@ -723,13 +724,15 @@ if MCP_AVAILABLE: split_server_prefix_from_name, ) - # Check if the full name is in the list - if tool_name in filter_list: + # Normalize filter list to lowercase for case-insensitive comparison + filter_list_lower = [f.lower() for f in filter_list] + + if tool_name.lower() in filter_list_lower: return True - # Check if the unprefixed name is in the list + # Check if the unprefixed name is in the list (case-insensitive) unprefixed_name, _ = split_server_prefix_from_name(tool_name) - return unprefixed_name in filter_list + return unprefixed_name.lower() in filter_list_lower def filter_tools_by_allowed_tools( tools: List[MCPTool], diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index de2ec13b4a..a104ac2257 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2093,3 +2093,150 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab assert spend_meta["tool_count_total"] == 1 assert spend_meta["allowed_server_count"] == 1 assert spend_meta["per_server_tool_counts"]["server_a"] == 1 + + +def test_tool_name_matches_case_insensitive(): + """Test that _tool_name_matches performs case-insensitive comparison. + + This is critical for OpenAPI-based MCP servers where: + 1. operationIds are often in camelCase (e.g., 'addPet', 'updatePet') + 2. Tool names are lowercased during registration (e.g., 'addpet', 'updatepet') + 3. allowed_tools configuration may use the original camelCase names + + Without case-insensitive matching, all tools would be filtered out. + """ + try: + from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + except ImportError: + pytest.skip("MCP server not available") + + # Test case 1: Unprefixed tool name with camelCase in filter list + assert _tool_name_matches("addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("deletepet", ["addPet", "updatePet"]) is False + + # Test case 2: Prefixed tool name with camelCase in filter list + assert _tool_name_matches("per_store-addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-deletepet", ["addPet", "updatePet"]) is False + + # Test case 3: Mixed case variations + assert _tool_name_matches("findPetsByStatus", ["findpetsbystatus"]) is True + assert _tool_name_matches("findpetsbystatus", ["findPetsByStatus"]) is True + assert _tool_name_matches("FINDPETSBYSTATUS", ["findPetsByStatus"]) is True + + # Test case 4: Full prefixed name in filter list (case-insensitive) + assert _tool_name_matches("server-addPet", ["server-addpet"]) is True + assert _tool_name_matches("server-addpet", ["server-addPet"]) is True + + # Test case 5: Ensure non-matching names still don't match + assert _tool_name_matches("addpet", ["deletePet", "updatePet"]) is False + assert _tool_name_matches("server-addpet", ["deletePet", "updatePet"]) is False + + +def test_filter_tools_by_allowed_tools_case_insensitive(): + """Test that filter_tools_by_allowed_tools handles case-insensitive matching. + + Ensures that OpenAPI tools with lowercase names can be filtered using + camelCase allowed_tools configuration from the OpenAPI spec. + """ + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + # Create mock tools with lowercase names (as registered from OpenAPI) + tools = [ + MCPTool( + name="per_store-addpet", + description="Add a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-updatepet", + description="Update a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-deletepet", + description="Delete a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-findpetsbystatus", + description="Find pets by status", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Create mock server with camelCase allowed_tools (as from OpenAPI spec) + server = MCPServer( + server_id="test-server", + name="per_store", + transport=MCPTransport.http, + allowed_tools=["addPet", "updatePet", "findPetsByStatus"], + ) + + # Filter tools + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return 3 tools (case-insensitive match) + assert len(filtered_tools) == 3 + assert any(t.name == "per_store-addpet" for t in filtered_tools) + assert any(t.name == "per_store-updatepet" for t in filtered_tools) + assert any(t.name == "per_store-findpetsbystatus" for t in filtered_tools) + assert not any(t.name == "per_store-deletepet" for t in filtered_tools) + + +def test_filter_tools_by_allowed_tools_no_filter(): + """Test that filter_tools_by_allowed_tools returns all tools when no filter is set.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + tools = [ + MCPTool( + name="fusion_litellm_mcp-model_list", + description="List models", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="fusion_litellm_mcp-chat_completion", + description="Chat completion", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Server with no allowed_tools filter + server = MCPServer( + server_id="test-server", + name="fusion_litellm_mcp", + transport=MCPTransport.http, + allowed_tools=None, + ) + + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return all tools when no filter is configured + assert len(filtered_tools) == 2 From 200b001633610341f0c032103ea52c40ca2daf7f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 11:53:36 +0530 Subject: [PATCH 32/40] fix(bedrock): strip output_config from Converse requests; fix spend tracking redaction test Made-with: Cursor --- .../bedrock/chat/converse_transformation.py | 1 + .../chat/test_converse_transformation.py | 27 +++++++++++++++++++ .../test_spend_tracking_utils.py | 5 ++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d210f294c6..4fa407701c 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1206,6 +1206,7 @@ class AmazonConverseConfig(BaseConfig): self._validate_request_metadata(request_metadata) output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None) + inference_params.pop("output_config", None) # Bedrock Converse doesn't support it # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 345f3ae7c5..7e1f235c49 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3170,6 +3170,33 @@ def test_transform_request_with_output_config(): assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" +def test_output_config_snake_case_stripped_from_bedrock_converse_request(): + """Test that output_config (snake_case) is stripped from Bedrock Converse requests. + + Bedrock Converse API doesn't support the output_config parameter (Anthropic-only). + Nova and other Converse models reject requests with extraneous output_config. + """ + config = AmazonConverseConfig() + messages = [{"role": "user", "content": "test"}] + optional_params = { + "output_config": {"effort": "high"}, + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # output_config must not appear in additionalModelRequestFields + additional = result.get("additionalModelRequestFields", {}) + assert "output_config" not in additional, ( + f"output_config should be stripped for Bedrock Converse, got: {list(additional.keys())}" + ) + + def test_transform_response_native_structured_output(): """Test response handling when model returns JSON as text content (native structured output).""" response_json = { diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9a64e641b5..3249a7ec79 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1071,9 +1071,10 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs) # When redaction is enabled and response is a dict (not ModelResponse), - # perform_redaction returns {"text": "redacted-by-litellm"} + # perform_redaction redacts content in-place within the choices structure parsed_response = json.loads(response_result) - assert parsed_response == {"text": "redacted-by-litellm"} + assert parsed_response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert parsed_response["choices"][0]["message"]["role"] == "assistant" @patch("litellm.secret_managers.main.get_secret_bool") From 9ee489863d3e9f37ea805a09155e61dd86a16d9e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 23:30:55 -0700 Subject: [PATCH 33/40] [Feature] UI - Virtual Keys: Add refetch button and keep stale data during refetch Show a Fetch/Fetching button next to "Showing X of Y results" that acts as both a manual refetch trigger and a loading indicator. The "Loading keys..." message now only appears on initial load; subsequent refetches keep the table visible with stale data (via React Query's keepPreviousData). Co-Authored-By: Claude Opus 4.6 --- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 66 ++++++++++++++++++- .../VirtualKeysPage/VirtualKeysTable.tsx | 53 ++++++++++----- 2 files changed, 100 insertions(+), 19 deletions(-) diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 4fd513b0d2..d7322d6d76 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -262,8 +262,8 @@ it("should display user email correctly", async () => { }); }); -it("should show skeleton loaders when isLoading is true", () => { - // Mock loading state +it("should show loading message only on initial load (isPending)", () => { + // Mock initial loading state mockUseKeys.mockReturnValue({ data: null, isPending: true, @@ -283,7 +283,7 @@ it("should show skeleton loaders when isLoading is true", () => { renderWithProviders(); - // Check that loading message is shown + // Check that loading message is shown on initial load expect(screen.getByText("🚅 Loading keys...")).toBeInTheDocument(); // Check that actual key data is not shown @@ -795,3 +795,63 @@ describe("pagination display – total count and page count", () => { }); }); }); + +describe("refetch button", () => { + it("should show Fetch button in normal state", () => { + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeInTheDocument(); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); + + it("should show Fetching state and keep table data visible during refetch", () => { + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + // Button should show "Fetching" and be disabled + expect(screen.getByText("Fetching")).toBeInTheDocument(); + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeDisabled(); + + // Table data should still be visible (stale data) + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); + + // "Loading keys..." should NOT appear during refetch + expect(screen.queryByText("🚅 Loading keys...")).not.toBeInTheDocument(); + }); + + it("should call refetch when Fetch button is clicked", () => { + const mockRefetch = vi.fn(); + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: false, + refetch: mockRefetch, + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + fireEvent.click(fetchButton); + + expect(mockRefetch).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index fe9d58b979..b4588a899d 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -24,9 +24,9 @@ import { TableRow, Text, } from "@tremor/react"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Popover, Skeleton, Tooltip } from "antd"; -import React, { useEffect, useMemo, useState } from "react"; +import { InfoCircleOutlined, SyncOutlined } from "@ant-design/icons"; +import { Button as AntButton, Popover, Skeleton, Tooltip } from "antd"; +import React, { useEffect, useDeferredValue, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { useFilterLogic } from "../key_team_helpers/filter_logic"; import { PaginatedKeyAliasSelect } from "../KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect"; @@ -97,6 +97,15 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo organizations, }); + // Defer the transition so the button stays in loading state until the table + // has rendered with the new data (mirrors the spend-logs pattern) + const isFetchingDeferred = useDeferredValue(isFetching); + const isButtonLoading = isFetching || isFetchingDeferred; + + const handleRefresh = () => { + refetch(); + }; + const totalCount = filteredTotalCount ?? keys?.total_count ?? 0; // Add a useEffect to call refresh when a key is created @@ -606,16 +615,28 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo
- {isLoading || isFetching ? ( - - ) : ( - - Showing {rangeLabel} of {totalCount} results - - )} +
+ {isLoading ? ( + + ) : ( + + Showing {rangeLabel} of {totalCount} results + + )} + + } + onClick={handleRefresh} + disabled={isButtonLoading} + title="Fetch data" + > + {isButtonLoading ? "Fetching" : "Fetch"} + +
- {isLoading || isFetching ? ( + {isLoading ? ( ) : ( @@ -623,24 +644,24 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : ( )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : (