From a5bec4911fdd1e44bcae1c661a17424eb7d15aac Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 13 Mar 2026 13:29:39 +0530 Subject: [PATCH 1/7] Fix _supports_reasoning_effort_level for responses bridge --- litellm/main.py | 15 ++++++++------- tests/llm_translation/test_openai.py | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index f2ce894ba3..395a8b0838 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1356,6 +1356,13 @@ def completion( # type: ignore # noqa: PLR0915 api_key=api_key, ) + ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name + responses_api_model_info, model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + web_search_options=web_search_options, + ) + if not _should_allow_input_examples( custom_llm_provider=custom_llm_provider, model=model ): @@ -1591,14 +1598,8 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, ) - ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map - model_info, model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - if model_info.get("mode") == "responses": + if responses_api_model_info.get("mode") == "responses": from litellm.completion_extras import responses_api_bridge return responses_api_bridge.completion( diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 9e6e5bb369..acbb9c5136 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -1454,3 +1454,29 @@ def test_gpt_5_web_search(): for chunk in response: print("chunk: ", chunk) + + +def test_responses_gpt54_with_xhigh_reasoning(): + """ + Ensure chat->responses bridge sends the correct request payload for + openai/responses/gpt-5.4 with reasoning_effort="xhigh". + """ + with patch("litellm.responses") as mock_responses: + # Stop execution right after request generation to avoid external API calls. + mock_responses.side_effect = RuntimeError("stop_after_request_build") + + with pytest.raises(Exception): + litellm.completion( + model="openai/responses/gpt-5.4", + messages=[{"role": "user", "content": "What is 2+2?"}], + reasoning_effort="xhigh", + max_tokens=100, + ) + + mock_responses.assert_called_once() + request_body = mock_responses.call_args.kwargs + + # The responses prefix should be stripped before routing. + assert request_body["model"] == "gpt-5.4" + # chat-completions reasoning_effort must map to Responses API reasoning. + assert request_body["reasoning"] == {"effort": "xhigh"} From a0a951b34fd023ad4a8d8136f501055175a86997 Mon Sep 17 00:00:00 2001 From: RoomWithOutRoof <166608075+Jah-yee@users.noreply.github.com> Date: Sat, 14 Mar 2026 12:54:43 +0800 Subject: [PATCH 2/7] fix: forward extra_headers to HuggingFace embedding calls (#23525) Fixes #23502 The huggingface_embed.embedding() call was not receiving the headers parameter, causing extra_headers (e.g., X-HF-Bill-To) to be silently dropped. Other providers (openrouter, vercel_ai_gateway, bedrock) already pass headers correctly. This fix adds headers=headers to match the behavior of other providers. Co-authored-by: Jah-yee --- litellm/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/main.py b/litellm/main.py index 722b4a7aae..5a1b7f0fe1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5122,6 +5122,7 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, litellm_params=litellm_params_dict, + headers=headers, ) elif custom_llm_provider == "bedrock": if isinstance(input, str): From 7bb2d78394d395b74363115aa6f33b4f11ed3888 Mon Sep 17 00:00:00 2001 From: "Ethan T." Date: Sat, 14 Mar 2026 12:58:58 +0800 Subject: [PATCH 3/7] fix: add getPopupContainer to Select components in fallback modal to fix z-index issue (#23516) The model dropdown menus in the Add Fallbacks modal were rendering behind the modal overlay because Ant Design portals Select dropdowns to document.body by default. By setting getPopupContainer to attach the dropdown to its parent element, the dropdown inherits the modal's stacking context and renders above the modal. Fixes #17895 --- .../Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx index 24ce80f0a0..86d70048b6 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx @@ -78,6 +78,7 @@ export function FallbackGroupConfig({ value={group.primaryModel} onChange={handlePrimaryChange} showSearch + getPopupContainer={(trigger) => trigger.parentElement || document.body} filterOption={(input, option) => (option?.label ?? "").toLowerCase().includes(input.toLowerCase()) } @@ -125,6 +126,7 @@ export function FallbackGroupConfig({ value={group.fallbackModels} onChange={handleFallbackSelect} disabled={!group.primaryModel} + getPopupContainer={(trigger) => trigger.parentElement || document.body} options={availableFallbackOptions.map((m) => ({ label: m, value: m, From c7ba7948bc79606f96382f4f334a528ba12ad28a Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Sat, 14 Mar 2026 10:41:25 +0500 Subject: [PATCH 4/7] =?UTF-8?q?PR=20#22867=20added=20=5Fremove=5Fscope=5Ff?= =?UTF-8?q?rom=5Fcache=5Fcontrol=20for=20Bedrock=20and=20Azur=E2=80=A6=20(?= =?UTF-8?q?#23183)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * PR #22867 added _remove_scope_from_cache_control for Bedrock and Azure AI but omitted Vertex AI. This applies the same pattern to VertexAIPartnerModelsAnthropicMessagesConfig." * PR #22867 added _remove_scope_from_cache_control for Bedrock and Azure AI but omitted Vertex AI. This applies the same pattern to VertexAIPartnerModelsAnthropicMessagesConfig." * PR #22867 added _remove_scope_from_cache_control to AzureAnthropicMessagesConfig but missed VertexAIPartnerModelsAnthropicMessagesConfi Rather than duplicating the method again, moved it up to the base AnthropicMessagesConfig so all providers inherit it, and removed the now-redundant copy from the Azure AI subclass. * PR #22867 added _remove_scope_from_cache_control to AzureAnthropicMessagesConfig but missed VertexAIPartnerModelsAnthropicMessagesConfi Rather than duplicating the method again, moved it up to the base AnthropicMessagesConfig so all providers inherit it, and removed the now-redundant copy from the Azure AI subclass. --------- Co-authored-by: Krish Dholakia --- .../messages/transformation.py | 32 +++++++++++++ .../transformation.py | 2 + ...artner_models_anthropic_messages_config.py | 45 +++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 4d0a2cd829..e9ceea4822 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -50,6 +50,38 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # "metadata", ] + def _remove_scope_from_cache_control( + self, anthropic_messages_request: Dict + ) -> None: + """ + Remove `scope` field from cache_control blocks. + + Some providers (Vertex AI, Azure AI Foundry) do not support the `scope` + field in cache_control (e.g. "global" for cross-request caching). + Processes both `system` and `messages` content blocks. + """ + + def _sanitize(cache_control: Any) -> None: + if isinstance(cache_control, dict): + cache_control.pop("scope", None) + + def _process_content_list(content: list) -> None: + for item in content: + if isinstance(item, dict) and "cache_control" in item: + _sanitize(item["cache_control"]) + + if "system" in anthropic_messages_request: + system = anthropic_messages_request["system"] + if isinstance(system, list): + _process_content_list(system) + + if "messages" in anthropic_messages_request: + for message in anthropic_messages_request["messages"]: + if isinstance(message, dict) and "content" in message: + content = message["content"] + if isinstance(content, list): + _process_content_list(content) + @staticmethod def _filter_billing_headers_from_system(system_param): """ diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index d3b0217d04..5c3bbf61ee 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -150,6 +150,8 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers=headers, ) + self._remove_scope_from_cache_control(anthropic_messages_request) + anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16" anthropic_messages_request.pop( diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index b5f076262d..391daa24f4 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -5,6 +5,7 @@ import pytest from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) +from litellm.types.router import GenericLiteLLMParams def test_validate_environment_uses_vertex_ai_location(): @@ -248,3 +249,47 @@ def test_validate_environment_with_authorization_header_calculates_api_base(): # Verify Authorization header is still present assert "Authorization" in updated_headers, \ "Authorization header should be preserved" + + +def test_transform_anthropic_messages_request_removes_scope_from_cache_control(): + """Ensure scope field is removed from cache_control for Vertex AI (not supported).""" + config = VertexAIPartnerModelsAnthropicMessagesConfig() + + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": {"type": "ephemeral", "scope": "global"}, + } + ], + } + ] + anthropic_messages_optional_request_params = { + "max_tokens": 1024, + "system": [ + { + "type": "text", + "text": "You are an AI assistant.", + "cache_control": {"type": "ephemeral", "scope": "global"}, + } + ], + } + + result = config.transform_anthropic_messages_request( + model="claude-sonnet-4-6", + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + # scope removed from system + assert "scope" not in result["system"][0]["cache_control"] + assert result["system"][0]["cache_control"]["type"] == "ephemeral" + + # scope removed from message content + assert "scope" not in result["messages"][0]["content"][0]["cache_control"] + assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" From a94b961c18995eb51bf07bbfb6386eb7eac91e33 Mon Sep 17 00:00:00 2001 From: Pradyumna Yadav Date: Sat, 14 Mar 2026 11:18:45 +0530 Subject: [PATCH 5/7] fix: auto-fill reasoning_content for moonshot kimi reasoning models in multi-turn tool calling (#23580) --- litellm/llms/moonshot/chat/transformation.py | 70 +++++++-- ...odel_prices_and_context_window_backup.json | 3 + model_prices_and_context_window.json | 3 + .../test_moonshot_chat_transformation.py | 148 +++++++++++++++++- 4 files changed, 207 insertions(+), 17 deletions(-) diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index 3ed08f51c8..40096be05c 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -2,13 +2,15 @@ Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions` """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, ) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues +from litellm.utils import supports_reasoning from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -17,8 +19,7 @@ class MoonshotChatConfig(OpenAIGPTConfig): @overload def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, List[AllMessageValues]]: - ... + ) -> Coroutine[Any, Any, List[AllMessageValues]]: ... @overload def _transform_messages( @@ -26,8 +27,7 @@ class MoonshotChatConfig(OpenAIGPTConfig): messages: List[AllMessageValues], model: str, is_async: Literal[False] = False, - ) -> List[AllMessageValues]: - ... + ) -> List[AllMessageValues]: ... def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: bool = False @@ -53,22 +53,14 @@ class MoonshotChatConfig(OpenAIGPTConfig): messages = handle_messages_with_content_list_to_str_conversion(messages) if is_async: - return super()._transform_messages( - messages=messages, model=model, is_async=True - ) + return super()._transform_messages(messages=messages, model=model, is_async=True) else: - return super()._transform_messages( - messages=messages, model=model, is_async=False - ) + return super()._transform_messages(messages=messages, model=model, is_async=False) def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: - api_base = ( - api_base - or get_secret_str("MOONSHOT_API_BASE") - or "https://api.moonshot.ai/v1" - ) # type: ignore + api_base = api_base or get_secret_str("MOONSHOT_API_BASE") or "https://api.moonshot.ai/v1" # type: ignore dynamic_api_key = api_key or get_secret_str("MOONSHOT_API_KEY") return api_base, dynamic_api_key @@ -149,6 +141,48 @@ class MoonshotChatConfig(OpenAIGPTConfig): optional_params["temperature"] = 0.3 return optional_params + def fill_reasoning_content(self, messages: List[AllMessageValues]) -> List[AllMessageValues]: + """ + Moonshot reasoning models require `reasoning_content` on every assistant + message that contains tool_calls (multi-turn tool-calling flows). + + For each such message that is missing the field: + 1. Promote provider_specific_fields["reasoning_content"] if present and non-empty + (this is where LiteLLM stores it from a previous response) + 2. Otherwise inject a single space — the minimum value the API accepts + Messages that already carry the field, or are not assistant/tool-call messages, + are appended as-is (no copy made). + """ + result: List[AllMessageValues] = [] + for msg in messages: + if ( + msg.get("role") == "assistant" + and msg.get("tool_calls") + and "reasoning_content" not in msg + ): + patched = dict(cast(dict, msg)) + provider_fields = patched.get("provider_specific_fields") or {} + stored = provider_fields.get("reasoning_content") + if stored: + patched["reasoning_content"] = stored + # Remove the promoted key from provider_specific_fields to + # avoid sending the value twice in the serialised request body + cleaned_provider_fields = dict(provider_fields) + cleaned_provider_fields.pop("reasoning_content", None) + patched["provider_specific_fields"] = cleaned_provider_fields + else: + litellm.verbose_logger.warning( + "Moonshot reasoning model: assistant tool-call message is missing " + "`reasoning_content`. Injecting a placeholder to satisfy API validation. " + "For best results, preserve `reasoning_content` from the original " + "assistant response when building multi-turn conversation history." + ) + patched["reasoning_content"] = " " + result.append(cast(AllMessageValues, patched)) + else: + result.append(msg) + return result + def transform_request( self, model: str, @@ -169,6 +203,10 @@ class MoonshotChatConfig(OpenAIGPTConfig): optional_params=optional_params, ) + # Moonshot reasoning models: fill in reasoning_content before the API call + if supports_reasoning(model=model, custom_llm_provider="moonshot"): + messages = self.fill_reasoning_content(messages) + # Call parent transform_request which handles _transform_messages return super().transform_request( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9b1d81fee4..6786fc3359 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22083,6 +22083,7 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true @@ -22166,6 +22167,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, @@ -22180,6 +22182,7 @@ "output_cost_per_token": 8e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9b1d81fee4..6786fc3359 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22083,6 +22083,7 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true @@ -22166,6 +22167,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, @@ -22180,6 +22182,7 @@ "output_cost_per_token": 8e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, diff --git a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py index 345186e8a6..c557fb395f 100644 --- a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py @@ -7,6 +7,7 @@ Moonshot AI is an OpenAI-compatible provider with minor customizations. import os import sys +from unittest.mock import patch sys.path.insert( 0, os.path.abspath("../../../../..") @@ -404,4 +405,149 @@ class TestMoonshotConfig: # Content should be flattened to a plain string assert isinstance(result["messages"][0]["content"], str) - assert result["messages"][0]["content"] == "Hello, how are you?" \ No newline at end of file + assert result["messages"][0]["content"] == "Hello, how are you?" + + # ------------------------------------------------------------------ # + # Tests for fill_reasoning_content # + # ------------------------------------------------------------------ # + + def test_reasoning_content_space_injected_when_absent(self): + """Assistant tool-call message with no reasoning_content gets a space injected.""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 22°C"}, + ] + + result = config.fill_reasoning_content(messages) + + assert result[1].get("reasoning_content") == " " + # Non-assistant messages are untouched + assert "reasoning_content" not in result[0] + assert "reasoning_content" not in result[2] + + def test_empty_tool_calls_list_not_injected(self): + """Assistant message with tool_calls: [] should not get reasoning_content injected.""" + config = MoonshotChatConfig() + + original_msg = { + "role": "assistant", + "content": "Here is the answer.", + "tool_calls": [], + } + messages = [original_msg] + + result = config.fill_reasoning_content(messages) + + assert "reasoning_content" not in result[0] + assert result[0] is original_msg + + def test_existing_reasoning_content_not_overwritten(self): + """Message that already has reasoning_content is passed through unchanged.""" + config = MoonshotChatConfig() + + original_msg = { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + "reasoning_content": "", + } + messages = [original_msg] + + result = config.fill_reasoning_content(messages) + + assert result[0].get("reasoning_content") == "" + # Same object — no copy was made + assert result[0] is original_msg + + def test_provider_specific_fields_reasoning_content_promoted(self): + """reasoning_content stored in provider_specific_fields is promoted to top level.""" + config = MoonshotChatConfig() + + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + "provider_specific_fields": {"reasoning_content": "stored thinking"}, + } + ] + + result = config.fill_reasoning_content(messages) + + assert result[0].get("reasoning_content") == "stored thinking" + # The promoted key must be removed from provider_specific_fields to + # avoid sending the value twice in the serialised request body + assert "reasoning_content" not in (result[0].get("provider_specific_fields") or {}) + + def test_reasoning_model_fill_called_from_transform_request(self): + """transform_request injects reasoning_content end-to-end for reasoning models.""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "Call a tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + }, + ] + + with patch( + "litellm.llms.moonshot.chat.transformation.supports_reasoning", + return_value=True, + ): + result = config.transform_request( + model="kimi-k2-thinking", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["messages"][1].get("reasoning_content") == " " + + def test_non_reasoning_model_messages_untouched(self): + """For non-reasoning models, transform_request leaves messages unchanged.""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "Hello"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + }, + ] + + with patch( + "litellm.llms.moonshot.chat.transformation.supports_reasoning", + return_value=False, + ): + result = config.transform_request( + model="moonshot-v1-8k", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # reasoning_content must not have been injected + for msg in result["messages"]: + assert "reasoning_content" not in msg \ No newline at end of file From 81474c17fec6578044861ca2c5ff351bc73dbe1e Mon Sep 17 00:00:00 2001 From: xianzongxie-stripe <87151258+xianzongxie-stripe@users.noreply.github.com> Date: Fri, 13 Mar 2026 23:02:09 -0700 Subject: [PATCH 6/7] Handle response.failed, response.incomplete, and response.cancelled (#23492) * Handle response.failed, response.incomplete, and response.cancelled terminal events in background streaming Previously the background streaming task only handled response.completed and hardcoded the final status to "completed". This missed three other terminal event types from the OpenAI streaming spec, causing failed/incomplete/cancelled responses to be incorrectly marked as completed. Co-Authored-By: Claude Opus 4.6 Committed-By-Agent: claude * Remove unused terminal_response_data variable Co-Authored-By: Claude Opus 4.6 Committed-By-Agent: claude * Address code review: derive fallback status from event type, rewrite tests as integration tests 1. Replace hardcoded "completed" fallback in response_data.get("status") with _event_to_status lookup so that response.incomplete and response.cancelled events get the correct fallback if the response body ever omits the status field. 2. Replace duplicated-logic unit tests with integration tests that exercise background_streaming_task directly using mocked streaming responses and assert on the final update_state call arguments. Co-Authored-By: Claude Opus 4.6 Committed-By-Agent: claude * Remove dead mock_processor and unused mock_response parameter from test helper Co-Authored-By: Claude Opus 4.6 Committed-By-Agent: claude * Remove FastAPI and UserAPIKeyAuth imports from test file These types were only used as Mock(spec=...) arguments. Drop the spec constraints and remove the top-level imports to avoid pulling FastAPI into test files outside litellm/proxy/. Co-Authored-By: Claude Opus 4.6 Committed-By-Agent: claude * Log warning when streaming response has no body_iterator If base_process_llm_request returns a non-streaming response (no body_iterator), log a warning since this likely indicates a misconfiguration or provider error rather than a successful completion. Co-Authored-By: Claude Opus 4.6 Committed-By-Agent: claude --------- Co-authored-by: Claude Opus 4.6 --- .../response_polling/background_streaming.py | 44 ++- .../test_response_polling_handler.py | 261 ++++++++++++++++++ 2 files changed, 299 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index f1b2493976..682ad4943b 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -113,6 +113,16 @@ async def background_streaming_task( # noqa: PLR0915 last_update_time = asyncio.get_event_loop().time() UPDATE_INTERVAL = 0.150 # 150ms batching interval + # Track the terminal event from the stream (may not be "completed") + terminal_status = None # Will be set by response.completed/failed/incomplete/cancelled + terminal_error = None + _event_to_status = { + "response.completed": "completed", + "response.failed": "failed", + "response.incomplete": "incomplete", + "response.cancelled": "cancelled", + } + async def flush_state_if_needed(force: bool = False) -> None: """Flush accumulated state to Redis if interval elapsed or forced""" nonlocal state_dirty, last_update_time @@ -131,6 +141,12 @@ async def background_streaming_task( # noqa: PLR0915 last_update_time = current_time # Handle StreamingResponse + if not hasattr(response, "body_iterator"): + verbose_proxy_logger.warning( + f"background_streaming_task: response for {polling_id} has no " + "body_iterator; this may indicate a misconfiguration or provider error" + ) + if hasattr(response, "body_iterator"): async for chunk in response.body_iterator: # Parse chunk @@ -224,10 +240,23 @@ async def background_streaming_task( # noqa: PLR0915 status="in_progress", ) - elif event_type == "response.completed": - # Response completed - extract all ResponsesAPIResponse fields - # https://platform.openai.com/docs/api-reference/responses-streaming/response-completed + elif event_type in ( + "response.completed", + "response.failed", + "response.incomplete", + "response.cancelled", + ): + # Terminal event - extract all ResponsesAPIResponse fields + # https://platform.openai.com/docs/api-reference/responses-streaming response_data = event.get("response", {}) + terminal_status = response_data.get( + "status", + _event_to_status.get(event_type, "completed"), + ) + + # Extract error for failed responses + if event_type == "response.failed": + terminal_error = response_data.get("error") # Core response fields usage_data = response_data.get("usage") @@ -278,11 +307,14 @@ async def background_streaming_task( # noqa: PLR0915 # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) - # Mark as completed with all ResponsesAPIResponse fields + # Use the terminal status from the stream, default to "completed" + final_status = terminal_status or "completed" + await polling_handler.update_state( polling_id=polling_id, - status="completed", + status=final_status, usage=usage_data, + error=terminal_error, reasoning=reasoning_data, tool_choice=tool_choice_data, tools=tools_data, @@ -301,7 +333,7 @@ async def background_streaming_task( # noqa: PLR0915 ) verbose_proxy_logger.info( - f"Completed background streaming for {polling_id}, output_items={len(output_items)}" + f"Finished background streaming for {polling_id}, status={final_status}, output_items={len(output_items)}" ) except Exception as e: diff --git a/tests/proxy_unit_tests/test_response_polling_handler.py b/tests/proxy_unit_tests/test_response_polling_handler.py index 83e7e26728..6235dde847 100644 --- a/tests/proxy_unit_tests/test_response_polling_handler.py +++ b/tests/proxy_unit_tests/test_response_polling_handler.py @@ -1318,6 +1318,267 @@ class TestStreamingEventParsing: assert output_items["item_123"]["content"][0]["type"] == "text" +def _make_sse_stream(events: list) -> Mock: + """Create a mock StreamingResponse with body_iterator from a list of event dicts.""" + + async def _body_iterator(): + for event in events: + yield f"data: {json.dumps(event)}" + yield "data: [DONE]" + + mock_response = Mock() + mock_response.body_iterator = _body_iterator() + return mock_response + + +def _make_background_streaming_kwargs( + polling_id: str, + polling_handler: ResponsePollingHandler, +) -> dict: + """Build kwargs for background_streaming_task with all required mocks.""" + return dict( + polling_id=polling_id, + data={"model": "gpt-4o", "stream": False, "background": True}, + polling_handler=polling_handler, + request=Mock(), + fastapi_response=Mock(), + user_api_key_dict=Mock(), + general_settings={}, + llm_router=None, + proxy_config=Mock(), + proxy_logging_obj=Mock(), + select_data_generator=Mock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + version=None, + ) + + +@pytest.mark.xdist_group("heavy_imports") +class TestBackgroundStreamingTerminalEvents: + """ + Integration tests that exercise background_streaming_task with mocked + streaming responses, verifying the final update_state call for each + terminal event type. + """ + + @pytest.mark.asyncio + async def test_response_failed_sets_failed_status_and_error(self): + """Test that a response.failed stream event results in failed status with error""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + error_payload = { + "type": "server_error", + "message": "The model encountered an error", + "code": "model_error", + } + events = [ + {"type": "response.in_progress"}, + { + "type": "response.failed", + "response": { + "id": "resp_123", + "status": "failed", + "error": error_payload, + "model": "gpt-4o", + "output": [], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_1", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + # Find the final update_state call (last one) + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"] == error_payload + + @pytest.mark.asyncio + async def test_response_incomplete_sets_incomplete_status_and_details(self): + """Test that a response.incomplete stream event results in incomplete status""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + events = [ + {"type": "response.in_progress"}, + { + "type": "response.incomplete", + "response": { + "id": "resp_123", + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 10, "output_tokens": 4096}, + "model": "gpt-4o", + "output": [{"id": "item_1", "type": "message"}], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_2", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "incomplete" + assert final_call.kwargs["incomplete_details"] == {"reason": "max_output_tokens"} + assert final_call.kwargs["usage"] == {"input_tokens": 10, "output_tokens": 4096} + + @pytest.mark.asyncio + async def test_response_cancelled_sets_cancelled_status(self): + """Test that a response.cancelled stream event results in cancelled status""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + events = [ + {"type": "response.in_progress"}, + { + "type": "response.cancelled", + "response": { + "id": "resp_123", + "status": "cancelled", + "model": "gpt-4o", + "output": [], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_3", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "cancelled" + + @pytest.mark.asyncio + async def test_response_completed_sets_completed_status(self): + """Test that a response.completed stream event results in completed status""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + events = [ + {"type": "response.in_progress"}, + { + "type": "response.completed", + "response": { + "id": "resp_123", + "status": "completed", + "usage": {"input_tokens": 10, "output_tokens": 50}, + "model": "gpt-4o", + "output": [{"id": "item_1", "type": "message"}], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_4", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "completed" + assert final_call.kwargs["usage"] == {"input_tokens": 10, "output_tokens": 50} + + @pytest.mark.asyncio + async def test_fallback_status_derived_from_event_type_when_status_field_missing(self): + """Test that when the response body lacks a status field, the fallback + is derived from the event type, not hardcoded to 'completed'.""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + # response.incomplete event with NO status field in the response body + events = [ + {"type": "response.in_progress"}, + { + "type": "response.incomplete", + "response": { + "id": "resp_123", + # "status" deliberately omitted + "incomplete_details": {"reason": "max_output_tokens"}, + "model": "gpt-4o", + "output": [], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_5", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "incomplete" + + @pytest.mark.asyncio + async def test_no_terminal_event_defaults_to_completed(self): + """Test that when no terminal event is received, status defaults to completed""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + # Stream with only in_progress, no terminal event + events = [ + {"type": "response.in_progress"}, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_6", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "completed" + + class TestEdgeCases: """Test edge cases and error scenarios""" From 25ee2fb3f9d8b90f7927b3e8816424f1f3669fbc Mon Sep 17 00:00:00 2001 From: Joe Reyna Date: Fri, 13 Mar 2026 23:08:14 -0700 Subject: [PATCH 7/7] fix(security): bump tar to 7.5.11 and tornado to 6.5.5 (#23602) * fix(security): bump tar to 7.5.11 and tornado to 6.5.5 - tar >=7.5.11: fixes CVE-2026-31802 (HIGH) in node-pkg - tornado >=6.5.5: fixes CVE-2026-31958 (HIGH) and GHSA-78cv-mqj4-43f7 (MEDIUM) in python-pkg Addresses vulnerabilities found in ghcr.io/berriai/litellm:main-v1.82.0-stable Trivy scan. Co-Authored-By: Claude Sonnet 4.6 * fix: document tar override is enforced via Dockerfile, not npm * fix: revert invalid JSON comment in package.json tar override --------- Co-authored-by: Claude Sonnet 4.6 --- requirements.txt | 2 +- ui/litellm-dashboard/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f7b72b6f0c..bf2bf2c47a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # LITELLM PROXY DEPENDENCIES # # Security: explicit pins for transitive deps (CVE fixes) urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441 -tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724 +tornado>=6.5.5 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724, CVE-2026-31958, GHSA-78cv-mqj4-43f7 filelock>=3.20.1 # CVE-2025-68146 h11>=0.16.0 # CVE-2025-43859, GHSA-vqfr-h8mv-ghfj — HTTP request smuggling wheel>=0.46.2 # CVE-2026-24049 — path traversal diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 5cbe1ead88..2a9bb3e5e2 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -88,7 +88,7 @@ "mermaid": ">=11.10.0", "js-yaml": ">=4.1.1", "glob": ">=11.1.0", - "tar": ">=7.5.10", + "tar": ">=7.5.11", "minimatch": ">=10.2.4", "@isaacs/brace-expansion": ">=5.0.1", "node-forge": ">=1.3.2",