diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 4d0a2cd829..e9ceea4822 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -50,6 +50,38 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # "metadata", ] + def _remove_scope_from_cache_control( + self, anthropic_messages_request: Dict + ) -> None: + """ + Remove `scope` field from cache_control blocks. + + Some providers (Vertex AI, Azure AI Foundry) do not support the `scope` + field in cache_control (e.g. "global" for cross-request caching). + Processes both `system` and `messages` content blocks. + """ + + def _sanitize(cache_control: Any) -> None: + if isinstance(cache_control, dict): + cache_control.pop("scope", None) + + def _process_content_list(content: list) -> None: + for item in content: + if isinstance(item, dict) and "cache_control" in item: + _sanitize(item["cache_control"]) + + if "system" in anthropic_messages_request: + system = anthropic_messages_request["system"] + if isinstance(system, list): + _process_content_list(system) + + if "messages" in anthropic_messages_request: + for message in anthropic_messages_request["messages"]: + if isinstance(message, dict) and "content" in message: + content = message["content"] + if isinstance(content, list): + _process_content_list(content) + @staticmethod def _filter_billing_headers_from_system(system_param): """ diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index 3ed08f51c8..40096be05c 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -2,13 +2,15 @@ Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions` """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, ) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues +from litellm.utils import supports_reasoning from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -17,8 +19,7 @@ class MoonshotChatConfig(OpenAIGPTConfig): @overload def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, List[AllMessageValues]]: - ... + ) -> Coroutine[Any, Any, List[AllMessageValues]]: ... @overload def _transform_messages( @@ -26,8 +27,7 @@ class MoonshotChatConfig(OpenAIGPTConfig): messages: List[AllMessageValues], model: str, is_async: Literal[False] = False, - ) -> List[AllMessageValues]: - ... + ) -> List[AllMessageValues]: ... def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: bool = False @@ -53,22 +53,14 @@ class MoonshotChatConfig(OpenAIGPTConfig): messages = handle_messages_with_content_list_to_str_conversion(messages) if is_async: - return super()._transform_messages( - messages=messages, model=model, is_async=True - ) + return super()._transform_messages(messages=messages, model=model, is_async=True) else: - return super()._transform_messages( - messages=messages, model=model, is_async=False - ) + return super()._transform_messages(messages=messages, model=model, is_async=False) def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: - api_base = ( - api_base - or get_secret_str("MOONSHOT_API_BASE") - or "https://api.moonshot.ai/v1" - ) # type: ignore + api_base = api_base or get_secret_str("MOONSHOT_API_BASE") or "https://api.moonshot.ai/v1" # type: ignore dynamic_api_key = api_key or get_secret_str("MOONSHOT_API_KEY") return api_base, dynamic_api_key @@ -149,6 +141,48 @@ class MoonshotChatConfig(OpenAIGPTConfig): optional_params["temperature"] = 0.3 return optional_params + def fill_reasoning_content(self, messages: List[AllMessageValues]) -> List[AllMessageValues]: + """ + Moonshot reasoning models require `reasoning_content` on every assistant + message that contains tool_calls (multi-turn tool-calling flows). + + For each such message that is missing the field: + 1. Promote provider_specific_fields["reasoning_content"] if present and non-empty + (this is where LiteLLM stores it from a previous response) + 2. Otherwise inject a single space — the minimum value the API accepts + Messages that already carry the field, or are not assistant/tool-call messages, + are appended as-is (no copy made). + """ + result: List[AllMessageValues] = [] + for msg in messages: + if ( + msg.get("role") == "assistant" + and msg.get("tool_calls") + and "reasoning_content" not in msg + ): + patched = dict(cast(dict, msg)) + provider_fields = patched.get("provider_specific_fields") or {} + stored = provider_fields.get("reasoning_content") + if stored: + patched["reasoning_content"] = stored + # Remove the promoted key from provider_specific_fields to + # avoid sending the value twice in the serialised request body + cleaned_provider_fields = dict(provider_fields) + cleaned_provider_fields.pop("reasoning_content", None) + patched["provider_specific_fields"] = cleaned_provider_fields + else: + litellm.verbose_logger.warning( + "Moonshot reasoning model: assistant tool-call message is missing " + "`reasoning_content`. Injecting a placeholder to satisfy API validation. " + "For best results, preserve `reasoning_content` from the original " + "assistant response when building multi-turn conversation history." + ) + patched["reasoning_content"] = " " + result.append(cast(AllMessageValues, patched)) + else: + result.append(msg) + return result + def transform_request( self, model: str, @@ -169,6 +203,10 @@ class MoonshotChatConfig(OpenAIGPTConfig): optional_params=optional_params, ) + # Moonshot reasoning models: fill in reasoning_content before the API call + if supports_reasoning(model=model, custom_llm_provider="moonshot"): + messages = self.fill_reasoning_content(messages) + # Call parent transform_request which handles _transform_messages return super().transform_request( model=model, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index d3b0217d04..5c3bbf61ee 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -150,6 +150,8 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers=headers, ) + self._remove_scope_from_cache_control(anthropic_messages_request) + anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16" anthropic_messages_request.pop( diff --git a/litellm/main.py b/litellm/main.py index 722b4a7aae..3b1f5dc96f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1370,6 +1370,13 @@ def completion( # type: ignore # noqa: PLR0915 api_key=api_key, ) + ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name + responses_api_model_info, model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + web_search_options=web_search_options, + ) + if not _should_allow_input_examples( custom_llm_provider=custom_llm_provider, model=model ): @@ -1614,7 +1621,7 @@ def completion( # type: ignore # noqa: PLR0915 reasoning_effort=reasoning_effort, ) - if model_info.get("mode") == "responses": + if responses_api_model_info.get("mode") == "responses": from litellm.completion_extras import responses_api_bridge if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: @@ -5122,6 +5129,7 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, litellm_params=litellm_params_dict, + headers=headers, ) elif custom_llm_provider == "bedrock": if isinstance(input, str): diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9b1d81fee4..6786fc3359 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22083,6 +22083,7 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true @@ -22166,6 +22167,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, @@ -22180,6 +22182,7 @@ "output_cost_per_token": 8e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index f1b2493976..682ad4943b 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -113,6 +113,16 @@ async def background_streaming_task( # noqa: PLR0915 last_update_time = asyncio.get_event_loop().time() UPDATE_INTERVAL = 0.150 # 150ms batching interval + # Track the terminal event from the stream (may not be "completed") + terminal_status = None # Will be set by response.completed/failed/incomplete/cancelled + terminal_error = None + _event_to_status = { + "response.completed": "completed", + "response.failed": "failed", + "response.incomplete": "incomplete", + "response.cancelled": "cancelled", + } + async def flush_state_if_needed(force: bool = False) -> None: """Flush accumulated state to Redis if interval elapsed or forced""" nonlocal state_dirty, last_update_time @@ -131,6 +141,12 @@ async def background_streaming_task( # noqa: PLR0915 last_update_time = current_time # Handle StreamingResponse + if not hasattr(response, "body_iterator"): + verbose_proxy_logger.warning( + f"background_streaming_task: response for {polling_id} has no " + "body_iterator; this may indicate a misconfiguration or provider error" + ) + if hasattr(response, "body_iterator"): async for chunk in response.body_iterator: # Parse chunk @@ -224,10 +240,23 @@ async def background_streaming_task( # noqa: PLR0915 status="in_progress", ) - elif event_type == "response.completed": - # Response completed - extract all ResponsesAPIResponse fields - # https://platform.openai.com/docs/api-reference/responses-streaming/response-completed + elif event_type in ( + "response.completed", + "response.failed", + "response.incomplete", + "response.cancelled", + ): + # Terminal event - extract all ResponsesAPIResponse fields + # https://platform.openai.com/docs/api-reference/responses-streaming response_data = event.get("response", {}) + terminal_status = response_data.get( + "status", + _event_to_status.get(event_type, "completed"), + ) + + # Extract error for failed responses + if event_type == "response.failed": + terminal_error = response_data.get("error") # Core response fields usage_data = response_data.get("usage") @@ -278,11 +307,14 @@ async def background_streaming_task( # noqa: PLR0915 # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) - # Mark as completed with all ResponsesAPIResponse fields + # Use the terminal status from the stream, default to "completed" + final_status = terminal_status or "completed" + await polling_handler.update_state( polling_id=polling_id, - status="completed", + status=final_status, usage=usage_data, + error=terminal_error, reasoning=reasoning_data, tool_choice=tool_choice_data, tools=tools_data, @@ -301,7 +333,7 @@ async def background_streaming_task( # noqa: PLR0915 ) verbose_proxy_logger.info( - f"Completed background streaming for {polling_id}, output_items={len(output_items)}" + f"Finished background streaming for {polling_id}, status={final_status}, output_items={len(output_items)}" ) except Exception as e: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9b1d81fee4..6786fc3359 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22083,6 +22083,7 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true @@ -22166,6 +22167,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, @@ -22180,6 +22182,7 @@ "output_cost_per_token": 8e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, + "supports_reasoning": true, "supports_tool_choice": true, "supports_web_search": true }, diff --git a/requirements.txt b/requirements.txt index f7b72b6f0c..bf2bf2c47a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # LITELLM PROXY DEPENDENCIES # # Security: explicit pins for transitive deps (CVE fixes) urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441 -tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724 +tornado>=6.5.5 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724, CVE-2026-31958, GHSA-78cv-mqj4-43f7 filelock>=3.20.1 # CVE-2025-68146 h11>=0.16.0 # CVE-2025-43859, GHSA-vqfr-h8mv-ghfj — HTTP request smuggling wheel>=0.46.2 # CVE-2026-24049 — path traversal diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 9e6e5bb369..acbb9c5136 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -1454,3 +1454,29 @@ def test_gpt_5_web_search(): for chunk in response: print("chunk: ", chunk) + + +def test_responses_gpt54_with_xhigh_reasoning(): + """ + Ensure chat->responses bridge sends the correct request payload for + openai/responses/gpt-5.4 with reasoning_effort="xhigh". + """ + with patch("litellm.responses") as mock_responses: + # Stop execution right after request generation to avoid external API calls. + mock_responses.side_effect = RuntimeError("stop_after_request_build") + + with pytest.raises(Exception): + litellm.completion( + model="openai/responses/gpt-5.4", + messages=[{"role": "user", "content": "What is 2+2?"}], + reasoning_effort="xhigh", + max_tokens=100, + ) + + mock_responses.assert_called_once() + request_body = mock_responses.call_args.kwargs + + # The responses prefix should be stripped before routing. + assert request_body["model"] == "gpt-5.4" + # chat-completions reasoning_effort must map to Responses API reasoning. + assert request_body["reasoning"] == {"effort": "xhigh"} diff --git a/tests/proxy_unit_tests/test_response_polling_handler.py b/tests/proxy_unit_tests/test_response_polling_handler.py index 83e7e26728..6235dde847 100644 --- a/tests/proxy_unit_tests/test_response_polling_handler.py +++ b/tests/proxy_unit_tests/test_response_polling_handler.py @@ -1318,6 +1318,267 @@ class TestStreamingEventParsing: assert output_items["item_123"]["content"][0]["type"] == "text" +def _make_sse_stream(events: list) -> Mock: + """Create a mock StreamingResponse with body_iterator from a list of event dicts.""" + + async def _body_iterator(): + for event in events: + yield f"data: {json.dumps(event)}" + yield "data: [DONE]" + + mock_response = Mock() + mock_response.body_iterator = _body_iterator() + return mock_response + + +def _make_background_streaming_kwargs( + polling_id: str, + polling_handler: ResponsePollingHandler, +) -> dict: + """Build kwargs for background_streaming_task with all required mocks.""" + return dict( + polling_id=polling_id, + data={"model": "gpt-4o", "stream": False, "background": True}, + polling_handler=polling_handler, + request=Mock(), + fastapi_response=Mock(), + user_api_key_dict=Mock(), + general_settings={}, + llm_router=None, + proxy_config=Mock(), + proxy_logging_obj=Mock(), + select_data_generator=Mock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + version=None, + ) + + +@pytest.mark.xdist_group("heavy_imports") +class TestBackgroundStreamingTerminalEvents: + """ + Integration tests that exercise background_streaming_task with mocked + streaming responses, verifying the final update_state call for each + terminal event type. + """ + + @pytest.mark.asyncio + async def test_response_failed_sets_failed_status_and_error(self): + """Test that a response.failed stream event results in failed status with error""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + error_payload = { + "type": "server_error", + "message": "The model encountered an error", + "code": "model_error", + } + events = [ + {"type": "response.in_progress"}, + { + "type": "response.failed", + "response": { + "id": "resp_123", + "status": "failed", + "error": error_payload, + "model": "gpt-4o", + "output": [], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_1", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + # Find the final update_state call (last one) + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"] == error_payload + + @pytest.mark.asyncio + async def test_response_incomplete_sets_incomplete_status_and_details(self): + """Test that a response.incomplete stream event results in incomplete status""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + events = [ + {"type": "response.in_progress"}, + { + "type": "response.incomplete", + "response": { + "id": "resp_123", + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 10, "output_tokens": 4096}, + "model": "gpt-4o", + "output": [{"id": "item_1", "type": "message"}], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_2", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "incomplete" + assert final_call.kwargs["incomplete_details"] == {"reason": "max_output_tokens"} + assert final_call.kwargs["usage"] == {"input_tokens": 10, "output_tokens": 4096} + + @pytest.mark.asyncio + async def test_response_cancelled_sets_cancelled_status(self): + """Test that a response.cancelled stream event results in cancelled status""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + events = [ + {"type": "response.in_progress"}, + { + "type": "response.cancelled", + "response": { + "id": "resp_123", + "status": "cancelled", + "model": "gpt-4o", + "output": [], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_3", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "cancelled" + + @pytest.mark.asyncio + async def test_response_completed_sets_completed_status(self): + """Test that a response.completed stream event results in completed status""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + events = [ + {"type": "response.in_progress"}, + { + "type": "response.completed", + "response": { + "id": "resp_123", + "status": "completed", + "usage": {"input_tokens": 10, "output_tokens": 50}, + "model": "gpt-4o", + "output": [{"id": "item_1", "type": "message"}], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_4", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "completed" + assert final_call.kwargs["usage"] == {"input_tokens": 10, "output_tokens": 50} + + @pytest.mark.asyncio + async def test_fallback_status_derived_from_event_type_when_status_field_missing(self): + """Test that when the response body lacks a status field, the fallback + is derived from the event type, not hardcoded to 'completed'.""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + # response.incomplete event with NO status field in the response body + events = [ + {"type": "response.in_progress"}, + { + "type": "response.incomplete", + "response": { + "id": "resp_123", + # "status" deliberately omitted + "incomplete_details": {"reason": "max_output_tokens"}, + "model": "gpt-4o", + "output": [], + }, + }, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_5", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "incomplete" + + @pytest.mark.asyncio + async def test_no_terminal_event_defaults_to_completed(self): + """Test that when no terminal event is received, status defaults to completed""" + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + # Stream with only in_progress, no terminal event + events = [ + {"type": "response.in_progress"}, + ] + mock_response = _make_sse_stream(events) + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_6", handler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "completed" + + class TestEdgeCases: """Test edge cases and error scenarios""" diff --git a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py index 345186e8a6..c557fb395f 100644 --- a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py @@ -7,6 +7,7 @@ Moonshot AI is an OpenAI-compatible provider with minor customizations. import os import sys +from unittest.mock import patch sys.path.insert( 0, os.path.abspath("../../../../..") @@ -404,4 +405,149 @@ class TestMoonshotConfig: # Content should be flattened to a plain string assert isinstance(result["messages"][0]["content"], str) - assert result["messages"][0]["content"] == "Hello, how are you?" \ No newline at end of file + assert result["messages"][0]["content"] == "Hello, how are you?" + + # ------------------------------------------------------------------ # + # Tests for fill_reasoning_content # + # ------------------------------------------------------------------ # + + def test_reasoning_content_space_injected_when_absent(self): + """Assistant tool-call message with no reasoning_content gets a space injected.""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 22°C"}, + ] + + result = config.fill_reasoning_content(messages) + + assert result[1].get("reasoning_content") == " " + # Non-assistant messages are untouched + assert "reasoning_content" not in result[0] + assert "reasoning_content" not in result[2] + + def test_empty_tool_calls_list_not_injected(self): + """Assistant message with tool_calls: [] should not get reasoning_content injected.""" + config = MoonshotChatConfig() + + original_msg = { + "role": "assistant", + "content": "Here is the answer.", + "tool_calls": [], + } + messages = [original_msg] + + result = config.fill_reasoning_content(messages) + + assert "reasoning_content" not in result[0] + assert result[0] is original_msg + + def test_existing_reasoning_content_not_overwritten(self): + """Message that already has reasoning_content is passed through unchanged.""" + config = MoonshotChatConfig() + + original_msg = { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + "reasoning_content": "", + } + messages = [original_msg] + + result = config.fill_reasoning_content(messages) + + assert result[0].get("reasoning_content") == "" + # Same object — no copy was made + assert result[0] is original_msg + + def test_provider_specific_fields_reasoning_content_promoted(self): + """reasoning_content stored in provider_specific_fields is promoted to top level.""" + config = MoonshotChatConfig() + + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + "provider_specific_fields": {"reasoning_content": "stored thinking"}, + } + ] + + result = config.fill_reasoning_content(messages) + + assert result[0].get("reasoning_content") == "stored thinking" + # The promoted key must be removed from provider_specific_fields to + # avoid sending the value twice in the serialised request body + assert "reasoning_content" not in (result[0].get("provider_specific_fields") or {}) + + def test_reasoning_model_fill_called_from_transform_request(self): + """transform_request injects reasoning_content end-to-end for reasoning models.""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "Call a tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + }, + ] + + with patch( + "litellm.llms.moonshot.chat.transformation.supports_reasoning", + return_value=True, + ): + result = config.transform_request( + model="kimi-k2-thinking", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["messages"][1].get("reasoning_content") == " " + + def test_non_reasoning_model_messages_untouched(self): + """For non-reasoning models, transform_request leaves messages unchanged.""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "Hello"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}} + ], + }, + ] + + with patch( + "litellm.llms.moonshot.chat.transformation.supports_reasoning", + return_value=False, + ): + result = config.transform_request( + model="moonshot-v1-8k", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + # reasoning_content must not have been injected + for msg in result["messages"]: + assert "reasoning_content" not in msg \ No newline at end of file diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index b5f076262d..391daa24f4 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -5,6 +5,7 @@ import pytest from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) +from litellm.types.router import GenericLiteLLMParams def test_validate_environment_uses_vertex_ai_location(): @@ -248,3 +249,47 @@ def test_validate_environment_with_authorization_header_calculates_api_base(): # Verify Authorization header is still present assert "Authorization" in updated_headers, \ "Authorization header should be preserved" + + +def test_transform_anthropic_messages_request_removes_scope_from_cache_control(): + """Ensure scope field is removed from cache_control for Vertex AI (not supported).""" + config = VertexAIPartnerModelsAnthropicMessagesConfig() + + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": {"type": "ephemeral", "scope": "global"}, + } + ], + } + ] + anthropic_messages_optional_request_params = { + "max_tokens": 1024, + "system": [ + { + "type": "text", + "text": "You are an AI assistant.", + "cache_control": {"type": "ephemeral", "scope": "global"}, + } + ], + } + + result = config.transform_anthropic_messages_request( + model="claude-sonnet-4-6", + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + # scope removed from system + assert "scope" not in result["system"][0]["cache_control"] + assert result["system"][0]["cache_control"]["type"] == "ephemeral" + + # scope removed from message content + assert "scope" not in result["messages"][0]["content"][0]["cache_control"] + assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 5cbe1ead88..2a9bb3e5e2 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -88,7 +88,7 @@ "mermaid": ">=11.10.0", "js-yaml": ">=4.1.1", "glob": ">=11.1.0", - "tar": ">=7.5.10", + "tar": ">=7.5.11", "minimatch": ">=10.2.4", "@isaacs/brace-expansion": ">=5.0.1", "node-forge": ">=1.3.2", diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx index 24ce80f0a0..86d70048b6 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx @@ -78,6 +78,7 @@ export function FallbackGroupConfig({ value={group.primaryModel} onChange={handlePrimaryChange} showSearch + getPopupContainer={(trigger) => trigger.parentElement || document.body} filterOption={(input, option) => (option?.label ?? "").toLowerCase().includes(input.toLowerCase()) } @@ -125,6 +126,7 @@ export function FallbackGroupConfig({ value={group.fallbackModels} onChange={handleFallbackSelect} disabled={!group.primaryModel} + getPopupContainer={(trigger) => trigger.parentElement || document.body} options={availableFallbackOptions.map((m) => ({ label: m, value: m,