diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index d0cebcc78d..9d5bccecdf 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1,4 +1,5 @@ import asyncio +from typing import Any, AsyncIterator, cast from fastapi import APIRouter, Depends, HTTPException, Request, Response @@ -80,7 +81,9 @@ async def responses_api( data = await _read_request_body(request=request) # Check if polling via cache should be used for this request - from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request + from litellm.proxy.response_polling.polling_handler import ( + should_use_polling_for_request, + ) should_use_polling = should_use_polling_for_request( background_mode=data.get("background", False), @@ -92,12 +95,12 @@ async def responses_api( # If polling is enabled, use polling mode if should_use_polling: - from litellm.proxy.response_polling.polling_handler import ( - ResponsePollingHandler, - ) from litellm.proxy.response_polling.background_streaming import ( background_streaming_task, ) + from litellm.proxy.response_polling.polling_handler import ( + ResponsePollingHandler, + ) verbose_proxy_logger.info( f"Starting background response with polling for model={data.get('model')}" @@ -222,8 +225,15 @@ async def cursor_chat_completions( ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import ModelResponse data = await _read_request_body(request=request) + + # Convert 'messages' to 'input' for Responses API compatibility + # Cursor sends 'messages' but Responses API expects 'input' + if "messages" in data and "input" not in data: + data["input"] = data.pop("messages") + processor = ProxyBaseLLMRequestProcessing(data=data) def cursor_data_generator(response, user_api_key_dict, request_data): @@ -244,8 +254,9 @@ async def cursor_chat_completions( # If response is a BaseResponsesAPIStreamingIterator, transform it first if isinstance(response, BaseResponsesAPIStreamingIterator): # Transform Responses API iterator to chat completion iterator + # Cast to AsyncIterator[str] since BaseResponsesAPIStreamingIterator implements __aiter__/__anext__ completion_stream = responses_api_bridge.transformation_handler.get_model_response_iterator( - streaming_response=response, + streaming_response=cast(AsyncIterator[str], response), sync_stream=False, json_mode=False, ) @@ -296,8 +307,8 @@ async def cursor_chat_completions( transformed_response = responses_api_bridge.transformation_handler.transform_response( model=processor.data.get("model", ""), raw_response=response, - model_response=None, - logging_obj=logging_obj, + model_response=ModelResponse(), + logging_obj=cast(Any, logging_obj), request_data=processor.data, messages=processor.data.get("input", []), optional_params={}, @@ -375,7 +386,7 @@ async def get_response( version, ) from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler - + # Check if this is a polling ID if ResponsePollingHandler.is_polling_id(response_id): # Handle polling response @@ -483,7 +494,7 @@ async def delete_response( version, ) from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler - + # Check if this is a polling ID if ResponsePollingHandler.is_polling_id(response_id): # Handle polling response deletion @@ -675,7 +686,7 @@ async def cancel_response( version, ) from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler - + # Check if this is a polling ID if ResponsePollingHandler.is_polling_id(response_id): # Handle polling response cancellation diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 7c79c575a5..def2f72437 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -18,6 +18,7 @@ from litellm.types.llms.openai import ( ContentPartDonePartReasoningText, OutputItemAddedEvent, OutputItemDoneEvent, + OutputTextAnnotationAddedEvent, OutputTextDeltaEvent, OutputTextDoneEvent, ReasoningSummaryTextDeltaEvent, @@ -29,7 +30,6 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, - OutputTextAnnotationAddedEvent ) from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( @@ -104,9 +104,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if "text" in self.responses_api_request: response_created_event_data["text"] = self.responses_api_request["text"] if "tool_choice" in self.responses_api_request: - response_created_event_data["tool_choice"] = self.responses_api_request[ - "tool_choice" - ] + # Transform tool_choice from dict format (e.g., {"type": "auto"}) to string format + response_created_event_data["tool_choice"] = LiteLLMCompletionResponsesConfig._transform_tool_choice( + self.responses_api_request["tool_choice"] + ) or "auto" else: response_created_event_data["tool_choice"] = "auto" if "tools" in self.responses_api_request: @@ -348,14 +349,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # Get the next chunk from the stream try: chunk = await self.litellm_custom_stream_wrapper.__anext__() - self.collected_chat_completion_chunks.append(chunk) - response_api_chunk = ( - self._transform_chat_completion_chunk_to_response_api_chunk( - chunk + if chunk is not None: + chunk = cast(ModelResponseStream, chunk) + self.collected_chat_completion_chunks.append(chunk) + response_api_chunk = ( + self._transform_chat_completion_chunk_to_response_api_chunk( + chunk + ) ) - ) - if response_api_chunk: - return response_api_chunk + if response_api_chunk: + return response_api_chunk except StopAsyncIteration: return self.common_done_event_logic(sync_mode=False) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 49a8ffc725..9149d0269b 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -36,6 +36,8 @@ from litellm.types.llms.openai import ( ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStatus, + ValidChatCompletionMessageContentTypes, + ValidChatCompletionMessageContentTypesLiteral, ) from litellm.types.responses.main import ( GenericResponseOutputItem, @@ -97,6 +99,58 @@ class LiteLLMCompletionResponsesConfig: "user", ] + @staticmethod + def _transform_tool_choice( + tool_choice: Any, + ) -> Optional[Union[str, Dict[str, Any]]]: + """ + Transform tool_choice from various formats to OpenAI Chat Completion format. + + Handles: + - String values: "auto", "none", "required" -> pass through as-is + - Dict with type only (Cursor IDE format): + - {"type": "auto"} -> "auto" + - {"type": "none"} -> "none" + - {"type": "required"} -> "required" + - {"type": "tool"} -> "required" (force tool use without specific tool) + - Dict with function (OpenAI format): + - {"type": "function", "function": {"name": "..."}} -> pass through as-is + + This normalization is needed because some clients (like Cursor IDE) send + tool_choice in a dict format like {"type": "tool"} which is not valid for + providers like Anthropic that require a tool name when forcing tool use. + """ + if tool_choice is None: + return None + + if isinstance(tool_choice, str): + return tool_choice + + if isinstance(tool_choice, dict): + tool_choice_type = tool_choice.get("type") + + # If it has a function with name, it's standard OpenAI format - pass through + if tool_choice.get("function") and tool_choice.get("function", {}).get( + "name" + ): + return tool_choice + + # Handle Cursor IDE dict formats without function name + if tool_choice_type == "auto": + return "auto" + elif tool_choice_type == "none": + return "none" + elif tool_choice_type in ["required", "tool", "any"]: + # "tool" without a specific function name means "use any tool" + # which is equivalent to "required" in OpenAI format + return "required" + elif tool_choice_type == "function": + # function type without name - fall back to required + return "required" + + # Return as-is for unknown formats + return tool_choice + @staticmethod def transform_responses_api_request_to_chat_completion_request( model: str, @@ -130,7 +184,9 @@ class LiteLLMCompletionResponsesConfig: responses_api_request=responses_api_request, ), "model": model, - "tool_choice": responses_api_request.get("tool_choice"), + "tool_choice": LiteLLMCompletionResponsesConfig._transform_tool_choice( + responses_api_request.get("tool_choice") + ), "tools": tools, "top_p": responses_api_request.get("top_p"), "user": responses_api_request.get("user"), @@ -352,6 +408,7 @@ class LiteLLMCompletionResponsesConfig: "function_call_output", "web_search_call", "computer_call_output", + "tool_result", # Anthropic/MCP format ] @staticmethod @@ -542,12 +599,16 @@ class LiteLLMCompletionResponsesConfig: ) ) else: + # Skip text blocks with None text to avoid downstream errors + text_value = item.get("text") + if text_value is None: + continue content_list.append( { "type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type( item.get("type") or "text" ), - "text": item.get("text"), + "text": text_value, } ) return content_list @@ -555,15 +616,37 @@ class LiteLLMCompletionResponsesConfig: raise ValueError(f"Invalid content type: {type(content)}") @staticmethod - def _get_chat_completion_request_content_type(content_type: str) -> str: + def _get_chat_completion_request_content_type( + content_type: str, + ) -> ValidChatCompletionMessageContentTypesLiteral: """ - Get the Chat Completion request content type + Transform Responses API content type to valid Chat Completion content type. + + Returns one of ValidChatCompletionMessageContentTypes: + - User: "text", "image_url", "input_audio", "audio_url", "document", + "guarded_text", "video_url", "file" + - Assistant: "text", "thinking", "redacted_thinking" """ # Responses API content has `input_` prefix, if it exists, remove it if content_type.startswith("input_"): - return content_type[len("input_") :] - else: - return content_type + stripped = content_type[len("input_") :] + # Validate stripped type is valid, otherwise default to "text" + if stripped in ValidChatCompletionMessageContentTypes: + return stripped # type: ignore + # Handle input_audio -> input_audio (it's already valid) + if stripped == "audio": + return "input_audio" + return "text" + + # Map Responses API specific types to valid Chat Completion types + if content_type in ["tool_result", "output_text"]: + return "text" + + # Return as-is if it's a valid type, otherwise default to "text" + if content_type in ValidChatCompletionMessageContentTypes: + return content_type # type: ignore + + return "text" @staticmethod def transform_instructions_to_system_message( @@ -610,13 +693,17 @@ class LiteLLMCompletionResponsesConfig: ) else: typed_tool = cast(FunctionToolParam, tool) + # Ensure parameters has "type": "object" as required by providers like Anthropic + parameters = dict(typed_tool.get("parameters", {}) or {}) + if not parameters or "type" not in parameters: + parameters["type"] = "object" chat_completion_tools.append( ChatCompletionToolParam( type="function", function=ChatCompletionToolParamFunctionChunk( name=typed_tool.get("name") or "", description=typed_tool.get("description") or "", - parameters=dict(typed_tool.get("parameters", {}) or {}), + parameters=parameters, strict=typed_tool.get("strict", False) or False, ), ) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index fc2bbb37ad..ebfb49dfc6 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -759,6 +759,68 @@ ValidUserMessageContentTypes = [ "file", ] # used for validating user messages. Prevent users from accidentally sending anthropic messages. +ValidUserMessageContentTypesLiteral = Literal[ + "text", + "image_url", + "input_audio", + "audio_url", + "document", + "guarded_text", + "video_url", + "file", +] + +ValidUserMessageContentTypes = [ + "text", + "image_url", + "input_audio", + "audio_url", + "document", + "guarded_text", + "video_url", + "file", +] # used for validating user messages. Prevent users from accidentally sending anthropic messages. + +# Assistant message content types (text, thinking, redacted_thinking) +ValidAssistantMessageContentTypesLiteral = Literal[ + "text", + "thinking", + "redacted_thinking", +] + +ValidAssistantMessageContentTypes = [ + "text", + "thinking", + "redacted_thinking", +] + +# Combined valid content types for chat completion messages +ValidChatCompletionMessageContentTypesLiteral = Literal[ + "text", + "image_url", + "input_audio", + "audio_url", + "document", + "guarded_text", + "video_url", + "file", + "thinking", + "redacted_thinking", +] + +ValidChatCompletionMessageContentTypes = [ + "text", + "image_url", + "input_audio", + "audio_url", + "document", + "guarded_text", + "video_url", + "file", + "thinking", + "redacted_thinking", +] + AllMessageValues = Union[ ChatCompletionUserMessage, ChatCompletionAssistantMessage, diff --git a/litellm/utils.py b/litellm/utils.py index 5702d8a4d2..9c3fa503ee 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -809,7 +809,8 @@ def function_setup( # noqa: PLR0915 call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value ): - messages = args[0] if len(args) > 0 else kwargs["input"] + # Handle both 'input' (standard Responses API) and 'messages' (Cursor chat format) + messages = args[0] if len(args) > 0 else kwargs.get("input") or kwargs.get("messages", "default-message-value") else: messages = "default-message-value" stream = False @@ -6822,10 +6823,12 @@ def is_cached_message(message: AllMessageValues) -> bool: if not isinstance(content_item, dict): continue + cache_control = content_item.get("cache_control") if ( content_item.get("type") == "text" - and content_item.get("cache_control") is not None - and content_item.get("cache_control", {}).get("type") == "ephemeral" + and cache_control is not None + and isinstance(cache_control, dict) + and cache_control.get("type") == "ephemeral" ): return True @@ -7001,7 +7004,7 @@ def validate_chat_completion_user_messages(messages: List[AllMessageValues]): for item in user_content: if isinstance(item, dict): if item.get("type") not in ValidUserMessageContentTypes: - raise Exception("invalid content type") + raise Exception(f"invalid content type={item.get('type')}") except Exception as e: if isinstance(e, KeyError): raise Exception( diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 436afd2e89..71ce57f425 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -375,6 +375,7 @@ def test_web_search_tool_result_in_provider_specific_fields(): response.choices[0].message.provider_specific_fields["web_search_results"] """ import httpx + from litellm.types.utils import ModelResponse config = AnthropicConfig() @@ -516,6 +517,37 @@ def test_map_tool_choice(): print(result) +def test_map_tool_choice_string_auto(): + """Test that string 'auto' maps to Anthropic type='auto'""" + config = AnthropicConfig() + result = config._map_tool_choice(tool_choice="auto", parallel_tool_use=None) + assert result is not None + assert result["type"] == "auto" + + +def test_map_tool_choice_string_required(): + """Test that string 'required' maps to Anthropic type='any'""" + config = AnthropicConfig() + result = config._map_tool_choice(tool_choice="required", parallel_tool_use=None) + assert result is not None + assert result["type"] == "any" + + +def test_map_tool_choice_dict_type_function_with_name(): + """ + Test that dict {"type": "function", "function": {"name": "my_tool"}} + (OpenAI format) maps to Anthropic type='tool' with name. + """ + config = AnthropicConfig() + result = config._map_tool_choice( + tool_choice={"type": "function", "function": {"name": "my_tool"}}, + parallel_tool_use=None, + ) + assert result is not None + assert result["type"] == "tool" + assert result["name"] == "my_tool" + + def test_transform_response_with_prefix_prompt(): import httpx diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 976a331297..3d6b47c758 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -685,6 +685,57 @@ class TestFunctionCallTransformation: assert tool_call.get("id") == "fallback_id" +class TestToolChoiceTransformation: + """Test the tool_choice transformation fix for Cursor IDE bug""" + + def test_transform_tool_choice_cursor_bug_fix(self): + """ + Test that {"type": "tool"} is transformed to "required". + This fixes the Anthropic error: "tool_choice.tool.name: Field required" + """ + result = LiteLLMCompletionResponsesConfig._transform_tool_choice({"type": "tool"}) + assert result == "required" + + def test_transform_tool_choice_preserves_function_with_name(self): + """Test that valid OpenAI format with function name passes through unchanged""" + tool_choice = {"type": "function", "function": {"name": "my_tool"}} + result = LiteLLMCompletionResponsesConfig._transform_tool_choice(tool_choice) + assert result == tool_choice + + +class TestContentTypeTransformation: + """Test content type transformation from Responses API to Chat Completion format""" + + def test_tool_result_content_type_transformed_to_text(self): + """ + Test that 'tool_result' content type is transformed to 'text'. + This fixes: Invalid user message - content type 'tool_result' not valid. + """ + result = LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type("tool_result") + assert result == "text" + + def test_input_text_content_type_transformed_to_text(self): + """Test that 'input_text' content type is transformed to 'text'""" + result = LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type("input_text") + assert result == "text" + + def test_none_text_blocks_filtered_out(self): + """ + Test that content blocks with None text are filtered out. + This fixes: TypeError: object of type 'NoneType' has no len() + in Anthropic transformation when text is None. + """ + content = [ + {"type": "text", "text": "valid text"}, + {"type": "text", "text": None}, # Should be filtered out + {"type": "text", "text": "another valid"}, + ] + result = LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content(content) + assert len(result) == 2 + assert result[0]["text"] == "valid text" + assert result[1]["text"] == "another valid" + + class TestUsageTransformation: """Test cases for usage transformation from Chat Completion to Responses API format"""