diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index ad910c9cd9..eaa80c6cfe 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2,7 +2,8 @@ Handles transforming from Responses API -> LiteLLM completion (Chat Completion API) """ -from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast +from collections.abc import Sequence +from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.tool_param import FunctionToolParam @@ -378,6 +379,7 @@ class LiteLLMCompletionResponsesConfig: if isinstance(input, str): messages.append(ChatCompletionUserMessage(role="user", content=input)) elif isinstance(input, list): + existing_tool_call_ids: Set[str] = set() for _input in input: chat_completion_messages = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( input_item=_input @@ -390,12 +392,98 @@ class LiteLLMCompletionResponsesConfig: input_item=_input ): tool_call_output_messages.extend(chat_completion_messages) - else: - messages.extend(chat_completion_messages) + continue - messages.extend(tool_call_output_messages) + if LiteLLMCompletionResponsesConfig._is_input_item_function_call( + input_item=_input + ): + call_id_raw = _input.get("call_id") or _input.get("id") or "" + if call_id_raw: + existing_tool_call_ids.add(str(call_id_raw)) + + messages.extend(chat_completion_messages) + + deduped_tool_call_messages = ( + LiteLLMCompletionResponsesConfig._deduplicate_tool_call_output_messages( + tool_call_output_messages=tool_call_output_messages, + existing_tool_call_ids=existing_tool_call_ids, + ) + ) + messages.extend(deduped_tool_call_messages) return messages + @staticmethod + def _deduplicate_tool_call_output_messages( + tool_call_output_messages: List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionMessageToolCall, + ChatCompletionResponseMessage, + ] + ], + existing_tool_call_ids: Set[str], + ) -> List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionMessageToolCall, + ChatCompletionResponseMessage, + ] + ]: + """Return tool call outputs after dropping assistant entries with duplicate call_ids.""" + if not tool_call_output_messages: + return [] + + filtered_messages: List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionMessageToolCall, + ChatCompletionResponseMessage, + ] + ] = [] + seen_tool_call_ids: Set[str] = set(existing_tool_call_ids) + + for tool_call_message in tool_call_output_messages: + if isinstance(tool_call_message, dict): + role = tool_call_message.get("role", "") + else: + role = getattr(tool_call_message, "role", "") + call_id = "" + + if role == "assistant": + tool_calls: Any = None + if isinstance(tool_call_message, dict): + tool_calls = tool_call_message.get("tool_calls") + else: + tool_calls = getattr(tool_call_message, "tool_calls", None) + + if ( + isinstance(tool_calls, Sequence) + and not isinstance(tool_calls, (str, bytes)) + and len(tool_calls) > 0 + ): + first_call = tool_calls[0] + call_id_raw = None + if isinstance(first_call, dict): + call_id_raw = first_call.get("id") + else: + call_id_raw = getattr(first_call, "id", None) + + if call_id_raw: + call_id = str(call_id_raw) + + if call_id and call_id in seen_tool_call_ids and role == "assistant": + continue + + if call_id and role == "assistant": + seen_tool_call_ids.add(call_id) + + filtered_messages.append(tool_call_message) + + return filtered_messages + @staticmethod def _ensure_tool_call_output_has_corresponding_tool_call( messages: List[Union[AllMessageValues, GenericChatCompletionMessage]], diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index c00c2a2f3b..ac040d3d6e 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -655,7 +655,6 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): follow_up_params.update( { "input": follow_up_input, - "previous_response_id": self.collected_response.id, # type: ignore[attr-defined] "stream": True, } ) diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/mcp_tests/test_aresponses_api_with_mcp.py index 865a580f0c..57c79039ee 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/mcp_tests/test_aresponses_api_with_mcp.py @@ -1,3 +1,4 @@ +import logging import os import sys import pytest @@ -660,16 +661,25 @@ async def test_streaming_mcp_events_validation(): @pytest.mark.asyncio -async def test_streaming_responses_api_with_mcp_tools(): +@pytest.mark.parametrize( + "model", + [ + pytest.param("gpt-4o-mini", id="openai"), + pytest.param("claude-haiku-4-5", id="anthropic"), + ], +) +async def test_streaming_responses_api_with_mcp_tools( + model: str, caplog: pytest.LogCaptureFixture +): """ Test the streaming responses API with MCP tools when using server_url="litellm_proxy" Under the hood the follow occurs - MCP: responses called litellm MCP manager.list_tools (MOCKED) - - Request 1: Made to gpt-4o with fetched tools (REAL LLM CALL) + - Request 1: Made to model under test with fetched tools (REAL LLM CALL) - MCP: Execute tool call from request 1 and returns result (MOCKED) - - Request 2: Made to gpt-4o with fetched tools and tool results (REAL LLM CALL) + - Request 2: Made to model under test with fetched tools and tool results (REAL LLM CALL) Return the user the result of request 2 """ @@ -693,75 +703,101 @@ async def test_streaming_responses_api_with_mcp_tools(): ] # Only mock the MCP-specific operations, let LLM responses be real - with patch.object(LiteLLM_Proxy_MCP_Handler, '_get_mcp_tools_from_manager', new_callable=AsyncMock) as mock_get_tools, \ - patch.object(LiteLLM_Proxy_MCP_Handler, '_execute_tool_calls', new_callable=AsyncMock) as mock_execute_tools: - - # Setup MCP mocks only - mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"]) - - # Create a dynamic mock that will match the actual tool call ID from the LLM response - def mock_execute_tool_calls_side_effect(tool_calls, user_api_key_auth): - """Mock function that returns results matching the actual tool call IDs from the LLM""" - results = [] - for tool_call in tool_calls: - # Extract call_id from the tool call - call_id = None - if isinstance(tool_call, dict): - call_id = tool_call.get("call_id") or tool_call.get("id") - elif hasattr(tool_call, 'call_id'): - call_id = tool_call.call_id - elif hasattr(tool_call, 'id'): - call_id = tool_call.id - - if call_id: - results.append({ - "tool_call_id": call_id, - "result": "LiteLLM is a unified interface for 100+ LLMs that translates inputs to provider-specific completion endpoints and provides consistent OpenAI-format output." - }) - return results - - mock_execute_tools.side_effect = mock_execute_tool_calls_side_effect - - # Make the actual call - LLM responses will be real - mcp_tool_config = cast(Any, { - "type": "mcp", - "server_url": "litellm_proxy", - "require_approval": "never" - }) - response = await litellm.aresponses( - model="gpt-4o-mini", - tools=[mcp_tool_config], - tool_choice="required", - input=[ + with caplog.at_level(logging.ERROR): + with patch.object( + LiteLLM_Proxy_MCP_Handler, + '_get_mcp_tools_from_manager', + new_callable=AsyncMock, + ) as mock_get_tools, patch.object( + LiteLLM_Proxy_MCP_Handler, + '_execute_tool_calls', + new_callable=AsyncMock, + ) as mock_execute_tools: + # Setup MCP mocks only + mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"]) + + # Create a dynamic mock that will match the actual tool call ID from the LLM response + def mock_execute_tool_calls_side_effect( + tool_calls, user_api_key_auth, **kwargs + ): + """Mock function that returns results matching the actual tool call IDs from the LLM""" + results = [] + for tool_call in tool_calls: + # Extract call_id from the tool call + call_id = None + if isinstance(tool_call, dict): + call_id = tool_call.get("call_id") or tool_call.get("id") + elif hasattr(tool_call, 'call_id'): + call_id = tool_call.call_id + elif hasattr(tool_call, 'id'): + call_id = tool_call.id + + if call_id: + results.append( + { + "tool_call_id": call_id, + "result": "LiteLLM is a unified interface for 100+ LLMs that translates inputs to provider-specific completion endpoints and provides consistent OpenAI-format output.", + } + ) + return results + + mock_execute_tools.side_effect = mock_execute_tool_calls_side_effect + + # Make the actual call - LLM responses will be real + mcp_tool_config = cast( + Any, { - "role": "user", - "type": "message", - "content": "give me a TLDR of what BerriAI/litellm is about" - } - ], - stream=True - ) - - print(f"📋 Response type: {type(response)}") - assert hasattr(response, '__aiter__'), "Response should be an async streaming response" - - # Collect streaming chunks - chunks = [] - async for chunk in response: - chunks.append(chunk) - print(f"📦 Chunk type: {getattr(chunk, 'type', 'unknown')}") - - print(f"📊 Total chunks received: {len(chunks)}") - - # Verify MCP mocks were called (may be called multiple times in streaming) - assert mock_get_tools.call_count >= 1, f"Expected MCP tools to be fetched at least once, got {mock_get_tools.call_count}" - print(f"MCP tools fetched: {len(mock_mcp_tools)}") - - # Verify we got a response - assert response is not None - assert len(chunks) > 0, "Should have received streaming chunks" - - print("Basic streaming responses API with MCP tools test passed!") + "type": "mcp", + "server_url": "litellm_proxy", + "require_approval": "never", + }, + ) + response = await litellm.aresponses( + model=model, + tools=[mcp_tool_config], + tool_choice="required", + input=[ + { + "role": "user", + "type": "message", + "content": "give me a TLDR of what BerriAI/litellm is about", + } + ], + stream=True, + ) + + print(f"📋 Response type: {type(response)}") + assert hasattr(response, '__aiter__'), "Response should be an async streaming response" + + # Collect streaming chunks + chunks = [] + async for chunk in response: + chunks.append(chunk) + print(f"📦 Chunk type: {getattr(chunk, 'type', 'unknown')}") + + print(f"📊 Total chunks received: {len(chunks)}") + + # Verify MCP mocks were called (may be called multiple times in streaming) + assert ( + mock_get_tools.call_count >= 1 + ), f"Expected MCP tools to be fetched at least once, got {mock_get_tools.call_count}" + print(f"MCP tools fetched: {len(mock_mcp_tools)}") + + # Verify we got a response + assert response is not None + assert len(chunks) > 0, "Should have received streaming chunks" + + print("Basic streaming responses API with MCP tools test passed!") + + lite_errors = [ + record + for record in caplog.records + if record.levelno >= logging.ERROR + and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage()) + ] + assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join( + record.getMessage() for record in lite_errors + ) @pytest.mark.asyncio