tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-11 01:05:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #19317 from BerriAI/litellm_fix-responses-api-non-openai-models

Browse Source 
[fix] responses api non OpenAI models
This commit is contained in:
YutaSaito
2026-01-19 11:21:39 +09:00
committed by GitHub
parent e326b397c5 60f40856e1
commit 14679cff3f
3 changed files with 199 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/responses/litellm_completion_transformation/transformation.py
+92 -4
View File
@@ -2,7 +2,8 @@
Handles transforming from Responses API -> LiteLLM completion (Chat Completion API)
"""
from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast
from collections.abc import Sequence
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast
from openai.types.responses import ResponseFunctionToolCall
from openai.types.responses.tool_param import FunctionToolParam
@@ -378,6 +379,7 @@ class LiteLLMCompletionResponsesConfig:
if isinstance(input, str):
messages.append(ChatCompletionUserMessage(role="user", content=input))
elif isinstance(input, list):
existing_tool_call_ids: Set[str] = set()
for _input in input:
chat_completion_messages = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
input_item=_input
@@ -390,12 +392,98 @@ class LiteLLMCompletionResponsesConfig:
input_item=_input
):
tool_call_output_messages.extend(chat_completion_messages)
else:
messages.extend(chat_completion_messages)
continue
messages.extend(tool_call_output_messages)
if LiteLLMCompletionResponsesConfig._is_input_item_function_call(
input_item=_input
):
call_id_raw = _input.get("call_id") or _input.get("id") or ""
if call_id_raw:
existing_tool_call_ids.add(str(call_id_raw))
messages.extend(chat_completion_messages)
deduped_tool_call_messages = (
LiteLLMCompletionResponsesConfig._deduplicate_tool_call_output_messages(
tool_call_output_messages=tool_call_output_messages,
existing_tool_call_ids=existing_tool_call_ids,
)
)
messages.extend(deduped_tool_call_messages)
return messages
@staticmethod
def _deduplicate_tool_call_output_messages(
tool_call_output_messages: List[
Union[
AllMessageValues,
GenericChatCompletionMessage,
ChatCompletionMessageToolCall,
ChatCompletionResponseMessage,
]
],
existing_tool_call_ids: Set[str],
) -> List[
Union[
AllMessageValues,
GenericChatCompletionMessage,
ChatCompletionMessageToolCall,
ChatCompletionResponseMessage,
]
]:
"""Return tool call outputs after dropping assistant entries with duplicate call_ids."""
if not tool_call_output_messages:
return []
filtered_messages: List[
Union[
AllMessageValues,
GenericChatCompletionMessage,
ChatCompletionMessageToolCall,
ChatCompletionResponseMessage,
]
] = []
seen_tool_call_ids: Set[str] = set(existing_tool_call_ids)
for tool_call_message in tool_call_output_messages:
if isinstance(tool_call_message, dict):
role = tool_call_message.get("role", "")
else:
role = getattr(tool_call_message, "role", "")
call_id = ""
if role == "assistant":
tool_calls: Any = None
if isinstance(tool_call_message, dict):
tool_calls = tool_call_message.get("tool_calls")
else:
tool_calls = getattr(tool_call_message, "tool_calls", None)
if (
isinstance(tool_calls, Sequence)
and not isinstance(tool_calls, (str, bytes))
and len(tool_calls) > 0
):
first_call = tool_calls[0]
call_id_raw = None
if isinstance(first_call, dict):
call_id_raw = first_call.get("id")
else:
call_id_raw = getattr(first_call, "id", None)
if call_id_raw:
call_id = str(call_id_raw)
if call_id and call_id in seen_tool_call_ids and role == "assistant":
continue
if call_id and role == "assistant":
seen_tool_call_ids.add(call_id)
filtered_messages.append(tool_call_message)
return filtered_messages
@staticmethod
def _ensure_tool_call_output_has_corresponding_tool_call(
messages: List[Union[AllMessageValues, GenericChatCompletionMessage]],
litellm/responses/mcp/mcp_streaming_iterator.py
-1
View File
@@ -655,7 +655,6 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
follow_up_params.update(
{
"input": follow_up_input,
"previous_response_id": self.collected_response.id, # type: ignore[attr-defined]
"stream": True,
}
)
tests/mcp_tests/test_aresponses_api_with_mcp.py
+107 -71
View File
@@ -1,3 +1,4 @@
import logging
import os
import sys
import pytest
@@ -660,16 +661,25 @@ async def test_streaming_mcp_events_validation():
@pytest.mark.asyncio
async def test_streaming_responses_api_with_mcp_tools():
@pytest.mark.parametrize(
"model",
[
pytest.param("gpt-4o-mini", id="openai"),
pytest.param("claude-haiku-4-5", id="anthropic"),
],
)
async def test_streaming_responses_api_with_mcp_tools(
model: str, caplog: pytest.LogCaptureFixture
):
"""
Test the streaming responses API with MCP tools when using server_url="litellm_proxy"
Under the hood the follow occurs
- MCP: responses called litellm MCP manager.list_tools (MOCKED)
- Request 1: Made to gpt-4o with fetched tools (REAL LLM CALL)
- Request 1: Made to model under test with fetched tools (REAL LLM CALL)
- MCP: Execute tool call from request 1 and returns result (MOCKED)
- Request 2: Made to gpt-4o with fetched tools and tool results (REAL LLM CALL)
- Request 2: Made to model under test with fetched tools and tool results (REAL LLM CALL)
Return the user the result of request 2
"""
@@ -693,75 +703,101 @@ async def test_streaming_responses_api_with_mcp_tools():
]
# Only mock the MCP-specific operations, let LLM responses be real
with patch.object(LiteLLM_Proxy_MCP_Handler, '_get_mcp_tools_from_manager', new_callable=AsyncMock) as mock_get_tools, \
patch.object(LiteLLM_Proxy_MCP_Handler, '_execute_tool_calls', new_callable=AsyncMock) as mock_execute_tools:
# Setup MCP mocks only
mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"])
# Create a dynamic mock that will match the actual tool call ID from the LLM response
def mock_execute_tool_calls_side_effect(tool_calls, user_api_key_auth):
"""Mock function that returns results matching the actual tool call IDs from the LLM"""
results = []
for tool_call in tool_calls:
# Extract call_id from the tool call
call_id = None
if isinstance(tool_call, dict):
call_id = tool_call.get("call_id") or tool_call.get("id")
elif hasattr(tool_call, 'call_id'):
call_id = tool_call.call_id
elif hasattr(tool_call, 'id'):
call_id = tool_call.id
if call_id:
results.append({
"tool_call_id": call_id,
"result": "LiteLLM is a unified interface for 100+ LLMs that translates inputs to provider-specific completion endpoints and provides consistent OpenAI-format output."
})
return results
mock_execute_tools.side_effect = mock_execute_tool_calls_side_effect
# Make the actual call - LLM responses will be real
mcp_tool_config = cast(Any, {
"type": "mcp",
"server_url": "litellm_proxy",
"require_approval": "never"
})
response = await litellm.aresponses(
model="gpt-4o-mini",
tools=[mcp_tool_config],
tool_choice="required",
input=[
with caplog.at_level(logging.ERROR):
with patch.object(
LiteLLM_Proxy_MCP_Handler,
'_get_mcp_tools_from_manager',
new_callable=AsyncMock,
) as mock_get_tools, patch.object(
LiteLLM_Proxy_MCP_Handler,
'_execute_tool_calls',
new_callable=AsyncMock,
) as mock_execute_tools:
# Setup MCP mocks only
mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"])
# Create a dynamic mock that will match the actual tool call ID from the LLM response
def mock_execute_tool_calls_side_effect(
tool_calls, user_api_key_auth, **kwargs
):
"""Mock function that returns results matching the actual tool call IDs from the LLM"""
results = []
for tool_call in tool_calls:
# Extract call_id from the tool call
call_id = None
if isinstance(tool_call, dict):
call_id = tool_call.get("call_id") or tool_call.get("id")
elif hasattr(tool_call, 'call_id'):
call_id = tool_call.call_id
elif hasattr(tool_call, 'id'):
call_id = tool_call.id
if call_id:
results.append(
{
"tool_call_id": call_id,
"result": "LiteLLM is a unified interface for 100+ LLMs that translates inputs to provider-specific completion endpoints and provides consistent OpenAI-format output.",
}
)
return results
mock_execute_tools.side_effect = mock_execute_tool_calls_side_effect
# Make the actual call - LLM responses will be real
mcp_tool_config = cast(
Any,
{
"role": "user",
"type": "message",
"content": "give me a TLDR of what BerriAI/litellm is about"
}
],
stream=True
)
print(f"📋 Response type: {type(response)}")
assert hasattr(response, '__aiter__'), "Response should be an async streaming response"
# Collect streaming chunks
chunks = []
async for chunk in response:
chunks.append(chunk)
print(f"📦 Chunk type: {getattr(chunk, 'type', 'unknown')}")
print(f"📊 Total chunks received: {len(chunks)}")
# Verify MCP mocks were called (may be called multiple times in streaming)
assert mock_get_tools.call_count >= 1, f"Expected MCP tools to be fetched at least once, got {mock_get_tools.call_count}"
print(f"MCP tools fetched: {len(mock_mcp_tools)}")
# Verify we got a response
assert response is not None
assert len(chunks) > 0, "Should have received streaming chunks"
print("Basic streaming responses API with MCP tools test passed!")
"type": "mcp",
"server_url": "litellm_proxy",
"require_approval": "never",
},
)
response = await litellm.aresponses(
model=model,
tools=[mcp_tool_config],
tool_choice="required",
input=[
{
"role": "user",
"type": "message",
"content": "give me a TLDR of what BerriAI/litellm is about",
}
],
stream=True,
)
print(f"📋 Response type: {type(response)}")
assert hasattr(response, '__aiter__'), "Response should be an async streaming response"
# Collect streaming chunks
chunks = []
async for chunk in response:
chunks.append(chunk)
print(f"📦 Chunk type: {getattr(chunk, 'type', 'unknown')}")
print(f"📊 Total chunks received: {len(chunks)}")
# Verify MCP mocks were called (may be called multiple times in streaming)
assert (
mock_get_tools.call_count >= 1
), f"Expected MCP tools to be fetched at least once, got {mock_get_tools.call_count}"
print(f"MCP tools fetched: {len(mock_mcp_tools)}")
# Verify we got a response
assert response is not None
assert len(chunks) > 0, "Should have received streaming chunks"
print("Basic streaming responses API with MCP tools test passed!")
lite_errors = [
record
for record in caplog.records
if record.levelno >= logging.ERROR
and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage())
]
assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join(
record.getMessage() for record in lite_errors
)
@pytest.mark.asyncio