tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-11 14:22:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(factory.py): support openai 'functions' messages

Browse Source
This commit is contained in:
Krrish Dholakia
2024-05-04 12:33:39 -07:00
parent d9d5149aa1
commit 8d49b3a84c
3 changed files with 130 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/prompt_templates/factory.py
+53 -90
View File
@@ -850,6 +850,13 @@ def convert_to_anthropic_tool_result(message: dict) -> dict:
"name": "get_current_weather",
"content": "function result goes here",
},
OpenAI message with a function call result looks like:
{
"role": "function",
"name": "get_current_weather",
"content": "function result goes here",
}
"""
"""
@@ -866,18 +873,42 @@ def convert_to_anthropic_tool_result(message: dict) -> dict:
]
}
"""
tool_call_id = message.get("tool_call_id")
content = message.get("content")
if message["role"] == "tool":
tool_call_id = message.get("tool_call_id")
content = message.get("content")
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
anthropic_tool_result = {
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": content,
}
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
anthropic_tool_result = {
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": content,
}
return anthropic_tool_result
elif message["role"] == "function":
content = message.get("content")
anthropic_tool_result = {
"type": "tool_result",
"tool_use_id": str(uuid.uuid4()),
"content": content,
}
return anthropic_tool_result
return {}
return anthropic_tool_result
def convert_function_to_anthropic_tool_invoke(function_call):
try:
anthropic_tool_invoke = [
{
"type": "tool_use",
"id": str(uuid.uuid4()),
"name": get_attribute_or_key(function_call, "name"),
"input": json.loads(get_attribute_or_key(function_call, "arguments")),
}
]
return anthropic_tool_invoke
except Exception as e:
raise e
def convert_to_anthropic_tool_invoke(tool_calls: list) -> list:
@@ -940,7 +971,7 @@ def convert_to_anthropic_tool_invoke(tool_calls: list) -> list:
def anthropic_messages_pt(messages: list):
"""
format messages for anthropic
1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant)
1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately)
2. The first message always needs to be of role "user"
3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm)
4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise)
@@ -948,7 +979,7 @@ def anthropic_messages_pt(messages: list):
6. Ensure we only accept role, content. (message.name is not supported)
"""
# add role=tool support to allow function call result/error submission
user_message_types = {"user", "tool"}
user_message_types = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.
new_messages = []
msg_i = 0
@@ -971,7 +1002,10 @@ def anthropic_messages_pt(messages: list):
)
elif m.get("type", "") == "text":
user_content.append({"type": "text", "text": m["text"]})
elif messages[msg_i]["role"] == "tool":
elif (
messages[msg_i]["role"] == "tool"
or messages[msg_i]["role"] == "function"
):
# OpenAI's tool message content will always be a string
user_content.append(convert_to_anthropic_tool_result(messages[msg_i]))
else:
@@ -1000,72 +1034,12 @@ def anthropic_messages_pt(messages: list):
convert_to_anthropic_tool_invoke(messages[msg_i]["tool_calls"])
)
msg_i += 1
## MERGE CONSECUTIVE FUNCTION CONTENT ##
while msg_i < len(messages) and messages[msg_i]["role"] == "function":
"""
Anthropic function message: "role", "name", "input", "id"
OpenAI function message: "content", "name", "role"
- Check if received message is a tool call input or model text response
"""
tool_use_param = True
_message = ChatCompletionFunctionMessageParam(**messages[msg_i]) # type: ignore
anthropic_function_message: Optional[
AnthropicMessagesAssistantMessageValues
] = None
try:
anthropic_function_message = (
AnthopicMessagesAssistantMessageToolCallParam(type="tool_use")
if messages[msg_i].get("function_call"):
assistant_content.extend(
convert_function_to_anthropic_tool_invoke(
messages[msg_i]["function_call"]
)
)
anthropic_function_message["input"] = json.loads(_message["content"])
anthropic_function_message["id"] = str(uuid.uuid4())
anthropic_function_message["name"] = _message["name"]
except Exception as e:
litellm.print_verbose(
"Invalid dictionary content. Treating as text instead."
)
anthropic_function_message = (
AnthopicMessagesAssistantMessageTextContentParam(type="text")
)
anthropic_function_message["text"] = _message["content"]
assistant_content.append(anthropic_function_message) # type: ignore
msg_i += 1
## MERGE CONSECUTIVE TOOL CONTENT ##
while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
"""
Anthropic function message: "role", "name", "input", "id"
OpenAI function message: "content", "name", "role"
- Check if received message is a tool call input or model text response
"""
tool_use_param = True
_message = ChatCompletionToolMessageParam(**messages[msg_i]) # type: ignore
anthropic_tool_message: Optional[
AnthropicMessagesAssistantMessageValues
] = None
try:
anthropic_tool_message = AnthopicMessagesAssistantMessageToolCallParam(
type="tool_use"
)
anthropic_tool_message["input"] = json.loads(_message["content"])
anthropic_tool_message["id"] = _message["tool_call_id"]
anthropic_tool_message["name"] = _message["name"]
except Exception as e:
litellm.print_verbose(
"Invalid dictionary content. Treating as text instead."
)
anthropic_tool_message = (
AnthopicMessagesAssistantMessageTextContentParam(type="text")
)
anthropic_tool_message["text"] = _message["content"]
assistant_content.append(anthropic_tool_message) # type: ignore
msg_i += 1
@@ -1089,18 +1063,6 @@ def anthropic_messages_pt(messages: list):
)
if new_messages[-1]["role"] == "assistant":
if tool_use_param == True:
"""
Final assistant message cannot be a tool use param.
"""
if litellm.modify_params:
new_messages.append(
{"role": "user", "content": [{"type": "text", "text": "."}]}
)
else:
raise Exception(
"AnthropicError: Invalid last message. Your API request included an `assistant` message in the final position, which would pre-fill the `assistant` response. When using tools, pre-filling the `assistant` response is not supported. set 'litellm.modify_params = True' or 'litellm_settings:modify_params = True' on proxy, to insert a placeholder user message - '.' as the last message, "
)
if isinstance(new_messages[-1]["content"], str):
new_messages[-1]["content"] = new_messages[-1]["content"].rstrip()
elif isinstance(new_messages[-1]["content"], list):
@@ -1109,6 +1071,7 @@ def anthropic_messages_pt(messages: list):
content["text"] = content[
"text"
].rstrip() # no trailing whitespace for final assistant message
return new_messages
litellm/tests/log.txt
+44 -30
View File
@@ -5,8 +5,48 @@ plugins: timeout-2.2.0, asyncio-0.23.2, anyio-3.7.1, xdist-3.3.1
asyncio: mode=Mode.STRICT
collected 1 item
test_image_generation.py . [100%]
test_completion.py F [100%]
=================================== FAILURES ===================================
______________________ test_completion_anthropic_hanging _______________________
def test_completion_anthropic_hanging():
litellm.set_verbose = True
litellm.modify_params = True
messages = [
{
"role": "user",
"content": "What's the capital of fictional country Ubabababababaaba? Use your tools.",
},
{
"role": "assistant",
"function_call": {
"name": "get_capital",
"arguments": '{"country": "Ubabababababaaba"}',
},
},
{"role": "function", "name": "get_capital", "content": "Kokoko"},
]
converted_messages = anthropic_messages_pt(messages)
print(f"converted_messages: {converted_messages}")
## ENSURE USER / ASSISTANT ALTERNATING
for i, msg in enumerate(converted_messages):
if i < len(converted_messages) - 1:
> assert msg["role"] != converted_messages[i + 1]["role"]
E AssertionError: assert 'user' != 'user'
test_completion.py:2406: AssertionError
---------------------------- Captured stdout setup -----------------------------
<module 'litellm' from '/Users/krrishdholakia/Documents/litellm/litellm/__init__.py'>
pytest fixture - resetting callbacks
----------------------------- Captured stdout call -----------------------------
message: {'role': 'user', 'content': "What's the capital of fictional country Ubabababababaaba? Use your tools."}
message: {'role': 'function', 'name': 'get_capital', 'content': 'Kokoko'}
converted_messages: [{'role': 'user', 'content': [{'type': 'text', 'text': "What's the capital of fictional country Ubabababababaaba? Use your tools."}]}, {'role': 'user', 'content': [{'type': 'tool_result', 'tool_use_id': '10e9f4d4-bdc9-4514-8b7a-c10bc555d67c', 'content': 'Kokoko'}]}]
=============================== warnings summary ===============================
../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 23 warnings
/opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
@@ -111,33 +151,7 @@ test_image_generation.py . [100%]
Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
declare_namespace(pkg)
test_image_generation.py::test_aimage_generation_bedrock_with_optional_params
/opt/homebrew/lib/python3.11/site-packages/_pytest/threadexception.py:73: PytestUnhandledThreadExceptionWarning: Exception in thread Thread-1 (success_handler)
Traceback (most recent call last):
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1412, in _success_handler_helper_fn
litellm.completion_cost(
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4442, in completion_cost
raise e
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4405, in completion_cost
raise Exception(
Exception: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 1045, in _bootstrap_inner
self.run()
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 982, in run
self._target(*self._args, **self._kwargs)
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1465, in success_handler
start_time, end_time, result = self._success_handler_helper_fn(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1459, in _success_handler_helper_fn
raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {str(e)}")
Exception: [Non-Blocking] LiteLLM.Success_Call Error: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map
warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
======================== 1 passed, 61 warnings in 3.00s ========================
=========================== short test summary info ============================
FAILED test_completion.py::test_completion_anthropic_hanging - AssertionError...
======================== 1 failed, 60 warnings in 0.15s ========================
litellm/tests/test_completion.py
+33 -45
View File
@@ -12,6 +12,7 @@ import pytest
import litellm
from litellm import embedding, completion, completion_cost, Timeout
from litellm import RateLimitError
from litellm.llms.prompt_templates.factory import anthropic_messages_pt
# litellm.num_retries=3
litellm.cache = None
@@ -2357,27 +2358,28 @@ def test_completion_with_fallbacks():
# test_completion_with_fallbacks()
@pytest.mark.parametrize(
"function_call",
[
[{"role": "function", "name": "get_capital", "content": "Kokoko"}],
[
{"role": "function", "name": "get_capital", "content": "Kokoko"},
{"role": "function", "name": "get_capital", "content": "Kokoko"},
],
],
)
@pytest.mark.parametrize(
"tool_call",
[
[{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}],
[
{"role": "tool", "tool_call_id": "12344", "content": "Kokoko"},
{"role": "tool", "tool_call_id": "1214", "content": "Kokoko"},
],
],
)
def test_completion_anthropic_hanging(function_call, tool_call):
# @pytest.mark.parametrize(
# "function_call",
# [
# [{"role": "function", "name": "get_capital", "content": "Kokoko"}],
# [
# {"role": "function", "name": "get_capital", "content": "Kokoko"},
# {"role": "function", "name": "get_capital", "content": "Kokoko"},
# ],
# ],
# )
# @pytest.mark.parametrize(
# "tool_call",
# [
# [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}],
# [
# {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"},
# {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"},
# ],
# ],
# )
def test_completion_anthropic_hanging():
litellm.set_verbose = True
litellm.modify_params = True
messages = [
{
@@ -2391,31 +2393,17 @@ def test_completion_anthropic_hanging(function_call, tool_call):
"arguments": '{"country": "Ubabababababaaba"}',
},
},
{"role": "function", "name": "get_capital", "content": "Kokoko"},
]
messages = messages + function_call + tool_call
litellm.completion(
model="claude-3-haiku-20240307",
messages=messages,
tools=[
{
"function": {
"name": "get_capital",
"description": "Get the capital of a country",
"parameters": {
"title": "GetCapitalToolArgs",
"type": "object",
"properties": {
"country": {"title": "Country", "type": "string"}
},
"required": ["country"],
},
},
"type": "function",
}
],
tool_choice="auto",
temperature=0.0,
)
converted_messages = anthropic_messages_pt(messages)
print(f"converted_messages: {converted_messages}")
## ENSURE USER / ASSISTANT ALTERNATING
for i, msg in enumerate(converted_messages):
if i < len(converted_messages) - 1:
assert msg["role"] != converted_messages[i + 1]["role"]
def test_completion_anyscale_api():