tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-18 22:17:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3448 from BerriAI/litellm_anthropic_fix

Browse Source 
fix(factory.py): support 'function' openai message role for anthropic
This commit is contained in:
Krish Dholakia
2024-05-04 16:36:05 -07:00
committed by GitHub
parent d45328dda6 09d7121af2
commit 601e8a1172
7 changed files with 273 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/bedrock.py
+10 -2
View File
@@ -4,7 +4,13 @@ from enum import Enum
import time, uuid
from typing import Callable, Optional, Any, Union, List
import litellm
from litellm.utils import ModelResponse, get_secret, Usage, ImageResponse
from litellm.utils import (
ModelResponse,
get_secret,
Usage,
ImageResponse,
map_finish_reason,
)
from .prompt_templates.factory import (
prompt_factory,
custom_prompt,
@@ -1050,7 +1056,9 @@ def completion(
logging_obj=logging_obj,
)
model_response["finish_reason"] = response_body["stop_reason"]
model_response["finish_reason"] = map_finish_reason(
response_body["stop_reason"]
)
_usage = litellm.Usage(
prompt_tokens=response_body["usage"]["input_tokens"],
completion_tokens=response_body["usage"]["output_tokens"],
litellm/llms/prompt_templates/factory.py
+75 -18
View File
@@ -16,7 +16,12 @@ from litellm.types.completion import (
ChatCompletionUserMessageParam,
ChatCompletionSystemMessageParam,
ChatCompletionMessageParam,
ChatCompletionFunctionMessageParam,
ChatCompletionMessageToolCallParam,
ChatCompletionToolMessageParam,
)
from litellm.types.llms.anthropic import *
import uuid
def default_pt(messages):
@@ -845,6 +850,13 @@ def convert_to_anthropic_tool_result(message: dict) -> dict:
"name": "get_current_weather",
"content": "function result goes here",
},
OpenAI message with a function call result looks like:
{
"role": "function",
"name": "get_current_weather",
"content": "function result goes here",
}
"""
"""
@@ -861,18 +873,42 @@ def convert_to_anthropic_tool_result(message: dict) -> dict:
]
}
"""
tool_call_id = message.get("tool_call_id")
content = message.get("content")
if message["role"] == "tool":
tool_call_id = message.get("tool_call_id")
content = message.get("content")
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
anthropic_tool_result = {
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": content,
}
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
anthropic_tool_result = {
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": content,
}
return anthropic_tool_result
elif message["role"] == "function":
content = message.get("content")
anthropic_tool_result = {
"type": "tool_result",
"tool_use_id": str(uuid.uuid4()),
"content": content,
}
return anthropic_tool_result
return {}
return anthropic_tool_result
def convert_function_to_anthropic_tool_invoke(function_call):
try:
anthropic_tool_invoke = [
{
"type": "tool_use",
"id": str(uuid.uuid4()),
"name": get_attribute_or_key(function_call, "name"),
"input": json.loads(get_attribute_or_key(function_call, "arguments")),
}
]
return anthropic_tool_invoke
except Exception as e:
raise e
def convert_to_anthropic_tool_invoke(tool_calls: list) -> list:
@@ -935,7 +971,7 @@ def convert_to_anthropic_tool_invoke(tool_calls: list) -> list:
def anthropic_messages_pt(messages: list):
"""
format messages for anthropic
1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant)
1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately)
2. The first message always needs to be of role "user"
3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm)
4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise)
@@ -943,12 +979,14 @@ def anthropic_messages_pt(messages: list):
6. Ensure we only accept role, content. (message.name is not supported)
"""
# add role=tool support to allow function call result/error submission
user_message_types = {"user", "tool"}
user_message_types = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.
new_messages = []
msg_i = 0
tool_use_param = False
while msg_i < len(messages):
user_content = []
init_msg_i = msg_i
## MERGE CONSECUTIVE USER CONTENT ##
while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types:
if isinstance(messages[msg_i]["content"], list):
@@ -964,7 +1002,10 @@ def anthropic_messages_pt(messages: list):
)
elif m.get("type", "") == "text":
user_content.append({"type": "text", "text": m["text"]})
elif messages[msg_i]["role"] == "tool":
elif (
messages[msg_i]["role"] == "tool"
or messages[msg_i]["role"] == "function"
):
# OpenAI's tool message content will always be a string
user_content.append(convert_to_anthropic_tool_result(messages[msg_i]))
else:
@@ -993,11 +1034,24 @@ def anthropic_messages_pt(messages: list):
convert_to_anthropic_tool_invoke(messages[msg_i]["tool_calls"])
)
if messages[msg_i].get("function_call"):
assistant_content.extend(
convert_function_to_anthropic_tool_invoke(
messages[msg_i]["function_call"]
)
)
msg_i += 1
if assistant_content:
new_messages.append({"role": "assistant", "content": assistant_content})
if msg_i == init_msg_i: # prevent infinite loops
raise Exception(
"Invalid Message passed in - {}. File an issue https://github.com/BerriAI/litellm/issues".format(
messages[msg_i]
)
)
if not new_messages or new_messages[0]["role"] != "user":
if litellm.modify_params:
new_messages.insert(
@@ -1009,11 +1063,14 @@ def anthropic_messages_pt(messages: list):
)
if new_messages[-1]["role"] == "assistant":
for content in new_messages[-1]["content"]:
if isinstance(content, dict) and content["type"] == "text":
content["text"] = content[
"text"
].rstrip() # no trailing whitespace for final assistant message
if isinstance(new_messages[-1]["content"], str):
new_messages[-1]["content"] = new_messages[-1]["content"].rstrip()
elif isinstance(new_messages[-1]["content"], list):
for content in new_messages[-1]["content"]:
if isinstance(content, dict) and content["type"] == "text":
content["text"] = content[
"text"
].rstrip() # no trailing whitespace for final assistant message
return new_messages
litellm/tests/log.txt
+44 -30
View File
@@ -5,8 +5,48 @@ plugins: timeout-2.2.0, asyncio-0.23.2, anyio-3.7.1, xdist-3.3.1
asyncio: mode=Mode.STRICT
collected 1 item
test_image_generation.py . [100%]
test_completion.py F [100%]
=================================== FAILURES ===================================
______________________ test_completion_anthropic_hanging _______________________
def test_completion_anthropic_hanging():
litellm.set_verbose = True
litellm.modify_params = True
messages = [
{
"role": "user",
"content": "What's the capital of fictional country Ubabababababaaba? Use your tools.",
},
{
"role": "assistant",
"function_call": {
"name": "get_capital",
"arguments": '{"country": "Ubabababababaaba"}',
},
},
{"role": "function", "name": "get_capital", "content": "Kokoko"},
]
converted_messages = anthropic_messages_pt(messages)
print(f"converted_messages: {converted_messages}")
## ENSURE USER / ASSISTANT ALTERNATING
for i, msg in enumerate(converted_messages):
if i < len(converted_messages) - 1:
> assert msg["role"] != converted_messages[i + 1]["role"]
E AssertionError: assert 'user' != 'user'
test_completion.py:2406: AssertionError
---------------------------- Captured stdout setup -----------------------------
<module 'litellm' from '/Users/krrishdholakia/Documents/litellm/litellm/__init__.py'>
pytest fixture - resetting callbacks
----------------------------- Captured stdout call -----------------------------
message: {'role': 'user', 'content': "What's the capital of fictional country Ubabababababaaba? Use your tools."}
message: {'role': 'function', 'name': 'get_capital', 'content': 'Kokoko'}
converted_messages: [{'role': 'user', 'content': [{'type': 'text', 'text': "What's the capital of fictional country Ubabababababaaba? Use your tools."}]}, {'role': 'user', 'content': [{'type': 'tool_result', 'tool_use_id': '10e9f4d4-bdc9-4514-8b7a-c10bc555d67c', 'content': 'Kokoko'}]}]
=============================== warnings summary ===============================
../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 23 warnings
/opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
@@ -111,33 +151,7 @@ test_image_generation.py . [100%]
Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
declare_namespace(pkg)
test_image_generation.py::test_aimage_generation_bedrock_with_optional_params
/opt/homebrew/lib/python3.11/site-packages/_pytest/threadexception.py:73: PytestUnhandledThreadExceptionWarning: Exception in thread Thread-1 (success_handler)
Traceback (most recent call last):
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1412, in _success_handler_helper_fn
litellm.completion_cost(
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4442, in completion_cost
raise e
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4405, in completion_cost
raise Exception(
Exception: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 1045, in _bootstrap_inner
self.run()
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 982, in run
self._target(*self._args, **self._kwargs)
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1465, in success_handler
start_time, end_time, result = self._success_handler_helper_fn(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1459, in _success_handler_helper_fn
raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {str(e)}")
Exception: [Non-Blocking] LiteLLM.Success_Call Error: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map
warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
======================== 1 passed, 61 warnings in 3.00s ========================
=========================== short test summary info ============================
FAILED test_completion.py::test_completion_anthropic_hanging - AssertionError...
======================== 1 failed, 60 warnings in 0.15s ========================
litellm/tests/test_bedrock_completion.py
+27 -3
View File
@@ -210,15 +210,39 @@ def test_completion_bedrock_claude_sts_client_auth():
def test_bedrock_claude_3():
try:
litellm.set_verbose = True
data = {
"max_tokens": 2000,
"stream": False,
"temperature": 0.3,
"messages": [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi"},
{
"role": "user",
"content": [
{"text": "describe this image", "type": "text"},
{
"image_url": {
"detail": "high",
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAL0AAAC9CAMAAADRCYwCAAAAh1BMVEX///8AAAD8/Pz5+fkEBAT39/cJCQn09PRNTU3y8vIMDAwzMzPe3t7v7+8QEBCOjo7FxcXR0dHn5+elpaWGhoYYGBivr686OjocHBy0tLQtLS1TU1PY2Ni6urpaWlpERER3d3ecnJxoaGiUlJRiYmIlJSU4ODhBQUFycnKAgIDBwcFnZ2chISE7EjuwAAAI/UlEQVR4nO1caXfiOgz1bhJIyAJhX1JoSzv8/9/3LNlpYd4rhX6o4/N8Z2lKM2cURZau5JsQEhERERERERERERERERERERHx/wBjhDPC3OGN8+Cc5JeMuheaETSdO8vZFyCScHtmz2CsktoeMn7rLM1u3h0PMAEhyYX7v/Q9wQvoGdB0hlbzm45lEq/wd6y6G9aezvBk9AXwp1r3LHJIRsh6s2maxaJpmvqgvkC7WFS3loUnaFJtKRVUCEoV/RpCnHRvAsesVQ1hw+vd7Mpo+424tLs72NplkvQgcdrsvXkW/zJWqH/fA0FT84M/xnQJt4to3+ZLuanbM6X5lfXKHosO9COgREqpCR5i86pf2zPS7j9tTj+9nO7bQz3+xGEyGW9zqgQ1tyQ/VsxEDvce/4dcUPNb5OD9yXvR4Z2QisuP0xiGWPnemgugU5q/troHhGEjIF5sTOyW648aC0TssuaaCEsYEIkGzjWXOp3A0vVsf6kgRyqaDk+T7DIVWrb58b2tT5xpUucKwodOD/5LbrZC1ws6YSaBZJ/8xlh+XZSYXaMJ2ezNqjB3IPXuehPcx2U6b4t1dS/xNdFzguUt8ie7arnPeyCZroxLHzGgGdqVcspwafizPWEXBee+9G1OaufGdvNng/9C+gwgZ3PH3r87G6zXTZ5D5De2G2DeFoANXfbACkT+fxBQ22YFsTTJF9hjFVO6VbqxZXko4WJ8s52P4PnuxO5KRzu0/hlix1ySt8iXjgaQ+4IHPA9nVzNkdduM9LFT/Aacj4FtKrHA7iAw602Vnht6R8Vq1IOS+wNMKLYqayAYfRuufQPGeGb7sZogQQoLZrGPgZ6KoYn70Iw30O92BNEDpvwouCFn6wH2uS+EhRb3WF/HObZk3HuxfRQM3Y/Of/VH0n4MKNHZDiZvO9+m/ABALfkOcuar/7nOo7B95ACGVAFaz4jMiJwJhdaHBkySmzlGTu82gr6FSTik2kJvLnY9nOd/D90qcH268m3I/cgI1xg1maE5CuZYaWLH+UHANCIck0yt7Mx5zBm5vVHXHwChsZ35kKqUpmo5Svq5/fzfAI5g2vDtFPYo1HiEA85QrDeGm9g//LG7K0scO3sdpj2CBDgCa+0OFs0bkvVgnnM/QBDwllOMm+cN7vMSHlB7Uu4haHKaTwgGkv8tlK+hP8fzmFuK/RQTpaLPWvbd58yWIo66HHM0OsPoPhVqmtaEVL7N+wYcTLTbb0DLdgp23Eyy2VYJ2N7bkLFAAibtoLPe5sLt6Oa2bvU+zyeMa8wrixO0gRTn9tO9NCSThTLGqcqtsDvphlfmx/cPBZVvw24jg1LE2lPuEo35Mhi58U0I/Ga8n5w+NS8i34MAQLos5B1u0xL1ZvCVYVRw/Fs2q53KLaXJMWwOZZ/4MPYV19bAHmgGDKB6f01xoeJKFbl63q9J34KdaVNPJWztQyRkzA3KNs1AdAEDowMxh10emXTCx75CkurtbY/ZpdNDGdsn2UcHKHsQ8Ai3WZi48IfkvtjOhsLpuIRSKZTX9FA4o+0d6o/zOWqQzVJMynL9NsxhSJOaourq6nBVQBueMSyubsX2xHrmuABZN2Ns9jr5nwLFlLF/2R6atjW/67Yd11YQ1Z+kA9Zk9dPTM/o6dVo6HHVgC0JR8oUfmI93T9u3gvTG94bAH02Y5xeqRcjuwnKCK6Q2+ajl8KXJ3GSh22P3Zfx6S+n008ROhJn+JRIUVu6o7OXl8w1SeyhuqNDwNI7SjbK08QrqPxS95jy4G7nCXVq6G3HNu0LtK5J0e226CfC005WKK9sVvfxI0eUbcnzutfhWe3rpZHM0nZ/ny/N8tanKYlQ6VEW5Xuym8yV1zZX58vwGhZp/5tFfhybZabdbrQYOs8F+xEhmPsb0/nki6kIyVvzZzUASiOrTfF+Sj9bXC7DoJxeiV8tjQL6loSd0yCx7YyB6rPdLx31U2qCG3F/oXIuDuqd6LFO+4DNIJuxFZqSsU0ea88avovFnWKRYFYRQDfCfcGaBCLn4M4A1ntJ5E57vicwqq2enaZEF5nokCYu9TbKqCC5yCDfL+GhLxT4w4xEJs+anqgou8DOY2q8FMryjb2MehC1dRJ9s4g9NXeTwPkWON4RH+FhIe0AWR/S9ekvQ+t70XHeimGF78LzuU7d7PwrswdIG2VpgF8C53qVQsTDtBJc4CdnkQPbnZY9mbPdDFra3PCXBBQ5QBn2aQqtyhvlyYM4Hb2/mdhsxCUen04GZVvIJZw5PAamMOmjzq8Q+dzAKLXDQ3RUZItWsg4t7W2DP+JDrJDymoMH7E5zQtuEpG03GTIjGCW3LQqOYEsXgFc78x76NeRwY6SNM+IfQoh6myJKRBIcLYxZcwscJ/gI2isTBty2Po9IkYzP0/SS4hGlxRjFAG5z1Jt1LckiB57yWvo35EaolbvA+6fBa24xodL2YjsPpTnj3JgJOqhcgOeLVsYYwoK0wjY+m1D3rGc40CukkaHnkEjarlXrF1B9M6ECQ6Ow0V7R7N4G3LfOHAXtymoyXOb4QhaYHJ/gNBJUkxclpSs7DNcgWWDDmM7Ke5MJpGuioe7w5EOvfTunUKRzOh7G2ylL+6ynHrD54oQO3//cN3yVO+5qMVsPZq0CZIOx4TlcJ8+Vz7V5waL+7WekzUpRFMTnnTlSCq3X5usi8qmIleW/rit1+oQZn1WGSU/sKBYEqMNh1mBOc6PhK8yCfKHdUNQk8o/G19ZPTs5MYfai+DLs5vmee37zEyyH48WW3XA6Xw6+Az8lMhci7N/KleToo7PtTKm+RA887Kqc6E9dyqL/QPTugzMHLbLZtJKqKLFfzVWRNJ63c+95uWT/F7R0U5dDVvuS409AJXhJvD0EwWaWdW8UN11u/7+umaYjT8mJtzZwP/MD4r57fihiHlC5fylHfaqnJdro+Dr7DajvO+vi2EwyD70s8nCH71nzIO1l5Zl+v1DMCb5ebvCMkGHvobXy/hPumGLyX0218/3RyD1GRLOuf9u/OGQyDmto32yMiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIv7GP8YjWPR/czH2AAAAAElFTkSuQmCC",
},
"type": "image_url",
},
],
},
],
}
response: ModelResponse = completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=messages,
max_tokens=10,
temperature=0.78,
# messages=messages,
# max_tokens=10,
# temperature=0.78,
**data,
)
# Add any assertions here to check the response
assert len(response.choices) > 0
assert len(response.choices[0].message.content) > 0
except RateLimitError:
pass
except Exception as e:
litellm/tests/test_completion.py
+51
View File
@@ -12,6 +12,7 @@ import pytest
import litellm
from litellm import embedding, completion, completion_cost, Timeout
from litellm import RateLimitError
from litellm.llms.prompt_templates.factory import anthropic_messages_pt
# litellm.num_retries=3
litellm.cache = None
@@ -2355,6 +2356,56 @@ def test_completion_with_fallbacks():
# test_completion_with_fallbacks()
# @pytest.mark.parametrize(
# "function_call",
# [
# [{"role": "function", "name": "get_capital", "content": "Kokoko"}],
# [
# {"role": "function", "name": "get_capital", "content": "Kokoko"},
# {"role": "function", "name": "get_capital", "content": "Kokoko"},
# ],
# ],
# )
# @pytest.mark.parametrize(
# "tool_call",
# [
# [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}],
# [
# {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"},
# {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"},
# ],
# ],
# )
def test_completion_anthropic_hanging():
litellm.set_verbose = True
litellm.modify_params = True
messages = [
{
"role": "user",
"content": "What's the capital of fictional country Ubabababababaaba? Use your tools.",
},
{
"role": "assistant",
"function_call": {
"name": "get_capital",
"arguments": '{"country": "Ubabababababaaba"}',
},
},
{"role": "function", "name": "get_capital", "content": "Kokoko"},
]
converted_messages = anthropic_messages_pt(messages)
print(f"converted_messages: {converted_messages}")
## ENSURE USER / ASSISTANT ALTERNATING
for i, msg in enumerate(converted_messages):
if i < len(converted_messages) - 1:
assert msg["role"] != converted_messages[i + 1]["role"]
def test_completion_anyscale_api():
try:
# litellm.set_verbose=True
litellm/types/completion.py
+24
View File
@@ -93,6 +93,28 @@ class Function(TypedDict, total=False):
"""The name of the function to call."""
class ChatCompletionToolMessageParam(TypedDict, total=False):
content: Required[str]
"""The contents of the tool message."""
role: Required[Literal["tool"]]
"""The role of the messages author, in this case `tool`."""
tool_call_id: Required[str]
"""Tool call that this message is responding to."""
class ChatCompletionFunctionMessageParam(TypedDict, total=False):
content: Required[Optional[str]]
"""The contents of the function message."""
name: Required[str]
"""The name of the function to call."""
role: Required[Literal["function"]]
"""The role of the messages author, in this case `function`."""
class ChatCompletionMessageToolCallParam(TypedDict, total=False):
id: Required[str]
"""The ID of the tool call."""
@@ -136,6 +158,8 @@ ChatCompletionMessageParam = Union[
ChatCompletionSystemMessageParam,
ChatCompletionUserMessageParam,
ChatCompletionAssistantMessageParam,
ChatCompletionFunctionMessageParam,
ChatCompletionToolMessageParam,
]
litellm/types/llms/anthropic.py
+42
View File
@@ -0,0 +1,42 @@
from typing import List, Optional, Union, Iterable
from pydantic import BaseModel, validator
from typing_extensions import Literal, Required, TypedDict
class AnthopicMessagesAssistantMessageTextContentParam(TypedDict, total=False):
type: Required[Literal["text"]]
text: str
class AnthopicMessagesAssistantMessageToolCallParam(TypedDict, total=False):
type: Required[Literal["tool_use"]]
id: str
name: str
input: dict
AnthropicMessagesAssistantMessageValues = Union[
AnthopicMessagesAssistantMessageTextContentParam,
AnthopicMessagesAssistantMessageToolCallParam,
]
class AnthopicMessagesAssistantMessageParam(TypedDict, total=False):
content: Required[Union[str, Iterable[AnthropicMessagesAssistantMessageValues]]]
"""The contents of the system message."""
role: Required[Literal["assistant"]]
"""The role of the messages author, in this case `author`."""
name: str
"""An optional name for the participant.
Provides the model information to differentiate between participants of the same
role.
"""