diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 908aebef9a..0820303685 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -850,6 +850,13 @@ def convert_to_anthropic_tool_result(message: dict) -> dict: "name": "get_current_weather", "content": "function result goes here", }, + + OpenAI message with a function call result looks like: + { + "role": "function", + "name": "get_current_weather", + "content": "function result goes here", + } """ """ @@ -866,18 +873,42 @@ def convert_to_anthropic_tool_result(message: dict) -> dict: ] } """ - tool_call_id = message.get("tool_call_id") - content = message.get("content") + if message["role"] == "tool": + tool_call_id = message.get("tool_call_id") + content = message.get("content") - # We can't determine from openai message format whether it's a successful or - # error call result so default to the successful result template - anthropic_tool_result = { - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": content, - } + # We can't determine from openai message format whether it's a successful or + # error call result so default to the successful result template + anthropic_tool_result = { + "type": "tool_result", + "tool_use_id": tool_call_id, + "content": content, + } + return anthropic_tool_result + elif message["role"] == "function": + content = message.get("content") + anthropic_tool_result = { + "type": "tool_result", + "tool_use_id": str(uuid.uuid4()), + "content": content, + } + return anthropic_tool_result + return {} - return anthropic_tool_result + +def convert_function_to_anthropic_tool_invoke(function_call): + try: + anthropic_tool_invoke = [ + { + "type": "tool_use", + "id": str(uuid.uuid4()), + "name": get_attribute_or_key(function_call, "name"), + "input": json.loads(get_attribute_or_key(function_call, "arguments")), + } + ] + return anthropic_tool_invoke + except Exception as e: + raise e def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: @@ -940,7 +971,7 @@ def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: def anthropic_messages_pt(messages: list): """ format messages for anthropic - 1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant) + 1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately) 2. The first message always needs to be of role "user" 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) @@ -948,7 +979,7 @@ def anthropic_messages_pt(messages: list): 6. Ensure we only accept role, content. (message.name is not supported) """ # add role=tool support to allow function call result/error submission - user_message_types = {"user", "tool"} + user_message_types = {"user", "tool", "function"} # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. new_messages = [] msg_i = 0 @@ -971,7 +1002,10 @@ def anthropic_messages_pt(messages: list): ) elif m.get("type", "") == "text": user_content.append({"type": "text", "text": m["text"]}) - elif messages[msg_i]["role"] == "tool": + elif ( + messages[msg_i]["role"] == "tool" + or messages[msg_i]["role"] == "function" + ): # OpenAI's tool message content will always be a string user_content.append(convert_to_anthropic_tool_result(messages[msg_i])) else: @@ -1000,72 +1034,12 @@ def anthropic_messages_pt(messages: list): convert_to_anthropic_tool_invoke(messages[msg_i]["tool_calls"]) ) - msg_i += 1 - - ## MERGE CONSECUTIVE FUNCTION CONTENT ## - while msg_i < len(messages) and messages[msg_i]["role"] == "function": - """ - Anthropic function message: "role", "name", "input", "id" - OpenAI function message: "content", "name", "role" - - - Check if received message is a tool call input or model text response - """ - tool_use_param = True - _message = ChatCompletionFunctionMessageParam(**messages[msg_i]) # type: ignore - anthropic_function_message: Optional[ - AnthropicMessagesAssistantMessageValues - ] = None - try: - anthropic_function_message = ( - AnthopicMessagesAssistantMessageToolCallParam(type="tool_use") + if messages[msg_i].get("function_call"): + assistant_content.extend( + convert_function_to_anthropic_tool_invoke( + messages[msg_i]["function_call"] + ) ) - anthropic_function_message["input"] = json.loads(_message["content"]) - anthropic_function_message["id"] = str(uuid.uuid4()) - anthropic_function_message["name"] = _message["name"] - except Exception as e: - litellm.print_verbose( - "Invalid dictionary content. Treating as text instead." - ) - anthropic_function_message = ( - AnthopicMessagesAssistantMessageTextContentParam(type="text") - ) - anthropic_function_message["text"] = _message["content"] - - assistant_content.append(anthropic_function_message) # type: ignore - - msg_i += 1 - - ## MERGE CONSECUTIVE TOOL CONTENT ## - while msg_i < len(messages) and messages[msg_i]["role"] == "tool": - """ - Anthropic function message: "role", "name", "input", "id" - OpenAI function message: "content", "name", "role" - - - Check if received message is a tool call input or model text response - """ - tool_use_param = True - _message = ChatCompletionToolMessageParam(**messages[msg_i]) # type: ignore - anthropic_tool_message: Optional[ - AnthropicMessagesAssistantMessageValues - ] = None - - try: - anthropic_tool_message = AnthopicMessagesAssistantMessageToolCallParam( - type="tool_use" - ) - anthropic_tool_message["input"] = json.loads(_message["content"]) - anthropic_tool_message["id"] = _message["tool_call_id"] - anthropic_tool_message["name"] = _message["name"] - except Exception as e: - litellm.print_verbose( - "Invalid dictionary content. Treating as text instead." - ) - anthropic_tool_message = ( - AnthopicMessagesAssistantMessageTextContentParam(type="text") - ) - anthropic_tool_message["text"] = _message["content"] - - assistant_content.append(anthropic_tool_message) # type: ignore msg_i += 1 @@ -1089,18 +1063,6 @@ def anthropic_messages_pt(messages: list): ) if new_messages[-1]["role"] == "assistant": - if tool_use_param == True: - """ - Final assistant message cannot be a tool use param. - """ - if litellm.modify_params: - new_messages.append( - {"role": "user", "content": [{"type": "text", "text": "."}]} - ) - else: - raise Exception( - "AnthropicError: Invalid last message. Your API request included an `assistant` message in the final position, which would pre-fill the `assistant` response. When using tools, pre-filling the `assistant` response is not supported. set 'litellm.modify_params = True' or 'litellm_settings:modify_params = True' on proxy, to insert a placeholder user message - '.' as the last message, " - ) if isinstance(new_messages[-1]["content"], str): new_messages[-1]["content"] = new_messages[-1]["content"].rstrip() elif isinstance(new_messages[-1]["content"], list): @@ -1109,6 +1071,7 @@ def anthropic_messages_pt(messages: list): content["text"] = content[ "text" ].rstrip() # no trailing whitespace for final assistant message + return new_messages diff --git a/litellm/tests/log.txt b/litellm/tests/log.txt index a7d1233452..4092a961d1 100644 --- a/litellm/tests/log.txt +++ b/litellm/tests/log.txt @@ -5,8 +5,48 @@ plugins: timeout-2.2.0, asyncio-0.23.2, anyio-3.7.1, xdist-3.3.1 asyncio: mode=Mode.STRICT collected 1 item -test_image_generation.py . [100%] +test_completion.py F [100%] +=================================== FAILURES =================================== +______________________ test_completion_anthropic_hanging _______________________ + + def test_completion_anthropic_hanging(): + litellm.set_verbose = True + litellm.modify_params = True + messages = [ + { + "role": "user", + "content": "What's the capital of fictional country Ubabababababaaba? Use your tools.", + }, + { + "role": "assistant", + "function_call": { + "name": "get_capital", + "arguments": '{"country": "Ubabababababaaba"}', + }, + }, + {"role": "function", "name": "get_capital", "content": "Kokoko"}, + ] + + converted_messages = anthropic_messages_pt(messages) + + print(f"converted_messages: {converted_messages}") + + ## ENSURE USER / ASSISTANT ALTERNATING + for i, msg in enumerate(converted_messages): + if i < len(converted_messages) - 1: +> assert msg["role"] != converted_messages[i + 1]["role"] +E AssertionError: assert 'user' != 'user' + +test_completion.py:2406: AssertionError +---------------------------- Captured stdout setup ----------------------------- + + +pytest fixture - resetting callbacks +----------------------------- Captured stdout call ----------------------------- +message: {'role': 'user', 'content': "What's the capital of fictional country Ubabababababaaba? Use your tools."} +message: {'role': 'function', 'name': 'get_capital', 'content': 'Kokoko'} +converted_messages: [{'role': 'user', 'content': [{'type': 'text', 'text': "What's the capital of fictional country Ubabababababaaba? Use your tools."}]}, {'role': 'user', 'content': [{'type': 'tool_result', 'tool_use_id': '10e9f4d4-bdc9-4514-8b7a-c10bc555d67c', 'content': 'Kokoko'}]}] =============================== warnings summary =============================== ../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 23 warnings /opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @@ -111,33 +151,7 @@ test_image_generation.py . [100%] Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -test_image_generation.py::test_aimage_generation_bedrock_with_optional_params - /opt/homebrew/lib/python3.11/site-packages/_pytest/threadexception.py:73: PytestUnhandledThreadExceptionWarning: Exception in thread Thread-1 (success_handler) - - Traceback (most recent call last): - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1412, in _success_handler_helper_fn - litellm.completion_cost( - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4442, in completion_cost - raise e - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4405, in completion_cost - raise Exception( - Exception: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 1045, in _bootstrap_inner - self.run() - File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 982, in run - self._target(*self._args, **self._kwargs) - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1465, in success_handler - start_time, end_time, result = self._success_handler_helper_fn( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1459, in _success_handler_helper_fn - raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {str(e)}") - Exception: [Non-Blocking] LiteLLM.Success_Call Error: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map - - warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg)) - -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -======================== 1 passed, 61 warnings in 3.00s ======================== +=========================== short test summary info ============================ +FAILED test_completion.py::test_completion_anthropic_hanging - AssertionError... +======================== 1 failed, 60 warnings in 0.15s ======================== diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index c0b4e3136d..13f99bd571 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -12,6 +12,7 @@ import pytest import litellm from litellm import embedding, completion, completion_cost, Timeout from litellm import RateLimitError +from litellm.llms.prompt_templates.factory import anthropic_messages_pt # litellm.num_retries=3 litellm.cache = None @@ -2357,27 +2358,28 @@ def test_completion_with_fallbacks(): # test_completion_with_fallbacks() -@pytest.mark.parametrize( - "function_call", - [ - [{"role": "function", "name": "get_capital", "content": "Kokoko"}], - [ - {"role": "function", "name": "get_capital", "content": "Kokoko"}, - {"role": "function", "name": "get_capital", "content": "Kokoko"}, - ], - ], -) -@pytest.mark.parametrize( - "tool_call", - [ - [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}], - [ - {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"}, - {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"}, - ], - ], -) -def test_completion_anthropic_hanging(function_call, tool_call): +# @pytest.mark.parametrize( +# "function_call", +# [ +# [{"role": "function", "name": "get_capital", "content": "Kokoko"}], +# [ +# {"role": "function", "name": "get_capital", "content": "Kokoko"}, +# {"role": "function", "name": "get_capital", "content": "Kokoko"}, +# ], +# ], +# ) +# @pytest.mark.parametrize( +# "tool_call", +# [ +# [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}], +# [ +# {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"}, +# {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"}, +# ], +# ], +# ) +def test_completion_anthropic_hanging(): + litellm.set_verbose = True litellm.modify_params = True messages = [ { @@ -2391,31 +2393,17 @@ def test_completion_anthropic_hanging(function_call, tool_call): "arguments": '{"country": "Ubabababababaaba"}', }, }, + {"role": "function", "name": "get_capital", "content": "Kokoko"}, ] - messages = messages + function_call + tool_call - litellm.completion( - model="claude-3-haiku-20240307", - messages=messages, - tools=[ - { - "function": { - "name": "get_capital", - "description": "Get the capital of a country", - "parameters": { - "title": "GetCapitalToolArgs", - "type": "object", - "properties": { - "country": {"title": "Country", "type": "string"} - }, - "required": ["country"], - }, - }, - "type": "function", - } - ], - tool_choice="auto", - temperature=0.0, - ) + + converted_messages = anthropic_messages_pt(messages) + + print(f"converted_messages: {converted_messages}") + + ## ENSURE USER / ASSISTANT ALTERNATING + for i, msg in enumerate(converted_messages): + if i < len(converted_messages) - 1: + assert msg["role"] != converted_messages[i + 1]["role"] def test_completion_anyscale_api():