From 33472bfd2bca230a658b2855795469815c2f4d4d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 4 May 2024 10:03:30 -0700 Subject: [PATCH 1/4] fix(factory.py): support 'function' openai message role for anthropic Fixes https://github.com/BerriAI/litellm/issues/3446 --- litellm/llms/prompt_templates/factory.py | 71 ++++++++++++++++++++++-- litellm/tests/test_completion.py | 53 ++++++++++++++++++ litellm/types/completion.py | 24 ++++++++ litellm/types/llms/anthropic.py | 42 ++++++++++++++ 4 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 litellm/types/llms/anthropic.py diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 1a06123bd5..dc4c417cd3 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -16,7 +16,11 @@ from litellm.types.completion import ( ChatCompletionUserMessageParam, ChatCompletionSystemMessageParam, ChatCompletionMessageParam, + ChatCompletionFunctionMessageParam, + ChatCompletionMessageToolCallParam, ) +from litellm.types.llms.anthropic import * +import uuid def default_pt(messages): @@ -947,8 +951,10 @@ def anthropic_messages_pt(messages: list): # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. new_messages = [] msg_i = 0 + tool_use_param = False while msg_i < len(messages): user_content = [] + init_msg_i = msg_i ## MERGE CONSECUTIVE USER CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types: if isinstance(messages[msg_i]["content"], list): @@ -995,9 +1001,48 @@ def anthropic_messages_pt(messages: list): msg_i += 1 + ## MERGE CONSECUTIVE FUNCTION CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] == "function": + """ + Anthropic function message: "role", "name", "input", "id" + OpenAI function message: "content", "name", "role" + + - Check if received message is a tool call input or model text response + """ + tool_use_param = True + _message = ChatCompletionFunctionMessageParam(**messages[msg_i]) # type: ignore + anthropic_function_message: Optional[ + AnthropicMessagesAssistantMessageValues + ] = None + try: + anthropic_function_message = ( + AnthopicMessagesAssistantMessageToolCallParam(type="tool_use") + ) + anthropic_function_message["input"] = json.loads(_message["content"]) + anthropic_function_message["id"] = str(uuid.uuid4()) + anthropic_function_message["name"] = _message["name"] + except Exception as e: + litellm.print_verbose( + "Invalid dictionary content. Treating as text instead." + ) + anthropic_function_message = ( + AnthopicMessagesAssistantMessageTextContentParam(type="text") + ) + anthropic_function_message["text"] = _message["content"] + + assistant_content.append(anthropic_function_message) # type: ignore + + msg_i += 1 + if assistant_content: new_messages.append({"role": "assistant", "content": assistant_content}) + if msg_i == init_msg_i: # prevent infinite loops + raise Exception( + "Invalid Message passed in - {}. File an issue https://github.com/BerriAI/litellm/issues".format( + messages[msg_i] + ) + ) if not new_messages or new_messages[0]["role"] != "user": if litellm.modify_params: new_messages.insert( @@ -1009,12 +1054,26 @@ def anthropic_messages_pt(messages: list): ) if new_messages[-1]["role"] == "assistant": - for content in new_messages[-1]["content"]: - if isinstance(content, dict) and content["type"] == "text": - content["text"] = content[ - "text" - ].rstrip() # no trailing whitespace for final assistant message - + if tool_use_param == True: + """ + Final assistant message cannot be a tool use param. + """ + if litellm.modify_params: + new_messages.append( + {"role": "user", "content": [{"type": "text", "text": "."}]} + ) + else: + raise Exception( + "AnthropicError: Invalid last message. Your API request included an `assistant` message in the final position, which would pre-fill the `assistant` response. When using tools, pre-filling the `assistant` response is not supported. set 'litellm.modify_params = True' or 'litellm_settings:modify_params = True' on proxy, to insert a placeholder user message - '.' as the last message, " + ) + if isinstance(new_messages[-1]["content"], str): + new_messages[-1]["content"] = new_messages[-1]["content"].rstrip() + elif isinstance(new_messages[-1]["content"], list): + for content in new_messages[-1]["content"]: + if isinstance(content, dict) and content["type"] == "text": + content["text"] = content[ + "text" + ].rstrip() # no trailing whitespace for final assistant message return new_messages diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 2b6c05558a..1108db860d 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2355,6 +2355,59 @@ def test_completion_with_fallbacks(): # test_completion_with_fallbacks() + + +@pytest.mark.parametrize( + "function_call", + [ + [{"role": "function", "name": "get_capital", "content": "Kokoko"}], + [ + {"role": "function", "name": "get_capital", "content": "Kokoko"}, + {"role": "function", "name": "get_capital", "content": "Kokoko"}, + ], + ], +) +def test_completion_anthropic_hanging(function_call): + litellm.modify_params = True + messages = [ + { + "role": "user", + "content": "What's the capital of fictional country Ubabababababaaba? Use your tools.", + }, + { + "role": "assistant", + "function_call": { + "name": "get_capital", + "arguments": '{"country": "Ubabababababaaba"}', + }, + }, + ] + messages = messages + function_call + litellm.completion( + model="claude-3-haiku-20240307", + messages=messages, + tools=[ + { + "function": { + "name": "get_capital", + "description": "Get the capital of a country", + "parameters": { + "title": "GetCapitalToolArgs", + "type": "object", + "properties": { + "country": {"title": "Country", "type": "string"} + }, + "required": ["country"], + }, + }, + "type": "function", + } + ], + tool_choice="auto", + temperature=0.0, + ) + + def test_completion_anyscale_api(): try: # litellm.set_verbose=True diff --git a/litellm/types/completion.py b/litellm/types/completion.py index 9df860f585..c5148b1689 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -93,6 +93,28 @@ class Function(TypedDict, total=False): """The name of the function to call.""" +class ChatCompletionToolMessageParam(TypedDict, total=False): + content: Required[str] + """The contents of the tool message.""" + + role: Required[Literal["tool"]] + """The role of the messages author, in this case `tool`.""" + + tool_call_id: Required[str] + """Tool call that this message is responding to.""" + + +class ChatCompletionFunctionMessageParam(TypedDict, total=False): + content: Required[Optional[str]] + """The contents of the function message.""" + + name: Required[str] + """The name of the function to call.""" + + role: Required[Literal["function"]] + """The role of the messages author, in this case `function`.""" + + class ChatCompletionMessageToolCallParam(TypedDict, total=False): id: Required[str] """The ID of the tool call.""" @@ -136,6 +158,8 @@ ChatCompletionMessageParam = Union[ ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam, ChatCompletionAssistantMessageParam, + ChatCompletionFunctionMessageParam, + ChatCompletionToolMessageParam, ] diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py new file mode 100644 index 0000000000..faf4aa3564 --- /dev/null +++ b/litellm/types/llms/anthropic.py @@ -0,0 +1,42 @@ +from typing import List, Optional, Union, Iterable + +from pydantic import BaseModel, validator + +from typing_extensions import Literal, Required, TypedDict + + +class AnthopicMessagesAssistantMessageTextContentParam(TypedDict, total=False): + type: Required[Literal["text"]] + + text: str + + +class AnthopicMessagesAssistantMessageToolCallParam(TypedDict, total=False): + type: Required[Literal["tool_use"]] + + id: str + + name: str + + input: dict + + +AnthropicMessagesAssistantMessageValues = Union[ + AnthopicMessagesAssistantMessageTextContentParam, + AnthopicMessagesAssistantMessageToolCallParam, +] + + +class AnthopicMessagesAssistantMessageParam(TypedDict, total=False): + content: Required[Union[str, Iterable[AnthropicMessagesAssistantMessageValues]]] + """The contents of the system message.""" + + role: Required[Literal["assistant"]] + """The role of the messages author, in this case `author`.""" + + name: str + """An optional name for the participant. + + Provides the model information to differentiate between participants of the same + role. + """ From d9d5149aa148237291c2203ed2dd8f82e9a85fce Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 4 May 2024 10:14:52 -0700 Subject: [PATCH 2/4] fix(factory.py): support mapping openai 'tool' message to anthropic format --- litellm/llms/prompt_templates/factory.py | 35 ++++++++++++++++++++++++ litellm/tests/test_completion.py | 14 ++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index dc4c417cd3..908aebef9a 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -18,6 +18,7 @@ from litellm.types.completion import ( ChatCompletionMessageParam, ChatCompletionFunctionMessageParam, ChatCompletionMessageToolCallParam, + ChatCompletionToolMessageParam, ) from litellm.types.llms.anthropic import * import uuid @@ -1034,6 +1035,40 @@ def anthropic_messages_pt(messages: list): msg_i += 1 + ## MERGE CONSECUTIVE TOOL CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] == "tool": + """ + Anthropic function message: "role", "name", "input", "id" + OpenAI function message: "content", "name", "role" + + - Check if received message is a tool call input or model text response + """ + tool_use_param = True + _message = ChatCompletionToolMessageParam(**messages[msg_i]) # type: ignore + anthropic_tool_message: Optional[ + AnthropicMessagesAssistantMessageValues + ] = None + + try: + anthropic_tool_message = AnthopicMessagesAssistantMessageToolCallParam( + type="tool_use" + ) + anthropic_tool_message["input"] = json.loads(_message["content"]) + anthropic_tool_message["id"] = _message["tool_call_id"] + anthropic_tool_message["name"] = _message["name"] + except Exception as e: + litellm.print_verbose( + "Invalid dictionary content. Treating as text instead." + ) + anthropic_tool_message = ( + AnthopicMessagesAssistantMessageTextContentParam(type="text") + ) + anthropic_tool_message["text"] = _message["content"] + + assistant_content.append(anthropic_tool_message) # type: ignore + + msg_i += 1 + if assistant_content: new_messages.append({"role": "assistant", "content": assistant_content}) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 1108db860d..c0b4e3136d 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2367,7 +2367,17 @@ def test_completion_with_fallbacks(): ], ], ) -def test_completion_anthropic_hanging(function_call): +@pytest.mark.parametrize( + "tool_call", + [ + [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}], + [ + {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"}, + {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"}, + ], + ], +) +def test_completion_anthropic_hanging(function_call, tool_call): litellm.modify_params = True messages = [ { @@ -2382,7 +2392,7 @@ def test_completion_anthropic_hanging(function_call): }, }, ] - messages = messages + function_call + messages = messages + function_call + tool_call litellm.completion( model="claude-3-haiku-20240307", messages=messages, From 8d49b3a84c751ffedb8a30d6f90121f7e062983b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 4 May 2024 12:33:39 -0700 Subject: [PATCH 3/4] fix(factory.py): support openai 'functions' messages --- litellm/llms/prompt_templates/factory.py | 143 +++++++++-------------- litellm/tests/log.txt | 74 +++++++----- litellm/tests/test_completion.py | 78 ++++++------- 3 files changed, 130 insertions(+), 165 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 908aebef9a..0820303685 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -850,6 +850,13 @@ def convert_to_anthropic_tool_result(message: dict) -> dict: "name": "get_current_weather", "content": "function result goes here", }, + + OpenAI message with a function call result looks like: + { + "role": "function", + "name": "get_current_weather", + "content": "function result goes here", + } """ """ @@ -866,18 +873,42 @@ def convert_to_anthropic_tool_result(message: dict) -> dict: ] } """ - tool_call_id = message.get("tool_call_id") - content = message.get("content") + if message["role"] == "tool": + tool_call_id = message.get("tool_call_id") + content = message.get("content") - # We can't determine from openai message format whether it's a successful or - # error call result so default to the successful result template - anthropic_tool_result = { - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": content, - } + # We can't determine from openai message format whether it's a successful or + # error call result so default to the successful result template + anthropic_tool_result = { + "type": "tool_result", + "tool_use_id": tool_call_id, + "content": content, + } + return anthropic_tool_result + elif message["role"] == "function": + content = message.get("content") + anthropic_tool_result = { + "type": "tool_result", + "tool_use_id": str(uuid.uuid4()), + "content": content, + } + return anthropic_tool_result + return {} - return anthropic_tool_result + +def convert_function_to_anthropic_tool_invoke(function_call): + try: + anthropic_tool_invoke = [ + { + "type": "tool_use", + "id": str(uuid.uuid4()), + "name": get_attribute_or_key(function_call, "name"), + "input": json.loads(get_attribute_or_key(function_call, "arguments")), + } + ] + return anthropic_tool_invoke + except Exception as e: + raise e def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: @@ -940,7 +971,7 @@ def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: def anthropic_messages_pt(messages: list): """ format messages for anthropic - 1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant) + 1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately) 2. The first message always needs to be of role "user" 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) @@ -948,7 +979,7 @@ def anthropic_messages_pt(messages: list): 6. Ensure we only accept role, content. (message.name is not supported) """ # add role=tool support to allow function call result/error submission - user_message_types = {"user", "tool"} + user_message_types = {"user", "tool", "function"} # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. new_messages = [] msg_i = 0 @@ -971,7 +1002,10 @@ def anthropic_messages_pt(messages: list): ) elif m.get("type", "") == "text": user_content.append({"type": "text", "text": m["text"]}) - elif messages[msg_i]["role"] == "tool": + elif ( + messages[msg_i]["role"] == "tool" + or messages[msg_i]["role"] == "function" + ): # OpenAI's tool message content will always be a string user_content.append(convert_to_anthropic_tool_result(messages[msg_i])) else: @@ -1000,72 +1034,12 @@ def anthropic_messages_pt(messages: list): convert_to_anthropic_tool_invoke(messages[msg_i]["tool_calls"]) ) - msg_i += 1 - - ## MERGE CONSECUTIVE FUNCTION CONTENT ## - while msg_i < len(messages) and messages[msg_i]["role"] == "function": - """ - Anthropic function message: "role", "name", "input", "id" - OpenAI function message: "content", "name", "role" - - - Check if received message is a tool call input or model text response - """ - tool_use_param = True - _message = ChatCompletionFunctionMessageParam(**messages[msg_i]) # type: ignore - anthropic_function_message: Optional[ - AnthropicMessagesAssistantMessageValues - ] = None - try: - anthropic_function_message = ( - AnthopicMessagesAssistantMessageToolCallParam(type="tool_use") + if messages[msg_i].get("function_call"): + assistant_content.extend( + convert_function_to_anthropic_tool_invoke( + messages[msg_i]["function_call"] + ) ) - anthropic_function_message["input"] = json.loads(_message["content"]) - anthropic_function_message["id"] = str(uuid.uuid4()) - anthropic_function_message["name"] = _message["name"] - except Exception as e: - litellm.print_verbose( - "Invalid dictionary content. Treating as text instead." - ) - anthropic_function_message = ( - AnthopicMessagesAssistantMessageTextContentParam(type="text") - ) - anthropic_function_message["text"] = _message["content"] - - assistant_content.append(anthropic_function_message) # type: ignore - - msg_i += 1 - - ## MERGE CONSECUTIVE TOOL CONTENT ## - while msg_i < len(messages) and messages[msg_i]["role"] == "tool": - """ - Anthropic function message: "role", "name", "input", "id" - OpenAI function message: "content", "name", "role" - - - Check if received message is a tool call input or model text response - """ - tool_use_param = True - _message = ChatCompletionToolMessageParam(**messages[msg_i]) # type: ignore - anthropic_tool_message: Optional[ - AnthropicMessagesAssistantMessageValues - ] = None - - try: - anthropic_tool_message = AnthopicMessagesAssistantMessageToolCallParam( - type="tool_use" - ) - anthropic_tool_message["input"] = json.loads(_message["content"]) - anthropic_tool_message["id"] = _message["tool_call_id"] - anthropic_tool_message["name"] = _message["name"] - except Exception as e: - litellm.print_verbose( - "Invalid dictionary content. Treating as text instead." - ) - anthropic_tool_message = ( - AnthopicMessagesAssistantMessageTextContentParam(type="text") - ) - anthropic_tool_message["text"] = _message["content"] - - assistant_content.append(anthropic_tool_message) # type: ignore msg_i += 1 @@ -1089,18 +1063,6 @@ def anthropic_messages_pt(messages: list): ) if new_messages[-1]["role"] == "assistant": - if tool_use_param == True: - """ - Final assistant message cannot be a tool use param. - """ - if litellm.modify_params: - new_messages.append( - {"role": "user", "content": [{"type": "text", "text": "."}]} - ) - else: - raise Exception( - "AnthropicError: Invalid last message. Your API request included an `assistant` message in the final position, which would pre-fill the `assistant` response. When using tools, pre-filling the `assistant` response is not supported. set 'litellm.modify_params = True' or 'litellm_settings:modify_params = True' on proxy, to insert a placeholder user message - '.' as the last message, " - ) if isinstance(new_messages[-1]["content"], str): new_messages[-1]["content"] = new_messages[-1]["content"].rstrip() elif isinstance(new_messages[-1]["content"], list): @@ -1109,6 +1071,7 @@ def anthropic_messages_pt(messages: list): content["text"] = content[ "text" ].rstrip() # no trailing whitespace for final assistant message + return new_messages diff --git a/litellm/tests/log.txt b/litellm/tests/log.txt index a7d1233452..4092a961d1 100644 --- a/litellm/tests/log.txt +++ b/litellm/tests/log.txt @@ -5,8 +5,48 @@ plugins: timeout-2.2.0, asyncio-0.23.2, anyio-3.7.1, xdist-3.3.1 asyncio: mode=Mode.STRICT collected 1 item -test_image_generation.py . [100%] +test_completion.py F [100%] +=================================== FAILURES =================================== +______________________ test_completion_anthropic_hanging _______________________ + + def test_completion_anthropic_hanging(): + litellm.set_verbose = True + litellm.modify_params = True + messages = [ + { + "role": "user", + "content": "What's the capital of fictional country Ubabababababaaba? Use your tools.", + }, + { + "role": "assistant", + "function_call": { + "name": "get_capital", + "arguments": '{"country": "Ubabababababaaba"}', + }, + }, + {"role": "function", "name": "get_capital", "content": "Kokoko"}, + ] + + converted_messages = anthropic_messages_pt(messages) + + print(f"converted_messages: {converted_messages}") + + ## ENSURE USER / ASSISTANT ALTERNATING + for i, msg in enumerate(converted_messages): + if i < len(converted_messages) - 1: +> assert msg["role"] != converted_messages[i + 1]["role"] +E AssertionError: assert 'user' != 'user' + +test_completion.py:2406: AssertionError +---------------------------- Captured stdout setup ----------------------------- + + +pytest fixture - resetting callbacks +----------------------------- Captured stdout call ----------------------------- +message: {'role': 'user', 'content': "What's the capital of fictional country Ubabababababaaba? Use your tools."} +message: {'role': 'function', 'name': 'get_capital', 'content': 'Kokoko'} +converted_messages: [{'role': 'user', 'content': [{'type': 'text', 'text': "What's the capital of fictional country Ubabababababaaba? Use your tools."}]}, {'role': 'user', 'content': [{'type': 'tool_result', 'tool_use_id': '10e9f4d4-bdc9-4514-8b7a-c10bc555d67c', 'content': 'Kokoko'}]}] =============================== warnings summary =============================== ../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 23 warnings /opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @@ -111,33 +151,7 @@ test_image_generation.py . [100%] Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -test_image_generation.py::test_aimage_generation_bedrock_with_optional_params - /opt/homebrew/lib/python3.11/site-packages/_pytest/threadexception.py:73: PytestUnhandledThreadExceptionWarning: Exception in thread Thread-1 (success_handler) - - Traceback (most recent call last): - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1412, in _success_handler_helper_fn - litellm.completion_cost( - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4442, in completion_cost - raise e - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4405, in completion_cost - raise Exception( - Exception: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 1045, in _bootstrap_inner - self.run() - File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 982, in run - self._target(*self._args, **self._kwargs) - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1465, in success_handler - start_time, end_time, result = self._success_handler_helper_fn( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1459, in _success_handler_helper_fn - raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {str(e)}") - Exception: [Non-Blocking] LiteLLM.Success_Call Error: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map - - warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg)) - -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -======================== 1 passed, 61 warnings in 3.00s ======================== +=========================== short test summary info ============================ +FAILED test_completion.py::test_completion_anthropic_hanging - AssertionError... +======================== 1 failed, 60 warnings in 0.15s ======================== diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index c0b4e3136d..13f99bd571 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -12,6 +12,7 @@ import pytest import litellm from litellm import embedding, completion, completion_cost, Timeout from litellm import RateLimitError +from litellm.llms.prompt_templates.factory import anthropic_messages_pt # litellm.num_retries=3 litellm.cache = None @@ -2357,27 +2358,28 @@ def test_completion_with_fallbacks(): # test_completion_with_fallbacks() -@pytest.mark.parametrize( - "function_call", - [ - [{"role": "function", "name": "get_capital", "content": "Kokoko"}], - [ - {"role": "function", "name": "get_capital", "content": "Kokoko"}, - {"role": "function", "name": "get_capital", "content": "Kokoko"}, - ], - ], -) -@pytest.mark.parametrize( - "tool_call", - [ - [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}], - [ - {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"}, - {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"}, - ], - ], -) -def test_completion_anthropic_hanging(function_call, tool_call): +# @pytest.mark.parametrize( +# "function_call", +# [ +# [{"role": "function", "name": "get_capital", "content": "Kokoko"}], +# [ +# {"role": "function", "name": "get_capital", "content": "Kokoko"}, +# {"role": "function", "name": "get_capital", "content": "Kokoko"}, +# ], +# ], +# ) +# @pytest.mark.parametrize( +# "tool_call", +# [ +# [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}], +# [ +# {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"}, +# {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"}, +# ], +# ], +# ) +def test_completion_anthropic_hanging(): + litellm.set_verbose = True litellm.modify_params = True messages = [ { @@ -2391,31 +2393,17 @@ def test_completion_anthropic_hanging(function_call, tool_call): "arguments": '{"country": "Ubabababababaaba"}', }, }, + {"role": "function", "name": "get_capital", "content": "Kokoko"}, ] - messages = messages + function_call + tool_call - litellm.completion( - model="claude-3-haiku-20240307", - messages=messages, - tools=[ - { - "function": { - "name": "get_capital", - "description": "Get the capital of a country", - "parameters": { - "title": "GetCapitalToolArgs", - "type": "object", - "properties": { - "country": {"title": "Country", "type": "string"} - }, - "required": ["country"], - }, - }, - "type": "function", - } - ], - tool_choice="auto", - temperature=0.0, - ) + + converted_messages = anthropic_messages_pt(messages) + + print(f"converted_messages: {converted_messages}") + + ## ENSURE USER / ASSISTANT ALTERNATING + for i, msg in enumerate(converted_messages): + if i < len(converted_messages) - 1: + assert msg["role"] != converted_messages[i + 1]["role"] def test_completion_anyscale_api(): From 09d7121af235e38ab92251b6f13b11151c321f2e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 4 May 2024 12:45:40 -0700 Subject: [PATCH 4/4] fix(bedrock.py): map finish reason for bedrock --- litellm/llms/bedrock.py | 12 ++++++++-- litellm/tests/test_bedrock_completion.py | 30 +++++++++++++++++++++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 517e441469..1ce1184690 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -4,7 +4,13 @@ from enum import Enum import time, uuid from typing import Callable, Optional, Any, Union, List import litellm -from litellm.utils import ModelResponse, get_secret, Usage, ImageResponse +from litellm.utils import ( + ModelResponse, + get_secret, + Usage, + ImageResponse, + map_finish_reason, +) from .prompt_templates.factory import ( prompt_factory, custom_prompt, @@ -1050,7 +1056,9 @@ def completion( logging_obj=logging_obj, ) - model_response["finish_reason"] = response_body["stop_reason"] + model_response["finish_reason"] = map_finish_reason( + response_body["stop_reason"] + ) _usage = litellm.Usage( prompt_tokens=response_body["usage"]["input_tokens"], completion_tokens=response_body["usage"]["output_tokens"], diff --git a/litellm/tests/test_bedrock_completion.py b/litellm/tests/test_bedrock_completion.py index ca2ffea5f5..8eb4675429 100644 --- a/litellm/tests/test_bedrock_completion.py +++ b/litellm/tests/test_bedrock_completion.py @@ -210,15 +210,39 @@ def test_completion_bedrock_claude_sts_client_auth(): def test_bedrock_claude_3(): try: litellm.set_verbose = True + data = { + "max_tokens": 2000, + "stream": False, + "temperature": 0.3, + "messages": [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hi"}, + { + "role": "user", + "content": [ + {"text": "describe this image", "type": "text"}, + { + "image_url": { + "detail": "high", + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAL0AAAC9CAMAAADRCYwCAAAAh1BMVEX///8AAAD8/Pz5+fkEBAT39/cJCQn09PRNTU3y8vIMDAwzMzPe3t7v7+8QEBCOjo7FxcXR0dHn5+elpaWGhoYYGBivr686OjocHBy0tLQtLS1TU1PY2Ni6urpaWlpERER3d3ecnJxoaGiUlJRiYmIlJSU4ODhBQUFycnKAgIDBwcFnZ2chISE7EjuwAAAI/UlEQVR4nO1caXfiOgz1bhJIyAJhX1JoSzv8/9/3LNlpYd4rhX6o4/N8Z2lKM2cURZau5JsQEhERERERERERERERERERERHx/wBjhDPC3OGN8+Cc5JeMuheaETSdO8vZFyCScHtmz2CsktoeMn7rLM1u3h0PMAEhyYX7v/Q9wQvoGdB0hlbzm45lEq/wd6y6G9aezvBk9AXwp1r3LHJIRsh6s2maxaJpmvqgvkC7WFS3loUnaFJtKRVUCEoV/RpCnHRvAsesVQ1hw+vd7Mpo+424tLs72NplkvQgcdrsvXkW/zJWqH/fA0FT84M/xnQJt4to3+ZLuanbM6X5lfXKHosO9COgREqpCR5i86pf2zPS7j9tTj+9nO7bQz3+xGEyGW9zqgQ1tyQ/VsxEDvce/4dcUPNb5OD9yXvR4Z2QisuP0xiGWPnemgugU5q/troHhGEjIF5sTOyW648aC0TssuaaCEsYEIkGzjWXOp3A0vVsf6kgRyqaDk+T7DIVWrb58b2tT5xpUucKwodOD/5LbrZC1ws6YSaBZJ/8xlh+XZSYXaMJ2ezNqjB3IPXuehPcx2U6b4t1dS/xNdFzguUt8ie7arnPeyCZroxLHzGgGdqVcspwafizPWEXBee+9G1OaufGdvNng/9C+gwgZ3PH3r87G6zXTZ5D5De2G2DeFoANXfbACkT+fxBQ22YFsTTJF9hjFVO6VbqxZXko4WJ8s52P4PnuxO5KRzu0/hlix1ySt8iXjgaQ+4IHPA9nVzNkdduM9LFT/Aacj4FtKrHA7iAw602Vnht6R8Vq1IOS+wNMKLYqayAYfRuufQPGeGb7sZogQQoLZrGPgZ6KoYn70Iw30O92BNEDpvwouCFn6wH2uS+EhRb3WF/HObZk3HuxfRQM3Y/Of/VH0n4MKNHZDiZvO9+m/ABALfkOcuar/7nOo7B95ACGVAFaz4jMiJwJhdaHBkySmzlGTu82gr6FSTik2kJvLnY9nOd/D90qcH268m3I/cgI1xg1maE5CuZYaWLH+UHANCIck0yt7Mx5zBm5vVHXHwChsZ35kKqUpmo5Svq5/fzfAI5g2vDtFPYo1HiEA85QrDeGm9g//LG7K0scO3sdpj2CBDgCa+0OFs0bkvVgnnM/QBDwllOMm+cN7vMSHlB7Uu4haHKaTwgGkv8tlK+hP8fzmFuK/RQTpaLPWvbd58yWIo66HHM0OsPoPhVqmtaEVL7N+wYcTLTbb0DLdgp23Eyy2VYJ2N7bkLFAAibtoLPe5sLt6Oa2bvU+zyeMa8wrixO0gRTn9tO9NCSThTLGqcqtsDvphlfmx/cPBZVvw24jg1LE2lPuEo35Mhi58U0I/Ga8n5w+NS8i34MAQLos5B1u0xL1ZvCVYVRw/Fs2q53KLaXJMWwOZZ/4MPYV19bAHmgGDKB6f01xoeJKFbl63q9J34KdaVNPJWztQyRkzA3KNs1AdAEDowMxh10emXTCx75CkurtbY/ZpdNDGdsn2UcHKHsQ8Ai3WZi48IfkvtjOhsLpuIRSKZTX9FA4o+0d6o/zOWqQzVJMynL9NsxhSJOaourq6nBVQBueMSyubsX2xHrmuABZN2Ns9jr5nwLFlLF/2R6atjW/67Yd11YQ1Z+kA9Zk9dPTM/o6dVo6HHVgC0JR8oUfmI93T9u3gvTG94bAH02Y5xeqRcjuwnKCK6Q2+ajl8KXJ3GSh22P3Zfx6S+n008ROhJn+JRIUVu6o7OXl8w1SeyhuqNDwNI7SjbK08QrqPxS95jy4G7nCXVq6G3HNu0LtK5J0e226CfC005WKK9sVvfxI0eUbcnzutfhWe3rpZHM0nZ/ny/N8tanKYlQ6VEW5Xuym8yV1zZX58vwGhZp/5tFfhybZabdbrQYOs8F+xEhmPsb0/nki6kIyVvzZzUASiOrTfF+Sj9bXC7DoJxeiV8tjQL6loSd0yCx7YyB6rPdLx31U2qCG3F/oXIuDuqd6LFO+4DNIJuxFZqSsU0ea88avovFnWKRYFYRQDfCfcGaBCLn4M4A1ntJ5E57vicwqq2enaZEF5nokCYu9TbKqCC5yCDfL+GhLxT4w4xEJs+anqgou8DOY2q8FMryjb2MehC1dRJ9s4g9NXeTwPkWON4RH+FhIe0AWR/S9ekvQ+t70XHeimGF78LzuU7d7PwrswdIG2VpgF8C53qVQsTDtBJc4CdnkQPbnZY9mbPdDFra3PCXBBQ5QBn2aQqtyhvlyYM4Hb2/mdhsxCUen04GZVvIJZw5PAamMOmjzq8Q+dzAKLXDQ3RUZItWsg4t7W2DP+JDrJDymoMH7E5zQtuEpG03GTIjGCW3LQqOYEsXgFc78x76NeRwY6SNM+IfQoh6myJKRBIcLYxZcwscJ/gI2isTBty2Po9IkYzP0/SS4hGlxRjFAG5z1Jt1LckiB57yWvo35EaolbvA+6fBa24xodL2YjsPpTnj3JgJOqhcgOeLVsYYwoK0wjY+m1D3rGc40CukkaHnkEjarlXrF1B9M6ECQ6Ow0V7R7N4G3LfOHAXtymoyXOb4QhaYHJ/gNBJUkxclpSs7DNcgWWDDmM7Ke5MJpGuioe7w5EOvfTunUKRzOh7G2ylL+6ynHrD54oQO3//cN3yVO+5qMVsPZq0CZIOx4TlcJ8+Vz7V5waL+7WekzUpRFMTnnTlSCq3X5usi8qmIleW/rit1+oQZn1WGSU/sKBYEqMNh1mBOc6PhK8yCfKHdUNQk8o/G19ZPTs5MYfai+DLs5vmee37zEyyH48WW3XA6Xw6+Az8lMhci7N/KleToo7PtTKm+RA887Kqc6E9dyqL/QPTugzMHLbLZtJKqKLFfzVWRNJ63c+95uWT/F7R0U5dDVvuS409AJXhJvD0EwWaWdW8UN11u/7+umaYjT8mJtzZwP/MD4r57fihiHlC5fylHfaqnJdro+Dr7DajvO+vi2EwyD70s8nCH71nzIO1l5Zl+v1DMCb5ebvCMkGHvobXy/hPumGLyX0218/3RyD1GRLOuf9u/OGQyDmto32yMiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIv7GP8YjWPR/czH2AAAAAElFTkSuQmCC", + }, + "type": "image_url", + }, + ], + }, + ], + } response: ModelResponse = completion( model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - messages=messages, - max_tokens=10, - temperature=0.78, + # messages=messages, + # max_tokens=10, + # temperature=0.78, + **data, ) # Add any assertions here to check the response assert len(response.choices) > 0 assert len(response.choices[0].message.content) > 0 + except RateLimitError: pass except Exception as e: