From f1f2204c2aee4f9c2bf9945c8f0f325f6f7edc92 Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Thu, 18 Apr 2024 15:04:54 +0400 Subject: [PATCH 1/8] fix tool call errors using anthropic --- litellm/llms/prompt_templates/factory.py | 31 +++++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 8afda252ac..dbe27278e1 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -466,10 +466,11 @@ def construct_tool_use_system_prompt( ): # from https://github.com/anthropics/anthropic-cookbook/blob/main/function_calling/function_calling.ipynb tool_str_list = [] for tool in tools: + tool_function = get_attribute_or_key(tool, "function") tool_str = construct_format_tool_for_claude_prompt( - tool["function"]["name"], - tool["function"].get("description", ""), - tool["function"].get("parameters", {}), + get_attribute_or_key(tool_function, "name"), + get_attribute_or_key(tool_function, "description", ""), + get_attribute_or_key(tool_function, "parameters", {}), ) tool_str_list.append(tool_str) tool_use_system_prompt = ( @@ -614,13 +615,14 @@ def convert_to_anthropic_tool_result_xml(message: dict) -> str: def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: invokes = "" for tool in tool_calls: - if tool["type"] != "function": + if get_attribute_or_key(tool, "type") != "function": continue - - tool_name = tool["function"]["name"] + + tool_function = get_attribute_or_key(tool,"function") + tool_name = tool_function["name"] parameters = "".join( f"<{param}>{val}\n" - for param, val in json.loads(tool["function"]["arguments"]).items() + for param, val in json.loads(tool_function["arguments"]).items() ) invokes += ( "\n" @@ -705,7 +707,7 @@ def anthropic_messages_pt_xml(messages: list): if assistant_content: new_messages.append({"role": "assistant", "content": assistant_content}) - if not new_messages or new_messages[0]["role"] != "user": + if new_messages[0]["role"] != "user": if litellm.modify_params: new_messages.insert( 0, {"role": "user", "content": [{"type": "text", "text": "."}]} @@ -807,12 +809,12 @@ def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: anthropic_tool_invoke = [ { "type": "tool_use", - "id": tool["id"], - "name": tool["function"]["name"], - "input": json.loads(tool["function"]["arguments"]), + "id": get_attribute_or_key(tool, "id"), + "name": get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"), + "input": json.loads(get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments")), } for tool in tool_calls - if tool["type"] == "function" + if get_attribute_or_key(tool, "type") == "function" ] return anthropic_tool_invoke @@ -1355,3 +1357,8 @@ def prompt_factory( return default_pt( messages=messages ) # default that covers Bloom, T-5, any non-chat tuned model (e.g. base Llama2) + +def get_attribute_or_key(tool_or_function, attribute, default=None): + if hasattr(tool_or_function, attribute): + return getattr(tool_or_function, attribute) + return tool_or_function.get(attribute, default) From 7ca213e92f4257dc192e1ffe9641dfc1c0cefc14 Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Thu, 18 Apr 2024 15:12:31 +0400 Subject: [PATCH 2/8] update factory.py --- litellm/llms/prompt_templates/factory.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index dbe27278e1..bbf87ca49f 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -619,10 +619,11 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: continue tool_function = get_attribute_or_key(tool,"function") - tool_name = tool_function["name"] + tool_name = get_attribute_or_key(tool_function, "name") + tool_arguments = get_attribute_or_key(tool_function, "arguments") parameters = "".join( f"<{param}>{val}\n" - for param, val in json.loads(tool_function["arguments"]).items() + for param, val in json.loads(tool_arguments).items() ) invokes += ( "\n" @@ -707,7 +708,7 @@ def anthropic_messages_pt_xml(messages: list): if assistant_content: new_messages.append({"role": "assistant", "content": assistant_content}) - if new_messages[0]["role"] != "user": + if not new_messages or new_messages[0]["role"] != "user": if litellm.modify_params: new_messages.insert( 0, {"role": "user", "content": [{"type": "text", "text": "."}]} From 4c7d94b2b4a18413f54c77a8507064d57f6c677f Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Thu, 18 Apr 2024 17:35:52 +0400 Subject: [PATCH 3/8] update factory.py --- litellm/llms/prompt_templates/factory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index bbf87ca49f..6658671f8e 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -698,8 +698,10 @@ def anthropic_messages_pt_xml(messages: list): if messages[msg_i].get( "tool_calls", [] ): # support assistant tool invoke convertion - assistant_text += convert_to_anthropic_tool_invoke( # type: ignore - messages[msg_i]["tool_calls"] + assistant_content.extend( + convert_to_anthropic_tool_invoke( # type: ignore + messages[msg_i]["tool_calls"] + ) ) assistant_content.append({"type": "text", "text": assistant_text}) From c85018c780529579206ce2428db7a2bfa053923c Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Thu, 18 Apr 2024 18:13:20 +0400 Subject: [PATCH 4/8] update factory.py --- litellm/llms/prompt_templates/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 6658671f8e..5b8fc9a6f4 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -695,6 +695,8 @@ def anthropic_messages_pt_xml(messages: list): assistant_text = ( messages[msg_i].get("content") or "" ) # either string or none + if assistant_text: + assistant_content.append({"type": "text", "text": assistant_text}) if messages[msg_i].get( "tool_calls", [] ): # support assistant tool invoke convertion @@ -703,8 +705,6 @@ def anthropic_messages_pt_xml(messages: list): messages[msg_i]["tool_calls"] ) ) - - assistant_content.append({"type": "text", "text": assistant_text}) msg_i += 1 if assistant_content: From e1fd463f8cf414691d850a5bcbf999debc69d8af Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Thu, 18 Apr 2024 22:27:11 +0400 Subject: [PATCH 5/8] update factory.py --- litellm/llms/prompt_templates/factory.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 5b8fc9a6f4..b71857a8b2 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -617,7 +617,7 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: for tool in tool_calls: if get_attribute_or_key(tool, "type") != "function": continue - + tool_function = get_attribute_or_key(tool,"function") tool_name = get_attribute_or_key(tool_function, "name") tool_arguments = get_attribute_or_key(tool_function, "arguments") @@ -695,16 +695,14 @@ def anthropic_messages_pt_xml(messages: list): assistant_text = ( messages[msg_i].get("content") or "" ) # either string or none - if assistant_text: - assistant_content.append({"type": "text", "text": assistant_text}) if messages[msg_i].get( "tool_calls", [] ): # support assistant tool invoke convertion - assistant_content.extend( - convert_to_anthropic_tool_invoke( # type: ignore - messages[msg_i]["tool_calls"] - ) + assistant_text += convert_to_anthropic_tool_invoke_xml( # type: ignore + messages[msg_i]["tool_calls"] ) + + assistant_content.append({"type": "text", "text": assistant_text}) msg_i += 1 if assistant_content: From ecfae6d465e78f8960ff2dc3fd75197de5c2dd0b Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Thu, 18 Apr 2024 22:31:08 +0400 Subject: [PATCH 6/8] update factory.py --- litellm/llms/prompt_templates/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index b71857a8b2..7ba0ee0070 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -677,7 +677,7 @@ def anthropic_messages_pt_xml(messages: list): { "type": "text", "text": ( - convert_to_anthropic_tool_result(messages[msg_i]) + convert_to_anthropic_tool_result_xml(messages[msg_i]) if messages[msg_i]["role"] == "tool" else messages[msg_i]["content"] ), From ca3d2fea5694d0eb7cf4c3db354e12274df43ef3 Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Thu, 18 Apr 2024 22:42:32 +0400 Subject: [PATCH 7/8] fix for #2904, remove XML characters in content --- litellm/llms/prompt_templates/factory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 7ba0ee0070..218aa77fea 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -594,7 +594,8 @@ def convert_to_anthropic_tool_result_xml(message: dict) -> str: """ name = message.get("name") - content = message.get("content") + content = message.get("content", "") + content = content.replace("<", "<").replace(">", ">").replace("&", "&") # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template From 5113d470232f47cc213e131be18c2e7be7b47f95 Mon Sep 17 00:00:00 2001 From: Nilanjan De Date: Fri, 19 Apr 2024 00:42:48 +0400 Subject: [PATCH 8/8] add test --- litellm/tests/test_bedrock_completion.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/litellm/tests/test_bedrock_completion.py b/litellm/tests/test_bedrock_completion.py index 4b1781cd93..ca2ffea5f5 100644 --- a/litellm/tests/test_bedrock_completion.py +++ b/litellm/tests/test_bedrock_completion.py @@ -269,6 +269,30 @@ def test_bedrock_claude_3_tool_calling(): assert isinstance( response.choices[0].message.tool_calls[0].function.arguments, str ) + messages.append( + response.choices[0].message.model_dump() + ) # Add assistant tool invokes + tool_result = ( + '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' + ) + # Add user submitted tool results in the OpenAI format + messages.append( + { + "tool_call_id": response.choices[0].message.tool_calls[0].id, + "role": "tool", + "name": response.choices[0].message.tool_calls[0].function.name, + "content": tool_result, + } + ) + # In the second response, Claude should deduce answer from tool results + second_response = completion( + model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + messages=messages, + tools=tools, + tool_choice="auto", + ) + print(f"second response: {second_response}") + assert isinstance(second_response.choices[0].message.content, str) except RateLimitError: pass except Exception as e: