From c8dc4f3eecde12d458c3629cfe489cb170dd26d8 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Fri, 28 Feb 2025 18:47:07 -0800 Subject: [PATCH] converse_transformation: pass 'description' if set in response_format (#8907) * test(test_bedrock_completion.py): e2e test ensuring tool description is passed in * fix(converse_transformation.py): pass description, if set * fix(transformation.py): Fixes https://github.com/BerriAI/litellm/issues/8767#issuecomment-2689887663 --- litellm/llms/anthropic/chat/transformation.py | 16 ++++++++ .../bedrock/chat/converse_transformation.py | 15 +++++-- litellm/proxy/_new_secret_config.yaml | 5 ++- .../test_anthropic_completion.py | 39 +++++++++++++++++++ .../test_bedrock_completion.py | 39 +++++++++++++++++++ 5 files changed, 109 insertions(+), 5 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e4f87aa5b4..8d5440986e 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -123,6 +123,7 @@ class AnthropicConfig(BaseConfig): prompt_caching_set: bool = False, pdf_used: bool = False, is_vertex_request: bool = False, + user_anthropic_beta_headers: Optional[List[str]] = None, ) -> dict: betas = [] @@ -139,6 +140,9 @@ class AnthropicConfig(BaseConfig): "content-type": "application/json", } + if user_anthropic_beta_headers is not None: + betas.extend(user_anthropic_beta_headers) + # Don't send any beta headers to Vertex, Vertex has failed requests when they are sent if is_vertex_request is True: pass @@ -795,6 +799,13 @@ class AnthropicConfig(BaseConfig): headers=cast(httpx.Headers, headers), ) + def _get_user_anthropic_beta_headers( + self, anthropic_beta_header: Optional[str] + ) -> Optional[List[str]]: + if anthropic_beta_header is None: + return None + return anthropic_beta_header.split(",") + def validate_environment( self, headers: dict, @@ -815,13 +826,18 @@ class AnthropicConfig(BaseConfig): prompt_caching_set = self.is_cache_control_set(messages=messages) computer_tool_used = self.is_computer_tool_used(tools=tools) pdf_used = self.is_pdf_used(messages=messages) + user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( + anthropic_beta_header=headers.get("anthropic-beta") + ) anthropic_headers = self.get_anthropic_headers( computer_tool_used=computer_tool_used, prompt_caching_set=prompt_caching_set, pdf_used=pdf_used, api_key=api_key, is_vertex_request=optional_params.get("is_vertex_request", False), + user_anthropic_beta_headers=user_anthropic_beta_headers, ) headers = {**headers, **anthropic_headers} + return headers diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3837369a8e..ef5d5b56a8 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -167,6 +167,7 @@ class AmazonConverseConfig(BaseConfig): self, json_schema: Optional[dict] = None, schema_name: str = "json_tool_call", + description: Optional[str] = None, ) -> ChatCompletionToolParam: """ Handles creating a tool call for getting responses in JSON format. @@ -189,11 +190,15 @@ class AmazonConverseConfig(BaseConfig): else: _input_schema = json_schema + tool_param_function_chunk = ChatCompletionToolParamFunctionChunk( + name=schema_name, parameters=_input_schema + ) + if description: + tool_param_function_chunk["description"] = description + _tool = ChatCompletionToolParam( type="function", - function=ChatCompletionToolParamFunctionChunk( - name=schema_name, parameters=_input_schema - ), + function=tool_param_function_chunk, ) return _tool @@ -214,12 +219,14 @@ class AmazonConverseConfig(BaseConfig): json_schema: Optional[dict] = None schema_name: str = "" + description: Optional[str] = None if "response_schema" in value: json_schema = value["response_schema"] schema_name = "json_tool_call" elif "json_schema" in value: json_schema = value["json_schema"]["schema"] schema_name = value["json_schema"]["name"] + description = value["json_schema"].get("description") """ Follow similar approach to anthropic - translate to a single tool call. @@ -228,10 +235,10 @@ class AmazonConverseConfig(BaseConfig): - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective. """ - _tool_choice = {"name": schema_name, "type": "tool"} _tool = self._create_json_tool_call_for_response_format( json_schema=json_schema, schema_name=schema_name if schema_name != "" else "json_tool_call", + description=description, ) optional_params["tools"] = [_tool] if litellm.utils.supports_tool_choice( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index fe36f5708b..bf0d780c4d 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -19,7 +19,10 @@ model_list: model: bedrock/invoke/us.anthropic.claude-3-7-sonnet-20250219-v1:0 - model_name: bedrock-claude-3-5-sonnet litellm_params: - model: bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/invoke/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + - model_name: bedrock-nova + litellm_params: + model: bedrock/us.amazon.nova-pro-v1:0 litellm_settings: callbacks: ["langfuse"] \ No newline at end of file diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 4db1c8376c..37253a37e6 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1224,3 +1224,42 @@ def test_anthropic_thinking_output_stream(model): assert reasoning_content_exists except litellm.Timeout: pytest.skip("Model is timing out") + + +def test_anthropic_custom_headers(): + from litellm import completion + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + tools = [ + { + "type": "computer_20241022", + "function": { + "name": "get_current_weather", + "parameters": { + "display_height_px": 100, + "display_width_px": 100, + "display_number": 1, + }, + }, + } + ] + + with patch.object(client, "post") as mock_post: + try: + resp = completion( + model="claude-3-5-sonnet-20240620", + headers={"anthropic-beta": "structured-output-2024-03-01"}, + messages=[ + {"role": "user", "content": "What is the capital of France?"} + ], + client=client, + tools=tools, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + headers = mock_post.call_args[1]["headers"] + assert "structured-output-2024-03-01" in headers["anthropic-beta"] diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 534280c3ef..57710846b9 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2715,3 +2715,42 @@ def test_bedrock_top_k_param(model, expected_params): assert data["top_k"] == 2 else: assert data["additionalModelRequestFields"] == expected_params + + +def test_bedrock_description_param(): + from litellm import completion + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post") as mock_post: + try: + response = completion( + model="bedrock/us.amazon.nova-pro-v1:0", + messages=[ + {"role": "user", "content": "What is the meaning of this poem?"} + ], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "meaning_reasoning", + "description": "Find the meaning inside a poem", + "schema": { + "type": "object", + "properties": {"meaning": {"type": "string"}}, + }, + }, + }, + client=client, + ) + except Exception as e: + print(e) + mock_post.assert_called_once() + + request_body = json.loads(mock_post.call_args.kwargs["data"]) + request_body_str = json.dumps(request_body, indent=4, default=str) + print("request_body=", request_body_str) + + assert ( + "Find the meaning inside a poem" in request_body_str + ) # assert description is passed