From 3f3efea3016bb52d93601d32f326fd02e8ac4313 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 11 Sep 2025 15:12:21 -0700 Subject: [PATCH] test(test_gemini.py): add additional testing for additionalproperties case --- litellm/types/llms/vertex_ai.py | 1 - tests/llm_translation/test_gemini.py | 79 ++++++++++++++----- .../test_amazing_vertex_completion.py | 50 +++++++++--- 3 files changed, 96 insertions(+), 34 deletions(-) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 2687b79f72..f17a284ddf 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -113,7 +113,6 @@ class Schema(TypedDict, total=False): pattern: str example: Any anyOf: List["Schema"] - additionalProperties: Any class FunctionDeclaration(TypedDict, total=False): diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 47e3aaa814..122429a87a 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -269,7 +269,11 @@ def test_gemini_image_generation(): assert len(response.choices[0].message.images) > 0 assert response.choices[0].message.images[0]["image_url"] is not None assert response.choices[0].message.images[0]["image_url"]["url"] is not None - assert response.choices[0].message.images[0]["image_url"]["url"].startswith("data:image/png;base64,") + assert ( + response.choices[0] + .message.images[0]["image_url"]["url"] + .startswith("data:image/png;base64,") + ) def test_gemini_thinking(): @@ -661,7 +665,8 @@ def test_system_message_with_no_user_message(): assert response is not None assert response.choices[0].message.content is not None - + + def get_current_weather(location, unit="fahrenheit"): """Get the current weather in a given location""" if "tokyo" in location.lower(): @@ -778,9 +783,9 @@ def test_gemini_reasoning_effort_minimal(): # Test with different Gemini models to verify model-specific mapping test_cases = [ - ("gemini/gemini-2.5-flash", 1), # Flash: minimum 1 token - ("gemini/gemini-2.5-pro", 128), # Pro: minimum 128 tokens - ("gemini/gemini-2.5-flash-lite", 512), # Flash-Lite: minimum 512 tokens + ("gemini/gemini-2.5-flash", 1), # Flash: minimum 1 token + ("gemini/gemini-2.5-pro", 128), # Pro: minimum 128 tokens + ("gemini/gemini-2.5-flash-lite", 512), # Flash-Lite: minimum 512 tokens ] for model, expected_min_budget in test_cases: @@ -793,24 +798,32 @@ def test_gemini_reasoning_effort_minimal(): "reasoning_effort": "minimal", }, ) - + # Verify that the thinking config is set correctly request_body = raw_request["raw_request_body"] - assert "generationConfig" in request_body, f"Model {model} should have generationConfig" - + assert ( + "generationConfig" in request_body + ), f"Model {model} should have generationConfig" + generation_config = request_body["generationConfig"] - assert "thinkingConfig" in generation_config, f"Model {model} should have thinkingConfig" - + assert ( + "thinkingConfig" in generation_config + ), f"Model {model} should have thinkingConfig" + thinking_config = generation_config["thinkingConfig"] - assert "thinkingBudget" in thinking_config, f"Model {model} should have thinkingBudget" - + assert ( + "thinkingBudget" in thinking_config + ), f"Model {model} should have thinkingBudget" + actual_budget = thinking_config["thinkingBudget"] - assert actual_budget == expected_min_budget, \ - f"Model {model} should map 'minimal' to {expected_min_budget} tokens, got {actual_budget}" - + assert ( + actual_budget == expected_min_budget + ), f"Model {model} should map 'minimal' to {expected_min_budget} tokens, got {actual_budget}" + # Verify that includeThoughts is True for minimal reasoning effort - assert thinking_config.get("includeThoughts", True), \ - f"Model {model} should have includeThoughts=True for minimal reasoning effort" + assert thinking_config.get( + "includeThoughts", True + ), f"Model {model} should have includeThoughts=True for minimal reasoning effort" # Test with unknown model (should use generic fallback) try: @@ -822,15 +835,41 @@ def test_gemini_reasoning_effort_minimal(): "reasoning_effort": "minimal", }, ) - + request_body = raw_request["raw_request_body"] generation_config = request_body["generationConfig"] thinking_config = generation_config["thinkingConfig"] # Should use generic fallback (128 tokens) - assert thinking_config["thinkingBudget"] == 128, \ - "Unknown model should use generic fallback of 128 tokens" + assert ( + thinking_config["thinkingBudget"] == 128 + ), "Unknown model should use generic fallback of 128 tokens" except Exception as e: # If return_raw_request doesn't work for unknown models, that's okay # The important part is that our known models work correctly print(f"Note: Unknown model test skipped due to: {e}") pass + + +def test_gemini_additional_properties_bug(): + # Simple tool with additionalProperties (simulating the TypedDict issue) + tools = [ + { + "type": "function", + "function": { + "name": "test_tool", + "description": "Test tool", + "parameters": { + "type": "object", + "properties": {"param1": {"type": "string"}}, + # This causes the error - any non-False value + "additionalProperties": True, # Could also be None, {}, etc. + }, + }, + } + ] + + messages = [{"role": "user", "content": "Test message"}] + + response = litellm.completion( + model="gemini/gemini-2.5-flash", messages=messages, tools=tools + ) diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index a27fe738c7..d20f54bdd3 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -397,7 +397,7 @@ async def test_async_vertexai_response(): | litellm.vertex_text_models | litellm.vertex_code_text_models ) - + test_models = random.sample(list(test_models), 1) test_models += list(litellm.vertex_language_models) # always test gemini-pro for model in test_models: @@ -504,7 +504,6 @@ async def test_async_vertexai_streaming_response(): pytest.fail(f"An exception occurred: {e}") - @pytest.mark.parametrize("load_pdf", [False]) # True, @pytest.mark.flaky(retries=3, delay=1) def test_completion_function_plus_pdf(load_pdf): @@ -547,6 +546,7 @@ def test_completion_function_plus_pdf(load_pdf): except Exception as e: pytest.fail("Got={}".format(str(e))) + def encode_image(image_path): import base64 @@ -910,7 +910,10 @@ async def test_partner_models_httpx(model, region, sync_mode): [ ("vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas", "us-east5"), ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"), - ("vertex_ai/mistral-large-2411", "us-central1"), # critical - we had this issue: https://github.com/BerriAI/litellm/issues/13888 + ( + "vertex_ai/mistral-large-2411", + "us-central1", + ), # critical - we had this issue: https://github.com/BerriAI/litellm/issues/13888 ("vertex_ai/openai/gpt-oss-20b-maas", "us-central1"), ], ) @@ -3827,7 +3830,7 @@ def test_vertex_ai_gemini_audio_ogg(): @pytest.mark.asyncio async def test_vertex_ai_deepseek(): """Test that deepseek models use the correct v1 API endpoint instead of v1beta1.""" - #load_vertex_ai_credentials() + # load_vertex_ai_credentials() litellm._turn_on_debug() from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -3840,21 +3843,17 @@ async def test_vertex_ai_deepseek(): { "message": { "role": "assistant", - "content": "Hello! How can I help you today?" + "content": "Hello! How can I help you today?", }, "index": 0, - "finish_reason": "stop" + "finish_reason": "stop", } ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 20, - "total_tokens": 30 - }, - "model": "deepseek-ai/deepseek-r1-0528-maas" + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + "model": "deepseek-ai/deepseek-r1-0528-maas", } mock_response.status_code = 200 - + with patch.object(client, "post", return_value=mock_response) as mock_post: response = await acompletion( model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas", @@ -3900,3 +3899,28 @@ def test_gemini_grounding_on_streaming(): vertex_ai_grounding_metadata_shows_up = True print(chunk) assert vertex_ai_grounding_metadata_shows_up + + +def test_gemini_additional_properties_bug(): + # Simple tool with additionalProperties (simulating the TypedDict issue) + tools = [ + { + "type": "function", + "function": { + "name": "test_tool", + "description": "Test tool", + "parameters": { + "type": "object", + "properties": {"param1": {"type": "string"}}, + # This causes the error - any non-False value + "additionalProperties": True, # Could also be None, {}, etc. + }, + }, + } + ] + + messages = [{"role": "user", "content": "Test message"}] + + response = litellm.completion( + model="gemini/gemini-2.5-flash", messages=messages, tools=tools + )