tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-07 10:21:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(test_gemini.py): add additional testing for additionalproperties case

Browse Source
This commit is contained in:
Krrish Dholakia
2025-09-11 15:12:21 -07:00
parent 258b674dbb
commit 3f3efea301
3 changed files with 96 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/types/llms/vertex_ai.py
-1
View File
@@ -113,7 +113,6 @@ class Schema(TypedDict, total=False):
pattern: str
example: Any
anyOf: List["Schema"]
additionalProperties: Any
class FunctionDeclaration(TypedDict, total=False):
tests/llm_translation/test_gemini.py
+59 -20
View File
@@ -269,7 +269,11 @@ def test_gemini_image_generation():
assert len(response.choices[0].message.images) > 0
assert response.choices[0].message.images[0]["image_url"] is not None
assert response.choices[0].message.images[0]["image_url"]["url"] is not None
assert response.choices[0].message.images[0]["image_url"]["url"].startswith("data:image/png;base64,")
assert (
response.choices[0]
.message.images[0]["image_url"]["url"]
.startswith("data:image/png;base64,")
)
def test_gemini_thinking():
@@ -661,7 +665,8 @@ def test_system_message_with_no_user_message():
assert response is not None
assert response.choices[0].message.content is not None
def get_current_weather(location, unit="fahrenheit"):
"""Get the current weather in a given location"""
if "tokyo" in location.lower():
@@ -778,9 +783,9 @@ def test_gemini_reasoning_effort_minimal():
# Test with different Gemini models to verify model-specific mapping
test_cases = [
("gemini/gemini-2.5-flash", 1), # Flash: minimum 1 token
("gemini/gemini-2.5-pro", 128), # Pro: minimum 128 tokens
("gemini/gemini-2.5-flash-lite", 512), # Flash-Lite: minimum 512 tokens
("gemini/gemini-2.5-flash", 1), # Flash: minimum 1 token
("gemini/gemini-2.5-pro", 128), # Pro: minimum 128 tokens
("gemini/gemini-2.5-flash-lite", 512), # Flash-Lite: minimum 512 tokens
]
for model, expected_min_budget in test_cases:
@@ -793,24 +798,32 @@ def test_gemini_reasoning_effort_minimal():
"reasoning_effort": "minimal",
},
)
# Verify that the thinking config is set correctly
request_body = raw_request["raw_request_body"]
assert "generationConfig" in request_body, f"Model {model} should have generationConfig"
assert (
"generationConfig" in request_body
), f"Model {model} should have generationConfig"
generation_config = request_body["generationConfig"]
assert "thinkingConfig" in generation_config, f"Model {model} should have thinkingConfig"
assert (
"thinkingConfig" in generation_config
), f"Model {model} should have thinkingConfig"
thinking_config = generation_config["thinkingConfig"]
assert "thinkingBudget" in thinking_config, f"Model {model} should have thinkingBudget"
assert (
"thinkingBudget" in thinking_config
), f"Model {model} should have thinkingBudget"
actual_budget = thinking_config["thinkingBudget"]
assert actual_budget == expected_min_budget, \
f"Model {model} should map 'minimal' to {expected_min_budget} tokens, got {actual_budget}"
assert (
actual_budget == expected_min_budget
), f"Model {model} should map 'minimal' to {expected_min_budget} tokens, got {actual_budget}"
# Verify that includeThoughts is True for minimal reasoning effort
assert thinking_config.get("includeThoughts", True), \
f"Model {model} should have includeThoughts=True for minimal reasoning effort"
assert thinking_config.get(
"includeThoughts", True
), f"Model {model} should have includeThoughts=True for minimal reasoning effort"
# Test with unknown model (should use generic fallback)
try:
@@ -822,15 +835,41 @@ def test_gemini_reasoning_effort_minimal():
"reasoning_effort": "minimal",
},
)
request_body = raw_request["raw_request_body"]
generation_config = request_body["generationConfig"]
thinking_config = generation_config["thinkingConfig"]
# Should use generic fallback (128 tokens)
assert thinking_config["thinkingBudget"] == 128, \
"Unknown model should use generic fallback of 128 tokens"
assert (
thinking_config["thinkingBudget"] == 128
), "Unknown model should use generic fallback of 128 tokens"
except Exception as e:
# If return_raw_request doesn't work for unknown models, that's okay
# The important part is that our known models work correctly
print(f"Note: Unknown model test skipped due to: {e}")
pass
def test_gemini_additional_properties_bug():
# Simple tool with additionalProperties (simulating the TypedDict issue)
tools = [
{
"type": "function",
"function": {
"name": "test_tool",
"description": "Test tool",
"parameters": {
"type": "object",
"properties": {"param1": {"type": "string"}},
# This causes the error - any non-False value
"additionalProperties": True, # Could also be None, {}, etc.
},
},
}
]
messages = [{"role": "user", "content": "Test message"}]
response = litellm.completion(
model="gemini/gemini-2.5-flash", messages=messages, tools=tools
)
tests/local_testing/test_amazing_vertex_completion.py
+37 -13
View File
@@ -397,7 +397,7 @@ async def test_async_vertexai_response():
| litellm.vertex_text_models
| litellm.vertex_code_text_models
)
test_models = random.sample(list(test_models), 1)
test_models += list(litellm.vertex_language_models) # always test gemini-pro
for model in test_models:
@@ -504,7 +504,6 @@ async def test_async_vertexai_streaming_response():
pytest.fail(f"An exception occurred: {e}")
@pytest.mark.parametrize("load_pdf", [False]) # True,
@pytest.mark.flaky(retries=3, delay=1)
def test_completion_function_plus_pdf(load_pdf):
@@ -547,6 +546,7 @@ def test_completion_function_plus_pdf(load_pdf):
except Exception as e:
pytest.fail("Got={}".format(str(e)))
def encode_image(image_path):
import base64
@@ -910,7 +910,10 @@ async def test_partner_models_httpx(model, region, sync_mode):
[
("vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas", "us-east5"),
("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"),
("vertex_ai/mistral-large-2411", "us-central1"), # critical - we had this issue: https://github.com/BerriAI/litellm/issues/13888
(
"vertex_ai/mistral-large-2411",
"us-central1",
), # critical - we had this issue: https://github.com/BerriAI/litellm/issues/13888
("vertex_ai/openai/gpt-oss-20b-maas", "us-central1"),
],
)
@@ -3827,7 +3830,7 @@ def test_vertex_ai_gemini_audio_ogg():
@pytest.mark.asyncio
async def test_vertex_ai_deepseek():
"""Test that deepseek models use the correct v1 API endpoint instead of v1beta1."""
#load_vertex_ai_credentials()
# load_vertex_ai_credentials()
litellm._turn_on_debug()
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
@@ -3840,21 +3843,17 @@ async def test_vertex_ai_deepseek():
{
"message": {
"role": "assistant",
"content": "Hello! How can I help you today?"
"content": "Hello! How can I help you today?",
},
"index": 0,
"finish_reason": "stop"
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30
},
"model": "deepseek-ai/deepseek-r1-0528-maas"
"usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
"model": "deepseek-ai/deepseek-r1-0528-maas",
}
mock_response.status_code = 200
with patch.object(client, "post", return_value=mock_response) as mock_post:
response = await acompletion(
model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas",
@@ -3900,3 +3899,28 @@ def test_gemini_grounding_on_streaming():
vertex_ai_grounding_metadata_shows_up = True
print(chunk)
assert vertex_ai_grounding_metadata_shows_up
def test_gemini_additional_properties_bug():
# Simple tool with additionalProperties (simulating the TypedDict issue)
tools = [
{
"type": "function",
"function": {
"name": "test_tool",
"description": "Test tool",
"parameters": {
"type": "object",
"properties": {"param1": {"type": "string"}},
# This causes the error - any non-False value
"additionalProperties": True, # Could also be None, {}, etc.
},
},
}
]
messages = [{"role": "user", "content": "Test message"}]
response = litellm.completion(
model="gemini/gemini-2.5-flash", messages=messages, tools=tools
)