tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-14 00:23:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed media resoltion for gemini 3

Browse Source
This commit is contained in:
Sameer Kankute
2025-12-05 22:16:36 +05:30
parent ba59cebb57
commit 5f23d94b7e
5 changed files with 81 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/gemini/chat/transformation.py
+11 -4
View File
@@ -114,20 +114,27 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
img_element = element
_image_url: Optional[str] = None
format: Optional[str] = None
detail: Optional[str] = None
if isinstance(img_element.get("image_url"), dict):
_image_url = img_element["image_url"].get("url") # type: ignore
format = img_element["image_url"].get("format") # type: ignore
detail = img_element["image_url"].get("detail") # type: ignore
else:
_image_url = img_element.get("image_url") # type: ignore
if _image_url and "https://" in _image_url:
image_obj = convert_to_anthropic_image_obj(
_image_url, format=format
)
img_element["image_url"] = ( # type: ignore
convert_generic_image_chunk_to_openai_image_obj(
image_obj
)
converted_image_url = convert_generic_image_chunk_to_openai_image_obj(
image_obj
)
if detail is not None:
img_element["image_url"] = { # type: ignore
"url": converted_image_url,
"detail": detail
}
else:
img_element["image_url"] = converted_image_url # type: ignore
elif element.get("type") == "file":
file_element = cast(ChatCompletionFileObject, element)
file_id = file_element["file"].get("file_id")
litellm/llms/vertex_ai/common_utils.py
+10 -4
View File
@@ -199,18 +199,24 @@ def _get_gemini_url(
stream: Optional[bool],
gemini_api_key: Optional[str],
) -> Tuple[str, str]:
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
_gemini_model_name = "models/{}".format(model)
api_version = "v1alpha" if VertexGeminiConfig._is_gemini_3_or_newer(model) else "v1beta"
if mode == "chat":
endpoint = "generateContent"
if stream is True:
endpoint = "streamGenerateContent"
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}&alt=sse".format(
_gemini_model_name, endpoint, gemini_api_key
url = "https://generativelanguage.googleapis.com/{}/{}:{}?key={}&alt=sse".format(
api_version, _gemini_model_name, endpoint, gemini_api_key
)
else:
url = (
"https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format(
_gemini_model_name, endpoint, gemini_api_key
"https://generativelanguage.googleapis.com/{}/{}:{}?key={}".format(
api_version, _gemini_model_name, endpoint, gemini_api_key
)
)
elif mode == "embedding":
litellm/llms/vertex_ai/gemini/transformation.py
+28 -23
View File
@@ -5,7 +5,7 @@ Why separate file? Make it easy to see how transformation works
"""
import os
from typing import TYPE_CHECKING, List, Literal, Optional, Tuple, Union, cast
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union, cast
import httpx
from pydantic import BaseModel
@@ -63,24 +63,20 @@ else:
LiteLLMLoggingObj = Any
def _map_openai_detail_to_media_resolution(
def _convert_detail_to_media_resolution_enum(
detail: Optional[str],
) -> Optional[Literal["low", "medium", "high"]]:
"""
Map OpenAI's "detail" parameter to Gemini's "media_resolution" parameter.
"""
) -> Optional[Dict[str, str]]:
if detail == "low":
return "low"
return {"level": "MEDIA_RESOLUTION_LOW"}
elif detail == "high":
return "high"
# "auto" or None means let the model decide, so we don't set media_resolution
return {"level": "MEDIA_RESOLUTION_HIGH"}
return None
def _process_gemini_image(
image_url: str,
format: Optional[str] = None,
media_resolution: Optional[Literal["low", "medium", "high"]] = None,
media_resolution_enum: Optional[Dict[str, str]] = None,
model: Optional[str] = None,
) -> PartType:
"""
@@ -105,24 +101,33 @@ def _process_gemini_image(
else:
mime_type = format
file_data = FileDataType(mime_type=mime_type, file_uri=image_url)
return PartType(file_data=file_data)
part: PartType = {"file_data": file_data}
if media_resolution_enum is not None and model is not None:
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
if VertexGeminiConfig._is_gemini_3_or_newer(model):
part_dict = dict(part)
part_dict["media_resolution"] = media_resolution_enum
return cast(PartType, part_dict)
return part
elif (
"https://" in image_url
and (image_type := format or _get_image_mime_type_from_url(image_url))
is not None
):
file_data = FileDataType(file_uri=image_url, mime_type=image_type)
return PartType(file_data=file_data)
elif "http://" in image_url or "https://" in image_url or "base64" in image_url:
# https links for unsupported mime types and base64 images
image = convert_to_anthropic_image_obj(image_url, format=format)
_blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]}
# media_resolution on individual Part objects is exclusive to Gemini 3 models
if media_resolution is not None and model is not None:
part: PartType = {"file_data": file_data}
if media_resolution_enum is not None and model is not None:
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
if VertexGeminiConfig._is_gemini_3_or_newer(model):
_blob["media_resolution"] = media_resolution
part_dict = dict(part)
part_dict["media_resolution"] = media_resolution_enum
return cast(PartType, part_dict)
return part
elif "http://" in image_url or "https://" in image_url or "base64" in image_url:
image = convert_to_anthropic_image_obj(image_url, format=format)
_blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]}
return PartType(inline_data=cast(BlobType, _blob_dict))
raise Exception("Invalid image received - {}".format(image_url))
@@ -230,18 +235,18 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
element = cast(ChatCompletionImageObject, element)
img_element = element
format: Optional[str] = None
media_resolution: Optional[Literal["low", "medium", "high"]] = None
media_resolution_enum: Optional[Dict[str, str]] = None
if isinstance(img_element["image_url"], dict):
image_url = img_element["image_url"]["url"]
format = img_element["image_url"].get("format")
detail = img_element["image_url"].get("detail")
media_resolution = _map_openai_detail_to_media_resolution(detail)
media_resolution_enum = _convert_detail_to_media_resolution_enum(detail)
else:
image_url = img_element["image_url"]
_part = _process_gemini_image(
image_url=image_url,
format=format,
media_resolution=media_resolution,
media_resolution_enum=media_resolution_enum,
model=model,
)
_parts.append(_part)
litellm/types/llms/vertex_ai.py
+2 -2
View File
@@ -32,7 +32,6 @@ class FileDataType(TypedDict):
class BlobType(TypedDict, total=False):
mime_type: Required[str]
data: Required[str]
media_resolution: Literal["low", "medium", "high"]
class PartType(TypedDict, total=False):
@@ -43,6 +42,7 @@ class PartType(TypedDict, total=False):
function_response: FunctionResponse
thought: bool
thoughtSignature: str
media_resolution: Literal["low", "medium", "high"]
class HttpxFunctionCall(TypedDict):
@@ -63,7 +63,6 @@ class HttpxCodeExecutionResult(TypedDict):
class HttpxBlobType(TypedDict, total=False):
mimeType: str
data: str
mediaResolution: Literal["low", "medium", "high"]
class HttpxPartType(TypedDict, total=False):
@@ -76,6 +75,7 @@ class HttpxPartType(TypedDict, total=False):
codeExecutionResult: HttpxCodeExecutionResult
thought: bool
thoughtSignature: str
mediaResolution: Literal["low", "medium", "high"]
class HttpxContentType(TypedDict, total=False):
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
+30 -23
View File
@@ -1767,15 +1767,15 @@ def test_temperature_default_for_gemini_3():
def test_media_resolution_from_detail_parameter():
"""Test that OpenAI's detail parameter is correctly mapped to media_resolution"""
from litellm.llms.vertex_ai.gemini.transformation import (
_convert_detail_to_media_resolution_enum,
_gemini_convert_messages_with_history,
_map_openai_detail_to_media_resolution,
)
# Test detail -> media_resolution mapping
assert _map_openai_detail_to_media_resolution("low") == "low"
assert _map_openai_detail_to_media_resolution("high") == "high"
assert _map_openai_detail_to_media_resolution("auto") is None
assert _map_openai_detail_to_media_resolution(None) is None
# Test detail -> media_resolution enum mapping
assert _convert_detail_to_media_resolution_enum("low") == {"level": "MEDIA_RESOLUTION_LOW"}
assert _convert_detail_to_media_resolution_enum("high") == {"level": "MEDIA_RESOLUTION_HIGH"}
assert _convert_detail_to_media_resolution_enum("auto") is None
assert _convert_detail_to_media_resolution_enum(None) is None
# Test with actual message transformation using base64 image
# Using a minimal valid base64-encoded 1x1 PNG
@@ -1799,25 +1799,24 @@ def test_media_resolution_from_detail_parameter():
messages=messages, model="gemini-3-pro-preview"
)
# Verify media_resolution is set in the inline_data
# Note: Gemini adds a blank text part when there's no text, so we expect 2 parts
# Verify media_resolution is set at the Part level (not inside inline_data)
assert len(contents) == 1
assert len(contents[0]["parts"]) >= 1
# Find the part with inline_data
image_part = None
for part in contents[0]["parts"]:
if "inline_data" in part:
if "inline_data" in part or "inlineData" in part:
image_part = part
break
assert image_part is not None
assert "inline_data" in image_part
# The TypedDict uses snake_case internally, and we keep it as snake_case
assert "media_resolution" in image_part["inline_data"]
assert image_part["inline_data"]["media_resolution"] == "high"
# media_resolution should be at the Part level, not inside inline_data
assert "media_resolution" in image_part
media_res = image_part.get("media_resolution")
assert media_res == {"level": "MEDIA_RESOLUTION_HIGH"}
def test_media_resolution_low_detail():
"""Test that detail='low' maps to media_resolution='low'"""
"""Test that detail='low' maps to media_resolution enum with MEDIA_RESOLUTION_LOW"""
from litellm.llms.vertex_ai.gemini.transformation import (
_gemini_convert_messages_with_history,
)
@@ -1851,7 +1850,9 @@ def test_media_resolution_low_detail():
break
assert image_part is not None
assert "inline_data" in image_part
assert image_part["inline_data"]["media_resolution"] == "low"
# media_resolution should be at the Part level, not inside inline_data
assert "media_resolution" in image_part
assert image_part["media_resolution"] == {"level": "MEDIA_RESOLUTION_LOW"}
def test_media_resolution_auto_detail():
@@ -1888,8 +1889,8 @@ def test_media_resolution_auto_detail():
break
assert image_part is not None
assert "inline_data" in image_part
# media_resolution should not be set for auto
assert "media_resolution" not in image_part["inline_data"] or image_part["inline_data"].get("media_resolution") is None
# media_resolution should not be set for auto (check Part level, not inline_data)
assert "media_resolution" not in image_part
# Test with None
messages_none = [
@@ -1915,8 +1916,8 @@ def test_media_resolution_auto_detail():
break
assert image_part is not None
assert "inline_data" in image_part
# media_resolution should not be set
assert "media_resolution" not in image_part["inline_data"] or image_part["inline_data"].get("media_resolution") is None
# media_resolution should not be set (check Part level, not inline_data)
assert "media_resolution" not in image_part
def test_media_resolution_per_part():
@@ -1966,16 +1967,20 @@ def test_media_resolution_per_part():
# First image should have low resolution (first part is the image)
image1_part = contents[0]["parts"][0]
assert "inline_data" in image1_part
assert image1_part["inline_data"]["media_resolution"] == "low"
# media_resolution should be at the Part level, not inside inline_data
assert "media_resolution" in image1_part
assert image1_part["media_resolution"] == {"level": "MEDIA_RESOLUTION_LOW"}
# Second image should have high resolution (third part is the second image)
image2_part = contents[0]["parts"][2]
assert "inline_data" in image2_part
assert image2_part["inline_data"]["media_resolution"] == "high"
# media_resolution should be at the Part level, not inside inline_data
assert "media_resolution" in image2_part
assert image2_part["media_resolution"] == {"level": "MEDIA_RESOLUTION_HIGH"}
def test_media_resolution_only_for_gemini_3_models():
"""Ensure mediaResolution is not added for non-Gemini 3 models."""
"""Ensure media_resolution is not added for non-Gemini 3 models."""
from litellm.llms.vertex_ai.gemini.transformation import (
_gemini_convert_messages_with_history,
)
@@ -2006,7 +2011,9 @@ def test_media_resolution_only_for_gemini_3_models():
break
assert image_part is not None
assert "inline_data" in image_part
assert "mediaResolution" not in image_part["inline_data"]
# media_resolution should not be at the Part level for non-Gemini 3 models
assert "media_resolution" not in image_part
assert "mediaResolution" not in image_part
def test_gemini_3_image_models_no_thinking_config():