From 9a3d9b263241c97e58115bf7a0260ebb8166fe0f Mon Sep 17 00:00:00 2001 From: Vinh Pham Huu Date: Wed, 15 Apr 2026 17:11:12 +0700 Subject: [PATCH 1/2] feat: Enhance support for video metadata across all Gemini models in transformation logic and tests --- docs/my-website/docs/providers/vertex.md | 11 ++- .../llms/vertex_ai/gemini/transformation.py | 22 ++--- .../test_vertex_ai_gemini_transformation.py | 88 +++++++++++++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 46 +++------- 4 files changed, 117 insertions(+), 50 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 0079bd2f57..835e3bbcc2 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -2061,7 +2061,7 @@ assert isinstance( ## Media Resolution Control (Images & Videos) -For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types. +LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter for all Gemini models. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types. **Supported `detail` values:** - `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos) @@ -2146,12 +2146,12 @@ response = completion( :::info -**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models. +**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types across all Gemini models. ::: ## Video Metadata Control -For Gemini 3+ models, LiteLLM supports fine-grained video processing control through the `video_metadata` field. This allows you to specify frame extraction rates and time ranges for video analysis. +LiteLLM supports fine-grained video processing control through the `video_metadata` field for all Gemini models (1.x, 2.x, 3+). This allows you to specify frame extraction rates and time ranges for video analysis. **Supported `video_metadata` parameters:** @@ -2168,8 +2168,11 @@ For Gemini 3+ models, LiteLLM supports fine-grained video processing control thr - `fps` remains unchanged ::: +:::tip +Video clipping (`start_offset`/`end_offset`) and frame rate control (`fps`) are supported by all Gemini models, but analysis quality is significantly higher with the **Gemini 2.5 series** (e.g., `gemini-2.5-flash`, `gemini-2.5-pro`). +::: + :::warning -- **Gemini 3+ Only:** This feature is only available for Gemini 3.0 and newer models - **Video Files Recommended:** While `video_metadata` is designed for video files, error handling for other media types is delegated to the Vertex AI API - **File Formats Supported:** Works with `gs://`, `https://`, and base64-encoded video files ::: diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 6157a384dc..d49a16fa90 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -131,23 +131,19 @@ def _extract_max_media_resolution_from_messages( return max_resolution -def _apply_gemini_3_metadata( +def _apply_gemini_metadata( part: PartType, model: Optional[str], media_resolution_enum: Optional[Dict[str, str]], video_metadata: Optional[Dict[str, Any]], ) -> PartType: """ - Apply the unique media_resolution and video_metadata parameters of Gemini 3+ + Apply media_resolution and video_metadata parameters to a Gemini part. + Both are supported across all Gemini models (1.x, 2.x, 3+). """ if model is None: return part - from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig - - if not VertexGeminiConfig._is_gemini_3_or_newer(model): - return part - part_dict = dict(part) if media_resolution_enum is not None: @@ -205,7 +201,7 @@ def _process_gemini_media( mime_type = format file_data = FileDataType(mime_type=mime_type, file_uri=image_url) part: PartType = {"file_data": file_data} - return _apply_gemini_3_metadata( + return _apply_gemini_metadata( part, model, media_resolution_enum, video_metadata ) elif ( @@ -215,14 +211,14 @@ def _process_gemini_media( ): file_data = FileDataType(mime_type=image_type, file_uri=image_url) part = {"file_data": file_data} - return _apply_gemini_3_metadata( + return _apply_gemini_metadata( part, model, media_resolution_enum, video_metadata ) elif "http://" in image_url or "https://" in image_url or "base64" in image_url: image = convert_to_anthropic_image_obj(image_url, format=format) _blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]} part = {"inline_data": cast(BlobType, _blob)} - return _apply_gemini_3_metadata( + return _apply_gemini_metadata( part, model, media_resolution_enum, video_metadata ) raise Exception("Invalid image received - {}".format(image_url)) @@ -732,9 +728,9 @@ def _transform_request_body( # noqa: PLR0915 **filtered_params ) - # For Gemini 2.x models, add media_resolution to generation_config (global) - # Gemini 3+ supports per-part media_resolution, but 2.x only supports global - # Gemini 1.x does not support mediaResolution at all + # For Gemini 2.x models, also add media_resolution to generation_config (global) + # as a fallback, since some 2.x versions may not support per-part media_resolution. + # Gemini 1.x does not support mediaResolution at all. if "gemini-2" in model: max_media_resolution = _extract_max_media_resolution_from_messages(messages) if max_media_resolution: diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 98cdf83030..ab9ae4c167 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -930,6 +930,94 @@ class TestMediaResolution: assert "mediaResolution" not in result["generationConfig"] +# Tests for VideoMetadata support across all Gemini models (Issue #25474) +class TestVideoMetadataAllGeminiModels: + """Tests that video_metadata (fps, start_offset, end_offset) works for all Gemini models""" + + def _make_video_messages(self, video_metadata: dict) -> list: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Analyze this video"}, + { + "type": "file", + "file": { + "file_id": "gs://bucket/video.mp4", + "format": "video/mp4", + "video_metadata": video_metadata, + }, + }, + ], + } + ] + + def _get_file_part(self, contents: list) -> dict: + for part in contents[0]["parts"]: + if "file_data" in part: + return part + raise AssertionError("No file part found in contents") + + def test_video_metadata_fps_gemini_2_5_flash(self): + """Gemini 2.5 Flash: fps in video_metadata should be forwarded (Issue #25474)""" + messages = self._make_video_messages({"fps": 5}) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-flash" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + assert file_part["video_metadata"]["fps"] == 5 + + def test_video_metadata_fps_gemini_2_5_pro(self): + """Gemini 2.5 Pro: fps in video_metadata should be forwarded (Issue #25474)""" + messages = self._make_video_messages({"fps": 10}) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-pro" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + assert file_part["video_metadata"]["fps"] == 10 + + def test_video_metadata_offsets_gemini_2_5_flash(self): + """Gemini 2.5 Flash: start_offset/end_offset converted to camelCase (Issue #25474)""" + messages = self._make_video_messages( + {"start_offset": "5s", "end_offset": "30s"} + ) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-flash" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + vm = file_part["video_metadata"] + assert vm["startOffset"] == "5s" + assert vm["endOffset"] == "30s" + + def test_video_metadata_all_fields_gemini_2_5_flash(self): + """Gemini 2.5 Flash: all video_metadata fields forwarded correctly (Issue #25474)""" + messages = self._make_video_messages( + {"fps": 5, "start_offset": "10s", "end_offset": "60s"} + ) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-flash" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + vm = file_part["video_metadata"] + assert vm["fps"] == 5 + assert vm["startOffset"] == "10s" + assert vm["endOffset"] == "60s" + + def test_video_metadata_gemini_1_5_pro(self): + """Gemini 1.5 Pro: video_metadata should also be forwarded (Issue #25474)""" + messages = self._make_video_messages({"fps": 2}) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-1.5-pro" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + assert file_part["video_metadata"]["fps"] == 2 + + def test_convert_tool_response_with_base64_image(): """Test tool response with base64 data URI image.""" # Create a small test image (1x1 red pixel PNG) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index a097966494..f77bba56cb 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3476,8 +3476,8 @@ def test_new_detail_levels(): assert file_part["media_resolution"] == {"level": "MEDIA_RESOLUTION_MEDIUM"} -def test_video_metadata_only_for_gemini_3(): - """Test that video_metadata is only applied for Gemini 3+ models (Issue #19026)""" +def test_video_metadata_supported_for_all_gemini_models(): + """Test that video_metadata is applied for all Gemini models (Issue #25474)""" from litellm.llms.vertex_ai.gemini.transformation import ( _gemini_convert_messages_with_history, ) @@ -3499,39 +3499,19 @@ def test_video_metadata_only_for_gemini_3(): } ] - # Test with Gemini 1.5 (should not have video_metadata or media_resolution) - contents_1_5 = _gemini_convert_messages_with_history( - messages=messages, model="gemini-1.5-pro" - ) + for model in ["gemini-1.5-pro", "gemini-2.5-flash", "gemini-2.5-pro", "gemini-3-pro-preview"]: + contents = _gemini_convert_messages_with_history(messages=messages, model=model) - file_part_1_5 = None - for part in contents_1_5[0]["parts"]: - if "file_data" in part: - file_part_1_5 = part - break + file_part = None + for part in contents[0]["parts"]: + if "file_data" in part: + file_part = part + break - assert file_part_1_5 is not None - assert ( - "media_resolution" not in file_part_1_5 - ), "Gemini 1.5 should not have media_resolution" - assert ( - "video_metadata" not in file_part_1_5 - ), "Gemini 1.5 should not have video_metadata" - - # Test with Gemini 3 (should have both) - contents_3 = _gemini_convert_messages_with_history( - messages=messages, model="gemini-3-pro-preview" - ) - - file_part_3 = None - for part in contents_3[0]["parts"]: - if "file_data" in part: - file_part_3 = part - break - - assert file_part_3 is not None - assert "media_resolution" in file_part_3, "Gemini 3 should have media_resolution" - assert "video_metadata" in file_part_3, "Gemini 3 should have video_metadata" + assert file_part is not None, f"{model}: file part should exist" + assert "video_metadata" in file_part, f"{model}: video_metadata should be present" + assert file_part["video_metadata"]["fps"] == 5, f"{model}: fps should be 5" + assert "media_resolution" in file_part, f"{model}: media_resolution should be present" def test_chunk_parser_handles_prompt_feedback_block(): From 61aee29b41fa55ca039ca6a9eb64a525939afe9b Mon Sep 17 00:00:00 2001 From: Vinh Pham Huu Date: Wed, 15 Apr 2026 17:56:15 +0700 Subject: [PATCH 2/2] feat: Update video metadata handling and media resolution checks for Gemini models --- litellm/llms/vertex_ai/gemini/transformation.py | 10 ++++++++-- .../gemini/test_vertex_and_google_ai_studio_gemini.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index d49a16fa90..5eaac5e48c 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -139,14 +139,20 @@ def _apply_gemini_metadata( ) -> PartType: """ Apply media_resolution and video_metadata parameters to a Gemini part. - Both are supported across all Gemini models (1.x, 2.x, 3+). + + - Per-part media_resolution: Gemini 3+ only (2.x uses generation_config global). + - video_metadata (fps, startOffset, endOffset): all Gemini models (1.x, 2.x, 3+). """ if model is None: return part + from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig + part_dict = dict(part) - if media_resolution_enum is not None: + if media_resolution_enum is not None and VertexGeminiConfig._is_gemini_3_or_newer( + model + ): part_dict["media_resolution"] = media_resolution_enum if video_metadata is not None: diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index f77bba56cb..1ea7486a51 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3511,8 +3511,18 @@ def test_video_metadata_supported_for_all_gemini_models(): assert file_part is not None, f"{model}: file part should exist" assert "video_metadata" in file_part, f"{model}: video_metadata should be present" assert file_part["video_metadata"]["fps"] == 5, f"{model}: fps should be 5" + + # Per-part media_resolution is Gemini 3+ only; 2.x uses generation_config global + for model in ["gemini-3-pro-preview"]: + contents = _gemini_convert_messages_with_history(messages=messages, model=model) + file_part = next(p for p in contents[0]["parts"] if "file_data" in p) assert "media_resolution" in file_part, f"{model}: media_resolution should be present" + for model in ["gemini-1.5-pro", "gemini-2.5-flash", "gemini-2.5-pro"]: + contents = _gemini_convert_messages_with_history(messages=messages, model=model) + file_part = next(p for p in contents[0]["parts"] if "file_data" in p) + assert "media_resolution" not in file_part, f"{model}: per-part media_resolution should not be set" + def test_chunk_parser_handles_prompt_feedback_block(): """Test chunk_parser correctly handles promptFeedback.blockReason"""