Merge pull request #25767 from vinhphamhuu-ct/main

feat: Expand VideoMetadata support to all Gemini Models.

docs/my-website/docs/providers/vertex.md

@@ -2061,7 +2061,7 @@ assert isinstance(
 
 ## Media Resolution Control (Images & Videos)
 
-For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
+LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter for all Gemini models. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
 
 **Supported `detail` values:**
 - `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
@@ -2146,12 +2146,12 @@ response = completion(
 </Tabs>
 
 :::info
-**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
+**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types across all Gemini models.
 :::
 
 ## Video Metadata Control
 
-For Gemini 3+ models, LiteLLM supports fine-grained video processing control through the `video_metadata` field. This allows you to specify frame extraction rates and time ranges for video analysis.
+LiteLLM supports fine-grained video processing control through the `video_metadata` field for all Gemini models (1.x, 2.x, 3+). This allows you to specify frame extraction rates and time ranges for video analysis.
 
 **Supported `video_metadata` parameters:**
 
@@ -2168,8 +2168,11 @@ For Gemini 3+ models, LiteLLM supports fine-grained video processing control thr
 - `fps` remains unchanged
 :::
 
+:::tip
+Video clipping (`start_offset`/`end_offset`) and frame rate control (`fps`) are supported by all Gemini models, but analysis quality is significantly higher with the **Gemini 2.5 series** (e.g., `gemini-2.5-flash`, `gemini-2.5-pro`).
+:::
+
 :::warning
-- **Gemini 3+ Only:** This feature is only available for Gemini 3.0 and newer models
 - **Video Files Recommended:** While `video_metadata` is designed for video files, error handling for other media types is delegated to the Vertex AI API
 - **File Formats Supported:** Works with `gs://`, `https://`, and base64-encoded video files
 :::

litellm/llms/vertex_ai/gemini/transformation.py

@@ -132,26 +132,28 @@ def _extract_max_media_resolution_from_messages(
     return max_resolution
 
 
-def _apply_gemini_3_metadata(
+def _apply_gemini_metadata(
     part: PartType,
     model: Optional[str],
     media_resolution_enum: Optional[Dict[str, str]],
     video_metadata: Optional[Dict[str, Any]],
 ) -> PartType:
     """
-    Apply the unique media_resolution and video_metadata parameters of Gemini 3+
+    Apply media_resolution and video_metadata parameters to a Gemini part.
+
+    - Per-part media_resolution: Gemini 3+ only (2.x uses generation_config global).
+    - video_metadata (fps, startOffset, endOffset): all Gemini models (1.x, 2.x, 3+).
     """
     if model is None:
         return part
 
     from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
 
-    if not VertexGeminiConfig._is_gemini_3_or_newer(model):
-        return part
-
     part_dict = dict(part)
 
-    if media_resolution_enum is not None:
+    if media_resolution_enum is not None and VertexGeminiConfig._is_gemini_3_or_newer(
+        model
+    ):
         part_dict["media_resolution"] = media_resolution_enum
 
     if video_metadata is not None:
@@ -206,7 +208,7 @@ def _process_gemini_media(
                 mime_type = format
             file_data = FileDataType(mime_type=mime_type, file_uri=image_url)
             part: PartType = {"file_data": file_data}
-            return _apply_gemini_3_metadata(
+            return _apply_gemini_metadata(
                 part, model, media_resolution_enum, video_metadata
             )
         elif (
@@ -216,14 +218,14 @@ def _process_gemini_media(
         ):
             file_data = FileDataType(mime_type=image_type, file_uri=image_url)
             part = {"file_data": file_data}
-            return _apply_gemini_3_metadata(
+            return _apply_gemini_metadata(
                 part, model, media_resolution_enum, video_metadata
             )
         elif "http://" in image_url or "https://" in image_url or "base64" in image_url:
             image = convert_to_anthropic_image_obj(image_url, format=format)
             _blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]}
             part = {"inline_data": cast(BlobType, _blob)}
-            return _apply_gemini_3_metadata(
+            return _apply_gemini_metadata(
                 part, model, media_resolution_enum, video_metadata
             )
         raise Exception("Invalid image received - {}".format(image_url))
@@ -733,9 +735,9 @@ def _transform_request_body(  # noqa: PLR0915
             **filtered_params
         )
 
-        # For Gemini 2.x models, add media_resolution to generation_config (global)
-        # Gemini 3+ supports per-part media_resolution, but 2.x only supports global
-        # Gemini 1.x does not support mediaResolution at all
+        # For Gemini 2.x models, also add media_resolution to generation_config (global)
+        # as a fallback, since some 2.x versions may not support per-part media_resolution.
+        # Gemini 1.x does not support mediaResolution at all.
         if "gemini-2" in model:
             max_media_resolution = _extract_max_media_resolution_from_messages(messages)
             if max_media_resolution:

tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py

@@ -967,6 +967,94 @@ class TestMediaResolution:
             assert "mediaResolution" not in result["generationConfig"]
 
 
+# Tests for VideoMetadata support across all Gemini models (Issue #25474)
+class TestVideoMetadataAllGeminiModels:
+    """Tests that video_metadata (fps, start_offset, end_offset) works for all Gemini models"""
+
+    def _make_video_messages(self, video_metadata: dict) -> list:
+        return [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Analyze this video"},
+                    {
+                        "type": "file",
+                        "file": {
+                            "file_id": "gs://bucket/video.mp4",
+                            "format": "video/mp4",
+                            "video_metadata": video_metadata,
+                        },
+                    },
+                ],
+            }
+        ]
+
+    def _get_file_part(self, contents: list) -> dict:
+        for part in contents[0]["parts"]:
+            if "file_data" in part:
+                return part
+        raise AssertionError("No file part found in contents")
+
+    def test_video_metadata_fps_gemini_2_5_flash(self):
+        """Gemini 2.5 Flash: fps in video_metadata should be forwarded (Issue #25474)"""
+        messages = self._make_video_messages({"fps": 5})
+        contents = _gemini_convert_messages_with_history(
+            messages=messages, model="gemini-2.5-flash"
+        )
+        file_part = self._get_file_part(contents)
+        assert "video_metadata" in file_part
+        assert file_part["video_metadata"]["fps"] == 5
+
+    def test_video_metadata_fps_gemini_2_5_pro(self):
+        """Gemini 2.5 Pro: fps in video_metadata should be forwarded (Issue #25474)"""
+        messages = self._make_video_messages({"fps": 10})
+        contents = _gemini_convert_messages_with_history(
+            messages=messages, model="gemini-2.5-pro"
+        )
+        file_part = self._get_file_part(contents)
+        assert "video_metadata" in file_part
+        assert file_part["video_metadata"]["fps"] == 10
+
+    def test_video_metadata_offsets_gemini_2_5_flash(self):
+        """Gemini 2.5 Flash: start_offset/end_offset converted to camelCase (Issue #25474)"""
+        messages = self._make_video_messages(
+            {"start_offset": "5s", "end_offset": "30s"}
+        )
+        contents = _gemini_convert_messages_with_history(
+            messages=messages, model="gemini-2.5-flash"
+        )
+        file_part = self._get_file_part(contents)
+        assert "video_metadata" in file_part
+        vm = file_part["video_metadata"]
+        assert vm["startOffset"] == "5s"
+        assert vm["endOffset"] == "30s"
+
+    def test_video_metadata_all_fields_gemini_2_5_flash(self):
+        """Gemini 2.5 Flash: all video_metadata fields forwarded correctly (Issue #25474)"""
+        messages = self._make_video_messages(
+            {"fps": 5, "start_offset": "10s", "end_offset": "60s"}
+        )
+        contents = _gemini_convert_messages_with_history(
+            messages=messages, model="gemini-2.5-flash"
+        )
+        file_part = self._get_file_part(contents)
+        assert "video_metadata" in file_part
+        vm = file_part["video_metadata"]
+        assert vm["fps"] == 5
+        assert vm["startOffset"] == "10s"
+        assert vm["endOffset"] == "60s"
+
+    def test_video_metadata_gemini_1_5_pro(self):
+        """Gemini 1.5 Pro: video_metadata should also be forwarded (Issue #25474)"""
+        messages = self._make_video_messages({"fps": 2})
+        contents = _gemini_convert_messages_with_history(
+            messages=messages, model="gemini-1.5-pro"
+        )
+        file_part = self._get_file_part(contents)
+        assert "video_metadata" in file_part
+        assert file_part["video_metadata"]["fps"] == 2
+
+
 def test_convert_tool_response_with_base64_image():
     """Test tool response with base64 data URI image."""
     # Create a small test image (1x1 red pixel PNG)

tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py

@@ -3476,8 +3476,8 @@ def test_new_detail_levels():
     assert file_part["media_resolution"] == {"level": "MEDIA_RESOLUTION_MEDIUM"}
 
 
-def test_video_metadata_only_for_gemini_3():
-    """Test that video_metadata is only applied for Gemini 3+ models (Issue #19026)"""
+def test_video_metadata_supported_for_all_gemini_models():
+    """Test that video_metadata is applied for all Gemini models (Issue #25474)"""
     from litellm.llms.vertex_ai.gemini.transformation import (
         _gemini_convert_messages_with_history,
     )
@@ -3499,39 +3499,29 @@ def test_video_metadata_only_for_gemini_3():
         }
     ]
 
-    # Test with Gemini 1.5 (should not have video_metadata or media_resolution)
-    contents_1_5 = _gemini_convert_messages_with_history(
-        messages=messages, model="gemini-1.5-pro"
-    )
+    for model in ["gemini-1.5-pro", "gemini-2.5-flash", "gemini-2.5-pro", "gemini-3-pro-preview"]:
+        contents = _gemini_convert_messages_with_history(messages=messages, model=model)
 
-    file_part_1_5 = None
-    for part in contents_1_5[0]["parts"]:
-        if "file_data" in part:
-            file_part_1_5 = part
-            break
+        file_part = None
+        for part in contents[0]["parts"]:
+            if "file_data" in part:
+                file_part = part
+                break
 
-    assert file_part_1_5 is not None
-    assert (
-        "media_resolution" not in file_part_1_5
-    ), "Gemini 1.5 should not have media_resolution"
-    assert (
-        "video_metadata" not in file_part_1_5
-    ), "Gemini 1.5 should not have video_metadata"
+        assert file_part is not None, f"{model}: file part should exist"
+        assert "video_metadata" in file_part, f"{model}: video_metadata should be present"
+        assert file_part["video_metadata"]["fps"] == 5, f"{model}: fps should be 5"
 
-    # Test with Gemini 3 (should have both)
-    contents_3 = _gemini_convert_messages_with_history(
-        messages=messages, model="gemini-3-pro-preview"
-    )
+    # Per-part media_resolution is Gemini 3+ only; 2.x uses generation_config global
+    for model in ["gemini-3-pro-preview"]:
+        contents = _gemini_convert_messages_with_history(messages=messages, model=model)
+        file_part = next(p for p in contents[0]["parts"] if "file_data" in p)
+        assert "media_resolution" in file_part, f"{model}: media_resolution should be present"
 
-    file_part_3 = None
-    for part in contents_3[0]["parts"]:
-        if "file_data" in part:
-            file_part_3 = part
-            break
-
-    assert file_part_3 is not None
-    assert "media_resolution" in file_part_3, "Gemini 3 should have media_resolution"
-    assert "video_metadata" in file_part_3, "Gemini 3 should have video_metadata"
+    for model in ["gemini-1.5-pro", "gemini-2.5-flash", "gemini-2.5-pro"]:
+        contents = _gemini_convert_messages_with_history(messages=messages, model=model)
+        file_part = next(p for p in contents[0]["parts"] if "file_data" in p)
+        assert "media_resolution" not in file_part, f"{model}: per-part media_resolution should not be set"
 
 
 def test_chunk_parser_handles_prompt_feedback_block():