diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 1d4c57df42..9a4b158b62 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -133,6 +133,8 @@ _VIDEO_CALL_TYPES = frozenset( { CallTypes.create_video.value, CallTypes.acreate_video.value, + CallTypes.video_edit.value, + CallTypes.avideo_edit.value, CallTypes.video_remix.value, CallTypes.avideo_remix.value, } diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 87289ad6a0..9b4cf77728 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -321,6 +321,23 @@ class BaseVideoConfig(ABC): "video get character is not supported for this provider" ) + def get_video_edit_prefetch_params( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Optional[Tuple[str, Dict]]: + """ + Return (url, body) for a pre-fetch HTTP call that must be made before + transform_video_edit_request, or None if no pre-fetch is required. + + Providers that need to retrieve the source video before constructing the + edit request (e.g. Vertex AI) should override this method. The handler + uses the existing shared httpx client so the call is properly async. + """ + return None + def transform_video_edit_request( self, prompt: str, @@ -329,6 +346,7 @@ class BaseVideoConfig(ABC): litellm_params: GenericLiteLLMParams, headers: dict, extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """ Transform the video edit request into a URL and JSON data. @@ -343,6 +361,7 @@ class BaseVideoConfig(ABC): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, ) -> VideoObject: raise NotImplementedError("video edit is not supported for this provider") diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 36f6b3903e..941fe59e82 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6560,6 +6560,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: @@ -6642,6 +6643,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: @@ -6734,6 +6736,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6805,6 +6808,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6888,6 +6892,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6945,6 +6950,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7021,6 +7027,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7031,27 +7038,49 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url, data = video_provider_config.transform_video_edit_request( - prompt=prompt, + prefetched_source_data = None + prefetch_params = video_provider_config.get_video_edit_prefetch_params( video_id=video_id, api_base=api_base, litellm_params=litellm_params, headers=headers, - extra_body=extra_body, - ) - - logging_obj.pre_call( - input=prompt, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": url, - "headers": headers, - "video_id": video_id, - }, ) + if prefetch_params is not None: + prefetch_url, prefetch_body = prefetch_params + try: + prefetch_resp = sync_httpx_client.post( + url=prefetch_url, + headers=headers, + json=prefetch_body, + timeout=timeout, + ) + prefetch_resp.raise_for_status() + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + prefetched_source_data = prefetch_resp.json() try: + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + prefetched_source_data=prefetched_source_data, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + response = sync_httpx_client.post( url=url, headers=headers, @@ -7063,6 +7092,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + request_data=data, ) except Exception as e: raise self._handle_error(e=e, provider_config=video_provider_config) @@ -7093,6 +7123,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7103,27 +7134,49 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url, data = video_provider_config.transform_video_edit_request( - prompt=prompt, + prefetched_source_data = None + prefetch_params = video_provider_config.get_video_edit_prefetch_params( video_id=video_id, api_base=api_base, litellm_params=litellm_params, headers=headers, - extra_body=extra_body, - ) - - logging_obj.pre_call( - input=prompt, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": url, - "headers": headers, - "video_id": video_id, - }, ) + if prefetch_params is not None: + prefetch_url, prefetch_body = prefetch_params + try: + prefetch_resp = await async_httpx_client.post( + url=prefetch_url, + headers=headers, + json=prefetch_body, + timeout=timeout, + ) + prefetch_resp.raise_for_status() + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + prefetched_source_data = prefetch_resp.json() try: + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + prefetched_source_data=prefetched_source_data, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + response = await async_httpx_client.post( url=url, headers=headers, @@ -7135,6 +7188,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + request_data=data, ) except Exception as e: raise self._handle_error(e=e, provider_config=video_provider_config) @@ -7182,6 +7236,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7256,6 +7311,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7467,6 +7523,7 @@ class BaseLLMHTTPHandler: api_key=api_key, headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 9714c8a392..77a95bfa5a 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -581,12 +581,23 @@ class GeminiVideoConfig(BaseVideoConfig): raise NotImplementedError("video get character is not supported for Gemini") def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None + self, + prompt, + video_id, + api_base, + litellm_params, + headers, + extra_body=None, + prefetched_source_data=None, ): raise NotImplementedError("video edit is not supported for Gemini") def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None + self, + raw_response, + logging_obj, + custom_llm_provider=None, + request_data=None, ): raise NotImplementedError("video edit is not supported for Gemini") diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 2d165a7d7d..520a42e9dd 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -534,6 +534,7 @@ class OpenAIVideoConfig(BaseVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: original_video_id = extract_original_video_id(video_id) url = f"{api_base.rstrip('/')}/edits" @@ -547,6 +548,7 @@ class OpenAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: Any, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, ) -> VideoObject: video_obj = VideoObject(**raw_response.json()) if custom_llm_provider and video_obj.id: diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 4f84816a2b..b1723f494e 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -623,12 +623,23 @@ class RunwayMLVideoConfig(BaseVideoConfig): raise NotImplementedError("video get character is not supported for RunwayML") def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None + self, + prompt, + video_id, + api_base, + litellm_params, + headers, + extra_body=None, + prefetched_source_data=None, ): raise NotImplementedError("video edit is not supported for RunwayML") def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None + self, + raw_response, + logging_obj, + custom_llm_provider=None, + request_data=None, ): raise NotImplementedError("video edit is not supported for RunwayML") diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index ed6176cef0..b84966354b 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -40,6 +40,29 @@ else: BaseLLMException = Any +def _build_vertex_video_usage_from_request_data( + request_data: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """Build usage metadata (duration, resolution) for video cost calculation.""" + usage_data: Dict[str, Any] = {} + if not request_data: + return usage_data + + parameters = request_data.get("parameters", {}) + duration = ( + parameters.get("durationSeconds") or DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS + ) + if duration is not None: + try: + usage_data["duration_seconds"] = float(duration) + except (ValueError, TypeError): + pass + res = parameters.get("resolution") + if res is not None and str(res).strip() != "": + usage_data["video_resolution"] = str(res).strip().lower() + return usage_data + + def _convert_image_to_vertex_format(image_file) -> Dict[str, str]: """ Convert image file to Vertex AI format with base64 encoding and MIME type. @@ -363,23 +386,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): id=video_id, object="video", status="processing", model=model ) - usage_data: Dict[str, Any] = {} - if request_data: - parameters = request_data.get("parameters", {}) - duration = ( - parameters.get("durationSeconds") - or DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS - ) - if duration is not None: - try: - usage_data["duration_seconds"] = float(duration) - except (ValueError, TypeError): - pass - res = parameters.get("resolution") - if res is not None and str(res).strip() != "": - usage_data["video_resolution"] = str(res).strip().lower() - - video_obj.usage = usage_data + video_obj.usage = _build_vertex_video_usage_from_request_data(request_data) return video_obj def transform_video_status_retrieve_request( @@ -647,15 +654,123 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): def transform_video_get_character_response(self, raw_response, logging_obj): raise NotImplementedError("video get character is not supported for Vertex AI") + def get_video_edit_prefetch_params( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Return the fetchPredictOperation URL and body needed to retrieve the source video.""" + return self.transform_video_status_retrieve_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None - ): - raise NotImplementedError("video edit is not supported for Vertex AI") + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Build a predictLongRunning edit request from the pre-fetched source video. + + The actual fetchPredictOperation HTTP call is hoisted into the handler so + it can use the shared async/sync httpx client instead of blocking the loop. + """ + if prefetched_source_data is None: + raise ValueError( + "prefetched_source_data is required for Vertex AI video edit. " + "Ensure get_video_edit_prefetch_params is called by the handler." + ) + + if not prefetched_source_data.get("done", False): + raise ValueError( + "Source video generation is not complete yet. " + "Check the video status before editing." + ) + + videos = prefetched_source_data.get("response", {}).get("videos", []) + if not videos: + raise ValueError("No videos found in the completed operation. Cannot edit.") + + source_video = videos[0] + video_input: Dict[str, Any] = {} + if "gcsUri" in source_video: + video_input["gcsUri"] = source_video["gcsUri"] + elif "bytesBase64Encoded" in source_video: + video_input["bytesBase64Encoded"] = source_video["bytesBase64Encoded"] + video_input["mimeType"] = source_video.get("mimeType", "video/mp4") + else: + raise ValueError( + "Source video has neither gcsUri nor bytesBase64Encoded. Cannot edit." + ) + + operation_name = extract_original_video_id(video_id) + model = self.extract_model_from_operation_name(operation_name) or "" + + instance_dict: Dict[str, Any] = {"prompt": prompt, "video": video_input} + request_data: Dict[str, Any] = {"instances": [instance_dict]} + + if extra_body: + extra_body_copy = dict(extra_body) + nested_params = extra_body_copy.pop("parameters", None) + vertex_params: Dict[str, Any] = {} + if isinstance(nested_params, dict): + vertex_params.update(nested_params) + vertex_params.update(extra_body_copy) + if vertex_params: + request_data["parameters"] = vertex_params + + edit_url = f"{api_base.rstrip('/')}/{model}:predictLongRunning" + return edit_url, request_data def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None - ): - raise NotImplementedError("video edit is not supported for Vertex AI") + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, + ) -> VideoObject: + """ + Transform the Veo video edit response. + + Veo returns the same operation response as video generation: + {"name": "projects/.../operations/OPERATION_ID"} + + usage includes duration_seconds and optional video_resolution from the + edit request parameters for cost calculation. + """ + response_data = raw_response.json() + + operation_name = response_data.get("name") + if not operation_name: + raise ValueError(f"No operation name in Veo edit response: {response_data}") + + model = self.extract_model_from_operation_name(operation_name) or "" + + if custom_llm_provider: + video_id = encode_video_id_with_provider( + operation_name, custom_llm_provider, model + ) + else: + video_id = operation_name + + video_obj = VideoObject( + id=video_id, + object="video", + status="processing", + model=model, + ) + video_obj.usage = _build_vertex_video_usage_from_request_data(request_data) + return video_obj def transform_video_extension_request( self, diff --git a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py index 70583cfe61..55197d3165 100644 --- a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py @@ -456,6 +456,127 @@ class TestVertexAIVideoConfig: raw_response=mock_response, logging_obj=self.mock_logging_obj ) + def test_get_video_edit_prefetch_params(self): + """Test that prefetch params returns the fetchPredictOperation URL and body.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-123" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + + fetch_url, fetch_body = self.config.get_video_edit_prefetch_params( + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "fetchPredictOperation" in fetch_url + assert "veo-3.1-generate-001" in fetch_url + assert fetch_body == {"operationName": operation_name} + + def test_transform_video_edit_request_with_bytes(self): + """Test video edit request builds predictLongRunning body from pre-fetched bytes.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-123" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + fake_bytes = base64.b64encode(b"fake_video").decode() + + prefetched = { + "done": True, + "response": { + "videos": [{"bytesBase64Encoded": fake_bytes, "mimeType": "video/mp4"}] + }, + } + + url, data = self.config.transform_video_edit_request( + prompt="Make it brighter", + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={"Authorization": "Bearer token"}, + prefetched_source_data=prefetched, + ) + + assert url.endswith(":predictLongRunning") + assert "veo-3.1-generate-001" in url + instance = data["instances"][0] + assert instance["prompt"] == "Make it brighter" + assert instance["video"]["bytesBase64Encoded"] == fake_bytes + assert instance["video"]["mimeType"] == "video/mp4" + + def test_transform_video_edit_request_with_gcs_uri(self): + """Test that gcsUri is used when present in source video.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-456" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + + prefetched = { + "done": True, + "response": { + "videos": [{"gcsUri": "gs://bucket/video.mp4", "mimeType": "video/mp4"}] + }, + } + + _, data = self.config.transform_video_edit_request( + prompt="Make it darker", + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={}, + prefetched_source_data=prefetched, + ) + + assert data["instances"][0]["video"] == {"gcsUri": "gs://bucket/video.mp4"} + + def test_transform_video_edit_request_source_not_done_raises(self): + """Test that editing an in-progress video raises a clear error.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-789" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + + with pytest.raises(ValueError, match="not complete yet"): + self.config.transform_video_edit_request( + prompt="Make it brighter", + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={}, + prefetched_source_data={"done": False}, + ) + + def test_transform_video_edit_response(self): + """Test that edit response returns a processing VideoObject with encoded ID.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/new-op-123" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = {"name": operation_name} + + video_obj = self.config.transform_video_edit_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="vertex_ai", + ) + + assert isinstance(video_obj, VideoObject) + assert video_obj.status == "processing" + assert video_obj.id + assert video_obj.model == "veo-3.1-generate-001" + + def test_transform_video_edit_response_includes_usage_for_cost(self): + """Edit responses include duration/resolution usage for spend accounting.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/new-op-123" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = {"name": operation_name} + request_data = { + "instances": [{"prompt": "Make it brighter", "video": {}}], + "parameters": {"durationSeconds": 8, "resolution": "1080p"}, + } + + video_obj = self.config.transform_video_edit_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="vertex_ai", + request_data=request_data, + ) + + assert video_obj.usage is not None + assert video_obj.usage["duration_seconds"] == 8.0 + assert video_obj.usage["video_resolution"] == "1080p" + def test_transform_video_remix_request_not_supported(self): """Test that video remix raises NotImplementedError.""" with pytest.raises(NotImplementedError, match="Video remix is not supported"): diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index b0eb2438b9..3d0472ef96 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -398,6 +398,34 @@ class TestVideoGeneration: ) assert abs(cost - 0.8) < 0.001 + def test_completion_cost_video_edit_uses_video_calculator(self): + """video_edit is charged via the same video cost path as create_video.""" + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = MagicMock() + mock_response.usage.duration_seconds = 10.0 + type(mock_response)._hidden_params = {} + + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "metadata": { + "model_info": { + "output_cost_per_video_per_second": 0.05, + } + } + } + + cost = completion_cost( + completion_response=mock_response, + model="vertex_ai/veo-3.1-generate-001", + call_type="video_edit", + custom_llm_provider="vertex_ai", + custom_pricing=True, + litellm_logging_obj=mock_logging_obj, + ) + assert cost == 0.5 + def test_video_generation_with_files(self): """Test video generation with file uploads.""" config = OpenAIVideoConfig()