From 91010cda8f0f05cb845918219bb1aefc8b084d1c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Jun 2025 14:19:53 -0700 Subject: [PATCH] [Bug Fix] Add audio/ogg mapping for Audio MIME types (#11635) * Add audio/ogg mapping * test_vertex_ai_gemini_audio_ogg * test_vertex_ai_gemini_audio_ogg --- .../prompt_templates/common_utils.py | 8 ++-- .../test_amazing_vertex_completion.py | 38 ++++++++++++++++++- .../llms/vertex_ai/test_vertex.py | 4 ++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index acac97bd3e..a4ba25fc46 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -116,7 +116,7 @@ def strip_none_values_from_message(message: AllMessageValues) -> AllMessageValue def convert_content_list_to_str( - message: Union[AllMessageValues, ChatCompletionResponseMessage] + message: Union[AllMessageValues, ChatCompletionResponseMessage], ) -> str: """ - handles scenario where content is list and not string @@ -532,6 +532,7 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]: audio/mpeg audio/mp3 audio/wav + audio/ogg image/png image/jpeg image/webp @@ -565,6 +566,7 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]: (".mp3",): "audio/mp3", (".wav",): "audio/wav", (".mpeg",): "audio/mpeg", + (".ogg",): "audio/ogg", # Documents (".pdf",): "application/pdf", (".txt",): "text/plain", @@ -621,7 +623,6 @@ def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]: return file_ids - def check_is_function_call(logging_obj: "LoggingClass") -> bool: from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_function_call, @@ -635,6 +636,7 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: return False + def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: """ Filters a value from a dictionary @@ -686,5 +688,3 @@ def migrate_file_to_image_url( if format and isinstance(image_url_object["image_url"], dict): image_url_object["image_url"]["format"] = format return image_url_object - - diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 6247b9816b..f74307ae28 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -3958,4 +3958,40 @@ def test_vertex_ai_gemini_2_5_pro_streaming(): print(chunk) if chunk.choices[0].delta.content is not None and len(chunk.choices[0].delta.content) > 0: has_real_content = True - assert has_real_content \ No newline at end of file + assert has_real_content + + +def test_vertex_ai_gemini_audio_ogg(): + load_vertex_ai_credentials() + litellm._turn_on_debug() + response = completion( + model="vertex_ai/gemini-2.0-flash", + messages=[ + { + "content": [ + { + "text": "generate a transcript of the speech.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "file": { + "file_id": "https://upload.wikimedia.org/wikipedia/commons/5/5f/En-us-public.ogg" + }, + "type": "file" + } + ], + "role": "user" + } + ], + ) + print(response) + + + + + diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex.py b/tests/test_litellm/llms/vertex_ai/test_vertex.py index d4679b5ceb..07b0cbc623 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex.py @@ -1218,6 +1218,10 @@ def test_get_image_mime_type_from_url(): _get_image_mime_type_from_url("https://example.com/IMAGE.WEBP") == "image/webp" ) + # Test audio formats + assert _get_image_mime_type_from_url("https://example.com/audio.ogg") == "audio/ogg" + assert _get_image_mime_type_from_url("https://example.com/track.OGG") == "audio/ogg" + # Test unsupported formats assert _get_image_mime_type_from_url("https://example.com/image.gif") is None assert _get_image_mime_type_from_url("https://example.com/image.bmp") is None