diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index e50954b8f9..39edb9642e 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -16,6 +16,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( _get_image_mime_type_from_url, ) from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_generic_image_chunk_to_openai_image_obj, convert_to_anthropic_image_obj, convert_to_gemini_tool_call_invoke, convert_to_gemini_tool_call_result, @@ -45,6 +46,7 @@ from litellm.types.llms.vertex_ai import ( ToolConfig, Tools, ) +from litellm.types.utils import GenericImageParsingChunk from ..common_utils import ( _check_text_in_content, @@ -154,10 +156,26 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 _parts.append(_part) elif element["type"] == "input_audio": audio_element = cast(ChatCompletionAudioObject, element) - if audio_element["input_audio"].get("data") is not None: + audio_data = audio_element["input_audio"].get("data") + audio_format = audio_element["input_audio"].get("format") + if audio_data is not None and audio_format is not None: + audio_format_modified = ( + "audio/" + audio_format + if audio_format.startswith("audio/") is False + else audio_format + ) # Gemini expects audio/wav, audio/mp3, etc. + openai_image_str = ( + convert_generic_image_chunk_to_openai_image_obj( + image_chunk=GenericImageParsingChunk( + type="base64", + media_type=audio_format_modified, + data=audio_data, + ) + ) + ) _part = _process_gemini_image( - image_url=audio_element["input_audio"]["data"], - format=audio_element["input_audio"].get("format"), + image_url=openai_image_str, + format=audio_format_modified, ) _parts.append(_part) elif element["type"] == "file": diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ff4b44d21b..e57015a22e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -398,6 +398,19 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return params + def map_response_modalities(self, value: list) -> list: + response_modalities = [] + for modality in value: + if modality == "text": + response_modalities.append("TEXT") + elif modality == "image": + response_modalities.append("IMAGE") + elif modality == "audio": + response_modalities.append("AUDIO") + else: + response_modalities.append("MODALITY_UNSPECIFIED") + return response_modalities + def map_openai_params( self, non_default_params: Dict, @@ -465,14 +478,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): cast(AnthropicThinkingParam, value) ) elif param == "modalities" and isinstance(value, list): - response_modalities = [] - for modality in value: - if modality == "text": - response_modalities.append("TEXT") - elif modality == "image": - response_modalities.append("IMAGE") - else: - response_modalities.append("MODALITY_UNSPECIFIED") + response_modalities = self.map_response_modalities(value) optional_params["responseModalities"] = response_modalities if litellm.vertex_ai_safety_settings is not None: diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 6dc6fea96d..bb7c3a651c 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -525,6 +525,45 @@ class BaseLLMChatTest(ABC): except litellm.InternalServerError: pytest.skip("Model is overloaded") + @pytest.mark.flaky(retries=6, delay=1) + def test_audio_input(self): + """ + Test that audio input is supported by the LLM API + """ + from litellm.utils import supports_audio_input + litellm._turn_on_debug() + base_completion_call_args = self.get_base_completion_call_args() + if not supports_audio_input(base_completion_call_args["model"], None): + pytest.skip( + f"Model={base_completion_call_args['model']} does not support audio input" + ) + + url = "https://openaiassets.blob.core.windows.net/$web/API/docs/audio/alloy.wav" + response = httpx.get(url) + response.raise_for_status() + wav_data = response.content + encoded_string = base64.b64encode(wav_data).decode("utf-8") + + completion = self.completion_function( + **base_completion_call_args, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "input_audio", + "input_audio": {"data": encoded_string, "format": "wav"}, + }, + ], + }, + ], + ) + + print(completion.choices[0].message) + + + @pytest.mark.flaky(retries=6, delay=1) def test_json_response_format_stream(self): """ @@ -979,7 +1018,7 @@ class BaseLLMChatTest(ABC): assert response._hidden_params["response_cost"] > 0 @pytest.mark.parametrize("input_type", ["input_audio", "audio_url"]) - @pytest.mark.parametrize("format_specified", [True, False]) + @pytest.mark.parametrize("format_specified", [True]) def test_supports_audio_input(self, input_type, format_specified): from litellm.utils import return_raw_request, supports_audio_input from litellm.types.utils import CallTypes @@ -1010,16 +1049,10 @@ class BaseLLMChatTest(ABC): test_file_id = "gs://bucket/file.wav" if input_type == "input_audio": - if format_specified: - audio_content.append({ - "type": "input_audio", - "input_audio": {"data": encoded_string, "format": audio_format}, - }) - else: - audio_content.append({ - "type": "input_audio", - "input_audio": {"data": encoded_string}, - }) + audio_content.append({ + "type": "input_audio", + "input_audio": {"data": encoded_string, "format": audio_format}, + }) elif input_type == "audio_url": audio_content.append( {