From e223cadb9f55fbf8d680d8a68e7bd2f9679e3e9c Mon Sep 17 00:00:00 2001 From: nlineback <143428407+nlineback@users.noreply.github.com> Date: Fri, 12 Dec 2025 04:56:44 -0700 Subject: [PATCH] fix: add speechConfig to GenerationConfig for Gemini TTS (#17851) Moved speechConfig from RequestBody to GenerationConfig TypedDict so that TTS configuration survives the filtering in _transform_request_body(). This fixes the 400 INVALID_ARGUMENT error when using Gemini TTS models (gemini-2.5-flash-tts, gemini-2.5-flash-preview-tts, etc.) with both vertex_ai and gemini providers. Fixes: speechConfig was being created correctly in map_openai_params() but then filtered out because GenerationConfig.__annotations__.keys() didn't include it. Tested with both preview and non-preview TTS model names and both vertex_ai and gemini providers. --- litellm/types/llms/vertex_ai.py | 2 +- .../llms/gemini/test_gemini_tts.py | 125 +++++++++++++++++- 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 7c67332b53..9bc4ca1703 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -213,6 +213,7 @@ class GenerationConfig(TypedDict, total=False): responseModalities: List[GeminiResponseModalities] imageConfig: GeminiImageConfig thinkingConfig: GeminiThinkingConfig + speechConfig: SpeechConfig class VertexToolName(str, Enum): @@ -301,7 +302,6 @@ class RequestBody(TypedDict, total=False): generationConfig: GenerationConfig cachedContent: str labels: Dict[str, str] - speechConfig: SpeechConfig class CachedContentRequestBody(TypedDict, total=False): diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index 3012b79a42..0820456f87 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -23,9 +23,11 @@ class TestGeminiTTSTransformation: """Test that TTS models are correctly identified""" config = GoogleAIStudioGeminiConfig() - # Test TTS models + # Test TTS models (both preview and non-preview versions) assert config.is_model_gemini_audio_model("gemini-2.5-flash-preview-tts") == True assert config.is_model_gemini_audio_model("gemini-2.5-pro-preview-tts") == True + assert config.is_model_gemini_audio_model("gemini-2.5-flash-tts") == True + assert config.is_model_gemini_audio_model("gemini-2.5-pro-tts") == True # Test non-TTS models assert config.is_model_gemini_audio_model("gemini-2.5-flash") == False @@ -217,5 +219,126 @@ def test_gemini_tts_completion_mock(): assert response.choices[0].message.content is not None +class TestGeminiTTSSpeechConfigInRequestBody: + """Test that speechConfig is properly included in the final request body. + + This tests the full transformation pipeline, not just map_openai_params(). + Previously, speechConfig was created but filtered out because it was missing + from the GenerationConfig TypedDict. + """ + + @pytest.mark.parametrize( + "model,custom_llm_provider", + [ + ("gemini-2.5-flash-tts", "vertex_ai"), + ("gemini-2.5-flash-tts", "gemini"), + ("gemini-2.5-flash-preview-tts", "vertex_ai"), + ("gemini-2.5-flash-preview-tts", "gemini"), + ("gemini-2.5-pro-tts", "vertex_ai"), + ], + ) + def test_speechconfig_in_generation_config_transform_request_body(self, model, custom_llm_provider): + """Test that speechConfig is included in generationConfig after _transform_request_body()""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _transform_request_body, + ) + + # Simulate optional_params after map_openai_params() has run + optional_params = { + "speechConfig": { + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Kore" + } + } + }, + "responseModalities": ["AUDIO"], + } + + messages = [{"role": "user", "content": "Say hello"}] + + # Call _transform_request_body which applies the filtering + request_body = _transform_request_body( + messages=messages, + model=model, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params={}, + cached_content=None, + ) + + # Verify speechConfig is in generationConfig (not filtered out) + assert "generationConfig" in request_body + generation_config = request_body["generationConfig"] + assert "speechConfig" in generation_config, ( + f"speechConfig was filtered out of generationConfig for model={model}, provider={custom_llm_provider}. " + "Ensure speechConfig is in the GenerationConfig TypedDict." + ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + + @pytest.mark.parametrize( + "model,custom_llm_provider", + [ + ("gemini-2.5-flash-tts", "vertex_ai"), + ("gemini-2.5-flash-tts", "gemini"), + ("gemini-2.5-flash-preview-tts", "vertex_ai"), + ], + ) + def test_speechconfig_end_to_end_mapping(self, model, custom_llm_provider): + """Test full pipeline: audio param -> map_openai_params -> _transform_request_body""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.llms.vertex_ai.gemini.transformation import ( + _transform_request_body, + ) + + config = VertexGeminiConfig() + + # Step 1: Map OpenAI audio param to speechConfig + non_default_params = { + "audio": { + "voice": "Puck", + "format": "pcm16" + } + } + optional_params = {} + + mapped_params = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=False + ) + + # Verify map_openai_params creates speechConfig + assert "speechConfig" in mapped_params + + messages = [{"role": "user", "content": "Hello world"}] + + # Step 2: Transform to request body (this is where the bug was) + request_body = _transform_request_body( + messages=messages, + model=model, + optional_params=mapped_params, + custom_llm_provider=custom_llm_provider, + litellm_params={}, + cached_content=None, + ) + + # Verify speechConfig survives the transformation + assert "generationConfig" in request_body + generation_config = request_body["generationConfig"] + assert "speechConfig" in generation_config, ( + f"speechConfig was filtered out during _transform_request_body() for model={model}, provider={custom_llm_provider}. " + "This breaks Gemini TTS - speechConfig must be in GenerationConfig TypedDict." + ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Puck" + + # Also verify responseModalities is present + assert "responseModalities" in generation_config + assert "AUDIO" in generation_config["responseModalities"] + + if __name__ == "__main__": pytest.main([__file__])