From 08e115ecff7e8d917883991fe2495f7febe1d09d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 17 Nov 2025 15:41:22 -0800 Subject: [PATCH] [Feat] Add SSML Support for Azure Text-to-Speech (AVA) (#16747) * detect SSML in input * transform_text_to_speech_request * test_litellm_speech_with_ssml_passthrough * add Passing Raw SSML * fix is_ssml_input --- .../docs/providers/azure_ai_speech.md | 83 ++++++++++ .../azure/text_to_speech/transformation.py | 22 ++- .../test_azure_tts_transformation.py | 143 +++++++++++++++++- 3 files changed, 246 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/providers/azure_ai_speech.md b/docs/my-website/docs/providers/azure_ai_speech.md index 434a796a2f..22db98cfac 100644 --- a/docs/my-website/docs/providers/azure_ai_speech.md +++ b/docs/my-website/docs/providers/azure_ai_speech.md @@ -136,6 +136,89 @@ response = speech( | `wav` | riff-24khz-16bit-mono-pcm | 24kHz | | `pcm` | raw-24khz-16bit-mono-pcm | 24kHz | +## Passing Raw SSML + +LiteLLM automatically detects when your `input` contains SSML (by checking for `` tags) and passes it through to Azure without any transformation. This gives you complete control over speech synthesis. + +**When to use raw SSML:** +- Using the `` element with multilingual voices to translate text (e.g., English text → Spanish speech) +- Complex SSML structures with multiple voices or prosody changes +- Fine-grained control over pronunciation, breaks, emphasis, and other speech features + +### LiteLLM SDK + +```python showLineNumbers title="Raw SSML for Multilingual Translation" +from litellm import speech + +# Use element to convert English text to Spanish speech +# The element forces the output language regardless of input text language +language_code = "es-ES" +text = "Hello, how are you today?" # English text +voice = "en-US-AvaMultilingualNeural" + +ssml = f""" + + {text} + +""" + +response = speech( + model="azure/speech/azure-tts", + voice=voice, + input=ssml, # LiteLLM auto-detects SSML and sends as-is + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], +) +response.stream_to_file("speech.mp3") +``` + +```python showLineNumbers title="Raw SSML with Complex Features" +from litellm import speech + +# Complex SSML with multiple prosody adjustments +ssml = """ + + + + Welcome to our service! + + + + + How can I help you today? + + +""" + +response = speech( + model="azure/speech/azure-tts", + voice="en-US-JennyNeural", + input=ssml, # LiteLLM detects and passes through unchanged + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], +) +response.stream_to_file("speech.mp3") +``` + +### LiteLLM Proxy + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "azure-speech", + "voice": "en-US-AvaMultilingualNeural", + "input": "Hello, how are you today?" + }' \ + --output speech.mp3 +``` + + ## Sending Azure-Specific Params Azure AI Speech supports advanced SSML features through optional parameters: diff --git a/litellm/llms/azure/text_to_speech/transformation.py b/litellm/llms/azure/text_to_speech/transformation.py index 0f8911ac2b..df582c3c09 100644 --- a/litellm/llms/azure/text_to_speech/transformation.py +++ b/litellm/llms/azure/text_to_speech/transformation.py @@ -382,6 +382,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): return f"https://{region}.{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}" return f"https://{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}" + + def is_ssml_input(self, input: str) -> bool: + """ + Returns True if input is SSML, False otherwise + + Based on https://www.w3.org/TR/speech-synthesis/ all SSML must start with + """ + return "" in input or ", it's passed through as-is without transformation + Returns: TextToSpeechRequestData: Contains SSML body and Azure-specific headers """ @@ -414,7 +426,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): ) headers["X-Microsoft-OutputFormat"] = output_format - # Build SSML + # Auto-detect SSML: if input contains , pass it through as-is + # Similar to Vertex AI behavior - check if input looks like SSML + if self.is_ssml_input(input=input): + return TextToSpeechRequestData( + ssml_body=input, + headers=headers, + ) + + # Build SSML from plain text rate = optional_params.get("rate", "0%") style = optional_params.get("style") styledegree = optional_params.get("styledegree") diff --git a/tests/test_litellm/llms/azure/text_to_speech/test_azure_tts_transformation.py b/tests/test_litellm/llms/azure/text_to_speech/test_azure_tts_transformation.py index ed3c785c1f..1102306756 100644 --- a/tests/test_litellm/llms/azure/text_to_speech/test_azure_tts_transformation.py +++ b/tests/test_litellm/llms/azure/text_to_speech/test_azure_tts_transformation.py @@ -1,8 +1,10 @@ -from unittest.mock import Mock +import json +from unittest.mock import Mock, patch import httpx import pytest +import litellm from litellm.llms.azure.text_to_speech.transformation import AzureAVATextToSpeechConfig @@ -554,3 +556,142 @@ def test_transform_text_to_speech_request_without_lang(azure_tts_config: AzureAV assert "" in ssml + +def test_transform_text_to_speech_request_with_raw_ssml(azure_tts_config: AzureAVATextToSpeechConfig): + """ + Test that raw SSML input is auto-detected and passed through without transformation + """ + raw_ssml = """ + + + This is custom SSML with specific settings! + + +""" + + result = azure_tts_config.transform_text_to_speech_request( + model="azure-tts", + input=raw_ssml, + voice="en-US-AriaNeural", + optional_params={"voice": "en-US-AriaNeural"}, + litellm_params={}, + headers={} + ) + + ssml = result["ssml_body"] + + # The SSML should be passed through as-is + assert ssml == raw_ssml + assert "en-US-JennyNeural" in ssml + assert "fast" in ssml + assert "high" in ssml + assert "This is custom SSML with specific settings!" in ssml + + # Should NOT have been wrapped or transformed + assert ssml.count("") == 1 + + +def test_transform_text_to_speech_request_with_raw_ssml_header(azure_tts_config: AzureAVATextToSpeechConfig): + """ + Test that raw SSML preserves output format headers + """ + raw_ssml = """ + + Hello from raw SSML + +""" + + result = azure_tts_config.transform_text_to_speech_request( + model="azure-tts", + input=raw_ssml, + voice="en-US-AriaNeural", + optional_params={ + "voice": "en-US-AriaNeural", + "output_format": "audio-16khz-32kbitrate-mono-mp3" + }, + litellm_params={}, + headers={} + ) + + # SSML should be passed through + assert result["ssml_body"] == raw_ssml + + # Headers should still be set correctly + assert result["headers"]["X-Microsoft-OutputFormat"] == "audio-16khz-32kbitrate-mono-mp3" + + +def test_transform_text_to_speech_request_ssml_with_mstts_namespace(azure_tts_config: AzureAVATextToSpeechConfig): + """ + Test that raw SSML with Azure-specific mstts namespace is passed through + """ + raw_ssml = """ + + + + This is custom SSML with Azure-specific features! + + + +""" + + result = azure_tts_config.transform_text_to_speech_request( + model="azure-tts", + input=raw_ssml, + voice="en-US-AriaNeural", + optional_params={"voice": "en-US-AriaNeural"}, + litellm_params={}, + headers={} + ) + + ssml = result["ssml_body"] + + # The SSML should be passed through as-is with all Azure-specific features + assert ssml == raw_ssml + assert "mstts:express-as" in ssml + assert "style='cheerful'" in ssml + assert "styledegree='2'" in ssml + assert "rate='+20%'" in ssml + + +@patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") +def test_litellm_speech_with_ssml_passthrough(mock_post): + """ + Test that litellm.speech passes SSML through to Azure AVA without transformation + """ + raw_ssml = """ + + + Custom SSML content! + + +""" + + mock_response = Mock(spec=httpx.Response) + mock_response.content = b"fake_audio_data" + mock_response.status_code = 200 + mock_response.headers = {"content-type": "audio/mpeg"} + mock_post.return_value = mock_response + + litellm.speech( + model="azure/speech/tts", + input=raw_ssml, + voice="en-US-AriaNeural", + api_key="test-key", + api_base="https://eastus.api.cognitive.microsoft.com" + ) + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + + # Verify the SSML was sent in the request body + assert "data" in call_kwargs + assert call_kwargs["data"] == raw_ssml + print("REQUEST BODY: ", json.dumps(call_kwargs["data"], indent=4)) + + # Verify the SSML contains the original content + assert "en-US-JennyNeural" in call_kwargs["data"] + assert "fast" in call_kwargs["data"] + assert "high" in call_kwargs["data"] + assert "Custom SSML content!" in call_kwargs["data"] +