tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-11 00:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Feat] Add SSML Support for Azure Text-to-Speech (AVA) (#16747)

Browse Source 
* detect SSML in input

* transform_text_to_speech_request

* test_litellm_speech_with_ssml_passthrough

* add Passing Raw SSML

* fix is_ssml_input
This commit is contained in:
Ishaan Jaff
2025-11-17 15:41:22 -08:00
committed by GitHub
parent 88f5110e11
commit 08e115ecff
3 changed files with 246 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/my-website/docs/providers/azure_ai_speech.md
+83
View File
@@ -136,6 +136,89 @@ response = speech(
| `wav` | riff-24khz-16bit-mono-pcm | 24kHz |
| `pcm` | raw-24khz-16bit-mono-pcm | 24kHz |
## Passing Raw SSML
LiteLLM automatically detects when your `input` contains SSML (by checking for `<speak>` tags) and passes it through to Azure without any transformation. This gives you complete control over speech synthesis.
**When to use raw SSML:**
- Using the `<lang>` element with multilingual voices to translate text (e.g., English text → Spanish speech)
- Complex SSML structures with multiple voices or prosody changes
- Fine-grained control over pronunciation, breaks, emphasis, and other speech features
### LiteLLM SDK
```python showLineNumbers title="Raw SSML for Multilingual Translation"
from litellm import speech
# Use <lang> element to convert English text to Spanish speech
# The <lang> element forces the output language regardless of input text language
language_code = "es-ES"
text = "Hello, how are you today?" # English text
voice = "en-US-AvaMultilingualNeural"
ssml = f"""<speak version="1.0"
xmlns="http://www.w3.org/2001/10/synthesis"
xmlns:mstts="http://www.w3.org/2001/mstts"
xml:lang="{language_code}">
<voice name="{voice}">
<lang xml:lang="{language_code}">{text}</lang>
</voice>
</speak>"""
response = speech(
model="azure/speech/azure-tts",
voice=voice,
input=ssml, # LiteLLM auto-detects SSML and sends as-is
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
response.stream_to_file("speech.mp3")
```
```python showLineNumbers title="Raw SSML with Complex Features"
from litellm import speech
# Complex SSML with multiple prosody adjustments
ssml = """<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis'
xmlns:mstts='https://www.w3.org/2001/mstts' xml:lang='en-US'>
<voice name='en-US-JennyNeural'>
<mstts:express-as style='cheerful' styledegree='2'>
<prosody rate='+20%' pitch='high'>
Welcome to our service!
</prosody>
</mstts:express-as>
<break time='500ms'/>
<prosody rate='-10%'>
How can I help you today?
</prosody>
</voice>
</speak>"""
response = speech(
model="azure/speech/azure-tts",
voice="en-US-JennyNeural",
input=ssml, # LiteLLM detects <speak> and passes through unchanged
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
response.stream_to_file("speech.mp3")
```
### LiteLLM Proxy
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "azure-speech",
"voice": "en-US-AvaMultilingualNeural",
"input": "<speak version=\"1.0\" xmlns=\"http://www.w3.org/2001/10/synthesis\" xmlns:mstts=\"http://www.w3.org/2001/mstts\" xml:lang=\"es-ES\"><voice name=\"en-US-AvaMultilingualNeural\"><lang xml:lang=\"es-ES\">Hello, how are you today?</lang></voice></speak>"
}' \
--output speech.mp3
```
## Sending Azure-Specific Params
Azure AI Speech supports advanced SSML features through optional parameters:
litellm/llms/azure/text_to_speech/transformation.py
+21 -1
View File
@@ -382,6 +382,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig):
return f"https://{region}.{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}"
return f"https://{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}"
def is_ssml_input(self, input: str) -> bool:
"""
Returns True if input is SSML, False otherwise
Based on https://www.w3.org/TR/speech-synthesis/ all SSML must start with <speak>
"""
return "<speak>" in input or "<speak " in input
def transform_text_to_speech_request(
self,
model: str,
@@ -402,6 +411,9 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig):
- role: Voice role (e.g., "Girl", "Boy", "SeniorFemale", "SeniorMale")
- lang: Language code for multilingual voices (e.g., "es-ES", "fr-FR")
Auto-detects SSML:
- If input contains <speak>, it's passed through as-is without transformation
Returns:
TextToSpeechRequestData: Contains SSML body and Azure-specific headers
"""
@@ -414,7 +426,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig):
)
headers["X-Microsoft-OutputFormat"] = output_format
# Build SSML
# Auto-detect SSML: if input contains <speak>, pass it through as-is
# Similar to Vertex AI behavior - check if input looks like SSML
if self.is_ssml_input(input=input):
return TextToSpeechRequestData(
ssml_body=input,
headers=headers,
)
# Build SSML from plain text
rate = optional_params.get("rate", "0%")
style = optional_params.get("style")
styledegree = optional_params.get("styledegree")
tests/test_litellm/llms/azure/text_to_speech/test_azure_tts_transformation.py
+142 -1
View File
@@ -1,8 +1,10 @@
from unittest.mock import Mock
import json
from unittest.mock import Mock, patch
import httpx
import pytest
import litellm
from litellm.llms.azure.text_to_speech.transformation import AzureAVATextToSpeechConfig
@@ -554,3 +556,142 @@ def test_transform_text_to_speech_request_without_lang(azure_tts_config: AzureAV
assert "<voice name='en-US-AriaNeural' xml:lang=" not in ssml
assert "<voice name='en-US-AriaNeural'>" in ssml
def test_transform_text_to_speech_request_with_raw_ssml(azure_tts_config: AzureAVATextToSpeechConfig):
"""
Test that raw SSML input is auto-detected and passed through without transformation
"""
raw_ssml = """<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>
<voice name='en-US-JennyNeural'>
<prosody rate='fast' pitch='high'>
This is custom SSML with specific settings!
</prosody>
</voice>
</speak>"""
result = azure_tts_config.transform_text_to_speech_request(
model="azure-tts",
input=raw_ssml,
voice="en-US-AriaNeural",
optional_params={"voice": "en-US-AriaNeural"},
litellm_params={},
headers={}
)
ssml = result["ssml_body"]
# The SSML should be passed through as-is
assert ssml == raw_ssml
assert "en-US-JennyNeural" in ssml
assert "fast" in ssml
assert "high" in ssml
assert "This is custom SSML with specific settings!" in ssml
# Should NOT have been wrapped or transformed
assert ssml.count("<speak") == 1
assert ssml.count("</speak>") == 1
def test_transform_text_to_speech_request_with_raw_ssml_header(azure_tts_config: AzureAVATextToSpeechConfig):
"""
Test that raw SSML preserves output format headers
"""
raw_ssml = """<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>
<voice name='en-US-GuyNeural'>
Hello from raw SSML
</voice>
</speak>"""
result = azure_tts_config.transform_text_to_speech_request(
model="azure-tts",
input=raw_ssml,
voice="en-US-AriaNeural",
optional_params={
"voice": "en-US-AriaNeural",
"output_format": "audio-16khz-32kbitrate-mono-mp3"
},
litellm_params={},
headers={}
)
# SSML should be passed through
assert result["ssml_body"] == raw_ssml
# Headers should still be set correctly
assert result["headers"]["X-Microsoft-OutputFormat"] == "audio-16khz-32kbitrate-mono-mp3"
def test_transform_text_to_speech_request_ssml_with_mstts_namespace(azure_tts_config: AzureAVATextToSpeechConfig):
"""
Test that raw SSML with Azure-specific mstts namespace is passed through
"""
raw_ssml = """<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xmlns:mstts='https://www.w3.org/2001/mstts' xml:lang='en-US'>
<voice name='en-US-AriaNeural'>
<mstts:express-as style='cheerful' styledegree='2'>
<prosody rate='+20%'>
This is custom SSML with Azure-specific features!
</prosody>
</mstts:express-as>
</voice>
</speak>"""
result = azure_tts_config.transform_text_to_speech_request(
model="azure-tts",
input=raw_ssml,
voice="en-US-AriaNeural",
optional_params={"voice": "en-US-AriaNeural"},
litellm_params={},
headers={}
)
ssml = result["ssml_body"]
# The SSML should be passed through as-is with all Azure-specific features
assert ssml == raw_ssml
assert "mstts:express-as" in ssml
assert "style='cheerful'" in ssml
assert "styledegree='2'" in ssml
assert "rate='+20%'" in ssml
@patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post")
def test_litellm_speech_with_ssml_passthrough(mock_post):
"""
Test that litellm.speech passes SSML through to Azure AVA without transformation
"""
raw_ssml = """<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>
<voice name='en-US-JennyNeural'>
<prosody rate='fast' pitch='high'>
Custom SSML content!
</prosody>
</voice>
</speak>"""
mock_response = Mock(spec=httpx.Response)
mock_response.content = b"fake_audio_data"
mock_response.status_code = 200
mock_response.headers = {"content-type": "audio/mpeg"}
mock_post.return_value = mock_response
litellm.speech(
model="azure/speech/tts",
input=raw_ssml,
voice="en-US-AriaNeural",
api_key="test-key",
api_base="https://eastus.api.cognitive.microsoft.com"
)
mock_post.assert_called_once()
call_kwargs = mock_post.call_args.kwargs
# Verify the SSML was sent in the request body
assert "data" in call_kwargs
assert call_kwargs["data"] == raw_ssml
print("REQUEST BODY: ", json.dumps(call_kwargs["data"], indent=4))
# Verify the SSML contains the original content
assert "en-US-JennyNeural" in call_kwargs["data"]
assert "fast" in call_kwargs["data"]
assert "high" in call_kwargs["data"]
assert "Custom SSML content!" in call_kwargs["data"]