diff --git a/docs/my-website/docs/audio_transcription.md b/docs/my-website/docs/audio_transcription.md index 5853b5c187..7452a7007b 100644 --- a/docs/my-website/docs/audio_transcription.md +++ b/docs/my-website/docs/audio_transcription.md @@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem'; | Fallbacks | ✅ | Works between supported models | | Loadbalancing | ✅ | Works between supported models | | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) | -| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | | +| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud`, `mistral` | | ## Quick Start @@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create( - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription) - [Groq](./providers/groq.md#speech-to-text---whisper) - [Deepgram](./providers/deepgram.md) +- [Mistral (Voxtral)](./providers/mistral.md#audio-transcription) - [OVHcloud AI Endpoints](./providers/ovhcloud.md) --- diff --git a/docs/my-website/docs/providers/mistral.md b/docs/my-website/docs/providers/mistral.md index e0fccba786..8355cd2464 100644 --- a/docs/my-website/docs/providers/mistral.md +++ b/docs/my-website/docs/providers/mistral.md @@ -311,6 +311,79 @@ print(response) - **Model Compatibility**: Reasoning parameters only work with magistral models - **Backward Compatibility**: Non-magistral models will ignore reasoning parameters and work normally +## Audio Transcription + +Use Mistral's Voxtral models for audio transcription via `litellm.transcription()`. + +### SDK Usage + +```python +from litellm import transcription +import os + +os.environ["MISTRAL_API_KEY"] = "" + +audio_file = open("path/to/audio.wav", "rb") + +response = transcription( + model="mistral/voxtral-mini-latest", + file=audio_file, +) + +print(response.text) +``` + +### With Optional Parameters + +```python +response = transcription( + model="mistral/voxtral-mini-latest", + file=audio_file, + language="en", + temperature=0.0, + response_format="json", +) +``` + +### Mistral-Specific Parameters + +Mistral supports additional parameters beyond the OpenAI-compatible ones: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `diarize` | `bool` | Enable speaker diarization | + +```python +response = transcription( + model="mistral/voxtral-mini-latest", + file=audio_file, + diarize=True, +) +``` + +### Usage with LiteLLM Proxy + +```yaml +model_list: + - model_name: voxtral + litellm_params: + model: mistral/voxtral-mini-latest + api_key: os.environ/MISTRAL_API_KEY + model_info: + mode: audio_transcription +``` + +```bash +litellm --config /path/to/config.yaml +``` + +```bash +curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \ +--header 'Authorization: Bearer sk-1234' \ +--form 'file=@"audio.wav"' \ +--form 'model="voxtral"' +``` + ## Sample Usage - Embedding ```python from litellm import embedding diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 4b40f44cbc..534ec730b0 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -142,6 +142,14 @@ def get_supported_openai_params( # noqa: PLR0915 return litellm.MistralConfig().get_supported_openai_params(model=model) elif request_type == "embeddings": return litellm.MistralEmbeddingConfig().get_supported_openai_params() + elif request_type == "transcription": + from litellm.llms.mistral.audio_transcription.transformation import ( + MistralAudioTranscriptionConfig, + ) + + return MistralAudioTranscriptionConfig().get_supported_openai_params( + model=model + ) elif custom_llm_provider == "text-completion-codestral": return litellm.CodestralTextCompletionConfig().get_supported_openai_params( model=model diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py new file mode 100644 index 0000000000..fd84d63c4f --- /dev/null +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -0,0 +1,152 @@ +""" +Support for Mistral Voxtral audio transcription via ``/v1/audio/transcriptions``. + +API reference: https://docs.mistral.ai/api/#tag/audio/operation/audio_transcriptions_v1_audio_transcriptions_post +""" + +from typing import List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIAudioTranscriptionOptionalParams, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + + +class MistralAudioTranscriptionException(BaseLLMException): + pass + + +class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIAudioTranscriptionOptionalParams]: + return [ + "language", + "temperature", + "timestamp_granularities", + "response_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for k, v in non_default_params.items(): + if k in supported_params: + optional_params[k] = v + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = ( + "https://api.mistral.ai/v1" + if api_base is None + else api_base.rstrip("/") + ) + return f"{api_base}/audio/transcriptions" + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return MistralAudioTranscriptionException( + message=error_message, + status_code=status_code, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if api_key is None: + api_key = get_secret_str("MISTRAL_API_KEY") + + default_headers = { + "Authorization": f"Bearer {api_key}", + "accept": "application/json", + } + default_headers.update(headers or {}) + return default_headers + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + processed_audio = process_audio_file(audio_file) + + form_fields: dict = { + "model": model, + } + + # OpenAI-compatible params + for key in self.get_supported_openai_params(model): + value = optional_params.get(key) + if value is not None: + form_fields[key] = value + + # Mistral-specific params (e.g. diarize) + provider_specific_params = self.get_provider_specific_params( + model=model, + optional_params=optional_params, + openai_params=self.get_supported_openai_params(model), + ) + for key, value in provider_specific_params.items(): + form_fields[key] = str(value).lower() if isinstance(value, bool) else str(value) + + files = { + "file": ( + processed_audio.filename, + processed_audio.file_content, + processed_audio.content_type, + ) + } + + return AudioTranscriptionRequestData(data=form_fields, files=files) + + def transform_audio_transcription_response( + self, + raw_response: httpx.Response, + ) -> TranscriptionResponse: + try: + response_json = raw_response.json() + except Exception: + raise MistralAudioTranscriptionException( + message=raw_response.text, + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + text = response_json.get("text") or "" + response = TranscriptionResponse(text=text) + response._hidden_params = response_json + return response diff --git a/litellm/utils.py b/litellm/utils.py index 375bea724b..caab91c1c7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8288,6 +8288,12 @@ class ProviderConfigManager: ) return OVHCloudAudioTranscriptionConfig() + elif litellm.LlmProviders.MISTRAL == provider: + from litellm.llms.mistral.audio_transcription.transformation import ( + MistralAudioTranscriptionConfig, + ) + + return MistralAudioTranscriptionConfig() return None @staticmethod diff --git a/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py new file mode 100644 index 0000000000..7ef50dede0 --- /dev/null +++ b/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -0,0 +1,170 @@ +import os +from typing import Dict +from unittest.mock import MagicMock + +import httpx +import litellm +import pytest + +from litellm.llms.base_llm.audio_transcription.transformation import ( + BaseAudioTranscriptionConfig, +) +from litellm.llms.mistral.audio_transcription.transformation import ( + MistralAudioTranscriptionConfig, +) +from litellm.types.utils import TranscriptionResponse +from litellm.utils import ProviderConfigManager +from tests.llm_translation.base_audio_transcription_unit_tests import ( + BaseLLMAudioTranscriptionTest, +) + + +@pytest.mark.skipif( + not os.getenv("MISTRAL_API_KEY"), + reason="MISTRAL_API_KEY not set, skipping Mistral audio transcription tests", +) +class TestMistralAudioTranscription(BaseLLMAudioTranscriptionTest): + def get_base_audio_transcription_call_args(self) -> Dict: + return { + "model": "mistral/voxtral-mini-latest", + } + + def get_custom_llm_provider(self) -> litellm.LlmProviders: + return litellm.LlmProviders.MISTRAL + + def test_audio_transcription_async(self): # type: ignore[override] + pytest.skip( + "Async audio transcription test for Mistral is skipped in this suite; " + "async test plugins (e.g. pytest-asyncio/anyio) are not configured here." + ) + + +def test_mistral_audio_transcription_config_installed(): + """Ensure Mistral audio transcription config is registered with ProviderConfigManager.""" + config = ProviderConfigManager.get_provider_audio_transcription_config( + model="mistral/voxtral-mini-latest", + provider=litellm.LlmProviders.MISTRAL, + ) + assert config is not None + assert isinstance(config, BaseAudioTranscriptionConfig) + assert isinstance(config, MistralAudioTranscriptionConfig) + + +def test_mistral_audio_transcription_get_complete_url(): + config = MistralAudioTranscriptionConfig() + url = config.get_complete_url( + api_base=None, + api_key="fake-key", + model="voxtral-mini-latest", + optional_params={}, + litellm_params={}, + ) + assert url == "https://api.mistral.ai/v1/audio/transcriptions" + + +def test_mistral_audio_transcription_get_complete_url_custom_base(): + config = MistralAudioTranscriptionConfig() + url = config.get_complete_url( + api_base="https://custom.api.example.com/v1/", + api_key="fake-key", + model="voxtral-mini-latest", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.api.example.com/v1/audio/transcriptions" + + +def test_mistral_audio_transcription_validate_environment(): + config = MistralAudioTranscriptionConfig() + headers = config.validate_environment( + headers={}, + model="voxtral-mini-latest", + messages=[], + optional_params={}, + litellm_params={}, + api_key="test-key-123", + ) + assert headers["Authorization"] == "Bearer test-key-123" + assert headers["accept"] == "application/json" + + +def test_mistral_audio_transcription_supported_params(): + config = MistralAudioTranscriptionConfig() + params = config.get_supported_openai_params("voxtral-mini-latest") + assert "language" in params + assert "temperature" in params + assert "response_format" in params + assert "timestamp_granularities" in params + + +def test_mistral_audio_transcription_request_transform(): + config = MistralAudioTranscriptionConfig() + + wav_path = os.path.join( + os.path.dirname(__file__), "../../../../..", "tests", "llm_translation", "gettysburg.wav" + ) + audio_file = open(wav_path, "rb") + + result = config.transform_audio_transcription_request( + model="voxtral-mini-latest", + audio_file=audio_file, + optional_params={"language": "en", "temperature": 0.0}, + litellm_params={}, + ) + + audio_file.close() + + assert isinstance(result.data, dict) + assert result.data["model"] == "voxtral-mini-latest" + assert result.data["language"] == "en" + assert result.data["temperature"] == 0.0 + assert result.files is not None + assert "file" in result.files + + +def test_mistral_audio_transcription_request_with_diarize(): + """Test that Mistral-specific params like diarize are passed through.""" + config = MistralAudioTranscriptionConfig() + + wav_path = os.path.join( + os.path.dirname(__file__), "../../../../..", "tests", "llm_translation", "gettysburg.wav" + ) + audio_file = open(wav_path, "rb") + + result = config.transform_audio_transcription_request( + model="voxtral-mini-latest", + audio_file=audio_file, + optional_params={"diarize": True}, + litellm_params={}, + ) + + audio_file.close() + + assert isinstance(result.data, dict) + assert result.data["diarize"] == "true" + + +def test_mistral_audio_transcription_response_transform(): + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = { + "text": "Four score and seven years ago..." + } + + response = config.transform_audio_transcription_response(mock_response) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "Four score and seven years ago..." + + +def test_mistral_audio_transcription_response_transform_empty(): + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = {} + + response = config.transform_audio_transcription_response(mock_response) + + assert isinstance(response, TranscriptionResponse) + assert response.text == ""