Merge pull request #22801 from Chesars/feat/mistral-audio-transcription

feat(mistral): add Voxtral audio transcription support

docs/my-website/docs/audio_transcription.md

@@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
+| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud`, `mistral` | |
 
 ## Quick Start
 
@@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
 - [Deepgram](./providers/deepgram.md)
+- [Mistral (Voxtral)](./providers/mistral.md#audio-transcription)
 - [OVHcloud AI Endpoints](./providers/ovhcloud.md)
 
 ---

docs/my-website/docs/providers/mistral.md

@@ -311,6 +311,79 @@ print(response)
 - **Model Compatibility**: Reasoning parameters only work with magistral models
 - **Backward Compatibility**: Non-magistral models will ignore reasoning parameters and work normally
 
+## Audio Transcription
+
+Use Mistral's Voxtral models for audio transcription via `litellm.transcription()`.
+
+### SDK Usage
+
+```python
+from litellm import transcription
+import os
+
+os.environ["MISTRAL_API_KEY"] = ""
+
+audio_file = open("path/to/audio.wav", "rb")
+
+response = transcription(
+    model="mistral/voxtral-mini-latest",
+    file=audio_file,
+)
+
+print(response.text)
+```
+
+### With Optional Parameters
+
+```python
+response = transcription(
+    model="mistral/voxtral-mini-latest",
+    file=audio_file,
+    language="en",
+    temperature=0.0,
+    response_format="json",
+)
+```
+
+### Mistral-Specific Parameters
+
+Mistral supports additional parameters beyond the OpenAI-compatible ones:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `diarize` | `bool` | Enable speaker diarization |
+
+```python
+response = transcription(
+    model="mistral/voxtral-mini-latest",
+    file=audio_file,
+    diarize=True,
+)
+```
+
+### Usage with LiteLLM Proxy
+
+```yaml
+model_list:
+  - model_name: voxtral
+    litellm_params:
+      model: mistral/voxtral-mini-latest
+      api_key: os.environ/MISTRAL_API_KEY
+    model_info:
+      mode: audio_transcription
+```
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"audio.wav"' \
+--form 'model="voxtral"'
+```
+
 ## Sample Usage - Embedding
 ```python
 from litellm import embedding

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -142,6 +142,14 @@ def get_supported_openai_params(  # noqa: PLR0915
             return litellm.MistralConfig().get_supported_openai_params(model=model)
         elif request_type == "embeddings":
             return litellm.MistralEmbeddingConfig().get_supported_openai_params()
+        elif request_type == "transcription":
+            from litellm.llms.mistral.audio_transcription.transformation import (
+                MistralAudioTranscriptionConfig,
+            )
+
+            return MistralAudioTranscriptionConfig().get_supported_openai_params(
+                model=model
+            )
     elif custom_llm_provider == "text-completion-codestral":
         return litellm.CodestralTextCompletionConfig().get_supported_openai_params(
             model=model

litellm/llms/mistral/audio_transcription/transformation.py

@@ -0,0 +1,152 @@
+"""
+Support for Mistral Voxtral audio transcription via ``/v1/audio/transcriptions``.
+
+API reference: https://docs.mistral.ai/api/#tag/audio/operation/audio_transcriptions_v1_audio_transcriptions_post
+"""
+
+from typing import List, Optional, Union
+
+import httpx
+
+from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIAudioTranscriptionOptionalParams,
+)
+from litellm.types.utils import FileTypes, TranscriptionResponse
+
+
+class MistralAudioTranscriptionException(BaseLLMException):
+    pass
+
+
+class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
+    def get_supported_openai_params(
+        self, model: str
+    ) -> List[OpenAIAudioTranscriptionOptionalParams]:
+        return [
+            "language",
+            "temperature",
+            "timestamp_granularities",
+            "response_format",
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        supported_params = self.get_supported_openai_params(model)
+        for k, v in non_default_params.items():
+            if k in supported_params:
+                optional_params[k] = v
+        return optional_params
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        api_base = (
+            "https://api.mistral.ai/v1"
+            if api_base is None
+            else api_base.rstrip("/")
+        )
+        return f"{api_base}/audio/transcriptions"
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+    ) -> BaseLLMException:
+        return MistralAudioTranscriptionException(
+            message=error_message,
+            status_code=status_code,
+            headers=headers,
+        )
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        if api_key is None:
+            api_key = get_secret_str("MISTRAL_API_KEY")
+
+        default_headers = {
+            "Authorization": f"Bearer {api_key}",
+            "accept": "application/json",
+        }
+        default_headers.update(headers or {})
+        return default_headers
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> AudioTranscriptionRequestData:
+        processed_audio = process_audio_file(audio_file)
+
+        form_fields: dict = {
+            "model": model,
+        }
+
+        # OpenAI-compatible params
+        for key in self.get_supported_openai_params(model):
+            value = optional_params.get(key)
+            if value is not None:
+                form_fields[key] = value
+
+        # Mistral-specific params (e.g. diarize)
+        provider_specific_params = self.get_provider_specific_params(
+            model=model,
+            optional_params=optional_params,
+            openai_params=self.get_supported_openai_params(model),
+        )
+        for key, value in provider_specific_params.items():
+            form_fields[key] = str(value).lower() if isinstance(value, bool) else str(value)
+
+        files = {
+            "file": (
+                processed_audio.filename,
+                processed_audio.file_content,
+                processed_audio.content_type,
+            )
+        }
+
+        return AudioTranscriptionRequestData(data=form_fields, files=files)
+
+    def transform_audio_transcription_response(
+        self,
+        raw_response: httpx.Response,
+    ) -> TranscriptionResponse:
+        try:
+            response_json = raw_response.json()
+        except Exception:
+            raise MistralAudioTranscriptionException(
+                message=raw_response.text,
+                status_code=raw_response.status_code,
+                headers=raw_response.headers,
+            )
+
+        text = response_json.get("text") or ""
+        response = TranscriptionResponse(text=text)
+        response._hidden_params = response_json
+        return response

litellm/utils.py

@@ -8288,6 +8288,12 @@ class ProviderConfigManager:
             )
 
             return OVHCloudAudioTranscriptionConfig()
+        elif litellm.LlmProviders.MISTRAL == provider:
+            from litellm.llms.mistral.audio_transcription.transformation import (
+                MistralAudioTranscriptionConfig,
+            )
+
+            return MistralAudioTranscriptionConfig()
         return None
 
     @staticmethod

tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py

@@ -0,0 +1,170 @@
+import os
+from typing import Dict
+from unittest.mock import MagicMock
+
+import httpx
+import litellm
+import pytest
+
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.mistral.audio_transcription.transformation import (
+    MistralAudioTranscriptionConfig,
+)
+from litellm.types.utils import TranscriptionResponse
+from litellm.utils import ProviderConfigManager
+from tests.llm_translation.base_audio_transcription_unit_tests import (
+    BaseLLMAudioTranscriptionTest,
+)
+
+
+@pytest.mark.skipif(
+    not os.getenv("MISTRAL_API_KEY"),
+    reason="MISTRAL_API_KEY not set, skipping Mistral audio transcription tests",
+)
+class TestMistralAudioTranscription(BaseLLMAudioTranscriptionTest):
+    def get_base_audio_transcription_call_args(self) -> Dict:
+        return {
+            "model": "mistral/voxtral-mini-latest",
+        }
+
+    def get_custom_llm_provider(self) -> litellm.LlmProviders:
+        return litellm.LlmProviders.MISTRAL
+
+    def test_audio_transcription_async(self):  # type: ignore[override]
+        pytest.skip(
+            "Async audio transcription test for Mistral is skipped in this suite; "
+            "async test plugins (e.g. pytest-asyncio/anyio) are not configured here."
+        )
+
+
+def test_mistral_audio_transcription_config_installed():
+    """Ensure Mistral audio transcription config is registered with ProviderConfigManager."""
+    config = ProviderConfigManager.get_provider_audio_transcription_config(
+        model="mistral/voxtral-mini-latest",
+        provider=litellm.LlmProviders.MISTRAL,
+    )
+    assert config is not None
+    assert isinstance(config, BaseAudioTranscriptionConfig)
+    assert isinstance(config, MistralAudioTranscriptionConfig)
+
+
+def test_mistral_audio_transcription_get_complete_url():
+    config = MistralAudioTranscriptionConfig()
+    url = config.get_complete_url(
+        api_base=None,
+        api_key="fake-key",
+        model="voxtral-mini-latest",
+        optional_params={},
+        litellm_params={},
+    )
+    assert url == "https://api.mistral.ai/v1/audio/transcriptions"
+
+
+def test_mistral_audio_transcription_get_complete_url_custom_base():
+    config = MistralAudioTranscriptionConfig()
+    url = config.get_complete_url(
+        api_base="https://custom.api.example.com/v1/",
+        api_key="fake-key",
+        model="voxtral-mini-latest",
+        optional_params={},
+        litellm_params={},
+    )
+    assert url == "https://custom.api.example.com/v1/audio/transcriptions"
+
+
+def test_mistral_audio_transcription_validate_environment():
+    config = MistralAudioTranscriptionConfig()
+    headers = config.validate_environment(
+        headers={},
+        model="voxtral-mini-latest",
+        messages=[],
+        optional_params={},
+        litellm_params={},
+        api_key="test-key-123",
+    )
+    assert headers["Authorization"] == "Bearer test-key-123"
+    assert headers["accept"] == "application/json"
+
+
+def test_mistral_audio_transcription_supported_params():
+    config = MistralAudioTranscriptionConfig()
+    params = config.get_supported_openai_params("voxtral-mini-latest")
+    assert "language" in params
+    assert "temperature" in params
+    assert "response_format" in params
+    assert "timestamp_granularities" in params
+
+
+def test_mistral_audio_transcription_request_transform():
+    config = MistralAudioTranscriptionConfig()
+
+    wav_path = os.path.join(
+        os.path.dirname(__file__), "../../../../..", "tests", "llm_translation", "gettysburg.wav"
+    )
+    audio_file = open(wav_path, "rb")
+
+    result = config.transform_audio_transcription_request(
+        model="voxtral-mini-latest",
+        audio_file=audio_file,
+        optional_params={"language": "en", "temperature": 0.0},
+        litellm_params={},
+    )
+
+    audio_file.close()
+
+    assert isinstance(result.data, dict)
+    assert result.data["model"] == "voxtral-mini-latest"
+    assert result.data["language"] == "en"
+    assert result.data["temperature"] == 0.0
+    assert result.files is not None
+    assert "file" in result.files
+
+
+def test_mistral_audio_transcription_request_with_diarize():
+    """Test that Mistral-specific params like diarize are passed through."""
+    config = MistralAudioTranscriptionConfig()
+
+    wav_path = os.path.join(
+        os.path.dirname(__file__), "../../../../..", "tests", "llm_translation", "gettysburg.wav"
+    )
+    audio_file = open(wav_path, "rb")
+
+    result = config.transform_audio_transcription_request(
+        model="voxtral-mini-latest",
+        audio_file=audio_file,
+        optional_params={"diarize": True},
+        litellm_params={},
+    )
+
+    audio_file.close()
+
+    assert isinstance(result.data, dict)
+    assert result.data["diarize"] == "true"
+
+
+def test_mistral_audio_transcription_response_transform():
+    config = MistralAudioTranscriptionConfig()
+
+    mock_response = MagicMock(spec=httpx.Response)
+    mock_response.json.return_value = {
+        "text": "Four score and seven years ago..."
+    }
+
+    response = config.transform_audio_transcription_response(mock_response)
+
+    assert isinstance(response, TranscriptionResponse)
+    assert response.text == "Four score and seven years ago..."
+
+
+def test_mistral_audio_transcription_response_transform_empty():
+    config = MistralAudioTranscriptionConfig()
+
+    mock_response = MagicMock(spec=httpx.Response)
+    mock_response.json.return_value = {}
+
+    response = config.transform_audio_transcription_response(mock_response)
+
+    assert isinstance(response, TranscriptionResponse)
+    assert response.text == ""