From e3142b4294cfd5b0b5219607f99d1b554a2a11ff Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 17:22:57 -0700 Subject: [PATCH 1/4] fix whisper health check with litellm --- litellm/llms/openai.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 25e2e518c5..2c7a7a4df1 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -1,5 +1,6 @@ import hashlib import json +import os import time import traceback import types @@ -1870,6 +1871,16 @@ class OpenAIChatCompletion(BaseLLM): model=model, # type: ignore prompt=prompt, # type: ignore ) + elif mode == "audio_transcription": + # Get the current directory of the file being run + pwd = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(pwd, "../tests/gettysburg.wav") + audio_file = open(file_path, "rb") + completion = await client.audio.transcriptions.with_raw_response.create( + file=audio_file, + model=model, # type: ignore + prompt=prompt, # type: ignore + ) else: raise Exception("mode not set") response = {} From 2432c90515229da4d80d9ec298c315e7c9040a57 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 17:26:14 -0700 Subject: [PATCH 2/4] feat - support health check audio_speech --- litellm/llms/openai.py | 9 ++++++++- litellm/proxy/proxy_config.yaml | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 2c7a7a4df1..fae8a448ad 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -1881,8 +1881,15 @@ class OpenAIChatCompletion(BaseLLM): model=model, # type: ignore prompt=prompt, # type: ignore ) + elif mode == "audio_speech": + # Get the current directory of the file being run + completion = await client.audio.speech.with_raw_response.create( + model=model, # type: ignore + input=prompt, # type: ignore + voice="alloy", + ) else: - raise Exception("mode not set") + raise ValueError("mode not set, passed in mode: " + mode) response = {} if completion is None or not hasattr(completion, "headers"): diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 0e3f0826e2..bd8f5bfd0a 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -8,6 +8,12 @@ model_list: litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct api_key: "os.environ/FIREWORKS" + - model_name: tts + litellm_params: + model: openai/tts-1 + api_key: "os.environ/OPENAI_API_KEY" + model_info: + mode: audio_speech general_settings: master_key: sk-1234 alerting: ["slack"] From 3573b47098c52b1dc506e8918b46f5ee471bca28 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 17:29:28 -0700 Subject: [PATCH 3/4] docs add example on using text to speech models --- docs/my-website/docs/proxy/health.md | 57 +++++++++++++++++----------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/my-website/docs/proxy/health.md b/docs/my-website/docs/proxy/health.md index 6d383fc416..632702b914 100644 --- a/docs/my-website/docs/proxy/health.md +++ b/docs/my-website/docs/proxy/health.md @@ -41,28 +41,6 @@ litellm --health } ``` -### Background Health Checks - -You can enable model health checks being run in the background, to prevent each model from being queried too frequently via `/health`. - -Here's how to use it: -1. in the config.yaml add: -``` -general_settings: - background_health_checks: True # enable background health checks - health_check_interval: 300 # frequency of background health checks -``` - -2. Start server -``` -$ litellm /path/to/config.yaml -``` - -3. Query health endpoint: -``` -curl --location 'http://0.0.0.0:4000/health' -``` - ### Embedding Models We need some way to know if the model is an embedding model when running checks, if you have this in your config, specifying mode it makes an embedding health check @@ -124,6 +102,41 @@ model_list: mode: audio_transcription ``` + +### Text to Speech Models + +```yaml +# OpenAI Text to Speech Models + - model_name: tts + litellm_params: + model: openai/tts-1 + api_key: "os.environ/OPENAI_API_KEY" + model_info: + mode: audio_speech +``` + +## Background Health Checks + +You can enable model health checks being run in the background, to prevent each model from being queried too frequently via `/health`. + +Here's how to use it: +1. in the config.yaml add: +``` +general_settings: + background_health_checks: True # enable background health checks + health_check_interval: 300 # frequency of background health checks +``` + +2. Start server +``` +$ litellm /path/to/config.yaml +``` + +3. Query health endpoint: +``` +curl --location 'http://0.0.0.0:4000/health' +``` + ### Hide details The health check response contains details like endpoint URLs, error messages, From f2443996d82d50e88ecfbca4efb045fc0522aa84 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 17:30:15 -0700 Subject: [PATCH 4/4] feat support audio health checks for azure --- litellm/llms/azure.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index a2928cf208..ec143f3fec 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -1864,6 +1864,23 @@ class AzureChatCompletion(BaseLLM): model=model, # type: ignore prompt=prompt, # type: ignore ) + elif mode == "audio_transcription": + # Get the current directory of the file being run + pwd = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(pwd, "../tests/gettysburg.wav") + audio_file = open(file_path, "rb") + completion = await client.audio.transcriptions.with_raw_response.create( + file=audio_file, + model=model, # type: ignore + prompt=prompt, # type: ignore + ) + elif mode == "audio_speech": + # Get the current directory of the file being run + completion = await client.audio.speech.with_raw_response.create( + model=model, # type: ignore + input=prompt, # type: ignore + voice="alloy", + ) else: raise Exception("mode not set") response = {}