From 46134d3fba2894d8d76680c394d5982b11ec62d4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 08:14:24 -0800 Subject: [PATCH 1/8] (docs) using mistral azure ai studio --- docs/my-website/docs/providers/azure_ai.md | 29 +++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index 3504cd6b81..4d9ee1c8a8 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -2,16 +2,35 @@ ## Using Mistral models deployed on Azure AI Studio -**Ensure you have the `/v1` in your api_base** +### Sample Usage - setting env vars + +Set `MISTRAL_API_KEY` and `MISTRAL_API_BASE` in your env +```shell +MISTRAL_API_KEY = "zE************" +MISTRAL_API_BASE = "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com" +``` -### Sample Usage ```python from litellm import completion import os response = completion( model="mistral/Mistral-large-dfgfj", - api_base="https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com/v1", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], +) +print(response) +``` + +### Sample Usage - passing `api_base` and `api_key` to `litellm.completion` +```python +from litellm import completion +import os + +response = completion( + model="mistral/Mistral-large-dfgfj", + api_base="https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com", api_key = "JGbKodRcTp****" messages=[ {"role": "user", "content": "hello from litellm"} @@ -23,14 +42,12 @@ print(response) ### [LiteLLM Proxy] Using Mistral Models Set this on your litellm proxy config.yaml - -**Ensure you have the `/v1` in your api_base** ```yaml model_list: - model_name: mistral litellm_params: model: mistral/Mistral-large-dfgfj - api_base: https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com/v1 + api_base: https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com api_key: JGbKodRcTp**** ``` From 7c3141a66c73452216cc091e801e18fa1c332f44 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 08:15:47 -0800 Subject: [PATCH 2/8] (feat) mistral allow setting API base in env --- litellm/utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 1106d2fec9..83764961af 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5003,8 +5003,15 @@ def get_llm_provider( dynamic_api_key = get_secret("GROQ_API_KEY") elif custom_llm_provider == "mistral": # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai - api_base = api_base or "https://api.mistral.ai/v1" - dynamic_api_key = get_secret("MISTRAL_API_KEY") + api_base = ( + api_base + or get_secret("MISTRAL_API_BASE") + or "https://api.mistral.ai/v1" + ) + # if api_base does not end with /v1 we add it + if api_base is not None and not api_base.endswith("/v1"): + api_base = api_base + "/v1" + dynamic_api_key = api_key or get_secret("MISTRAL_API_KEY") elif custom_llm_provider == "voyage": # voyage is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.voyageai.com/v1 api_base = "https://api.voyageai.com/v1" From f881b12570378be3a359f2e611f3e0cf3437b94b Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 08:27:25 -0800 Subject: [PATCH 3/8] (test) mistral azure ai studio --- litellm/tests/test_completion.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 634f277590..fcfa9b3c30 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -107,6 +107,26 @@ def test_completion_mistral_api(): pytest.fail(f"Error occurred: {e}") +def test_completion_mistral_azure(): + try: + litellm.set_verbose = True + response = completion( + model="mistral/Mistral-large-nmefg", + max_tokens=5, + messages=[ + { + "role": "user", + "content": "Hi from litellm", + } + ], + ) + # Add any assertions here to check the response + print(response) + + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + # test_completion_mistral_api() From 817fa2483a3f2f6ac7b8c2f817540b4cb8f16c30 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 08:28:19 -0800 Subject: [PATCH 4/8] (docs) mistral on azure ai studio --- docs/my-website/docs/providers/azure_ai.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index 4d9ee1c8a8..71776e0fb7 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -4,10 +4,11 @@ ### Sample Usage - setting env vars -Set `MISTRAL_API_KEY` and `MISTRAL_API_BASE` in your env +Set `MISTRAL_AZURE_API_KEY` and `MISTRAL_AZURE_API_BASE` in your env + ```shell -MISTRAL_API_KEY = "zE************" -MISTRAL_API_BASE = "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com" +MISTRAL_AZURE_API_KEY = "zE************"" +MISTRAL_AZURE_API_BASE = "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com" ``` ```python From ec70cdc558504b5f32aef543b3c503d789405c37 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 08:28:46 -0800 Subject: [PATCH 5/8] (feat) use mistral azure with env vars --- litellm/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 83764961af..16900c617a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5005,13 +5005,13 @@ def get_llm_provider( # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai api_base = ( api_base - or get_secret("MISTRAL_API_BASE") + or get_secret("MISTRAL_AZURE_API_BASE") or "https://api.mistral.ai/v1" ) # if api_base does not end with /v1 we add it if api_base is not None and not api_base.endswith("/v1"): api_base = api_base + "/v1" - dynamic_api_key = api_key or get_secret("MISTRAL_API_KEY") + dynamic_api_key = api_key or get_secret("MISTRAL_AZURE_API_KEY") elif custom_llm_provider == "voyage": # voyage is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.voyageai.com/v1 api_base = "https://api.voyageai.com/v1" From 43b66d562e2e9876bac1e57abe438d73c4006001 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 08:33:34 -0800 Subject: [PATCH 6/8] (feat) add azure/mistral-large-2402 pricing --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ model_prices_and_context_window.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c3649ac4f2..15b9699808 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -424,6 +424,22 @@ "mode": "chat", "supports_function_calling": true }, + "azure/mistral-large-latest": { + "max_tokens": 32000, + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true + }, + "azure/mistral-large-2402": { + "max_tokens": 32000, + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true + }, "azure/ada": { "max_tokens": 8191, "input_cost_per_token": 0.0000001, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c3649ac4f2..15b9699808 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -424,6 +424,22 @@ "mode": "chat", "supports_function_calling": true }, + "azure/mistral-large-latest": { + "max_tokens": 32000, + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true + }, + "azure/mistral-large-2402": { + "max_tokens": 32000, + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true + }, "azure/ada": { "max_tokens": 8191, "input_cost_per_token": 0.0000001, From 38e003bcc0006100247681ae565f0f673acd02b9 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 12:01:01 -0800 Subject: [PATCH 7/8] (test) fix mistral tests --- litellm/tests/test_completion.py | 5 +++++ litellm/utils.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index fcfa9b3c30..02917cf885 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -107,11 +107,16 @@ def test_completion_mistral_api(): pytest.fail(f"Error occurred: {e}") +@pytest.mark.skip( + reason="Since we already test mistral/mistral-tiny in test_completion_mistral_api. This is only for locally verifying azure mistral works" +) def test_completion_mistral_azure(): try: litellm.set_verbose = True response = completion( model="mistral/Mistral-large-nmefg", + api_key=os.environ["MISTRAL_AZURE_API_KEY"], + api_base=os.environ["MISTRAL_AZURE_API_BASE"], max_tokens=5, messages=[ { diff --git a/litellm/utils.py b/litellm/utils.py index 16900c617a..a7b68ad122 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5011,7 +5011,11 @@ def get_llm_provider( # if api_base does not end with /v1 we add it if api_base is not None and not api_base.endswith("/v1"): api_base = api_base + "/v1" - dynamic_api_key = api_key or get_secret("MISTRAL_AZURE_API_KEY") + dynamic_api_key = ( + api_key + or get_secret("MISTRAL_AZURE_API_KEY") + or get_secret("MISTRAL_API_KEY") + ) elif custom_llm_provider == "voyage": # voyage is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.voyageai.com/v1 api_base = "https://api.voyageai.com/v1" From 3a661b209abfb8f205877b99c807fe0eb69f17e4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 29 Feb 2024 12:04:16 -0800 Subject: [PATCH 8/8] (chore) add mistral azure ai comments --- litellm/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index a7b68ad122..23feaf2b24 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5005,15 +5005,17 @@ def get_llm_provider( # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai api_base = ( api_base - or get_secret("MISTRAL_AZURE_API_BASE") + or get_secret("MISTRAL_AZURE_API_BASE") # for Azure AI Mistral or "https://api.mistral.ai/v1" ) # if api_base does not end with /v1 we add it - if api_base is not None and not api_base.endswith("/v1"): + if api_base is not None and not api_base.endswith( + "/v1" + ): # Mistral always needs a /v1 at the end api_base = api_base + "/v1" dynamic_api_key = ( api_key - or get_secret("MISTRAL_AZURE_API_KEY") + or get_secret("MISTRAL_AZURE_API_KEY") # for Azure AI Mistral or get_secret("MISTRAL_API_KEY") ) elif custom_llm_provider == "voyage":