From ecf6d22dc2a88ac43b0575d1e19d76a466517c87 Mon Sep 17 00:00:00 2001 From: ProphetJeremy Date: Mon, 13 Jan 2025 14:27:27 +0100 Subject: [PATCH 01/80] (docs) Update vertex.md old code example Complete imports Remove invalid parameter `disable_atributon` --- docs/my-website/docs/providers/vertex.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index cb8c031c06..0c741b6483 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -404,14 +404,16 @@ curl http://localhost:4000/v1/chat/completions \ If this was your initial VertexAI Grounding code, ```python -import vertexai +import vertexai +from vertexai.generative_models import GenerativeModel, GenerationConfig, Tool, grounding + vertexai.init(project=project_id, location="us-central1") model = GenerativeModel("gemini-1.5-flash-001") # Use Google Search for grounding -tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval(disable_attributon=False)) +tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) prompt = "When is the next total solar eclipse in US?" response = model.generate_content( @@ -428,7 +430,7 @@ print(response) then, this is what it looks like now ```python -from litellm import completion +from litellm import completion # !gcloud auth application-default login - run this to add vertex credentials to your env From 0c30909fe9b1dcbd263bc3132f1c2886aab3642a Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Sat, 8 Feb 2025 12:31:01 +0900 Subject: [PATCH 02/80] Reimplement methods required for triton streaming --- .../llms/triton/completion/transformation.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 0cd6940063..9b100ff1f8 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/chat/completions` endpoint to Triton's `/generate` """ import json -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, AsyncIterator, Dict, Iterator, List, Literal, Optional, Union from httpx import Headers, Response @@ -52,6 +52,17 @@ class TritonConfig(BaseConfig): ) -> Dict: return {"Content-Type": "application/json"} + def get_complete_url( + self, + api_base: str, + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + if stream: + return api_base + "_stream" + return api_base + def get_supported_openai_params(self, model: str) -> List: return ["max_tokens", "max_completion_tokens"] @@ -149,6 +160,18 @@ class TritonConfig(BaseConfig): else: raise ValueError(f"Invalid Triton API base: {api_base}") + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + return TritonResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) + class TritonGenerateConfig(TritonConfig): """ From 268702722504ec2f4bf8f72f7bb15cb6da6843b3 Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:40:56 +0900 Subject: [PATCH 03/80] Apply streaming-related transformations only for generate config --- .../llms/triton/completion/transformation.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 9b100ff1f8..b09f7b0444 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -52,17 +52,6 @@ class TritonConfig(BaseConfig): ) -> Dict: return {"Content-Type": "application/json"} - def get_complete_url( - self, - api_base: str, - model: str, - optional_params: dict, - stream: Optional[bool] = None, - ) -> str: - if stream: - return api_base + "_stream" - return api_base - def get_supported_openai_params(self, model: str) -> List: return ["max_tokens", "max_completion_tokens"] @@ -178,6 +167,17 @@ class TritonGenerateConfig(TritonConfig): Transformations for triton /generate endpoint (This is a trtllm model) """ + def get_complete_url( + self, + api_base: str, + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + if stream: + return api_base + "_stream" + return api_base + def transform_request( self, model: str, @@ -227,7 +227,7 @@ class TritonGenerateConfig(TritonConfig): return model_response -class TritonInferConfig(TritonGenerateConfig): +class TritonInferConfig(TritonConfig): """ Transformations for triton /infer endpoint (his is an infer model with a custom model on triton) """ From c1f2ae97c5e3573cbfff173c05cf88ad3b35a249 Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:43:42 +0900 Subject: [PATCH 04/80] Add streaming test --- tests/llm_translation/test_triton.py | 40 +++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/llm_translation/test_triton.py b/tests/llm_translation/test_triton.py index 0835d09fab..7e4ba92f23 100644 --- a/tests/llm_translation/test_triton.py +++ b/tests/llm_translation/test_triton.py @@ -49,16 +49,26 @@ def test_split_embedding_by_shape_fails_with_shape_value_error(): ) -def test_completion_triton_generate_api(): +@pytest.mark.parametrize("stream", [True, False]) +def test_completion_triton_generate_api(stream): try: mock_response = MagicMock() + if stream: + def mock_iter_lines(): + mock_output = ''.join([ + 'data: {"model_name":"ensemble","model_version":"1","sequence_end":false,"sequence_id":0,"sequence_start":false,"text_output":"' + t + '"}\n\n' + for t in ["I", " am", " an", " AI", " assistant"] + ]) + for out in mock_output.split('\n'): + yield out + mock_response.iter_lines = mock_iter_lines + else: + def return_val(): + return { + "text_output": "I am an AI assistant", + } - def return_val(): - return { - "text_output": "I am an AI assistant", - } - - mock_response.json = return_val + mock_response.json = return_val mock_response.status_code = 200 with patch( @@ -71,6 +81,7 @@ def test_completion_triton_generate_api(): max_tokens=10, timeout=5, api_base="http://localhost:8000/generate", + stream=stream, ) # Verify the call was made @@ -81,7 +92,10 @@ def test_completion_triton_generate_api(): call_kwargs = mock_post.call_args.kwargs # Access kwargs directly # Verify URL - assert call_kwargs["url"] == "http://localhost:8000/generate" + if stream: + assert call_kwargs["url"] == "http://localhost:8000/generate_stream" + else: + assert call_kwargs["url"] == "http://localhost:8000/generate" # Parse the request data from the JSON string request_data = json.loads(call_kwargs["data"]) @@ -91,7 +105,15 @@ def test_completion_triton_generate_api(): assert request_data["parameters"]["max_tokens"] == 10 # Verify response - assert response.choices[0].message.content == "I am an AI assistant" + if stream: + tokens = ["I", " am", " an", " AI", " assistant", None] + idx = 0 + for chunk in response: + assert chunk.choices[0].delta.content == tokens[idx] + idx += 1 + assert idx == len(tokens) + else: + assert response.choices[0].message.content == "I am an AI assistant" except Exception as e: print("exception", e) From c62be184c2e9228ad321384f1c385be1ff4f882b Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Thu, 13 Feb 2025 16:41:50 +0900 Subject: [PATCH 05/80] Fix get_complete_url --- .../llms/triton/completion/transformation.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index b09f7b0444..0a65e216df 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -67,6 +67,18 @@ class TritonConfig(BaseConfig): optional_params[param] = value return optional_params + def get_complete_url( + self, + api_base: str, + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + llm_type = self._get_triton_llm_type(api_base) + if llm_type == "generate" and stream: + return api_base + "_stream" + return api_base + def transform_response( self, model: str, @@ -167,17 +179,6 @@ class TritonGenerateConfig(TritonConfig): Transformations for triton /generate endpoint (This is a trtllm model) """ - def get_complete_url( - self, - api_base: str, - model: str, - optional_params: dict, - stream: Optional[bool] = None, - ) -> str: - if stream: - return api_base + "_stream" - return api_base - def transform_request( self, model: str, From e3455cd0451d051cd388e346773a930d4ad865d2 Mon Sep 17 00:00:00 2001 From: Nitin Patel Date: Mon, 24 Feb 2025 01:00:07 +0530 Subject: [PATCH 06/80] fix missing comma --- litellm/llms/perplexity/chat/transformation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index 8f71cc153f..dab64283ec 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -37,6 +37,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): "response_format", "stream", "temperature", - "top_p" "max_retries", + "top_p", + "max_retries", "extra_headers", ] From 4b8db4ec347f1950453c4c94d8ecaff794fbd076 Mon Sep 17 00:00:00 2001 From: Yazan Agha-Schrader Date: Mon, 24 Feb 2025 11:18:18 +0100 Subject: [PATCH 07/80] Update model_prices_and_context_window.json fix mistral/mistral-small from 1$/3$ per million tokens to -> 0.1$/0.3$ per million tokens cave: azure_ai and bedrock still show 1$/3$ for input/output cost per million - i dont have knowledge about azure and bedrock prices, but looks like wrong values as well. **please check** --- model_prices_and_context_window.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5e8d9353ad..932393c261 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1943,8 +1943,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000003, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat", @@ -1955,8 +1955,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000003, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat", From 57faa623e3fd3c301f737a2ad79ebd6f8df112fb Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 25 Feb 2025 10:44:10 -0600 Subject: [PATCH 08/80] Adding Azure Phi-4 --- litellm/model_prices_and_context_window_backup.json | 13 +++++++++++++ model_prices_and_context_window.json | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a11930cc7f..442d5fa776 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1732,6 +1732,19 @@ "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, + "azure_ai/Phi-4": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000000125, + "output_cost_per_token": 0.0000005, + "litellm_provider": "azure_ai", + "mode": "chat", + "supports_vision": false, + "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/affordable-innovation-unveiling-the-pricing-of-phi-3-slms-on-models-as-a-service/4156495", + "supports_function_calling": true, + "supports_tool_choice": true + }, "azure_ai/Phi-3.5-mini-instruct": { "max_tokens": 4096, "max_input_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a11930cc7f..442d5fa776 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1732,6 +1732,19 @@ "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, + "azure_ai/Phi-4": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000000125, + "output_cost_per_token": 0.0000005, + "litellm_provider": "azure_ai", + "mode": "chat", + "supports_vision": false, + "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/affordable-innovation-unveiling-the-pricing-of-phi-3-slms-on-models-as-a-service/4156495", + "supports_function_calling": true, + "supports_tool_choice": true + }, "azure_ai/Phi-3.5-mini-instruct": { "max_tokens": 4096, "max_input_tokens": 128000, From eeee61db658de558a914569567c048fb51278c0f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 25 Feb 2025 14:50:10 -0800 Subject: [PATCH 09/80] can_team_access_model --- litellm/proxy/auth/auth_checks.py | 92 +++++++++++++------------------ 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 0590bcb50a..c922599f86 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -38,6 +38,7 @@ from litellm.proxy._types import ( ProxyErrorTypes, ProxyException, RoleBasedPermissions, + SpecialModelNames, UserAPIKeyAuth, ) from litellm.proxy.auth.route_checks import RouteChecks @@ -97,12 +98,23 @@ async def common_checks( ) # 2. If team can call model - _team_model_access_check( - team_object=team_object, - model=_model, - llm_router=llm_router, - team_model_aliases=valid_token.team_model_aliases if valid_token else None, - ) + if ( + team_object is not None + and _model is not None + and can_team_access_model( + model=_model, + team_object=team_object, + llm_router=llm_router, + team_model_aliases=valid_token.team_model_aliases if valid_token else None, + ) + is False + ): + raise ProxyException( + message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", + type=ProxyErrorTypes.team_model_access_denied, + param="model", + code=status.HTTP_401_UNAUTHORIZED, + ) ## 2.1 If user can call model (if personal key) if team_object is None and user_object is not None: @@ -1017,6 +1029,9 @@ async def _can_object_call_model( if (len(filtered_models) == 0 and len(models) == 0) or "*" in filtered_models: all_model_access = True + if SpecialModelNames.all_proxy_models in filtered_models: + all_model_access = True + if model is not None and model not in filtered_models and all_model_access is False: raise ProxyException( message=f"API Key not allowed to access model. This token can only access models={models}. Tried to access {model}", @@ -1074,6 +1089,24 @@ async def can_key_call_model( ) +async def can_team_access_model( + model: str, + team_object: Optional[LiteLLM_TeamTable], + llm_router: Optional[Router], + team_model_aliases: Optional[Dict[str, str]] = None, +) -> Literal[True]: + """ + Returns True if the team can access a specific model. + + """ + return await _can_object_call_model( + model=model, + llm_router=llm_router, + models=team_object.models if team_object else [], + team_model_aliases=team_model_aliases, + ) + + async def can_user_call_model( model: str, llm_router: Optional[Router], @@ -1239,53 +1272,6 @@ async def _team_max_budget_check( ) -def _team_model_access_check( - model: Optional[str], - team_object: Optional[LiteLLM_TeamTable], - llm_router: Optional[Router], - team_model_aliases: Optional[Dict[str, str]] = None, -): - """ - Access check for team models - Raises: - Exception if the team is not allowed to call the`model` - """ - if ( - model is not None - and team_object is not None - and team_object.models is not None - and len(team_object.models) > 0 - and model not in team_object.models - ): - # this means the team has access to all models on the proxy - if "all-proxy-models" in team_object.models or "*" in team_object.models: - # this means the team has access to all models on the proxy - pass - # check if the team model is an access_group - elif ( - model_in_access_group( - model=model, team_models=team_object.models, llm_router=llm_router - ) - is True - ): - pass - elif model and "*" in model: - pass - elif _model_in_team_aliases(model=model, team_model_aliases=team_model_aliases): - pass - elif _model_matches_any_wildcard_pattern_in_list( - model=model, allowed_model_list=team_object.models - ): - pass - else: - raise ProxyException( - message=f"Team not allowed to access model. Team={team_object.team_id}, Model={model}. Allowed team models = {team_object.models}", - type=ProxyErrorTypes.team_model_access_denied, - param="model", - code=status.HTTP_401_UNAUTHORIZED, - ) - - def is_model_allowed_by_pattern(model: str, allowed_model_pattern: str) -> bool: """ Check if a model matches an allowed pattern. From b6d6e270b49e72a49c3f8a6496a8e965e8eb55c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 25 Feb 2025 14:51:57 -0800 Subject: [PATCH 10/80] can_team_access_model --- litellm/proxy/auth/handle_jwt.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 29f4b31f9c..248d553662 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -33,6 +33,7 @@ from litellm.proxy._types import ( ScopeMapping, Span, ) +from litellm.proxy.auth.auth_checks import can_team_access_model from litellm.proxy.utils import PrismaClient, ProxyLogging from .auth_checks import ( @@ -723,8 +724,12 @@ class JWTAuthManager: team_models = team_object.models if isinstance(team_models, list) and ( not requested_model - or requested_model in team_models - or "*" in team_models + or can_team_access_model( + model=requested_model, + team_object=team_object, + llm_router=None, + team_model_aliases=None, + ) ): is_allowed = allowed_routes_check( user_role=LitellmUserRoles.TEAM, From 3d0b56e8a34b73baf5057e8b45fd6dcb2a558920 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 25 Feb 2025 14:57:13 -0800 Subject: [PATCH 11/80] test_can_team_access_model --- tests/proxy_unit_tests/test_auth_checks.py | 28 ++++++++-------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index 0a8ebbe018..5b79ace1b9 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -27,7 +27,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.utils import PrismaClient from litellm.proxy.auth.auth_checks import ( - _team_model_access_check, + can_team_access_model, _virtual_key_soft_budget_check, ) from litellm.proxy.utils import ProxyLogging @@ -427,9 +427,9 @@ async def test_virtual_key_max_budget_check( ], ) @pytest.mark.asyncio -async def test_team_model_access_check(model, team_models, expect_to_work): +async def test_can_team_access_model(model, team_models, expected_result): """ - Test cases for _team_model_access_check: + Test cases for can_team_access_model: 1. Exact model match 2. all-proxy-models access 3. Wildcard (*) access @@ -443,21 +443,13 @@ async def test_team_model_access_check(model, team_models, expect_to_work): models=team_models, ) - try: - _team_model_access_check( - model=model, - team_object=team_object, - llm_router=None, - ) - if not expect_to_work: - pytest.fail( - f"Expected model access check to fail for model={model}, team_models={team_models}" - ) - except Exception as e: - if expect_to_work: - pytest.fail( - f"Expected model access check to work for model={model}, team_models={team_models}. Got error: {str(e)}" - ) + result = await can_team_access_model( + model=model, + team_object=team_object, + llm_router=None, + team_model_aliases=None, + ) + assert result == expected_result @pytest.mark.parametrize( From 7eaf0039193363fd562349d08df2a7744646a7f2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 25 Feb 2025 15:25:51 -0800 Subject: [PATCH 12/80] expected_result --- tests/proxy_unit_tests/test_auth_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index 5b79ace1b9..a5782653ad 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -394,7 +394,7 @@ async def test_virtual_key_max_budget_check( @pytest.mark.parametrize( - "model, team_models, expect_to_work", + "model, team_models, expected_result", [ ("gpt-4", ["gpt-4"], True), # exact match ("gpt-4", ["all-proxy-models"], True), # all-proxy-models access From c40d45ae093489c599bc023912e7fcf5f5dadcc8 Mon Sep 17 00:00:00 2001 From: Vivek Aditya Date: Wed, 26 Feb 2025 21:00:56 +0530 Subject: [PATCH 13/80] Added tags to additional keys that can be sent to athina --- docs/my-website/docs/observability/athina_integration.md | 1 + litellm/integrations/athina.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/observability/athina_integration.md b/docs/my-website/docs/observability/athina_integration.md index 4994d553c6..2e2141169a 100644 --- a/docs/my-website/docs/observability/athina_integration.md +++ b/docs/my-website/docs/observability/athina_integration.md @@ -78,6 +78,7 @@ Following are the allowed fields in metadata, their types, and their description * `context: Optional[Union[dict, str]]` - This is the context used as information for the prompt. For RAG applications, this is the "retrieved" data. You may log context as a string or as an object (dictionary). * `expected_response: Optional[str]` - This is the reference response to compare against for evaluation purposes. This is useful for segmenting inference calls by expected response. * `user_query: Optional[str]` - This is the user's query. For conversational applications, this is the user's last message. +* `tags: Optional[list]` - This is a list of tags. This is useful for segmenting inference calls by tags. * `custom_attributes: Optional[dict]` - This is a dictionary of custom attributes. This is useful for additional information about the inference. ## Using a self hosted deployment of Athina diff --git a/litellm/integrations/athina.py b/litellm/integrations/athina.py index 754e980c2a..f416b30f8e 100644 --- a/litellm/integrations/athina.py +++ b/litellm/integrations/athina.py @@ -23,6 +23,7 @@ class AthinaLogger: "context", "expected_response", "user_query", + "tags", "custom_attributes", ] @@ -78,10 +79,12 @@ class AthinaLogger: # Add additional metadata keys metadata = kwargs.get("litellm_params", {}).get("metadata", {}) if metadata: + print("additional_keys", self.additional_keys) for key in self.additional_keys: + print("key", key) if key in metadata: + print("key is being added", key) data[key] = metadata[key] - response = litellm.module_level_client.post( self.athina_logging_url, headers=self.headers, From ed75dd61c2d982895e5caef25bb7ce58d1248536 Mon Sep 17 00:00:00 2001 From: Vivek Aditya Date: Fri, 28 Feb 2025 21:48:13 +0530 Subject: [PATCH 14/80] Removed prints and added unit tests --- .../docs/observability/athina_integration.md | 2 + litellm/integrations/athina.py | 5 +- tests/litellm/integrations/test_athina.py | 207 ++++++++++++++++++ 3 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 tests/litellm/integrations/test_athina.py diff --git a/docs/my-website/docs/observability/athina_integration.md b/docs/my-website/docs/observability/athina_integration.md index 2e2141169a..ba93ea4c98 100644 --- a/docs/my-website/docs/observability/athina_integration.md +++ b/docs/my-website/docs/observability/athina_integration.md @@ -79,6 +79,8 @@ Following are the allowed fields in metadata, their types, and their description * `expected_response: Optional[str]` - This is the reference response to compare against for evaluation purposes. This is useful for segmenting inference calls by expected response. * `user_query: Optional[str]` - This is the user's query. For conversational applications, this is the user's last message. * `tags: Optional[list]` - This is a list of tags. This is useful for segmenting inference calls by tags. +* `user_feedback: Optional[str]` - The end user’s feedback. +* `model_options: Optional[dict]` - This is a dictionary of model options. This is useful for getting insights into how model behavior affects your end users. * `custom_attributes: Optional[dict]` - This is a dictionary of custom attributes. This is useful for additional information about the inference. ## Using a self hosted deployment of Athina diff --git a/litellm/integrations/athina.py b/litellm/integrations/athina.py index f416b30f8e..705dc11f1d 100644 --- a/litellm/integrations/athina.py +++ b/litellm/integrations/athina.py @@ -24,6 +24,8 @@ class AthinaLogger: "expected_response", "user_query", "tags", + "user_feedback", + "model_options", "custom_attributes", ] @@ -79,11 +81,8 @@ class AthinaLogger: # Add additional metadata keys metadata = kwargs.get("litellm_params", {}).get("metadata", {}) if metadata: - print("additional_keys", self.additional_keys) for key in self.additional_keys: - print("key", key) if key in metadata: - print("key is being added", key) data[key] = metadata[key] response = litellm.module_level_client.post( self.athina_logging_url, diff --git a/tests/litellm/integrations/test_athina.py b/tests/litellm/integrations/test_athina.py new file mode 100644 index 0000000000..fd660a036e --- /dev/null +++ b/tests/litellm/integrations/test_athina.py @@ -0,0 +1,207 @@ +import unittest +from unittest.mock import patch, MagicMock, ANY +import json +import datetime +import sys +import os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system-path + +from litellm.integrations.athina import AthinaLogger + +class TestAthinaLogger(unittest.TestCase): + + def setUp(self): + # Set up environment variables for testing + self.env_patcher = patch.dict('os.environ', { + 'ATHINA_API_KEY': 'test-api-key', + 'ATHINA_BASE_URL': 'https://test.athina.ai' + }) + self.env_patcher.start() + self.logger = AthinaLogger() + + # Setup common test variables + self.start_time = datetime.datetime(2023, 1, 1, 12, 0, 0) + self.end_time = datetime.datetime(2023, 1, 1, 12, 0, 1) + self.print_verbose = MagicMock() + + def tearDown(self): + self.env_patcher.stop() + + def test_init(self): + """Test the initialization of AthinaLogger""" + self.assertEqual(self.logger.athina_api_key, 'test-api-key') + self.assertEqual(self.logger.athina_logging_url, 'https://test.athina.ai/api/v1/log/inference') + self.assertEqual(self.logger.headers, { + 'athina-api-key': 'test-api-key', + 'Content-Type': 'application/json' + }) + + @patch('litellm.module_level_client.post') + def test_log_event_success(self, mock_post): + """Test successful logging of an event""" + # Setup mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "Success" + mock_post.return_value = mock_response + + # Create test data + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': 'Hello'}], + 'stream': False, + 'litellm_params': { + 'metadata': { + 'environment': 'test-environment', + 'prompt_slug': 'test-prompt', + 'customer_id': 'test-customer', + 'customer_user_id': 'test-user', + 'session_id': 'test-session', + 'external_reference_id': 'test-ext-ref', + 'context': 'test-context', + 'expected_response': 'test-expected', + 'user_query': 'test-query', + 'tags': ['test-tag'], + 'user_feedback': 'test-feedback', + 'model_options': {'test-opt': 'test-val'}, + 'custom_attributes': {'test-attr': 'test-val'} + } + } + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = { + 'id': 'resp-123', + 'choices': [{'message': {'content': 'Hi there'}}], + 'usage': { + 'prompt_tokens': 10, + 'completion_tokens': 5, + 'total_tokens': 15 + } + } + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify the results + mock_post.assert_called_once() + call_args = mock_post.call_args + self.assertEqual(call_args[0][0], 'https://test.athina.ai/api/v1/log/inference') + self.assertEqual(call_args[1]['headers'], self.logger.headers) + + # Parse and verify the sent data + sent_data = json.loads(call_args[1]['data']) + self.assertEqual(sent_data['language_model_id'], 'gpt-4') + self.assertEqual(sent_data['prompt'], kwargs['messages']) + self.assertEqual(sent_data['prompt_tokens'], 10) + self.assertEqual(sent_data['completion_tokens'], 5) + self.assertEqual(sent_data['total_tokens'], 15) + self.assertEqual(sent_data['response_time'], 1000) # 1 second = 1000ms + self.assertEqual(sent_data['customer_id'], 'test-customer') + self.assertEqual(sent_data['session_id'], 'test-session') + self.assertEqual(sent_data['environment'], 'test-environment') + self.assertEqual(sent_data['prompt_slug'], 'test-prompt') + self.assertEqual(sent_data['external_reference_id'], 'test-ext-ref') + self.assertEqual(sent_data['context'], 'test-context') + self.assertEqual(sent_data['expected_response'], 'test-expected') + self.assertEqual(sent_data['user_query'], 'test-query') + self.assertEqual(sent_data['tags'], ['test-tag']) + self.assertEqual(sent_data['user_feedback'], 'test-feedback') + self.assertEqual(sent_data['model_options'], {'test-opt': 'test-val'}) + self.assertEqual(sent_data['custom_attributes'], {'test-attr': 'test-val'}) + # Verify the print_verbose was called + self.print_verbose.assert_called_once_with("Athina Logger Succeeded - Success") + + @patch('litellm.module_level_client.post') + def test_log_event_error_response(self, mock_post): + """Test handling of error response from the API""" + # Setup mock error response + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = "Bad Request" + mock_post.return_value = mock_response + + # Create test data + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': 'Hello'}], + 'stream': False + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = { + 'id': 'resp-123', + 'choices': [{'message': {'content': 'Hi there'}}], + 'usage': { + 'prompt_tokens': 10, + 'completion_tokens': 5, + 'total_tokens': 15 + } + } + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify print_verbose was called with error message + self.print_verbose.assert_called_once_with("Athina Logger Error - Bad Request, 400") + + @patch('litellm.module_level_client.post') + def test_log_event_exception(self, mock_post): + """Test handling of exceptions during logging""" + # Setup mock to raise exception + mock_post.side_effect = Exception("Test exception") + + # Create test data + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': 'Hello'}], + 'stream': False + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = {} + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify print_verbose was called with exception info + self.print_verbose.assert_called_once() + self.assertIn("Athina Logger Error - Test exception", self.print_verbose.call_args[0][0]) + + @patch('litellm.module_level_client.post') + def test_log_event_with_tools(self, mock_post): + """Test logging with tools/functions data""" + # Setup mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + # Create test data with tools + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': "What's the weather?"}], + 'stream': False, + 'optional_params': { + 'tools': [{'type': 'function', 'function': {'name': 'get_weather'}}] + } + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = { + 'id': 'resp-123', + 'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15} + } + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify the results + sent_data = json.loads(mock_post.call_args[1]['data']) + self.assertEqual(sent_data['tools'], [{'type': 'function', 'function': {'name': 'get_weather'}}]) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 5ead81786dea2322b614118d0f4eac7c7843e3f0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 1 Mar 2025 17:42:50 -0800 Subject: [PATCH 15/80] test_can_team_access_model --- tests/proxy_unit_tests/test_auth_checks.py | 36 +++++++++++++--------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index ec36823633..0eb1a38755 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -394,7 +394,7 @@ async def test_virtual_key_max_budget_check( @pytest.mark.parametrize( - "model, team_models, expected_result", + "model, team_models, expect_to_work", [ ("gpt-4", ["gpt-4"], True), # exact match ("gpt-4", ["all-proxy-models"], True), # all-proxy-models access @@ -427,7 +427,7 @@ async def test_virtual_key_max_budget_check( ], ) @pytest.mark.asyncio -async def test_can_team_access_model(model, team_models, expected_result): +async def test_can_team_access_model(model, team_models, expect_to_work): """ Test cases for can_team_access_model: 1. Exact model match @@ -438,18 +438,26 @@ async def test_can_team_access_model(model, team_models, expected_result): 6. Empty model list 7. None model list """ - team_object = LiteLLM_TeamTable( - team_id="test-team", - models=team_models, - ) - - result = await can_team_access_model( - model=model, - team_object=team_object, - llm_router=None, - team_model_aliases=None, - ) - assert result == expected_result + try: + team_object = LiteLLM_TeamTable( + team_id="test-team", + models=team_models, + ) + result = await can_team_access_model( + model=model, + team_object=team_object, + llm_router=None, + team_model_aliases=None, + ) + if not expect_to_work: + pytest.fail( + f"Expected model access check to fail for model={model}, team_models={team_models}" + ) + except Exception as e: + if expect_to_work: + pytest.fail( + f"Expected model access check to work for model={model}, team_models={team_models}. Got error: {str(e)}" + ) @pytest.mark.parametrize( From f5ccd7c5aa19e8f39207a612bd11a2cd69f03568 Mon Sep 17 00:00:00 2001 From: Cole McIntosh Date: Sun, 2 Mar 2025 20:41:30 -0700 Subject: [PATCH 16/80] build: Add Makefile for LiteLLM project with test targets --- Makefile | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..6bd3cb57d4 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +# LiteLLM Makefile +# Simple Makefile for running tests and basic development tasks + +.PHONY: help test test-unit test-integration + +# Default target +help: + @echo "Available commands:" + @echo " make test - Run all tests" + @echo " make test-unit - Run unit tests" + @echo " make test-integration - Run integration tests" + +# Testing +test: + poetry run pytest tests/ + +test-unit: + poetry run pytest tests/litellm/ + +test-integration: + poetry run pytest tests/ -k "not litellm" \ No newline at end of file From fa88bc96328fd8e313632fd80cb962297a056323 Mon Sep 17 00:00:00 2001 From: Utkash Dubey Date: Mon, 3 Mar 2025 04:16:12 -0800 Subject: [PATCH 17/80] changes --- litellm/__init__.py | 2 +- .../litellm_core_utils/get_model_cost_map.py | 21 ++-- model_prices_and_context_window.json | 2 +- tests/code_coverage_tests/bedrock_pricing.py | 3 +- tests/litellm_utils_tests/test_utils.py | 12 +- .../base_embedding_unit_tests.py | 3 +- tests/llm_translation/base_llm_unit_tests.py | 27 ++--- .../llm_translation/base_rerank_unit_tests.py | 3 +- .../test_anthropic_completion.py | 3 +- .../test_bedrock_completion.py | 15 +-- tests/llm_translation/test_openai_o1.py | 9 +- tests/llm_translation/test_rerank.py | 3 +- tests/llm_translation/test_together_ai.py | 3 +- .../test_amazing_vertex_completion.py | 6 +- tests/local_testing/test_completion_cost.py | 70 ++++------- tests/local_testing/test_embedding.py | 6 +- tests/local_testing/test_get_model_info.py | 21 ++-- tests/local_testing/test_router_utils.py | 12 +- ..._model_prices_and_context_window_schema.py | 111 ++++++++++++++++++ 19 files changed, 191 insertions(+), 141 deletions(-) create mode 100644 tests/test_model_prices_and_context_window_schema.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 60b8cf81a0..a3756251d1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -308,7 +308,7 @@ _key_management_settings: KeyManagementSettings = KeyManagementSettings() #### PII MASKING #### output_parse_pii: bool = False ############################################# -from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, get_locally_cached_model_cost_map model_cost = get_model_cost_map(url=model_cost_map_url) custom_prompt_dict: Dict[str, dict] = {} diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index b8bdaee19c..0e14457b2a 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -8,24 +8,29 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True ``` """ +from functools import cache import os import httpx +@cache +def get_locally_cached_model_cost_map(): + import importlib.resources + import json + + with importlib.resources.open_text( + "litellm", "model_prices_and_context_window_backup.json" + ) as f: + content = json.load(f) + return content + def get_model_cost_map(url: str): if ( os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) == "True" ): - import importlib.resources - import json - - with importlib.resources.open_text( - "litellm", "model_prices_and_context_window_backup.json" - ) as f: - content = json.load(f) - return content + return get_locally_cached_model_cost_map() try: response = httpx.get( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 96076fa3b8..961b55f49b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6,7 +6,7 @@ "input_cost_per_token": 0.0000, "output_cost_per_token": 0.000, "litellm_provider": "one of https://docs.litellm.ai/docs/providers", - "mode": "one of chat, embedding, completion, image_generation, audio_transcription, audio_speech", + "mode": "one of: chat, embedding, completion, image_generation, audio_transcription, audio_speech, image_generation, moderation, moderations, rerank", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, diff --git a/tests/code_coverage_tests/bedrock_pricing.py b/tests/code_coverage_tests/bedrock_pricing.py index b2c9e78b06..9984cb8b0e 100644 --- a/tests/code_coverage_tests/bedrock_pricing.py +++ b/tests/code_coverage_tests/bedrock_pricing.py @@ -191,8 +191,7 @@ def _check_if_model_name_in_pricing( input_cost_per_1k_tokens: str, output_cost_per_1k_tokens: str, ): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() for model, value in litellm.model_cost.items(): if model.startswith(bedrock_model_name): diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 2b1e78a681..fd8ad01c8b 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -907,8 +907,7 @@ def test_supports_response_schema(model, expected_bool): Should be true for gemini-1.5-pro on google ai studio / vertex ai AND predibase models Should be false otherwise """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.utils import supports_response_schema @@ -1066,8 +1065,7 @@ def test_async_http_handler_force_ipv4(mock_async_client): "model, expected_bool", [("gpt-3.5-turbo", False), ("gpt-4o-audio-preview", True)] ) def test_supports_audio_input(model, expected_bool): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.utils import supports_audio_input, supports_audio_output @@ -1165,8 +1163,7 @@ def test_models_by_provider(): """ Make sure all providers from model map are in the valid providers list """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm import models_by_provider @@ -1484,8 +1481,7 @@ def test_get_valid_models_default(monkeypatch): def test_supports_vision_gemini(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.utils import supports_vision assert supports_vision("gemini-1.5-pro") is True diff --git a/tests/llm_translation/base_embedding_unit_tests.py b/tests/llm_translation/base_embedding_unit_tests.py index 30a9dcc0da..1fcc825481 100644 --- a/tests/llm_translation/base_embedding_unit_tests.py +++ b/tests/llm_translation/base_embedding_unit_tests.py @@ -84,8 +84,7 @@ class BaseLLMEmbeddingTest(ABC): litellm.set_verbose = True from litellm.utils import supports_embedding_image_input - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() base_embedding_call_args = self.get_base_embedding_call_args() if not supports_embedding_image_input(base_embedding_call_args["model"], None): diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index f91ef0eae9..eb18cbce90 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -342,8 +342,7 @@ class BaseLLMChatTest(ABC): from pydantic import BaseModel from litellm.utils import supports_response_schema - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() class TestModel(BaseModel): first_response: str @@ -382,16 +381,14 @@ class BaseLLMChatTest(ABC): from pydantic import BaseModel from litellm.utils import supports_response_schema - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() @pytest.mark.flaky(retries=6, delay=1) def test_json_response_nested_pydantic_obj(self): from pydantic import BaseModel from litellm.utils import supports_response_schema - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() class CalendarEvent(BaseModel): name: str @@ -438,8 +435,7 @@ class BaseLLMChatTest(ABC): from litellm.utils import supports_response_schema from litellm.llms.base_llm.base_utils import type_to_response_format_param - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() class CalendarEvent(BaseModel): name: str @@ -560,8 +556,7 @@ class BaseLLMChatTest(ABC): litellm.set_verbose = True from litellm.utils import supports_vision - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() base_completion_call_args = self.get_base_completion_call_args() if not supports_vision(base_completion_call_args["model"], None): @@ -615,8 +610,7 @@ class BaseLLMChatTest(ABC): litellm.set_verbose = True from litellm.utils import supports_vision - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" @@ -656,8 +650,7 @@ class BaseLLMChatTest(ABC): litellm.set_verbose = True from litellm.utils import supports_prompt_caching - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() base_completion_call_args = self.get_base_completion_call_args() if not supports_prompt_caching(base_completion_call_args["model"], None): @@ -773,8 +766,7 @@ class BaseLLMChatTest(ABC): litellm._turn_on_debug() from litellm.utils import supports_function_calling - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() base_completion_call_args = self.get_base_completion_call_args() if not supports_function_calling(base_completion_call_args["model"], None): @@ -872,8 +864,7 @@ class BaseLLMChatTest(ABC): async def test_completion_cost(self): from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.set_verbose = True response = await self.async_completion_function( diff --git a/tests/llm_translation/base_rerank_unit_tests.py b/tests/llm_translation/base_rerank_unit_tests.py index cff4a02753..b3f56f7c64 100644 --- a/tests/llm_translation/base_rerank_unit_tests.py +++ b/tests/llm_translation/base_rerank_unit_tests.py @@ -87,8 +87,7 @@ class BaseLLMRerankTest(ABC): @pytest.mark.parametrize("sync_mode", [True, False]) async def test_basic_rerank(self, sync_mode): litellm._turn_on_debug() - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() rerank_call_args = self.get_base_rerank_call_args() custom_llm_provider = self.get_custom_llm_provider() if sync_mode is True: diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 37253a37e6..04158b4ab4 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -693,8 +693,7 @@ class TestAnthropicCompletion(BaseLLMChatTest): from pydantic import BaseModel from litellm.utils import supports_response_schema - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() class RFormat(BaseModel): question: str diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 2fb0ffb9e5..99e4e7ed1a 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -1975,8 +1975,7 @@ def test_bedrock_converse_route(): def test_bedrock_mapped_converse_models(): litellm.set_verbose = True - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.add_known_models() litellm.completion( model="bedrock/us.amazon.nova-pro-v1:0", @@ -2108,8 +2107,7 @@ def test_bedrock_supports_tool_call(model, expected_supports_tool_call): class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.add_known_models() return { "model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", @@ -2137,8 +2135,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): """ Test if region models info is correctly used for cost calculation. Using the base model info for cost calculation. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() bedrock_model = "us.anthropic.claude-3-5-sonnet-20241022-v2:0" litellm.model_cost.pop(bedrock_model, None) model = f"bedrock/{bedrock_model}" @@ -2155,8 +2152,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): class TestBedrockConverseChatNormal(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.add_known_models() return { "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", @@ -2325,8 +2321,7 @@ def test_bedrock_nova_topk(top_k_param): def test_bedrock_cross_region_inference(monkeypatch): from litellm.llms.custom_httpx.http_handler import HTTPHandler - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.add_known_models() litellm.set_verbose = True diff --git a/tests/llm_translation/test_openai_o1.py b/tests/llm_translation/test_openai_o1.py index 4208f1ae38..bcd7648f2a 100644 --- a/tests/llm_translation/test_openai_o1.py +++ b/tests/llm_translation/test_openai_o1.py @@ -29,8 +29,7 @@ async def test_o1_handle_system_role(model): from openai import AsyncOpenAI from litellm.utils import supports_system_messages - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.set_verbose = True @@ -83,8 +82,7 @@ async def test_o1_handle_tool_calling_optional_params( from litellm.utils import ProviderConfigManager from litellm.types.utils import LlmProviders - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() config = ProviderConfigManager.get_provider_chat_config( model=model, provider=LlmProviders.OPENAI @@ -190,8 +188,7 @@ class TestOpenAIO3(BaseOSeriesModelsTest, BaseLLMChatTest): def test_o1_supports_vision(): """Test that o1 supports vision""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() for k, v in litellm.model_cost.items(): if k.startswith("o1") and v.get("litellm_provider") == "openai": assert v.get("supports_vision") is True, f"{k} does not support vision" diff --git a/tests/llm_translation/test_rerank.py b/tests/llm_translation/test_rerank.py index d2cb2b6fea..ef5df795ab 100644 --- a/tests/llm_translation/test_rerank.py +++ b/tests/llm_translation/test_rerank.py @@ -274,8 +274,7 @@ class TestLogger(CustomLogger): @pytest.mark.asyncio() async def test_rerank_custom_callbacks(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() custom_logger = TestLogger() litellm.callbacks = [custom_logger] diff --git a/tests/llm_translation/test_together_ai.py b/tests/llm_translation/test_together_ai.py index b83a700002..f275500817 100644 --- a/tests/llm_translation/test_together_ai.py +++ b/tests/llm_translation/test_together_ai.py @@ -42,8 +42,7 @@ class TestTogetherAI(BaseLLMChatTest): def test_get_supported_response_format_together_ai( self, model: str, expected_bool: bool ) -> None: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() optional_params = litellm.get_supported_openai_params( model, custom_llm_provider="together_ai" ) diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 02e0c9b2f1..d59df956be 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -1433,8 +1433,7 @@ async def test_gemini_pro_json_schema_args_sent_httpx( enforce_validation, ): load_vertex_ai_credentials() - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.set_verbose = True messages = [{"role": "user", "content": "List 5 cookie recipes"}] @@ -1554,8 +1553,7 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( from pydantic import BaseModel load_vertex_ai_credentials() - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.set_verbose = True diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 200f2c012e..77d49961bd 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -634,8 +634,7 @@ def test_gemini_completion_cost(above_128k, provider): """ Check if cost correctly calculated for gemini models based on context window """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() if provider == "gemini": model_name = "gemini-1.5-flash-latest" else: @@ -690,8 +689,7 @@ def _count_characters(text): def test_vertex_ai_completion_cost(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() text = "The quick brown fox jumps over the lazy dog." characters = _count_characters(text=text) @@ -726,8 +724,7 @@ def test_vertex_ai_medlm_completion_cost(): model=model, messages=messages, custom_llm_provider="vertex_ai" ) - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() model = "vertex_ai/medlm-medium" messages = [{"role": "user", "content": "Test MedLM completion cost."}] @@ -746,8 +743,7 @@ def test_vertex_ai_claude_completion_cost(): from litellm import Choices, Message, ModelResponse from litellm.utils import Usage - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.set_verbose = True input_tokens = litellm.token_counter( @@ -796,8 +792,7 @@ def test_vertex_ai_embedding_completion_cost(caplog): """ Relevant issue - https://github.com/BerriAI/litellm/issues/4630 """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() text = "The quick brown fox jumps over the lazy dog." input_tokens = litellm.token_counter( @@ -839,8 +834,7 @@ def test_vertex_ai_embedding_completion_cost(caplog): # from test_amazing_vertex_completion import load_vertex_ai_credentials # load_vertex_ai_credentials() -# os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" -# litellm.model_cost = litellm.get_model_cost_map(url="") +# litellm.model_cost = litellm.get_locally_cached_model_cost_map() # text = "The quick brown fox jumps over the lazy dog." # input_tokens = litellm.token_counter( @@ -867,8 +861,7 @@ def test_vertex_ai_embedding_completion_cost(caplog): def test_completion_azure_ai(): try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.set_verbose = True response = litellm.completion( @@ -974,8 +967,7 @@ def test_vertex_ai_mistral_predict_cost(usage): @pytest.mark.parametrize("model", ["openai/tts-1", "azure/tts-1"]) def test_completion_cost_tts(model): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() cost = completion_cost( model=model, @@ -1171,8 +1163,7 @@ def test_completion_cost_azure_common_deployment_name(): ], ) def test_completion_cost_prompt_caching(model, custom_llm_provider): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.utils import Choices, Message, ModelResponse, Usage @@ -1273,8 +1264,7 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): ], ) def test_completion_cost_databricks(model): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() model, messages = model, [{"role": "user", "content": "What is 2+2?"}] resp = litellm.completion(model=model, messages=messages) # works fine @@ -1291,8 +1281,7 @@ def test_completion_cost_databricks(model): ], ) def test_completion_cost_databricks_embedding(model): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() resp = litellm.embedding(model=model, input=["hey, how's it going?"]) # works fine print(resp) @@ -1319,8 +1308,7 @@ def test_get_model_params_fireworks_ai(model, base_model): ["fireworks_ai/llama-v3p1-405b-instruct", "fireworks_ai/mixtral-8x7b-instruct"], ) def test_completion_cost_fireworks_ai(model): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() messages = [{"role": "user", "content": "Hey, how's it going?"}] resp = litellm.completion(model=model, messages=messages) # works fine @@ -1337,8 +1325,7 @@ def test_cost_azure_openai_prompt_caching(): ) from litellm import get_model_info - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() model = "azure/o1-mini" @@ -1427,8 +1414,7 @@ def test_cost_azure_openai_prompt_caching(): def test_completion_cost_vertex_llama3(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.utils import Choices, Message, ModelResponse, Usage @@ -1468,8 +1454,7 @@ def test_cost_openai_prompt_caching(): from litellm.utils import Choices, Message, ModelResponse, Usage from litellm import get_model_info - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() model = "gpt-4o-mini-2024-07-18" @@ -1559,8 +1544,7 @@ def test_cost_openai_prompt_caching(): def test_completion_cost_azure_ai_rerank(model): from litellm import RerankResponse, rerank - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() response = RerankResponse( id="b01dbf2e-63c8-4981-9e69-32241da559ed", @@ -1591,8 +1575,7 @@ def test_completion_cost_azure_ai_rerank(model): def test_together_ai_embedding_completion_cost(): from litellm.utils import Choices, EmbeddingResponse, Message, ModelResponse, Usage - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() response = EmbeddingResponse( model="togethercomputer/m2-bert-80M-8k-retrieval", data=[ @@ -2449,8 +2432,7 @@ def test_completion_cost_params_gemini_3(): from litellm.llms.vertex_ai.cost_calculator import cost_per_character - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() response = ModelResponse( id="chatcmpl-61043504-4439-48be-9996-e29bdee24dc3", @@ -2519,8 +2501,7 @@ def test_completion_cost_params_gemini_3(): # @pytest.mark.flaky(retries=3, delay=1) @pytest.mark.parametrize("stream", [False]) # True, async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.types.utils import ( Choices, Message, @@ -2617,8 +2598,7 @@ def test_completion_cost_model_response_cost(response_model, custom_llm_provider """ from litellm import ModelResponse - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.set_verbose = True response = { @@ -2718,8 +2698,7 @@ def test_select_model_name_for_cost_calc(): def test_moderations(): from litellm import moderation - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.add_known_models() assert "omni-moderation-latest" in litellm.model_cost @@ -2772,8 +2751,7 @@ def test_bedrock_cost_calc_with_region(): from litellm import ModelResponse - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() litellm.add_known_models() @@ -2972,9 +2950,7 @@ async def test_cost_calculator_with_custom_pricing_router(model_item, custom_pri def test_json_valid_model_cost_map(): import json - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - - model_cost = litellm.get_model_cost_map(url="") + model_cost = litellm.get_locally_cached_model_cost_map() try: # Attempt to serialize and deserialize the JSON diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index c85a830e5f..c369dd73eb 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -115,8 +115,7 @@ def test_openai_embedding_3(): @pytest.mark.asyncio async def test_openai_azure_embedding_simple(model, api_base, api_key, sync_mode): try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() # litellm.set_verbose = True if sync_mode: response = embedding( @@ -198,8 +197,7 @@ def _azure_ai_image_mock_response(*args, **kwargs): @pytest.mark.asyncio async def test_azure_ai_embedding_image(model, api_base, api_key, sync_mode): try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() input = base64_image if sync_mode: client = HTTPHandler() diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index c879332c7b..c40ac41be2 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -58,16 +58,14 @@ def test_get_model_info_shows_correct_supports_vision(): def test_get_model_info_shows_assistant_prefill(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() info = litellm.get_model_info("deepseek/deepseek-chat") print("info", info) assert info.get("supports_assistant_prefill") is True def test_get_model_info_shows_supports_prompt_caching(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() info = litellm.get_model_info("deepseek/deepseek-chat") print("info", info) assert info.get("supports_prompt_caching") is True @@ -116,8 +114,7 @@ def test_get_model_info_gemini(): """ Tests if ALL gemini models have 'tpm' and 'rpm' in the model info """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() model_map = litellm.model_cost for model, info in model_map.items(): @@ -127,8 +124,7 @@ def test_get_model_info_gemini(): def test_get_model_info_bedrock_region(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() args = { "model": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", "custom_llm_provider": "bedrock", @@ -212,8 +208,7 @@ def test_model_info_bedrock_converse(monkeypatch): This ensures they are automatically routed to the converse endpoint. """ - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() try: # Load whitelist models from file with open("whitelisted_bedrock_models.txt", "r") as file: @@ -231,8 +226,7 @@ def test_model_info_bedrock_converse_enforcement(monkeypatch): """ Test the enforcement of the whitelist by adding a fake model and ensuring the test fails. """ - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() # Add a fake unwhitelisted model litellm.model_cost["fake.bedrock-chat-model"] = { @@ -323,8 +317,7 @@ def test_get_model_info_bedrock_models(): """ from litellm.llms.bedrock.common_utils import BedrockModelInfo - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() for k, v in litellm.model_cost.items(): if v["litellm_provider"] == "bedrock": diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index 7de9707579..d0afc440d9 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -178,8 +178,7 @@ async def test_update_kwargs_before_fallbacks(call_type): def test_router_get_model_info_wildcard_routes(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() router = Router( model_list=[ { @@ -200,8 +199,7 @@ def test_router_get_model_info_wildcard_routes(): @pytest.mark.asyncio async def test_router_get_model_group_usage_wildcard_routes(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() router = Router( model_list=[ { @@ -297,8 +295,7 @@ async def test_call_router_callbacks_on_failure(): @pytest.mark.asyncio async def test_router_model_group_headers(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.types.utils import OPENAI_RESPONSE_HEADERS router = Router( @@ -330,8 +327,7 @@ async def test_router_model_group_headers(): @pytest.mark.asyncio async def test_get_remaining_model_group_usage(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.model_cost = litellm.get_locally_cached_model_cost_map() from litellm.types.utils import OPENAI_RESPONSE_HEADERS router = Router( diff --git a/tests/test_model_prices_and_context_window_schema.py b/tests/test_model_prices_and_context_window_schema.py new file mode 100644 index 0000000000..80d35f84b4 --- /dev/null +++ b/tests/test_model_prices_and_context_window_schema.py @@ -0,0 +1,111 @@ +import litellm +from jsonschema import validate + +def test_model_prices_and_context_window_json_is_valid(): + ''' + Validates the `model_prices_and_context_window.json` file. + + If this test fails after you update the json, you need to update the schema or correct the change you made. + ''' + + INTENDED_SCHEMA = { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "cache_creation_input_audio_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "deprecation_date": {"type": "string"}, + "input_cost_per_audio_per_second": {"type": "number"}, + "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, + "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_character": {"type": "number"}, + "input_cost_per_character_above_128k_tokens": {"type": "number"}, + "input_cost_per_image": {"type": "number"}, + "input_cost_per_image_above_128k_tokens": {"type": "number"}, + "input_cost_per_pixel": {"type": "number"}, + "input_cost_per_query": {"type": "number"}, + "input_cost_per_request": {"type": "number"}, + "input_cost_per_second": {"type": "number"}, + "input_cost_per_token": {"type": "number"}, + "input_cost_per_token_above_128k_tokens": {"type": "number"}, + "input_cost_per_token_batch_requests": {"type": "number"}, + "input_cost_per_token_batches": {"type": "number"}, + "input_cost_per_token_cache_hit": {"type": "number"}, + "input_cost_per_video_per_second": {"type": "number"}, + "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, + "input_dbu_cost_per_token": {"type": "number"}, + "litellm_provider": {"type": "string"}, + "max_audio_length_hours": {"type": "number"}, + "max_audio_per_prompt": {"type": "number"}, + "max_document_chunks_per_query": {"type": "number"}, + "max_images_per_prompt": {"type": "number"}, + "max_input_tokens": {"type": "number"}, + "max_output_tokens": {"type": "number"}, + "max_pdf_size_mb": {"type": "number"}, + "max_query_tokens": {"type": "number"}, + "max_tokens": {"type": "number"}, + "max_tokens_per_document_chunk": {"type": "number"}, + "max_video_length": {"type": "number"}, + "max_videos_per_prompt": {"type": "number"}, + "metadata": {"type": "object"}, + "mode": { + "type": "string", + "enum": [ + "audio_speech", + "audio_transcription", + "chat", + "completion", + "embedding", + "image_generation", + "moderation", + "moderations", + "rerank" + ], + }, + "output_cost_per_audio_token": {"type": "number"}, + "output_cost_per_character": {"type": "number"}, + "output_cost_per_character_above_128k_tokens": {"type": "number"}, + "output_cost_per_image": {"type": "number"}, + "output_cost_per_pixel": {"type": "number"}, + "output_cost_per_second": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "output_cost_per_token_above_128k_tokens": {"type": "number"}, + "output_cost_per_token_batches": {"type": "number"}, + "output_db_cost_per_token": {"type": "number"}, + "output_dbu_cost_per_token": {"type": "number"}, + "output_vector_size": {"type": "number"}, + "rpd": {"type": "number"}, + "rpm": {"type": "number"}, + "source": {"type": "string"}, + "supports_assistant_prefill": {"type": "boolean"}, + "supports_audio_input": {"type": "boolean"}, + "supports_audio_output": {"type": "boolean"}, + "supports_embedding_image_input": {"type": "boolean"}, + "supports_function_calling": {"type": "boolean"}, + "supports_image_input": {"type": "boolean"}, + "supports_parallel_function_calling": {"type": "boolean"}, + "supports_pdf_input": {"type": "boolean"}, + "supports_prompt_caching": {"type": "boolean"}, + "supports_response_schema": {"type": "boolean"}, + "supports_system_messages": {"type": "boolean"}, + "supports_tool_choice": {"type": "boolean"}, + "supports_video_input": {"type": "boolean"}, + "supports_vision": {"type": "boolean"}, + "tool_use_system_prompt_tokens": {"type": "number"}, + "tpm": {"type": "number"}, + }, + "additionalProperties": False, + }, + } + + actual_json = litellm.get_locally_cached_model_cost_map() + assert isinstance(actual_json, dict) + temporarily_removed = actual_json.pop('sample_spec', None) # remove the sample, whose schema is inconsistent with the real data + + validate(actual_json, INTENDED_SCHEMA) + + if temporarily_removed is not None: + # put back the sample spec that we removed + actual_json.update({'sample_spec': temporarily_removed}) From 8cd5d8d8f3c7bfd06db48f2bcd146ccb9630b380 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Mar 2025 22:46:46 +0000 Subject: [PATCH 18/80] build(deps): bump jinja2 from 3.1.4 to 3.1.6 Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.4 to 3.1.6. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.4...3.1.6) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d9b89cfa07..3d695d1766 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ tiktoken==0.8.0 # for calculating usage importlib-metadata==6.8.0 # for random utils tokenizers==0.20.2 # for calculating usage click==8.1.7 # for proxy cli -jinja2==3.1.4 # for prompt templates +jinja2==3.1.6 # for prompt templates aiohttp==3.10.2 # for network calls aioboto3==12.3.0 # for async sagemaker calls tenacity==8.2.3 # for retrying requests, when litellm.num_retries set From 9dee3e2e3f9105e6a9fc99b8cb5873255b3ca61e Mon Sep 17 00:00:00 2001 From: lucca Date: Thu, 6 Mar 2025 13:28:07 -0300 Subject: [PATCH 19/80] pricing --- ...odel_prices_and_context_window_backup.json | 20 +++++++++++++++++++ model_prices_and_context_window.json | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dd90f3dff9..3d180c3df9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6044,6 +6044,26 @@ "mode": "chat", "supports_tool_choice": true }, + "jamba-1.6-large": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "litellm_provider": "ai21", + "mode": "chat", + "supports_tool_choice": true + }, + "jamba-1.6-mini": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000004, + "litellm_provider": "ai21", + "mode": "chat", + "supports_tool_choice": true + }, "jamba-1.5-mini": { "max_tokens": 256000, "max_input_tokens": 256000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dd90f3dff9..1d7392ae7f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6064,6 +6064,26 @@ "mode": "chat", "supports_tool_choice": true }, + "jamba-1.6-large": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "litellm_provider": "ai21", + "mode": "chat", + "supports_tool_choice": true + }, + "jamba-1.6-mini": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000004, + "litellm_provider": "ai21", + "mode": "chat", + "supports_tool_choice": true + }, "j2-mid": { "max_tokens": 8192, "max_input_tokens": 8192, From af9f85e0d9e384e384a8b4618e779545b3e03fd5 Mon Sep 17 00:00:00 2001 From: lucca Date: Thu, 6 Mar 2025 15:57:29 -0300 Subject: [PATCH 20/80] fix --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3d180c3df9..853791989e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6044,7 +6044,7 @@ "mode": "chat", "supports_tool_choice": true }, - "jamba-1.6-large": { + "jamba-large-1.6": { "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -6054,7 +6054,7 @@ "mode": "chat", "supports_tool_choice": true }, - "jamba-1.6-mini": { + "jamba-mini-1.6": { "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1d7392ae7f..292c2a6193 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6064,7 +6064,7 @@ "mode": "chat", "supports_tool_choice": true }, - "jamba-1.6-large": { + "jamba-large-1.6": { "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -6074,7 +6074,7 @@ "mode": "chat", "supports_tool_choice": true }, - "jamba-1.6-mini": { + "jamba-mini-1.6": { "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, From 805679becccbee2107b1d0f07a6c811fa8418067 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 6 Mar 2025 23:05:54 -0800 Subject: [PATCH 21/80] feat(handle_jwt.py): support multiple jwt url's --- litellm/proxy/auth/handle_jwt.py | 50 +++++++++++++++------------ tests/proxy_unit_tests/test_jwt.py | 55 +++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 29f4b31f9c..61da9825e6 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -344,32 +344,38 @@ class JWTHandler: if keys_url is None: raise Exception("Missing JWT Public Key URL from environment.") - cached_keys = await self.user_api_key_cache.async_get_cache( - "litellm_jwt_auth_keys" - ) - if cached_keys is None: - response = await self.http_handler.get(keys_url) + keys_url_list = [url.strip() for url in keys_url.split(",")] - response_json = response.json() - if "keys" in response_json: - keys: JWKKeyValue = response.json()["keys"] + for key_url in keys_url_list: + + cache_key = f"litellm_jwt_auth_keys_{key_url}" + + cached_keys = await self.user_api_key_cache.async_get_cache(cache_key) + + if cached_keys is None: + response = await self.http_handler.get(key_url) + + response_json = response.json() + if "keys" in response_json: + keys: JWKKeyValue = response.json()["keys"] + else: + keys = response_json + + await self.user_api_key_cache.async_set_cache( + key=cache_key, + value=keys, + ttl=self.litellm_jwtauth.public_key_ttl, # cache for 10 mins + ) else: - keys = response_json + keys = cached_keys - await self.user_api_key_cache.async_set_cache( - key="litellm_jwt_auth_keys", - value=keys, - ttl=self.litellm_jwtauth.public_key_ttl, # cache for 10 mins - ) - else: - keys = cached_keys + public_key = self.parse_keys(keys=keys, kid=kid) + if public_key is not None: + return cast(dict, public_key) - public_key = self.parse_keys(keys=keys, kid=kid) - if public_key is None: - raise Exception( - f"No matching public key found. kid={kid}, keys_url={keys_url}, cached_keys={cached_keys}, len(keys)={len(keys)}" - ) - return cast(dict, public_key) + raise Exception( + f"No matching public key found. keys={keys_url_list}, kid={kid}" + ) def parse_keys(self, keys: JWKKeyValue, kid: Optional[str]) -> Optional[JWTKeyItem]: public_key: Optional[JWTKeyItem] = None diff --git a/tests/proxy_unit_tests/test_jwt.py b/tests/proxy_unit_tests/test_jwt.py index 7a9d2f0019..d96fb691f7 100644 --- a/tests/proxy_unit_tests/test_jwt.py +++ b/tests/proxy_unit_tests/test_jwt.py @@ -64,7 +64,7 @@ def test_load_config_with_custom_role_names(): @pytest.mark.asyncio -async def test_token_single_public_key(): +async def test_token_single_public_key(monkeypatch): import jwt jwt_handler = JWTHandler() @@ -80,10 +80,15 @@ async def test_token_single_public_key(): ] } + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://example.com/public-key") + # set cache cache = DualCache() - await cache.async_set_cache(key="litellm_jwt_auth_keys", value=backend_keys["keys"]) + await cache.async_set_cache( + key="litellm_jwt_auth_keys_https://example.com/public-key", + value=backend_keys["keys"], + ) jwt_handler.user_api_key_cache = cache @@ -99,7 +104,7 @@ async def test_token_single_public_key(): @pytest.mark.parametrize("audience", [None, "litellm-proxy"]) @pytest.mark.asyncio -async def test_valid_invalid_token(audience): +async def test_valid_invalid_token(audience, monkeypatch): """ Tests - valid token @@ -116,6 +121,8 @@ async def test_valid_invalid_token(audience): if audience: os.environ["JWT_AUDIENCE"] = audience + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://example.com/public-key") + # Generate a private / public key pair using RSA algorithm key = rsa.generate_private_key( public_exponent=65537, key_size=2048, backend=default_backend() @@ -145,7 +152,9 @@ async def test_valid_invalid_token(audience): # set cache cache = DualCache() - await cache.async_set_cache(key="litellm_jwt_auth_keys", value=[public_jwk]) + await cache.async_set_cache( + key="litellm_jwt_auth_keys_https://example.com/public-key", value=[public_jwk] + ) jwt_handler = JWTHandler() @@ -294,7 +303,7 @@ def team_token_tuple(): @pytest.mark.parametrize("audience", [None, "litellm-proxy"]) @pytest.mark.asyncio -async def test_team_token_output(prisma_client, audience): +async def test_team_token_output(prisma_client, audience, monkeypatch): import json import uuid @@ -316,6 +325,8 @@ async def test_team_token_output(prisma_client, audience): if audience: os.environ["JWT_AUDIENCE"] = audience + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://example.com/public-key") + # Generate a private / public key pair using RSA algorithm key = rsa.generate_private_key( public_exponent=65537, key_size=2048, backend=default_backend() @@ -345,7 +356,9 @@ async def test_team_token_output(prisma_client, audience): # set cache cache = DualCache() - await cache.async_set_cache(key="litellm_jwt_auth_keys", value=[public_jwk]) + await cache.async_set_cache( + key="litellm_jwt_auth_keys_https://example.com/public-key", value=[public_jwk] + ) jwt_handler = JWTHandler() @@ -463,7 +476,7 @@ async def test_team_token_output(prisma_client, audience): @pytest.mark.parametrize("user_id_upsert", [True, False]) @pytest.mark.asyncio async def aaaatest_user_token_output( - prisma_client, audience, team_id_set, default_team_id, user_id_upsert + prisma_client, audience, team_id_set, default_team_id, user_id_upsert, monkeypatch ): import uuid @@ -528,10 +541,14 @@ async def aaaatest_user_token_output( assert isinstance(public_jwk, dict) + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://example.com/public-key") + # set cache cache = DualCache() - await cache.async_set_cache(key="litellm_jwt_auth_keys", value=[public_jwk]) + await cache.async_set_cache( + key="litellm_jwt_auth_keys_https://example.com/public-key", value=[public_jwk] + ) jwt_handler = JWTHandler() @@ -699,7 +716,9 @@ async def aaaatest_user_token_output( @pytest.mark.parametrize("admin_allowed_routes", [None, ["ui_routes"]]) @pytest.mark.parametrize("audience", [None, "litellm-proxy"]) @pytest.mark.asyncio -async def test_allowed_routes_admin(prisma_client, audience, admin_allowed_routes): +async def test_allowed_routes_admin( + prisma_client, audience, admin_allowed_routes, monkeypatch +): """ Add a check to make sure jwt proxy admin scope can access all allowed admin routes @@ -723,6 +742,8 @@ async def test_allowed_routes_admin(prisma_client, audience, admin_allowed_route setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) await litellm.proxy.proxy_server.prisma_client.connect() + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://example.com/public-key") + os.environ.pop("JWT_AUDIENCE", None) if audience: os.environ["JWT_AUDIENCE"] = audience @@ -756,7 +777,9 @@ async def test_allowed_routes_admin(prisma_client, audience, admin_allowed_route # set cache cache = DualCache() - await cache.async_set_cache(key="litellm_jwt_auth_keys", value=[public_jwk]) + await cache.async_set_cache( + key="litellm_jwt_auth_keys_https://example.com/public-key", value=[public_jwk] + ) jwt_handler = JWTHandler() @@ -910,7 +933,9 @@ def mock_user_object(*args, **kwargs): "user_email, should_work", [("ishaan@berri.ai", True), ("krrish@tassle.xyz", False)] ) @pytest.mark.asyncio -async def test_allow_access_by_email(public_jwt_key, user_email, should_work): +async def test_allow_access_by_email( + public_jwt_key, user_email, should_work, monkeypatch +): """ Allow anyone with an `@xyz.com` email make a request to the proxy. @@ -925,10 +950,14 @@ async def test_allow_access_by_email(public_jwt_key, user_email, should_work): public_jwk = public_jwt_key["public_jwk"] private_key = public_jwt_key["private_key"] + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://example.com/public-key") + # set cache cache = DualCache() - await cache.async_set_cache(key="litellm_jwt_auth_keys", value=[public_jwk]) + await cache.async_set_cache( + key="litellm_jwt_auth_keys_https://example.com/public-key", value=[public_jwk] + ) jwt_handler = JWTHandler() @@ -1074,7 +1103,7 @@ async def test_end_user_jwt_auth(monkeypatch): ] cache.set_cache( - key="litellm_jwt_auth_keys", + key="litellm_jwt_auth_keys_https://example.com/public-key", value=keys, ) From 2c5b2da9558cbda394ff669785dc41ebf89f76d5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 9 Mar 2025 18:35:10 -0700 Subject: [PATCH 22/80] fix: make type object subscriptable --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index d1c410e786..7fe5c2fb94 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -618,7 +618,7 @@ class Router: @staticmethod def _create_redis_cache( - cache_config: dict[str, Any] + cache_config: Dict[str, Any] ) -> RedisCache | RedisClusterCache: if cache_config.get("startup_nodes"): return RedisClusterCache(**cache_config) From c08705517bfcca1ad48cb6029a4899f0820ef20c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 9 Mar 2025 19:40:03 -0700 Subject: [PATCH 23/80] test: fix test --- tests/proxy_unit_tests/test_user_api_key_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index dbe49a560d..e956a22282 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -826,7 +826,7 @@ async def test_jwt_user_api_key_auth_builder_enforce_rbac(enforce_rbac, monkeypa ] local_cache.set_cache( - key="litellm_jwt_auth_keys", + key="litellm_jwt_auth_keys_my-fake-url", value=keys, ) From 666690c31cc415317cb981cae00b5664bff40f38 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Mar 2025 10:18:03 -0700 Subject: [PATCH 24/80] fix atext_completion --- litellm/main.py | 46 +++++++++++----------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 846a908a8e..903e0e938b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3900,42 +3900,18 @@ async def atext_completion( ctx = contextvars.copy_context() func_with_context = partial(ctx.run, func) - _, custom_llm_provider, _, _ = get_llm_provider( - model=model, api_base=kwargs.get("api_base", None) - ) - - if ( - custom_llm_provider == "openai" - or custom_llm_provider == "azure" - or custom_llm_provider == "azure_text" - or custom_llm_provider == "custom_openai" - or custom_llm_provider == "anyscale" - or custom_llm_provider == "mistral" - or custom_llm_provider == "openrouter" - or custom_llm_provider == "deepinfra" - or custom_llm_provider == "perplexity" - or custom_llm_provider == "groq" - or custom_llm_provider == "nvidia_nim" - or custom_llm_provider == "cerebras" - or custom_llm_provider == "sambanova" - or custom_llm_provider == "ai21_chat" - or custom_llm_provider == "ai21" - or custom_llm_provider == "volcengine" - or custom_llm_provider == "text-completion-codestral" - or custom_llm_provider == "deepseek" - or custom_llm_provider == "text-completion-openai" - or custom_llm_provider == "huggingface" - or custom_llm_provider == "ollama" - or custom_llm_provider == "vertex_ai" - or custom_llm_provider in litellm.openai_compatible_providers - ): # currently implemented aiohttp calls for just azure and openai, soon all. - # Await normally - response = await loop.run_in_executor(None, func_with_context) - if asyncio.iscoroutine(response): - response = await response + init_response = await loop.run_in_executor(None, func_with_context) + if isinstance(init_response, dict) or isinstance( + init_response, TextCompletionResponse + ): ## CACHING SCENARIO + if isinstance(init_response, dict): + response = TextCompletionResponse(**init_response) + response = init_response + elif asyncio.iscoroutine(init_response): + response = await init_response else: - # Call the synchronous function using run_in_executor - response = await loop.run_in_executor(None, func_with_context) + response = init_response # type: ignore + if ( kwargs.get("stream", False) is True or isinstance(response, TextCompletionStreamWrapper) From 6d537aec48274e8482fbd46389714bef92e22c41 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Mar 2025 10:36:50 -0700 Subject: [PATCH 25/80] OpenAI_Text --- .../components/add_model/provider_specific_fields.tsx | 3 ++- .../src/components/provider_info_helpers.tsx | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index b3da80c715..365d75dbad 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -99,7 +99,8 @@ const ProviderSpecificFields: React.FC = ({ {(selectedProviderEnum === Providers.Azure || selectedProviderEnum === Providers.Azure_AI_Studio || - selectedProviderEnum === Providers.OpenAI_Compatible + selectedProviderEnum === Providers.OpenAI_Compatible || + selectedProviderEnum === Providers.OpenAI_Text_Compatible ) && ( = { OpenAI: "openai", + OpenAI_Text: "text-completion-openai", Azure: "azure", Azure_AI_Studio: "azure_ai", Anthropic: "anthropic", @@ -37,6 +41,7 @@ export const provider_map: Record = { MistralAI: "mistral", Cohere: "cohere_chat", OpenAI_Compatible: "openai", + OpenAI_Text_Compatible: "text-completion-openai", Vertex_AI: "vertex_ai", Databricks: "databricks", xAI: "xai", @@ -53,6 +58,9 @@ export const provider_map: Record = { export const providerLogoMap: Record = { [Providers.OpenAI]: "https://artificialanalysis.ai/img/logos/openai_small.svg", + [Providers.OpenAI_Text]: "https://artificialanalysis.ai/img/logos/openai_small.svg", + [Providers.OpenAI_Text_Compatible]: "https://artificialanalysis.ai/img/logos/openai_small.svg", + [Providers.OpenAI_Compatible]: "https://artificialanalysis.ai/img/logos/openai_small.svg", [Providers.Azure]: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", [Providers.Azure_AI_Studio]: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", [Providers.Anthropic]: "https://artificialanalysis.ai/img/logos/anthropic_small.svg", @@ -61,7 +69,6 @@ export const providerLogoMap: Record = { [Providers.Groq]: "https://artificialanalysis.ai/img/logos/groq_small.png", [Providers.MistralAI]: "https://artificialanalysis.ai/img/logos/mistral_small.png", [Providers.Cohere]: "https://artificialanalysis.ai/img/logos/cohere_small.png", - [Providers.OpenAI_Compatible]: "https://upload.wikimedia.org/wikipedia/commons/4/4e/OpenAI_Logo.svg", [Providers.Vertex_AI]: "https://artificialanalysis.ai/img/logos/google_small.svg", [Providers.Databricks]: "https://artificialanalysis.ai/img/logos/databricks_small.png", [Providers.Ollama]: "https://artificialanalysis.ai/img/logos/ollama_small.svg", From 51f074682f420e11df5a468c998add1b230a0b4b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Mar 2025 10:40:48 -0700 Subject: [PATCH 26/80] show eu api base on openai + text --- .../src/components/add_model/provider_specific_fields.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index 365d75dbad..b7565b0494 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -23,7 +23,7 @@ const ProviderSpecificFields: React.FC = ({ console.log(`type of selectedProviderEnum: ${typeof selectedProviderEnum}`); return ( <> - {selectedProviderEnum === Providers.OpenAI && ( + {selectedProviderEnum === Providers.OpenAI || selectedProviderEnum === Providers.OpenAI_Text && ( <> Date: Mon, 10 Mar 2025 12:20:37 -0700 Subject: [PATCH 27/80] fix linting error --- ui/litellm-dashboard/src/components/transform_request.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/transform_request.tsx b/ui/litellm-dashboard/src/components/transform_request.tsx index 879132ef50..5d405df78f 100644 --- a/ui/litellm-dashboard/src/components/transform_request.tsx +++ b/ui/litellm-dashboard/src/components/transform_request.tsx @@ -156,7 +156,7 @@ ${formattedBody} }}>

Original Request

-

The request you would send to LiteLLM's `/chat/completions` endpoint.

+

The request you would send to LiteLLM's /chat/completions endpoint.