From 3677d56e9e1ed5e0b28c1efcde919ab5f79cae1a Mon Sep 17 00:00:00 2001 From: Vince Loewe Date: Fri, 3 May 2024 17:42:50 +0100 Subject: [PATCH 01/56] Lunary: Fix tool calling --- litellm/integrations/lunary.py | 36 ++++++++++++++++++++++++------ litellm/tests/test_lunary.py | 40 ++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 6ddf2ca599..6b23f09875 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -4,7 +4,6 @@ from datetime import datetime, timezone import traceback import dotenv import importlib -import sys import packaging @@ -18,13 +17,33 @@ def parse_usage(usage): "prompt": usage["prompt_tokens"] if "prompt_tokens" in usage else 0, } +def parse_tool_calls(tool_calls): + if tool_calls is None: + return None + + def clean_tool_call(tool_call): + + serialized = { + "type": tool_call.type, + "id": tool_call.id, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + } + } + + return serialized + + return [clean_tool_call(tool_call) for tool_call in tool_calls] + def parse_messages(input): + if input is None: return None def clean_message(message): - # if is strin, return as is + # if is string, return as is if isinstance(message, str): return message @@ -38,9 +57,7 @@ def parse_messages(input): # Only add tool_calls and function_call to res if they are set if message.get("tool_calls"): - serialized["tool_calls"] = message.get("tool_calls") - if message.get("function_call"): - serialized["function_call"] = message.get("function_call") + serialized["tool_calls"] = parse_tool_calls(message.get("tool_calls")) return serialized @@ -93,8 +110,13 @@ class LunaryLogger: print_verbose(f"Lunary Logging - Logging request for model {model}") litellm_params = kwargs.get("litellm_params", {}) + optional_params = kwargs.get("optional_params", {}) metadata = litellm_params.get("metadata", {}) or {} + if optional_params: + # merge into extra + extra = {**extra, **optional_params} + tags = litellm_params.pop("tags", None) or [] if extra: @@ -104,7 +126,7 @@ class LunaryLogger: # keep only serializable types for param, value in extra.items(): - if not isinstance(value, (str, int, bool, float)): + if not isinstance(value, (str, int, bool, float)) and param != "tools": try: extra[param] = str(value) except: @@ -140,7 +162,7 @@ class LunaryLogger: metadata=metadata, runtime="litellm", tags=tags, - extra=extra, + params=extra, ) self.lunary_client.track_event( diff --git a/litellm/tests/test_lunary.py b/litellm/tests/test_lunary.py index cbf9364aff..c9a8afd57f 100644 --- a/litellm/tests/test_lunary.py +++ b/litellm/tests/test_lunary.py @@ -11,7 +11,6 @@ litellm.failure_callback = ["lunary"] litellm.success_callback = ["lunary"] litellm.set_verbose = True - def test_lunary_logging(): try: response = completion( @@ -59,9 +58,46 @@ def test_lunary_logging_with_metadata(): except Exception as e: print(e) +#test_lunary_logging_with_metadata() -# test_lunary_logging_with_metadata() +def test_lunary_with_tools(): + import litellm + + messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + + response = litellm.completion( + model="gpt-3.5-turbo-1106", + messages=messages, + tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit + ) + + response_message = response.choices[0].message + print("\nLLM Response:\n", response.choices[0].message) + + +#test_lunary_with_tools() def test_lunary_logging_with_streaming_and_metadata(): try: From 4b655d8b3343be0d5a537be59e58c12f29820c89 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 15:02:37 +0000 Subject: [PATCH 02/56] feat(util.py): Add OIDC support. --- litellm/tests/test_secret_manager.py | 33 +++++++++++++++++ litellm/utils.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/litellm/tests/test_secret_manager.py b/litellm/tests/test_secret_manager.py index 3ea38f8061..892a8831cf 100644 --- a/litellm/tests/test_secret_manager.py +++ b/litellm/tests/test_secret_manager.py @@ -23,3 +23,36 @@ def test_aws_secret_manager(): print(f"secret_val: {secret_val}") assert secret_val == "sk-1234" + + +def redact_oidc_signature(secret_val): + # remove the last part of `.` and replace it with "SIGNATURE_REMOVED" + return secret_val.split(".")[:-1] + ["SIGNATURE_REMOVED"] + + +@pytest.mark.skipif(os.environ.get('K_SERVICE') is None, reason="Cannot run without being in GCP Cloud Run") +def test_oidc_google(): + secret_val = get_secret("oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + +@pytest.mark.skipif(os.environ.get('ACTIONS_ID_TOKEN_REQUEST_TOKEN') is None, reason="Cannot run without being in GitHub Actions") +def test_oidc_github(): + secret_val = get_secret("oidc/github/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + +@pytest.mark.skipif(os.environ.get('CIRCLE_OIDC_TOKEN') is None, reason="Cannot run without being in a CircleCI Runner") +def test_oidc_circleci(): + secret_val = get_secret("oidc/circleci/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + +@pytest.mark.skipif(os.environ.get('CIRCLE_OIDC_TOKEN_V2') is None, reason="Cannot run without being in a CircleCI Runner") +def test_oidc_circleci_v2(): + secret_val = get_secret("oidc/circleci_v2/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") diff --git a/litellm/utils.py b/litellm/utils.py index a938a0ba84..652406f64e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -33,6 +33,7 @@ from dataclasses import ( ) import litellm._service_logger # for storing API inputs, outputs, and metadata +from litellm.llms.custom_httpx.http_handler import HTTPHandler try: # this works in python 3.8 @@ -9288,6 +9289,59 @@ def get_secret( if secret_name.startswith("os.environ/"): secret_name = secret_name.replace("os.environ/", "") + # Example: oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/stability.stable-diffusion-xl-v1/invoke + if secret_name.startswith("oidc/"): + secret_name = secret_name.replace("oidc/", "") + oidc_provider, oidc_aud = secret_name.split("/", 1) + # TODO: Add caching for HTTP requests + match oidc_provider: + case "google": + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature + response = client.get( + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity", + params={"audience": oidc_aud}, + headers={"Metadata-Flavor": "Google"}, + ) + if response.status_code == 200: + return response.text + else: + raise ValueError("Google OIDC provider failed") + case "circleci": + # https://circleci.com/docs/openid-connect-tokens/ + env_secret = os.getenv("CIRCLE_OIDC_TOKEN") + if env_secret is None: + raise ValueError("CIRCLE_OIDC_TOKEN not found in environment") + return env_secret + case "circleci_v2": + # https://circleci.com/docs/openid-connect-tokens/ + env_secret = os.getenv("CIRCLE_OIDC_TOKEN_V2") + if env_secret is None: + raise ValueError("CIRCLE_OIDC_TOKEN_V2 not found in environment") + return env_secret + case "github": + # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers#using-custom-actions + actions_id_token_request_url = os.getenv("ACTIONS_ID_TOKEN_REQUEST_URL") + actions_id_token_request_token = os.getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN") + if actions_id_token_request_url is None or actions_id_token_request_token is None: + raise ValueError("ACTIONS_ID_TOKEN_REQUEST_URL or ACTIONS_ID_TOKEN_REQUEST_TOKEN not found in environment") + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + response = client.get( + actions_id_token_request_url, + params={"audience": oidc_aud}, + headers={ + "Authorization": f"Bearer {actions_id_token_request_token}", + "Accept": "application/json; api-version=2.0", + }, + ) + if response.status_code == 200: + return response.text['value'] + else: + raise ValueError("Github OIDC provider failed") + case _: + raise ValueError("Unsupported OIDC provider") + + try: if litellm.secret_manager_client is not None: try: From 3ee0328b044e84e6bd7c3085681a468e69c3c646 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 15:01:46 +0000 Subject: [PATCH 03/56] feat(bedrock.py): Support using OIDC tokens. --- litellm/llms/bedrock.py | 42 +++++++++++++++++++++++- litellm/tests/test_bedrock_completion.py | 29 ++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 2f26ae4a9a..fe0aca44e7 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -550,6 +550,7 @@ def init_bedrock_client( aws_session_name: Optional[str] = None, aws_profile_name: Optional[str] = None, aws_role_name: Optional[str] = None, + aws_web_identity_token: Optional[str] = None, extra_headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, ): @@ -566,6 +567,7 @@ def init_bedrock_client( aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ] # Iterate over parameters and update if needed @@ -581,6 +583,7 @@ def init_bedrock_client( aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ) = params_to_check ### SET REGION NAME @@ -619,7 +622,38 @@ def init_bedrock_client( config = boto3.session.Config() ### CHECK STS ### - if aws_role_name is not None and aws_session_name is not None: + if aws_web_identity_token is not None and aws_role_name is not None and aws_session_name is not None: + oidc_token = get_secret(aws_web_identity_token) + + if oidc_token is None: + raise BedrockError( + message="OIDC token could not be retrieved from secret manager.", + status_code=401, + ) + + sts_client = boto3.client( + "sts" + ) + + # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html + sts_response = sts_client.assume_role_with_web_identity( + RoleArn=aws_role_name, + RoleSessionName=aws_session_name, + WebIdentityToken=oidc_token, + DurationSeconds=3600, + ) + + client = boto3.client( + service_name="bedrock-runtime", + aws_access_key_id=sts_response["Credentials"]["AccessKeyId"], + aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"], + aws_session_token=sts_response["Credentials"]["SessionToken"], + region_name=region_name, + endpoint_url=endpoint_url, + config=config, + ) + elif aws_role_name is not None and aws_session_name is not None: # use sts if role name passed in sts_client = boto3.client( "sts", @@ -752,6 +786,7 @@ def completion( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = optional_params.pop("aws_bedrock_client", None) @@ -766,6 +801,7 @@ def completion( aws_role_name=aws_role_name, aws_session_name=aws_session_name, aws_profile_name=aws_profile_name, + aws_web_identity_token=aws_web_identity_token, extra_headers=extra_headers, timeout=timeout, ) @@ -1288,6 +1324,7 @@ def embedding( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = init_bedrock_client( @@ -1295,6 +1332,7 @@ def embedding( aws_secret_access_key=aws_secret_access_key, aws_region_name=aws_region_name, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_web_identity_token=aws_web_identity_token, aws_role_name=aws_role_name, aws_session_name=aws_session_name, ) @@ -1377,6 +1415,7 @@ def image_generation( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = init_bedrock_client( @@ -1384,6 +1423,7 @@ def image_generation( aws_secret_access_key=aws_secret_access_key, aws_region_name=aws_region_name, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_web_identity_token=aws_web_identity_token, aws_role_name=aws_role_name, aws_session_name=aws_session_name, timeout=timeout, diff --git a/litellm/tests/test_bedrock_completion.py b/litellm/tests/test_bedrock_completion.py index 3f5c831d73..ef6774fd2f 100644 --- a/litellm/tests/test_bedrock_completion.py +++ b/litellm/tests/test_bedrock_completion.py @@ -206,6 +206,35 @@ def test_completion_bedrock_claude_sts_client_auth(): # test_completion_bedrock_claude_sts_client_auth() +@pytest.mark.skipif(os.environ.get('CIRCLE_OIDC_TOKEN_V2') is None, reason="CIRCLE_OIDC_TOKEN_V2 is not set") +def test_completion_bedrock_claude_sts_oidc_auth(): + print("\ncalling bedrock claude with oidc auth") + import os + + aws_web_identity_token = "oidc/circleci_v2/" + aws_region_name = os.environ["AWS_REGION_NAME"] + aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"] + + try: + litellm.set_verbose = True + + response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=messages, + max_tokens=10, + temperature=0.1, + aws_region_name=aws_region_name, + aws_web_identity_token=aws_web_identity_token, + aws_role_name=aws_role_name, + aws_session_name="my-test-session", + ) + # Add any assertions here to check the response + print(response) + except RateLimitError: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + def test_bedrock_extra_headers(): try: From d5767e940302f31defa9f640f59beacf060e666f Mon Sep 17 00:00:00 2001 From: Jean-Luc Duckworth Date: Tue, 7 May 2024 15:42:06 -0400 Subject: [PATCH 04/56] Expanding jwt access to other RS and PS algos. Updated to resolve merge conflicts. --- litellm/proxy/auth/handle_jwt.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 606ff68281..9c846e8f66 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -156,6 +156,11 @@ class JWTHandler: return public_key async def auth_jwt(self, token: str) -> dict: + # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html + # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret + # the key in different ways (e.g. HS* and RS*)." + algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512"], + audience = os.getenv("JWT_AUDIENCE") decode_options = None if audience is None: @@ -189,7 +194,7 @@ class JWTHandler: payload = jwt.decode( token, public_key_rsa, # type: ignore - algorithms=["RS256"], + algorithms=algorithms, options=decode_options, audience=audience, ) @@ -214,7 +219,7 @@ class JWTHandler: payload = jwt.decode( token, key, - algorithms=["RS256"], + algorithms=algorithms, audience=audience, options=decode_options ) From e268354acc3dbdbf0a2313aa7fd87fd485774b34 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 19:18:28 +0000 Subject: [PATCH 05/56] feat(azure.py): Support OIDC auth --- litellm/llms/azure.py | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index e7af9d43b6..c2bbe54c15 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -8,6 +8,7 @@ from litellm.utils import ( CustomStreamWrapper, convert_to_model_response_object, TranscriptionResponse, + get_secret, ) from typing import Callable, Optional, BinaryIO from litellm import OpenAIConfig @@ -16,6 +17,7 @@ import httpx from .custom_httpx.azure_dall_e_2 import CustomHTTPTransport, AsyncCustomHTTPTransport from openai import AzureOpenAI, AsyncAzureOpenAI import uuid +import os class AzureOpenAIError(Exception): @@ -126,6 +128,51 @@ def select_azure_base_url_or_endpoint(azure_client_params: dict): return azure_client_params +def get_azure_ad_token_from_oidc(azure_ad_token: str): + azure_client_id = os.getenv("AZURE_CLIENT_ID", None) + azure_tenant = os.getenv("AZURE_TENANT_ID", None) + + if azure_client_id is None or azure_tenant is None: + raise AzureOpenAIError( + status_code=422, + message="AZURE_CLIENT_ID and AZURE_TENANT_ID must be set", + ) + + oidc_token = get_secret(azure_ad_token) + + if oidc_token is None: + raise AzureOpenAIError( + status_code=401, + message="OIDC token could not be retrieved from secret manager.", + ) + + req_token = httpx.get( + f"https://login.microsoftonline.com/{azure_tenant}/oauth2/v2.0/token", + data={ + "client_id": azure_client_id, + "grant_type": "client_credentials", + "scope": "https://cognitiveservices.azure.com/.default", + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": oidc_token, + }, + ) + + if req_token.status_code != 200: + raise AzureOpenAIError( + status_code=req_token.status_code, + message=req_token.text, + ) + + possible_azure_ad_token = req_token.json().get("access_token", None) + + if possible_azure_ad_token is None: + raise AzureOpenAIError( + status_code=422, message="Azure AD Token not returned" + ) + + return possible_azure_ad_token + + class AzureChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() @@ -137,6 +184,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: headers["api-key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) headers["Authorization"] = f"Bearer {azure_ad_token}" return headers @@ -189,6 +238,9 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) + azure_client_params["azure_ad_token"] = azure_ad_token if acompletion is True: @@ -276,6 +328,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AzureOpenAI(**azure_client_params) @@ -351,6 +405,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token # setting Azure client @@ -422,6 +478,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AzureOpenAI(**azure_client_params) @@ -478,6 +536,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AsyncAzureOpenAI(**azure_client_params) @@ -599,6 +659,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token ## LOGGING @@ -755,6 +817,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if aimg_generation == True: @@ -833,6 +897,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if max_retries is not None: From 9a0bb36865a3ed10325eab67f5d74169fd152891 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 19:43:18 +0000 Subject: [PATCH 06/56] fix+feat(router.py): Fix missing azure_ad_token, and allow use OIDC auth --- litellm/router.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 4353da804b..24a926e5d9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -45,6 +45,7 @@ from litellm.types.router import ( RetryPolicy, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.azure import get_azure_ad_token_from_oidc class Router: @@ -2089,6 +2090,10 @@ class Router: raise ValueError( f"api_base is required for Azure OpenAI. Set it on your config. Model - {model}" ) + azure_ad_token = litellm_params.get("azure_ad_token") + if azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) if api_version is None: api_version = "2023-07-01-preview" if "gateway.ai.cloudflare.com" in api_base: @@ -2099,6 +2104,7 @@ class Router: cache_key = f"{model_id}_async_client" _client = openai.AsyncAzureOpenAI( api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=timeout, @@ -2123,6 +2129,7 @@ class Router: cache_key = f"{model_id}_client" _client = openai.AzureOpenAI( # type: ignore api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=timeout, @@ -2147,6 +2154,7 @@ class Router: cache_key = f"{model_id}_stream_async_client" _client = openai.AsyncAzureOpenAI( # type: ignore api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=stream_timeout, @@ -2171,6 +2179,7 @@ class Router: cache_key = f"{model_id}_stream_client" _client = openai.AzureOpenAI( # type: ignore api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=stream_timeout, From cb49fb004d7bba866d229760931256bf96971f8e Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 19:51:57 +0000 Subject: [PATCH 07/56] fix(azure.py): Correct invalid .get to a .post for OIDC --- litellm/llms/azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index c2bbe54c15..1e807b5e7d 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -146,7 +146,7 @@ def get_azure_ad_token_from_oidc(azure_ad_token: str): message="OIDC token could not be retrieved from secret manager.", ) - req_token = httpx.get( + req_token = httpx.post( f"https://login.microsoftonline.com/{azure_tenant}/oauth2/v2.0/token", data={ "client_id": azure_client_id, From d60aa8282eab4f1a80e900aafe21e13cc0ab17d7 Mon Sep 17 00:00:00 2001 From: Jean-Luc Duckworth Date: Tue, 7 May 2024 16:08:36 -0400 Subject: [PATCH 08/56] Fixed typo. test_jwt.py tests pass --- litellm/proxy/auth/handle_jwt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 9c846e8f66..18c0d7b2ce 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -159,7 +159,7 @@ class JWTHandler: # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret # the key in different ways (e.g. HS* and RS*)." - algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512"], + algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512"] audience = os.getenv("JWT_AUDIENCE") decode_options = None From 44b1b219115f9d4804ca9dec2b82441059a3c128 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 21:20:15 +0000 Subject: [PATCH 09/56] feat(utils.py) - Add OIDC caching for Google Cloud Run and GitHub Actions. --- litellm/utils.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 652406f64e..f5d3b974b6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -34,6 +34,8 @@ from dataclasses import ( import litellm._service_logger # for storing API inputs, outputs, and metadata from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.caching import DualCache +oidc_cache = DualCache() try: # this works in python 3.8 @@ -9291,11 +9293,15 @@ def get_secret( # Example: oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/stability.stable-diffusion-xl-v1/invoke if secret_name.startswith("oidc/"): - secret_name = secret_name.replace("oidc/", "") - oidc_provider, oidc_aud = secret_name.split("/", 1) + secret_name_split = secret_name.replace("oidc/", "") + oidc_provider, oidc_aud = secret_name_split.split("/", 1) # TODO: Add caching for HTTP requests match oidc_provider: case "google": + oidc_token = oidc_cache.get_cache(key=secret_name) + if oidc_token is not None: + return oidc_token + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature response = client.get( @@ -9304,7 +9310,9 @@ def get_secret( headers={"Metadata-Flavor": "Google"}, ) if response.status_code == 200: - return response.text + oidc_token = response.text + oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=3600 - 60) + return oidc_token else: raise ValueError("Google OIDC provider failed") case "circleci": @@ -9325,6 +9333,11 @@ def get_secret( actions_id_token_request_token = os.getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN") if actions_id_token_request_url is None or actions_id_token_request_token is None: raise ValueError("ACTIONS_ID_TOKEN_REQUEST_URL or ACTIONS_ID_TOKEN_REQUEST_TOKEN not found in environment") + + oidc_token = oidc_cache.get_cache(key=secret_name) + if oidc_token is not None: + return oidc_token + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) response = client.get( actions_id_token_request_url, @@ -9335,7 +9348,9 @@ def get_secret( }, ) if response.status_code == 200: - return response.text['value'] + oidc_token = response.text['value'] + oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=300 - 5) + return oidc_token else: raise ValueError("Github OIDC provider failed") case _: From 77b3acb396269ac31dc1916c17ff2b9e430d1585 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 8 May 2024 14:34:45 +0000 Subject: [PATCH 10/56] fix(router.py): Add missing azure_ad_token param. --- litellm/router.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/router.py b/litellm/router.py index 24a926e5d9..ea8d17286f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2212,6 +2212,7 @@ class Router: "api_key": api_key, "azure_endpoint": api_base, "api_version": api_version, + "azure_ad_token": azure_ad_token, } from litellm.llms.azure import select_azure_base_url_or_endpoint From fc51a3631e235c11fbbcee4fbb0e013c8b41f69a Mon Sep 17 00:00:00 2001 From: Merlinvt Date: Thu, 9 May 2024 15:16:34 +0200 Subject: [PATCH 11/56] add additional models from openrouter --- model_prices_and_context_window.json | 138 ++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 1 deletion(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 10c70a858d..dbd812038f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1571,6 +1571,142 @@ "litellm_provider": "replicate", "mode": "chat" }, + "openrouter/mistralai/mixtral-8x22b-instruct": { + "max_tokens": 65536, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000065, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/microsoft/wizardlm-2-8x22b:nitro": { + "max_tokens": 65536, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/google/gemini-pro-1.5": { + "max_tokens": 8192, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.0000075, + "input_cost_per_image": 0.00265, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/mistralai/mixtral-8x22b-instruct": { + "max_tokens": 65536, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000065, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/cohere/command-r-plus": { + "max_tokens": 128000, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/databricks/dbrx-instruct": { + "max_tokens": 32768, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000006, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/anthropic/claude-3-haiku": { + "max_tokens": 200000, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "input_cost_per_image": 0.0004, // Calculated for per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/anthropic/claude-3-sonnet": { + "max_tokens": 200000, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "input_cost_per_image": 0.0048, // Per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/mistralai/mistral-large": { + "max_tokens": 32000, + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { + "max_tokens": 32769, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000005, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/google/gemini-pro-vision": { + "max_tokens": 45875, + "input_cost_per_token": 0.000000125, + "output_cost_per_token": 0.000000375, + "input_cost_per_image": 0.0025, // Per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/fireworks/firellava-13b": { + "max_tokens": 4096, + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000002, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-8b-instruct:free": { + "max_tokens": 8200, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-8b-instruct:extended": { + "max_tokens": 16000, + "input_cost_per_token": 0.000000225, + "output_cost_per_token": 0.00000225, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-70b-instruct:nitro": { + "max_tokens": 8200, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-70b-instruct": { + "max_tokens": 8200, + "input_cost_per_token": 0.00000059, + "output_cost_per_token": 0.00000079, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/openai/gpt-4-vision-preview": { + "max_tokens": 130000, + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "input_cost_per_image": 0.01445, // Per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, "openrouter/openai/gpt-3.5-turbo": { "max_tokens": 4095, "input_cost_per_token": 0.0000015, @@ -3226,4 +3362,4 @@ "mode": "embedding" } -} +} \ No newline at end of file From ccdd2046af00e420983ee655d56aa84d38315ce4 Mon Sep 17 00:00:00 2001 From: Merlinvt Date: Thu, 9 May 2024 15:20:32 +0200 Subject: [PATCH 12/56] fixes --- model_prices_and_context_window.json | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dbd812038f..2f575757e5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1571,13 +1571,6 @@ "litellm_provider": "replicate", "mode": "chat" }, - "openrouter/mistralai/mixtral-8x22b-instruct": { - "max_tokens": 65536, - "input_cost_per_token": 0.00000065, - "output_cost_per_token": 0.00000065, - "litellm_provider": "openrouter", - "mode": "chat" - }, "openrouter/microsoft/wizardlm-2-8x22b:nitro": { "max_tokens": 65536, "input_cost_per_token": 0.000001, @@ -1622,7 +1615,7 @@ "max_tokens": 200000, "input_cost_per_token": 0.00000025, "output_cost_per_token": 0.00000125, - "input_cost_per_image": 0.0004, // Calculated for per 1000 images. + "input_cost_per_image": 0.0004, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1632,7 +1625,7 @@ "max_tokens": 200000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, - "input_cost_per_image": 0.0048, // Per 1000 images. + "input_cost_per_image": 0.0048, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1656,7 +1649,7 @@ "max_tokens": 45875, "input_cost_per_token": 0.000000125, "output_cost_per_token": 0.000000375, - "input_cost_per_image": 0.0025, // Per 1000 images. + "input_cost_per_image": 0.0025, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1701,7 +1694,7 @@ "max_tokens": 130000, "input_cost_per_token": 0.00001, "output_cost_per_token": 0.00003, - "input_cost_per_image": 0.01445, // Per 1000 images. + "input_cost_per_image": 0.01445, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1847,13 +1840,6 @@ "litellm_provider": "openrouter", "mode": "chat" }, - "openrouter/meta-llama/llama-3-70b-instruct": { - "max_tokens": 8192, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000008, - "litellm_provider": "openrouter", - "mode": "chat" - }, "j2-ultra": { "max_tokens": 8192, "max_input_tokens": 8192, From 265d777894a16a4daad03b18a91341229ecae788 Mon Sep 17 00:00:00 2001 From: Merlinvt Date: Thu, 9 May 2024 15:27:14 +0200 Subject: [PATCH 13/56] fixes 2 --- model_prices_and_context_window.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2f575757e5..1ade08fe35 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1663,28 +1663,28 @@ "mode": "chat" }, "openrouter/meta-llama/llama-3-8b-instruct:free": { - "max_tokens": 8200, + "max_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/meta-llama/llama-3-8b-instruct:extended": { - "max_tokens": 16000, + "max_tokens": 16384, "input_cost_per_token": 0.000000225, "output_cost_per_token": 0.00000225, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/meta-llama/llama-3-70b-instruct:nitro": { - "max_tokens": 8200, + "max_tokens": 8192, "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/meta-llama/llama-3-70b-instruct": { - "max_tokens": 8200, + "max_tokens": 8192, "input_cost_per_token": 0.00000059, "output_cost_per_token": 0.00000079, "litellm_provider": "openrouter", @@ -1750,14 +1750,14 @@ "tool_use_system_prompt_tokens": 395 }, "openrouter/google/palm-2-chat-bison": { - "max_tokens": 8000, + "max_tokens": 25804, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/google/palm-2-codechat-bison": { - "max_tokens": 8000, + "max_tokens": 20070, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", From 9a31f3d3d93b21eef8bcae45c0744a47c4c38ed9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 07:57:41 -0700 Subject: [PATCH 14/56] fix(main.py): support env var 'VERTEX_PROJECT' and 'VERTEX_LOCATION' --- litellm/main.py | 3 ++ .../tests/test_amazing_vertex_completion.py | 43 +++++++++++++++++++ litellm/utils.py | 4 +- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index aa078d322d..6fd4cdaab4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2960,17 +2960,20 @@ def embedding( or optional_params.pop("vertex_ai_project", None) or litellm.vertex_project or get_secret("VERTEXAI_PROJECT") + or get_secret("VERTEX_PROJECT") ) vertex_ai_location = ( optional_params.pop("vertex_location", None) or optional_params.pop("vertex_ai_location", None) or litellm.vertex_location or get_secret("VERTEXAI_LOCATION") + or get_secret("VERTEX_LOCATION") ) vertex_credentials = ( optional_params.pop("vertex_credentials", None) or optional_params.pop("vertex_ai_credentials", None) or get_secret("VERTEXAI_CREDENTIALS") + or get_secret("VERTEX_CREDENTIALS") ) response = vertex_ai.embedding( diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 1d79653ea6..91fd444742 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -113,6 +113,49 @@ async def get_response(): ], ) return response + + except litellm.UnprocessableEntityError as e: + pass + except Exception as e: + pytest.fail(f"An error occurred - {str(e)}") + + +@pytest.mark.asyncio +async def test_get_router_response(): + model = "claude-3-sonnet@20240229" + vertex_ai_project = "adroit-crow-413218" + vertex_ai_location = "asia-southeast1" + json_obj = get_vertex_ai_creds_json() + vertex_credentials = json.dumps(json_obj) + + prompt = '\ndef count_nums(arr):\n """\n Write a function count_nums which takes an array of integers and returns\n the number of elements which has a sum of digits > 0.\n If a number is negative, then its first signed digit will be negative:\n e.g. -123 has signed digits -1, 2, and 3.\n >>> count_nums([]) == 0\n >>> count_nums([-1, 11, -11]) == 1\n >>> count_nums([1, 1, 2]) == 3\n """\n' + try: + router = litellm.Router( + model_list=[ + { + "model_name": "sonnet", + "litellm_params": { + "model": "vertex_ai/claude-3-sonnet@20240229", + "vertex_ai_project": vertex_ai_project, + "vertex_ai_location": vertex_ai_location, + "vertex_credentials": vertex_credentials, + }, + } + ] + ) + response = await router.acompletion( + model="sonnet", + messages=[ + { + "role": "system", + "content": "Complete the given code with no more explanation. Remember that there is a 4-space indent before the first line of your generated code.", + }, + {"role": "user", "content": prompt}, + ], + ) + + print(f"\n\nResponse: {response}\n\n") + except litellm.UnprocessableEntityError as e: pass except Exception as e: diff --git a/litellm/utils.py b/litellm/utils.py index 206001dbb1..838d0fe553 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5769,9 +5769,7 @@ def get_optional_params( extra_body # openai client supports `extra_body` param ) else: # assume passing in params for openai/azure openai - print_verbose( - f"UNMAPPED PROVIDER, ASSUMING IT'S OPENAI/AZURE - model={model}, custom_llm_provider={custom_llm_provider}" - ) + supported_params = get_supported_openai_params( model=model, custom_llm_provider="openai" ) From f9a0364bffaa73b7cefb2d0375a9313b019cdf4e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 08:34:01 -0700 Subject: [PATCH 15/56] =?UTF-8?q?bump:=20version=201.37.0=20=E2=86=92=201.?= =?UTF-8?q?37.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9854cf692..835dfb3bf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.37.0" +version = "1.37.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.37.0" +version = "1.37.1" version_files = [ "pyproject.toml:^version" ] From 40e19a838cdf123922d4cd861b4372ad493a843f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 08:40:31 -0700 Subject: [PATCH 16/56] =?UTF-8?q?bump:=20version=201.37.1=20=E2=86=92=201.?= =?UTF-8?q?37.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 835dfb3bf7..d09489e62d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.37.1" +version = "1.37.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.37.1" +version = "1.37.2" version_files = [ "pyproject.toml:^version" ] From cdec7a414f6376fbde82816e4ac4dece6d4c1e7c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 09:58:40 -0700 Subject: [PATCH 17/56] test(test_router_fallbacks.py): fix test --- litellm/main.py | 1 + litellm/router.py | 1 + litellm/tests/test_custom_logger.py | 3 ++- litellm/tests/test_router_fallbacks.py | 20 +++++++++++--------- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 6fd4cdaab4..72f5b1dc63 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -662,6 +662,7 @@ def completion( "region_name", "allowed_model_region", ] + default_params = openai_params + litellm_params non_default_params = { k: v for k, v in kwargs.items() if k not in default_params diff --git a/litellm/router.py b/litellm/router.py index 68f49a0a0c..39d49a1474 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -102,6 +102,7 @@ class Router: "usage-based-routing", "latency-based-routing", "cost-based-routing", + "usage-based-routing-v2", ] = "simple-shuffle", routing_strategy_args: dict = {}, # just for latency-based routing semaphore: Optional[asyncio.Semaphore] = None, diff --git a/litellm/tests/test_custom_logger.py b/litellm/tests/test_custom_logger.py index 9c2afe5a3a..c7df312146 100644 --- a/litellm/tests/test_custom_logger.py +++ b/litellm/tests/test_custom_logger.py @@ -437,8 +437,9 @@ async def test_cost_tracking_with_caching(): max_tokens=40, temperature=0.2, caching=True, + mock_response="Hey, i'm doing well!", ) - await asyncio.sleep(1) # success callback is async + await asyncio.sleep(3) # success callback is async response_cost = customHandler_optional_params.response_cost assert response_cost > 0 response2 = await litellm.acompletion( diff --git a/litellm/tests/test_router_fallbacks.py b/litellm/tests/test_router_fallbacks.py index 0e001eeba9..0bce9894b7 100644 --- a/litellm/tests/test_router_fallbacks.py +++ b/litellm/tests/test_router_fallbacks.py @@ -754,6 +754,9 @@ async def test_async_fallbacks_max_retries_per_request(): def test_ausage_based_routing_fallbacks(): try: + import litellm + + litellm.set_verbose = False # [Prod Test] # IT tests Usage Based Routing with fallbacks # The Request should fail azure/gpt-4-fast. Then fallback -> "azure/gpt-4-basic" -> "openai-gpt-4" @@ -766,10 +769,10 @@ def test_ausage_based_routing_fallbacks(): load_dotenv() # Constants for TPM and RPM allocation - AZURE_FAST_RPM = 0 - AZURE_BASIC_RPM = 0 + AZURE_FAST_RPM = 1 + AZURE_BASIC_RPM = 1 OPENAI_RPM = 0 - ANTHROPIC_RPM = 2 + ANTHROPIC_RPM = 10 def get_azure_params(deployment_name: str): params = { @@ -832,9 +835,9 @@ def test_ausage_based_routing_fallbacks(): fallbacks=fallbacks_list, set_verbose=True, debug_level="DEBUG", - routing_strategy="usage-based-routing", + routing_strategy="usage-based-routing-v2", redis_host=os.environ["REDIS_HOST"], - redis_port=os.environ["REDIS_PORT"], + redis_port=int(os.environ["REDIS_PORT"]), num_retries=0, ) @@ -853,8 +856,8 @@ def test_ausage_based_routing_fallbacks(): # the token count of this message is > AZURE_FAST_TPM, > AZURE_BASIC_TPM assert response._hidden_params["model_id"] == "1" - # now make 100 mock requests to OpenAI - expect it to fallback to anthropic-claude-instant-1.2 - for i in range(3): + for i in range(10): + # now make 100 mock requests to OpenAI - expect it to fallback to anthropic-claude-instant-1.2 response = router.completion( model="azure/gpt-4-fast", messages=messages, @@ -863,8 +866,7 @@ def test_ausage_based_routing_fallbacks(): ) print("response: ", response) print("response._hidden_params: ", response._hidden_params) - if i == 2: - # by the 19th call we should have hit TPM LIMIT for OpenAI, it should fallback to anthropic-claude-instant-1.2 + if i == 9: assert response._hidden_params["model_id"] == "4" except Exception as e: From 781d5888c39107058ae96966d114ce70cfe99581 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 10:58:35 -0700 Subject: [PATCH 18/56] docs(predibase.md): add support for predibase to docs --- docs/my-website/docs/providers/predibase.md | 247 ++++++++++++++++++++ docs/my-website/sidebars.js | 4 +- litellm/tests/test_completion.py | 1 - 3 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 docs/my-website/docs/providers/predibase.md diff --git a/docs/my-website/docs/providers/predibase.md b/docs/my-website/docs/providers/predibase.md new file mode 100644 index 0000000000..3d5bbaef41 --- /dev/null +++ b/docs/my-website/docs/providers/predibase.md @@ -0,0 +1,247 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 🆕 Predibase + +LiteLLM supports all models on Predibase + + +## Usage + + + + +### API KEYS +```python +import os +os.environ["PREDIBASE_API_KEY"] = "" +``` + +### Example Call + +```python +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" +os.environ["PREDIBASE_TENANT_ID"] = "predibase tenant id" + +# predibase llama-3 call +response = completion( + model="predibase/llama-3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + tenant_id: os.environ/PREDIBASE_TENANT_ID + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + client = openai.OpenAI( + api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="llama-3", + messages = [ + { + "role": "system", + "content": "Be a good human!" + }, + { + "role": "user", + "content": "What do you know about earth?" + } + ] + ) + + print(response) + ``` + + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "llama-3", + "messages": [ + { + "role": "system", + "content": "Be a good human!" + }, + { + "role": "user", + "content": "What do you know about earth?" + } + ], + }' + ``` + + + + + + + + + +## Advanced Usage - Prompt Formatting + +LiteLLM has prompt template mappings for all `meta-llama` llama3 instruct models. [**See Code**](https://github.com/BerriAI/litellm/blob/4f46b4c3975cd0f72b8c5acb2cb429d23580c18a/litellm/llms/prompt_templates/factory.py#L1360) + +To apply a custom prompt template: + + + + +```python +import litellm + +import os +os.environ["PREDIBASE_API_KEY"] = "" + +# Create your own custom prompt template +litellm.register_prompt_template( + model="togethercomputer/LLaMA-2-7B-32K", + initial_prompt_value="You are a good assistant" # [OPTIONAL] + roles={ + "system": { + "pre_message": "[INST] <>\n", # [OPTIONAL] + "post_message": "\n<>\n [/INST]\n" # [OPTIONAL] + }, + "user": { + "pre_message": "[INST] ", # [OPTIONAL] + "post_message": " [/INST]" # [OPTIONAL] + }, + "assistant": { + "pre_message": "\n" # [OPTIONAL] + "post_message": "\n" # [OPTIONAL] + } + } + final_prompt_value="Now answer as best you can:" # [OPTIONAL] +) + +def predibase_custom_model(): + model = "predibase/togethercomputer/LLaMA-2-7B-32K" + response = completion(model=model, messages=messages) + print(response['choices'][0]['message']['content']) + return response + +predibase_custom_model() +``` + + + +```yaml +# Model-specific parameters +model_list: + - model_name: mistral-7b # model alias + litellm_params: # actual params for litellm.completion() + model: "predibase/mistralai/Mistral-7B-Instruct-v0.1" + api_key: os.environ/PREDIBASE_API_KEY + initial_prompt_value: "\n" + roles: {"system":{"pre_message":"<|im_start|>system\n", "post_message":"<|im_end|>"}, "assistant":{"pre_message":"<|im_start|>assistant\n","post_message":"<|im_end|>"}, "user":{"pre_message":"<|im_start|>user\n","post_message":"<|im_end|>"}} + final_prompt_value: "\n" + bos_token: "" + eos_token: "" + max_tokens: 4096 +``` + + + + + +## Passing additional params - max_tokens, temperature +See all litellm.completion supported params [here](https://docs.litellm.ai/docs/completion/input) + +```python +# !pip install litellm +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" + +# predibae llama-3 call +response = completion( + model="predibase/llama3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}], + max_tokens=20, + temperature=0.5 +) +``` + +**proxy** + +```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + max_tokens: 20 + temperature: 0.5 +``` + +## Passings Predibase specific params - adapter_id, adapter_source, +Send params [not supported by `litellm.completion()`](https://docs.litellm.ai/docs/completion/input) but supported by Predibase by passing them to `litellm.completion` + +Example `adapter_id`, `adapter_source` are Predibase specific param - [See List](https://github.com/BerriAI/litellm/blob/8a35354dd6dbf4c2fcefcd6e877b980fcbd68c58/litellm/llms/predibase.py#L54) + +```python +# !pip install litellm +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" + +# predibase llama3 call +response = completion( + model="predibase/llama-3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}], + adapter_id="my_repo/3", + adapter_soruce="pbase", +) +``` + +**proxy** + +```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + adapter_id: my_repo/3 + adapter_source: pbase +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3c968ea57f..4ce5870805 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -132,6 +132,8 @@ const sidebars = { "providers/cohere", "providers/anyscale", "providers/huggingface", + "providers/watsonx", + "providers/predibase", "providers/ollama", "providers/perplexity", "providers/groq", @@ -151,7 +153,7 @@ const sidebars = { "providers/openrouter", "providers/custom_openai_proxy", "providers/petals", - "providers/watsonx", + ], }, "proxy/custom_pricing", diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index f726ed95a7..630baf3465 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -96,7 +96,6 @@ async def test_completion_predibase(sync_mode): response = completion( model="predibase/llama-3-8b-instruct", tenant_id="c4768f95", - api_base="https://serving.app.predibase.com", api_key=os.getenv("PREDIBASE_API_KEY"), messages=[{"role": "user", "content": "What is the meaning of life?"}], ) From 9d3f01c6ae23e63f20ff1f066b5ca12a9a29d5e1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:32:16 -0700 Subject: [PATCH 19/56] fix - router add model logic --- litellm/router.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 39d49a1474..5e71fea583 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2558,20 +2558,20 @@ class Router: self.set_client(model=deployment.to_json(exclude_none=True)) # set region (if azure model) - try: - if "azure" in deployment.litellm_params.model: - region = litellm.utils.get_model_region( - litellm_params=deployment.litellm_params, mode=None - ) + # try: + # if "azure" in deployment.litellm_params.model: + # region = litellm.utils.get_model_region( + # litellm_params=deployment.litellm_params, mode=None + # ) - deployment.litellm_params.region_name = region - except Exception as e: - verbose_router_logger.error( - "Unable to get the region for azure model - {}, {}".format( - deployment.litellm_params.model, str(e) - ) - ) - pass # [NON-BLOCKING] + # deployment.litellm_params.region_name = region + # except Exception as e: + # verbose_router_logger.error( + # "Unable to get the region for azure model - {}, {}".format( + # deployment.litellm_params.model, str(e) + # ) + # ) + # pass # [NON-BLOCKING] return deployment @@ -2610,7 +2610,6 @@ class Router: - The added/updated deployment """ # check if deployment already exists - if deployment.model_info.id in self.get_model_ids(): # remove the previous deployment removal_idx: Optional[int] = None @@ -2620,16 +2619,9 @@ class Router: if removal_idx is not None: self.model_list.pop(removal_idx) - - # add to model list - _deployment = deployment.to_json(exclude_none=True) - self.model_list.append(_deployment) - - # initialize client - self._add_deployment(deployment=deployment) - - # add to model names - self.model_names.append(deployment.model_name) + else: + # if the model_id is not in router + self.add_deployment(deployment=deployment) return deployment def delete_deployment(self, id: str) -> Optional[Deployment]: From 6fd6490d6304043d9687723fb495a67e2749ac36 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:38:06 -0700 Subject: [PATCH 20/56] fix hide - _auto_infer_region behind a feature flag --- litellm/router.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5e71fea583..3c777e2f07 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2558,20 +2558,23 @@ class Router: self.set_client(model=deployment.to_json(exclude_none=True)) # set region (if azure model) - # try: - # if "azure" in deployment.litellm_params.model: - # region = litellm.utils.get_model_region( - # litellm_params=deployment.litellm_params, mode=None - # ) + _auto_infer_region = os.environ.get("AUTO_INFER_REGION", "true") + _auto_infer_region_value = bool(_auto_infer_region) + if _auto_infer_region_value == True: + try: + if "azure" in deployment.litellm_params.model: + region = litellm.utils.get_model_region( + litellm_params=deployment.litellm_params, mode=None + ) - # deployment.litellm_params.region_name = region - # except Exception as e: - # verbose_router_logger.error( - # "Unable to get the region for azure model - {}, {}".format( - # deployment.litellm_params.model, str(e) - # ) - # ) - # pass # [NON-BLOCKING] + deployment.litellm_params.region_name = region + except Exception as e: + verbose_router_logger.error( + "Unable to get the region for azure model - {}, {}".format( + deployment.litellm_params.model, str(e) + ) + ) + pass # [NON-BLOCKING] return deployment From 75d6658bbcb54e9a9d73f38ece0a0f9d7be81866 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:39:19 -0700 Subject: [PATCH 21/56] fix - explain why behind feature flag --- litellm/router.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 3c777e2f07..ac9b4cb479 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2561,6 +2561,10 @@ class Router: _auto_infer_region = os.environ.get("AUTO_INFER_REGION", "true") _auto_infer_region_value = bool(_auto_infer_region) if _auto_infer_region_value == True: + """ + Hiding behind a feature flag + When there is a large amount of LLM deployments this makes startup times blow up + """ try: if "azure" in deployment.litellm_params.model: region = litellm.utils.get_model_region( From 547976448f094af5875285cac400e3872446ee6a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:50:46 -0700 Subject: [PATCH 22/56] fix feature flag logic --- litellm/router.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac9b4cb479..d833c1a85d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2558,9 +2558,8 @@ class Router: self.set_client(model=deployment.to_json(exclude_none=True)) # set region (if azure model) - _auto_infer_region = os.environ.get("AUTO_INFER_REGION", "true") - _auto_infer_region_value = bool(_auto_infer_region) - if _auto_infer_region_value == True: + _auto_infer_region = os.environ.get("AUTO_INFER_REGION", False) + if _auto_infer_region == True: """ Hiding behind a feature flag When there is a large amount of LLM deployments this makes startup times blow up From 5c69515a1347751f25e99671a1260cdc96108be8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:41:51 -0700 Subject: [PATCH 23/56] fix - upsert_deployment logic --- litellm/router.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d833c1a85d..20566f255b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2606,7 +2606,7 @@ class Router: self.model_names.append(deployment.model_name) return deployment - def upsert_deployment(self, deployment: Deployment) -> Deployment: + def upsert_deployment(self, deployment: Deployment) -> Deployment | None: """ Add or update deployment Parameters: @@ -2616,7 +2616,17 @@ class Router: - The added/updated deployment """ # check if deployment already exists - if deployment.model_info.id in self.get_model_ids(): + _deployment_model_id = deployment.model_info.id or "" + _deployment_on_router: Optional[Deployment] = self.get_deployment( + model_id=_deployment_model_id + ) + if _deployment_on_router is not None: + # deployment with this model_id exists on the router + if deployment.litellm_params == _deployment_on_router.litellm_params: + # No need to update + return None + + # if there is a new litellm param -> then update the deployment # remove the previous deployment removal_idx: Optional[int] = None for idx, model in enumerate(self.model_list): From 414af64343e696196d0ee3ddad3e1f51996ddb58 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:43:19 -0700 Subject: [PATCH 24/56] test - OpenAI client is re-used for Azure, OpenAI --- tests/test_models.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index b009ded6c1..9696c52a23 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -246,6 +246,33 @@ async def get_model_info_v2(session, key): raise Exception(f"Request did not return a 200 status code: {status}") +async def get_specific_model_info_v2(session, key, model_name): + url = "http://0.0.0.0:4000/v2/model/info?debug=True&model=" + model_name + print("running /model/info check for model=", model_name) + + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + + async with session.get(url, headers=headers) as response: + status = response.status + response_text = await response.text() + print("response from v2/model/info") + print(response_text) + print() + + _json_response = await response.json() + print("JSON response from /v2/model/info?model=", model_name, _json_response) + + _model_info = _json_response["data"] + assert len(_model_info) == 1, f"Expected 1 model, got {len(_model_info)}" + + if status != 200: + raise Exception(f"Request did not return a 200 status code: {status}") + return _model_info[0] + + async def get_model_health(session, key, model_name): url = "http://0.0.0.0:4000/health?model=" + model_name headers = { @@ -285,6 +312,11 @@ async def test_add_model_run_health(): model_name = f"azure-model-health-check-{model_id}" print("adding model", model_name) await add_model_for_health_checking(session=session, model_id=model_id) + _old_model_info = await get_specific_model_info_v2( + session=session, key=key, model_name=model_name + ) + print("model info before test", _old_model_info) + await asyncio.sleep(30) print("calling /model/info") await get_model_info(session=session, key=key) @@ -305,5 +337,28 @@ async def test_add_model_run_health(): _healthy_endpooint["model"] == "azure/chatgpt-v-2" ) # this is the model that got added + # assert httpx client is is unchanges + + await asyncio.sleep(10) + + _model_info_after_test = await get_specific_model_info_v2( + session=session, key=key, model_name=model_name + ) + + print("model info after test", _model_info_after_test) + old_openai_client = _old_model_info["openai_client"] + new_openai_client = _model_info_after_test["openai_client"] + print("old openai client", old_openai_client) + print("new openai client", new_openai_client) + + """ + PROD TEST - This is extremly important + The OpenAI client used should be the same after 30 seconds + It is a serious bug if the openai client does not match here + """ + assert ( + old_openai_client == new_openai_client + ), "OpenAI client does not match for the same model after 30 seconds" + # cleanup await delete_model(session=session, model_id=model_id) From 933f8ed16be60ff38223425b30a360c057995716 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:47:35 -0700 Subject: [PATCH 25/56] fix - proxy_server.py --- litellm/proxy/proxy_server.py | 38 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af1be7f266..46c1327732 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7795,11 +7795,15 @@ async def update_model( ) async def model_info_v2( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + model: Optional[str] = fastapi.Query( + None, description="Specify the model name (optional)" + ), + debug: Optional[bool] = False, ): """ BETA ENDPOINT. Might change unexpectedly. Use `/v1/model/info` for now. """ - global llm_model_list, general_settings, user_config_file_path, proxy_config + global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router if llm_model_list is None or not isinstance(llm_model_list, list): raise HTTPException( @@ -7822,19 +7826,35 @@ async def model_info_v2( if len(user_api_key_dict.models) > 0: user_models = user_api_key_dict.models + if model is not None: + all_models = [m for m in all_models if m["model_name"] == model] + # fill in model info based on config.yaml and litellm model_prices_and_context_window.json - for model in all_models: + for _model in all_models: # provided model_info in config.yaml - model_info = model.get("model_info", {}) + model_info = _model.get("model_info", {}) + if debug == True: + _openai_client = "None" + if llm_router is not None: + _openai_client = ( + llm_router._get_client( + deployment=_model, kwargs={}, client_type="async" + ) + or "None" + ) + else: + _openai_client = "llm_router_is_None" + openai_client = str(_openai_client) + _model["openai_client"] = openai_client # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - litellm_model_info = get_litellm_model_info(model=model) + litellm_model_info = get_litellm_model_info(model=_model) # 2nd pass on the model, try seeing if we can find model in litellm model_cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = _model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) try: litellm_model_info = litellm.get_model_info(model=litellm_model) @@ -7843,7 +7863,7 @@ async def model_info_v2( # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = _model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) split_model = litellm_model.split("/") if len(split_model) > 0: @@ -7855,10 +7875,10 @@ async def model_info_v2( for k, v in litellm_model_info.items(): if k not in model_info: model_info[k] = v - model["model_info"] = model_info + _model["model_info"] = model_info # don't return the api key / vertex credentials - model["litellm_params"].pop("api_key", None) - model["litellm_params"].pop("vertex_credentials", None) + _model["litellm_params"].pop("api_key", None) + _model["litellm_params"].pop("vertex_credentials", None) verbose_proxy_logger.debug("all_models: %s", all_models) return {"data": all_models} From 9bbb13c373313107a21d62e6e051d65b0afce91b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:54:52 -0700 Subject: [PATCH 26/56] fix bug upsert_deployment --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 20566f255b..32c2b61d13 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2606,7 +2606,7 @@ class Router: self.model_names.append(deployment.model_name) return deployment - def upsert_deployment(self, deployment: Deployment) -> Deployment | None: + def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]: """ Add or update deployment Parameters: From c17f221b8966e4b3581702822e2b8e181976a8f2 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:07:01 -0700 Subject: [PATCH 27/56] test(test_completion.py): reintegrate testing for huggingface tgi + non-tgi --- litellm/llms/huggingface_restapi.py | 103 +++++++++++++++++++--- litellm/tests/test_completion.py | 129 ++++++++++++++++++++++++++-- litellm/utils.py | 44 ++-------- 3 files changed, 218 insertions(+), 58 deletions(-) diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index a2c4457c23..26591b95da 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -6,10 +6,12 @@ import httpx, requests from .base import BaseLLM import time import litellm -from typing import Callable, Dict, List, Any +from typing import Callable, Dict, List, Any, Literal from litellm.utils import ModelResponse, Choices, Message, CustomStreamWrapper, Usage from typing import Optional from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm.types.completion import ChatCompletionMessageToolCallParam +import enum class HuggingfaceError(Exception): @@ -39,11 +41,29 @@ class HuggingfaceError(Exception): ) # Call the base class constructor with the parameters it needs +hf_task_list = [ + "text-generation-inference", + "conversational", + "text-classification", + "text-generation", +] + +hf_tasks = Literal[ + "text-generation-inference", + "conversational", + "text-classification", + "text-generation", +] + + class HuggingfaceConfig: """ Reference: https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/compat_generate """ + hf_task: Optional[hf_tasks] = ( + None # litellm-specific param, used to know the api spec to use when calling huggingface api + ) best_of: Optional[int] = None decoder_input_details: Optional[bool] = None details: Optional[bool] = True # enables returning logprobs + best of @@ -101,6 +121,51 @@ class HuggingfaceConfig: and v is not None } + def get_supported_openai_params(self): + return [ + "stream", + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "echo", + ] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + for param, value in non_default_params.items(): + # temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None + if param == "temperature": + if value == 0.0 or value == 0: + # hugging face exception raised when temp==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive + value = 0.01 + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + if param == "n": + optional_params["best_of"] = value + optional_params["do_sample"] = ( + True # Need to sample if you want best of for hf inference endpoints + ) + if param == "stream": + optional_params["stream"] = value + if param == "stop": + optional_params["stop"] = value + if param == "max_tokens": + # HF TGI raises the following exception when max_new_tokens==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive + if value == 0: + value = 1 + optional_params["max_new_tokens"] = value + if param == "echo": + # https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details + # Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False + optional_params["decoder_input_details"] = True + return optional_params + def output_parser(generated_text: str): """ @@ -162,7 +227,7 @@ def read_tgi_conv_models(): return set(), set() -def get_hf_task_for_model(model): +def get_hf_task_for_model(model: str) -> hf_tasks: # read text file, cast it to set # read the file called "huggingface_llms_metadata/hf_text_generation_models.txt" tgi_models, conversational_models = read_tgi_conv_models() @@ -171,7 +236,7 @@ def get_hf_task_for_model(model): elif model in conversational_models: return "conversational" elif "roneneldan/TinyStories" in model: - return None + return "text-generation" else: return "text-generation-inference" # default to tgi @@ -202,7 +267,7 @@ class Huggingface(BaseLLM): self, completion_response, model_response, - task, + task: hf_tasks, optional_params, encoding, input_text, @@ -270,6 +335,10 @@ class Huggingface(BaseLLM): ) choices_list.append(choice_obj) model_response["choices"].extend(choices_list) + elif task == "text-classification": + model_response["choices"][0]["message"]["content"] = json.dumps( + completion_response + ) else: if len(completion_response[0]["generated_text"]) > 0: model_response["choices"][0]["message"]["content"] = output_parser( @@ -332,7 +401,16 @@ class Huggingface(BaseLLM): exception_mapping_worked = False try: headers = self.validate_environment(api_key, headers) - task = get_hf_task_for_model(model) + if optional_params.get("hf_task") is None: + task = get_hf_task_for_model(model) + else: + task = optional_params.get("hf_task") # type: ignore + ## VALIDATE API FORMAT + if task is None or not isinstance(task, str) or task not in hf_task_list: + raise Exception( + "Invalid hf task - {}. Valid formats - {}.".format(task, hf_tasks) + ) + print_verbose(f"{model}, {task}") completion_url = "" input_text = "" @@ -433,14 +511,15 @@ class Huggingface(BaseLLM): inference_params.pop("return_full_text") data = { "inputs": prompt, - "parameters": inference_params, - "stream": ( # type: ignore + } + if task == "text-generation-inference": + data["parameters"] = inference_params + data["stream"] = ( # type: ignore True if "stream" in optional_params and optional_params["stream"] == True else False - ), - } + ) input_text = prompt ## LOGGING logging_obj.pre_call( @@ -531,10 +610,10 @@ class Huggingface(BaseLLM): isinstance(completion_response, dict) and "error" in completion_response ): - print_verbose(f"completion error: {completion_response['error']}") + print_verbose(f"completion error: {completion_response['error']}") # type: ignore print_verbose(f"response.status_code: {response.status_code}") raise HuggingfaceError( - message=completion_response["error"], + message=completion_response["error"], # type: ignore status_code=response.status_code, ) return self.convert_to_model_response_object( @@ -563,7 +642,7 @@ class Huggingface(BaseLLM): data: dict, headers: dict, model_response: ModelResponse, - task: str, + task: hf_tasks, encoding: Any, input_text: str, model: str, diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 630baf3465..4da489cc53 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -13,6 +13,7 @@ import litellm from litellm import embedding, completion, completion_cost, Timeout from litellm import RateLimitError from litellm.llms.prompt_templates.factory import anthropic_messages_pt +from unittest.mock import patch, MagicMock # litellm.num_retries=3 litellm.cache = None @@ -1145,15 +1146,92 @@ def test_get_hf_task_for_model(): # ################### Hugging Face TGI models ######################## # # TGI model # # this is a TGI model https://huggingface.co/glaiveai/glaive-coder-7b -def hf_test_completion_tgi(): - # litellm.set_verbose=True +def tgi_mock_post(url, data=None, json=None, headers=None): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.json.return_value = [ + { + "generated_text": "<|assistant|>\nI'm", + "details": { + "finish_reason": "length", + "generated_tokens": 10, + "seed": None, + "prefill": [], + "tokens": [ + { + "id": 28789, + "text": "<", + "logprob": -0.025222778, + "special": False, + }, + { + "id": 28766, + "text": "|", + "logprob": -0.000003695488, + "special": False, + }, + { + "id": 489, + "text": "ass", + "logprob": -0.0000019073486, + "special": False, + }, + { + "id": 11143, + "text": "istant", + "logprob": -0.000002026558, + "special": False, + }, + { + "id": 28766, + "text": "|", + "logprob": -0.0000015497208, + "special": False, + }, + { + "id": 28767, + "text": ">", + "logprob": -0.0000011920929, + "special": False, + }, + { + "id": 13, + "text": "\n", + "logprob": -0.00009703636, + "special": False, + }, + {"id": 28737, "text": "I", "logprob": -0.1953125, "special": False}, + { + "id": 28742, + "text": "'", + "logprob": -0.88183594, + "special": False, + }, + { + "id": 28719, + "text": "m", + "logprob": -0.00032639503, + "special": False, + }, + ], + }, + } + ] + return mock_response + + +def test_hf_test_completion_tgi(): + litellm.set_verbose = True try: - response = completion( - model="huggingface/HuggingFaceH4/zephyr-7b-beta", - messages=[{"content": "Hello, how are you?", "role": "user"}], - ) - # Add any assertions here to check the response - print(response) + with patch("requests.post", side_effect=tgi_mock_post): + response = completion( + model="huggingface/HuggingFaceH4/zephyr-7b-beta", + messages=[{"content": "Hello, how are you?", "role": "user"}], + max_tokens=10, + ) + # Add any assertions here to check the response + print(response) except litellm.ServiceUnavailableError as e: pass except Exception as e: @@ -1191,6 +1269,41 @@ def hf_test_completion_tgi(): # except Exception as e: # pytest.fail(f"Error occurred: {e}") # hf_test_completion_none_task() + + +def mock_post(url, data=None, json=None, headers=None): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.json.return_value = [ + [ + {"label": "LABEL_0", "score": 0.9990691542625427}, + {"label": "LABEL_1", "score": 0.0009308889275416732}, + ] + ] + return mock_response + + +def test_hf_classifier_task(): + try: + with patch("requests.post", side_effect=mock_post): + litellm.set_verbose = True + user_message = "I like you. I love you" + messages = [{"content": user_message, "role": "user"}] + response = completion( + model="huggingface/shahrukhx01/question-vs-statement-classifier", + messages=messages, + hf_task="text-classification", + ) + print(f"response: {response}") + assert isinstance(response, litellm.ModelResponse) + assert isinstance(response.choices[0], litellm.Choices) + assert response.choices[0].message.content is not None + assert isinstance(response.choices[0].message.content, str) + except Exception as e: + pytest.fail(f"Error occurred: {str(e)}") + + ########################### End of Hugging Face Tests ############################################## # def test_completion_hf_api(): # # failing on circle ci commenting out diff --git a/litellm/utils.py b/litellm/utils.py index 838d0fe553..f20cd220ce 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4840,6 +4840,7 @@ def get_optional_params_embeddings( def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create + model: str, functions=None, function_call=None, temperature=None, @@ -4853,7 +4854,6 @@ def get_optional_params( frequency_penalty=None, logit_bias=None, user=None, - model=None, custom_llm_provider="", response_format=None, seed=None, @@ -4882,7 +4882,7 @@ def get_optional_params( passed_params[k] = v - optional_params = {} + optional_params: Dict = {} common_auth_dict = litellm.common_cloud_provider_auth_params if custom_llm_provider in common_auth_dict["providers"]: @@ -5156,41 +5156,9 @@ def get_optional_params( model=model, custom_llm_provider=custom_llm_provider ) _check_valid_arg(supported_params=supported_params) - # temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None - if temperature is not None: - if temperature == 0.0 or temperature == 0: - # hugging face exception raised when temp==0 - # Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive - temperature = 0.01 - optional_params["temperature"] = temperature - if top_p is not None: - optional_params["top_p"] = top_p - if n is not None: - optional_params["best_of"] = n - optional_params["do_sample"] = ( - True # Need to sample if you want best of for hf inference endpoints - ) - if stream is not None: - optional_params["stream"] = stream - if stop is not None: - optional_params["stop"] = stop - if max_tokens is not None: - # HF TGI raises the following exception when max_new_tokens==0 - # Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive - if max_tokens == 0: - max_tokens = 1 - optional_params["max_new_tokens"] = max_tokens - if n is not None: - optional_params["best_of"] = n - if presence_penalty is not None: - optional_params["repetition_penalty"] = presence_penalty - if "echo" in passed_params: - # https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details - # Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False - optional_params["decoder_input_details"] = special_params["echo"] - passed_params.pop( - "echo", None - ) # since we handle translating echo, we should not send it to TGI request + optional_params = litellm.HuggingfaceConfig().map_openai_params( + non_default_params=non_default_params, optional_params=optional_params + ) elif custom_llm_provider == "together_ai": ## check if unsupported param passed in supported_params = get_supported_openai_params( @@ -6150,7 +6118,7 @@ def get_supported_openai_params(model: str, custom_llm_provider: str): "seed", ] elif custom_llm_provider == "huggingface": - return ["stream", "temperature", "max_tokens", "top_p", "stop", "n"] + return litellm.HuggingfaceConfig().get_supported_openai_params() elif custom_llm_provider == "together_ai": return [ "stream", From c744851d139265932123e81447261a174c14b47d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 14:08:38 -0700 Subject: [PATCH 28/56] fix AUTO_INFER_REGION --- .circleci/config.yml | 1 + litellm/router.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ef8c0e33b..08bbad6fac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,6 +198,7 @@ jobs: -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION="True" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ diff --git a/litellm/router.py b/litellm/router.py index 32c2b61d13..e8b0f658fa 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2559,7 +2559,7 @@ class Router: # set region (if azure model) _auto_infer_region = os.environ.get("AUTO_INFER_REGION", False) - if _auto_infer_region == True: + if _auto_infer_region == True or _auto_infer_region == "True": """ Hiding behind a feature flag When there is a large amount of LLM deployments this makes startup times blow up From 50be25d11a05d7a4b15c43c35f35f969b678f324 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:08:47 -0700 Subject: [PATCH 29/56] test(test_optional_params.py): fix optional params --- litellm/tests/test_optional_params.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/tests/test_optional_params.py b/litellm/tests/test_optional_params.py index fac63ed67b..5c33cfa0e5 100644 --- a/litellm/tests/test_optional_params.py +++ b/litellm/tests/test_optional_params.py @@ -86,6 +86,7 @@ def test_azure_optional_params_embeddings(): def test_azure_gpt_optional_params_gpt_vision(): # for OpenAI, Azure all extra params need to get passed as extra_body to OpenAI python. We assert we actually set extra_body here optional_params = litellm.utils.get_optional_params( + model="", user="John", custom_llm_provider="azure", max_tokens=10, @@ -125,6 +126,7 @@ def test_azure_gpt_optional_params_gpt_vision(): def test_azure_gpt_optional_params_gpt_vision_with_extra_body(): # if user passes extra_body, we should not over write it, we should pass it along to OpenAI python optional_params = litellm.utils.get_optional_params( + model="", user="John", custom_llm_provider="azure", max_tokens=10, @@ -167,6 +169,7 @@ def test_azure_gpt_optional_params_gpt_vision_with_extra_body(): def test_openai_extra_headers(): optional_params = litellm.utils.get_optional_params( + model="", user="John", custom_llm_provider="openai", max_tokens=10, From d4d175030fbf81afed5b16ff523c5e240e3dc1b6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:39:14 -0700 Subject: [PATCH 30/56] docs(huggingface.md): add text-classification to huggingface docs --- docs/my-website/docs/providers/huggingface.md | 176 +++++++++++++++++- litellm/llms/huggingface_restapi.py | 7 +- litellm/tests/test_completion.py | 3 +- 3 files changed, 177 insertions(+), 9 deletions(-) diff --git a/docs/my-website/docs/providers/huggingface.md b/docs/my-website/docs/providers/huggingface.md index f8ebadfcfa..35befd3e20 100644 --- a/docs/my-website/docs/providers/huggingface.md +++ b/docs/my-website/docs/providers/huggingface.md @@ -21,6 +21,11 @@ This is done by adding the "huggingface/" prefix to `model`, example `completion +By default, LiteLLM will assume a huggingface call follows the TGI format. + + + + ```python import os from litellm import completion @@ -40,9 +45,58 @@ response = completion( print(response) ``` + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: wizard-coder + litellm_params: + model: huggingface/WizardLM/WizardCoder-Python-34B-V1.0 + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "wizard-coder", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + +Append `conversational` to the model name + +e.g. `huggingface/conversational/` + + + + ```python import os from litellm import completion @@ -54,7 +108,7 @@ messages = [{ "content": "There's a llama in my garden 😱 What should I do?"," # e.g. Call 'facebook/blenderbot-400M-distill' hosted on HF Inference endpoints response = completion( - model="huggingface/facebook/blenderbot-400M-distill", + model="huggingface/conversational/facebook/blenderbot-400M-distill", messages=messages, api_base="https://my-endpoint.huggingface.cloud" ) @@ -62,7 +116,123 @@ response = completion( print(response) ``` - + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: blenderbot + litellm_params: + model: huggingface/conversational/facebook/blenderbot-400M-distill + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "blenderbot", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + + + + +Append `text-classification` to the model name + +e.g. `huggingface/text-classification/` + + + + +```python +import os +from litellm import completion + +# [OPTIONAL] set env var +os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" + +messages = [{ "content": "I like you, I love you!","role": "user"}] + +# e.g. Call 'shahrukhx01/question-vs-statement-classifier' hosted on HF Inference endpoints +response = completion( + model="huggingface/text-classification/shahrukhx01/question-vs-statement-classifier", + messages=messages, + api_base="https://my-endpoint.endpoints.huggingface.cloud", +) + +print(response) +``` + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: bert-classifier + litellm_params: + model: huggingface/text-classification/shahrukhx01/question-vs-statement-classifier + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "bert-classifier", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + + + + +Append `text-generation` to the model name + +e.g. `huggingface/text-generation/` ```python import os @@ -75,7 +245,7 @@ messages = [{ "content": "There's a llama in my garden 😱 What should I do?"," # e.g. Call 'roneneldan/TinyStories-3M' hosted on HF Inference endpoints response = completion( - model="huggingface/roneneldan/TinyStories-3M", + model="huggingface/text-generation/roneneldan/TinyStories-3M", messages=messages, api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", ) diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 26591b95da..ad3c570e76 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -230,6 +230,8 @@ def read_tgi_conv_models(): def get_hf_task_for_model(model: str) -> hf_tasks: # read text file, cast it to set # read the file called "huggingface_llms_metadata/hf_text_generation_models.txt" + if model.split("/")[0] in hf_task_list: + return model.split("/")[0] # type: ignore tgi_models, conversational_models = read_tgi_conv_models() if model in tgi_models: return "text-generation-inference" @@ -401,10 +403,7 @@ class Huggingface(BaseLLM): exception_mapping_worked = False try: headers = self.validate_environment(api_key, headers) - if optional_params.get("hf_task") is None: - task = get_hf_task_for_model(model) - else: - task = optional_params.get("hf_task") # type: ignore + task = get_hf_task_for_model(model) ## VALIDATE API FORMAT if task is None or not isinstance(task, str) or task not in hf_task_list: raise Exception( diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 4da489cc53..b3e1928a8e 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -1291,9 +1291,8 @@ def test_hf_classifier_task(): user_message = "I like you. I love you" messages = [{"content": user_message, "role": "user"}] response = completion( - model="huggingface/shahrukhx01/question-vs-statement-classifier", + model="huggingface/text-classification/shahrukhx01/question-vs-statement-classifier", messages=messages, - hf_task="text-classification", ) print(f"response: {response}") assert isinstance(response, litellm.ModelResponse) From 500995696ad2d1fe450d40461cc2f75ce2f24a1b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:42:06 -0700 Subject: [PATCH 31/56] test: fix linting --- ...optional_params_functions_not_supported.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/litellm/tests/test_get_optional_params_functions_not_supported.py b/litellm/tests/test_get_optional_params_functions_not_supported.py index 2abfbc41f0..c4580adba2 100644 --- a/litellm/tests/test_get_optional_params_functions_not_supported.py +++ b/litellm/tests/test_get_optional_params_functions_not_supported.py @@ -3,7 +3,27 @@ from litellm import get_optional_params litellm.add_function_to_prompt = True optional_params = get_optional_params( - tools= [{'type': 'function', 'function': {'description': 'Get the current weather in a given location', 'name': 'get_current_weather', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city and state, e.g. San Francisco, CA'}, 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}}, 'required': ['location']}}}], - tool_choice= 'auto', + model="", + tools=[ + { + "type": "function", + "function": { + "description": "Get the current weather in a given location", + "name": "get_current_weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ], + tool_choice="auto", ) -assert optional_params is not None \ No newline at end of file +assert optional_params is not None From 759ff3f750729f840b934bb3af09dd9844ce99f5 Mon Sep 17 00:00:00 2001 From: Nick Wong Date: Fri, 10 May 2024 15:42:07 -0700 Subject: [PATCH 32/56] added code to enforce unique key and team aliases in the ui --- .../src/components/create_key_button.tsx | 383 ++++++++------- ui/litellm-dashboard/src/components/teams.tsx | 439 +++++++++++------- 2 files changed, 489 insertions(+), 333 deletions(-) diff --git a/ui/litellm-dashboard/src/components/create_key_button.tsx b/ui/litellm-dashboard/src/components/create_key_button.tsx index 648eee9cad..0e158778f9 100644 --- a/ui/litellm-dashboard/src/components/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/create_key_button.tsx @@ -2,8 +2,17 @@ import React, { useState, useEffect, useRef } from "react"; import { Button, TextInput, Grid, Col } from "@tremor/react"; -import { Card, Metric, Text, Title, Subtitle, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react"; -import { CopyToClipboard } from 'react-copy-to-clipboard'; +import { + Card, + Metric, + Text, + Title, + Subtitle, + Accordion, + AccordionHeader, + AccordionBody, +} from "@tremor/react"; +import { CopyToClipboard } from "react-copy-to-clipboard"; import { Button as Button2, Modal, @@ -13,7 +22,11 @@ import { Select, message, } from "antd"; -import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking"; +import { + keyCreateCall, + slackBudgetAlertsHealthCheck, + modelAvailableCall, +} from "./networking"; const { Option } = Select; @@ -59,7 +72,11 @@ const CreateKey: React.FC = ({ } if (accessToken !== null) { - const model_available = await modelAvailableCall(accessToken, userID, userRole); + const model_available = await modelAvailableCall( + accessToken, + userID, + userRole + ); let available_model_names = model_available["data"].map( (element: { id: string }) => element.id ); @@ -70,12 +87,25 @@ const CreateKey: React.FC = ({ console.error("Error fetching user models:", error); } }; - + fetchUserModels(); }, [accessToken, userID, userRole]); const handleCreate = async (formValues: Record) => { try { + const newKeyAlias = formValues?.key_alias ?? ""; + const newKeyTeamId = formValues?.team_id ?? null; + const existingKeyAliases = + data + ?.filter((k) => k.team_id === newKeyTeamId) + .map((k) => k.key_alias) ?? []; + + if (existingKeyAliases.includes(newKeyAlias)) { + throw new Error( + `Key alias ${newKeyAlias} already exists for team with ID ${newKeyTeamId}, please provide another key alias` + ); + } + message.info("Making API Call"); setIsModalVisible(true); const response = await keyCreateCall(accessToken, userID, formValues); @@ -89,12 +119,13 @@ const CreateKey: React.FC = ({ localStorage.removeItem("userData" + userID); } catch (error) { console.error("Error creating the key:", error); + message.error(`Error creating the key: ${error}`, 20); } }; const handleCopy = () => { - message.success('API Key copied to clipboard'); -}; + message.success("API Key copied to clipboard"); + }; useEffect(() => { let tempModelsToPick = []; @@ -119,7 +150,6 @@ const CreateKey: React.FC = ({ setModelsToPick(tempModelsToPick); }, [team, userModels]); - return (
@@ -141,140 +171,164 @@ const CreateKey: React.FC = ({ wrapperCol={{ span: 16 }} labelAlign="left" > - <> - - - - - - - - - - - Optional Settings - - - { - if (value && team && team.max_budget !== null && value > team.max_budget) { - throw new Error(`Budget cannot exceed team max budget: $${team.max_budget}`); - } - }, - }, - ]} - > - - - + + + + + + + - daily - monthly - - - + All Team Models + + {modelsToPick.map((model: string) => ( + + ))} + + + + + Optional Settings + + + { - if (value && team && team.tpm_limit !== null && value > team.tpm_limit) { - throw new Error(`TPM limit cannot exceed team TPM limit: ${team.tpm_limit}`); - } + if ( + value && + team && + team.max_budget !== null && + value > team.max_budget + ) { + throw new Error( + `Budget cannot exceed team max budget: $${team.max_budget}` + ); + } }, - }, - ]} + }, + ]} > - - - + + + + + { + if ( + value && + team && + team.tpm_limit !== null && + value > team.tpm_limit + ) { + throw new Error( + `TPM limit cannot exceed team TPM limit: ${team.tpm_limit}` + ); + } + }, + }, + ]} + > + + + { - if (value && team && team.rpm_limit !== null && value > team.rpm_limit) { - throw new Error(`RPM limit cannot exceed team RPM limit: ${team.rpm_limit}`); - } + if ( + value && + team && + team.rpm_limit !== null && + value > team.rpm_limit + ) { + throw new Error( + `RPM limit cannot exceed team RPM limit: ${team.rpm_limit}` + ); + } }, - }, - ]} - > - - - - - - - - + }, + ]} + > + + + + + + + + + + + - - - -
Create Key
@@ -288,36 +342,45 @@ const CreateKey: React.FC = ({ footer={null} > - - Save your Key - -

- Please save this secret key somewhere safe and accessible. For - security reasons, you will not be able to view it again{" "} - through your LiteLLM account. If you lose this secret key, you - will need to generate a new one. -

- - - {apiKey != null ? ( -
+ Save your Key + +

+ Please save this secret key somewhere safe and accessible. For + security reasons, you will not be able to view it again{" "} + through your LiteLLM account. If you lose this secret key, you + will need to generate a new one. +

+ + + {apiKey != null ? ( +
API Key: -
- 
{apiKey}
-
- - +
+ 
+                      {apiKey}
+
+
+ + - - {/* */} -
- ) : ( - Key being created, this might take 30s - )} - - +
+ ) : ( + Key being created, this might take 30s + )} + )} diff --git a/ui/litellm-dashboard/src/components/teams.tsx b/ui/litellm-dashboard/src/components/teams.tsx index b248017990..d22789420f 100644 --- a/ui/litellm-dashboard/src/components/teams.tsx +++ b/ui/litellm-dashboard/src/components/teams.tsx @@ -2,7 +2,13 @@ import React, { useState, useEffect } from "react"; import Link from "next/link"; import { Typography } from "antd"; import { teamDeleteCall, teamUpdateCall, teamInfoCall } from "./networking"; -import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon } from "@heroicons/react/outline"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, +} from "@heroicons/react/outline"; import { Button as Button2, Modal, @@ -46,8 +52,12 @@ interface EditTeamModalProps { onSubmit: (data: FormData) => void; // Assuming FormData is the type of data to be submitted } - -import { teamCreateCall, teamMemberAddCall, Member, modelAvailableCall } from "./networking"; +import { + teamCreateCall, + teamMemberAddCall, + Member, + modelAvailableCall, +} from "./networking"; const Team: React.FC = ({ teams, @@ -63,7 +73,6 @@ const Team: React.FC = ({ const [value, setValue] = useState(""); const [editModalVisible, setEditModalVisible] = useState(false); - const [selectedTeam, setSelectedTeam] = useState( teams ? teams[0] : null ); @@ -76,127 +85,125 @@ const Team: React.FC = ({ // store team info as {"team_id": team_info_object} const [perTeamInfo, setPerTeamInfo] = useState>({}); + const EditTeamModal: React.FC = ({ + visible, + onCancel, + team, + onSubmit, + }) => { + const [form] = Form.useForm(); - const EditTeamModal: React.FC = ({ visible, onCancel, team, onSubmit }) => { - const [form] = Form.useForm(); + const handleOk = () => { + form + .validateFields() + .then((values) => { + const updatedValues = { ...values, team_id: team.team_id }; + onSubmit(updatedValues); + form.resetFields(); + }) + .catch((error) => { + console.error("Validation failed:", error); + }); + }; - const handleOk = () => { - form - .validateFields() - .then((values) => { - const updatedValues = {...values, team_id: team.team_id}; - onSubmit(updatedValues); - form.resetFields(); - }) - .catch((error) => { - console.error("Validation failed:", error); - }); -}; - - return ( + return ( -
- <> - - - - - - - {"All Proxy Models"} - - {userModels && userModels.map((model) => ( - - {model} - - ))} - - - - - - - - - - - - - - -
- Edit Team -
-
- - ); -}; - -const handleEditClick = (team: any) => { - setSelectedTeam(team); - setEditModalVisible(true); -}; - -const handleEditCancel = () => { - setEditModalVisible(false); - setSelectedTeam(null); -}; - -const handleEditSubmit = async (formValues: Record) => { - // Call API to update team with teamId and values - const teamId = formValues.team_id; // get team_id - - console.log("handleEditSubmit:", formValues); - if (accessToken == null) { - return; - } - - let newTeamValues = await teamUpdateCall(accessToken, formValues); - - // Update the teams state with the updated team data - if (teams) { - const updatedTeams = teams.map((team) => - team.team_id === teamId ? newTeamValues.data : team +
+ <> + + + + + + + {"All Proxy Models"} + + {userModels && + userModels.map((model) => ( + + {model} + + ))} + + + + + + + + + + + + + +
+ Edit Team +
+
+ ); - setTeams(updatedTeams); - } - message.success("Team updated successfully"); + }; - setEditModalVisible(false); - setSelectedTeam(null); -}; + const handleEditClick = (team: any) => { + setSelectedTeam(team); + setEditModalVisible(true); + }; + + const handleEditCancel = () => { + setEditModalVisible(false); + setSelectedTeam(null); + }; + + const handleEditSubmit = async (formValues: Record) => { + // Call API to update team with teamId and values + const teamId = formValues.team_id; // get team_id + + console.log("handleEditSubmit:", formValues); + if (accessToken == null) { + return; + } + + let newTeamValues = await teamUpdateCall(accessToken, formValues); + + // Update the teams state with the updated team data + if (teams) { + const updatedTeams = teams.map((team) => + team.team_id === teamId ? newTeamValues.data : team + ); + setTeams(updatedTeams); + } + message.success("Team updated successfully"); + + setEditModalVisible(false); + setSelectedTeam(null); + }; const handleOk = () => { setIsTeamModalVisible(false); @@ -224,9 +231,6 @@ const handleEditSubmit = async (formValues: Record) => { setIsDeleteModalOpen(true); }; - - - const confirmDelete = async () => { if (teamToDelete == null || teams == null || accessToken == null) { return; @@ -235,7 +239,9 @@ const handleEditSubmit = async (formValues: Record) => { try { await teamDeleteCall(accessToken, teamToDelete); // Successfully completed the deletion. Update the state to trigger a rerender. - const filteredData = teams.filter((item) => item.team_id !== teamToDelete); + const filteredData = teams.filter( + (item) => item.team_id !== teamToDelete + ); setTeams(filteredData); } catch (error) { console.error("Error deleting the team:", error); @@ -253,8 +259,6 @@ const handleEditSubmit = async (formValues: Record) => { setTeamToDelete(null); }; - - useEffect(() => { const fetchUserModels = async () => { try { @@ -263,7 +267,11 @@ const handleEditSubmit = async (formValues: Record) => { } if (accessToken !== null) { - const model_available = await modelAvailableCall(accessToken, userID, userRole); + const model_available = await modelAvailableCall( + accessToken, + userID, + userRole + ); let available_model_names = model_available["data"].map( (element: { id: string }) => element.id ); @@ -275,7 +283,6 @@ const handleEditSubmit = async (formValues: Record) => { } }; - const fetchTeamInfo = async () => { try { if (userID === null || userRole === null || accessToken === null) { @@ -288,22 +295,21 @@ const handleEditSubmit = async (formValues: Record) => { console.log("fetching team info:"); - let _team_id_to_info: Record = {}; for (let i = 0; i < teams?.length; i++) { let _team_id = teams[i].team_id; const teamInfo = await teamInfoCall(accessToken, _team_id); console.log("teamInfo response:", teamInfo); if (teamInfo !== null) { - _team_id_to_info = {..._team_id_to_info, [_team_id]: teamInfo}; + _team_id_to_info = { ..._team_id_to_info, [_team_id]: teamInfo }; } } setPerTeamInfo(_team_id_to_info); - } catch (error) { - console.error("Error fetching team info:", error); - } - }; - + } catch (error) { + console.error("Error fetching team info:", error); + } + }; + fetchUserModels(); fetchTeamInfo(); }, [accessToken, userID, userRole, teams]); @@ -311,6 +317,15 @@ const handleEditSubmit = async (formValues: Record) => { const handleCreate = async (formValues: Record) => { try { if (accessToken != null) { + const newTeamAlias = formValues?.team_alias; + const existingTeamAliases = teams?.map((t) => t.team_alias) ?? []; + + if (existingTeamAliases.includes(newTeamAlias)) { + throw new Error( + `Team alias ${newTeamAlias} already exists, please pick another alias` + ); + } + message.info("Creating Team"); const response: any = await teamCreateCall(accessToken, formValues); if (teams !== null) { @@ -364,7 +379,7 @@ const handleEditSubmit = async (formValues: Record) => { console.error("Error creating the team:", error); } }; - console.log(`received teams ${teams}`); + console.log(`received teams ${JSON.stringify(teams)}`); return (
@@ -387,55 +402,124 @@ const handleEditSubmit = async (formValues: Record) => { {teams && teams.length > 0 ? teams.map((team: any) => ( - {team["team_alias"]} - {team["spend"]} - + + {team["team_alias"]} + + + {team["spend"]} + + {team["max_budget"] ? team["max_budget"] : "No limit"} - + {Array.isArray(team.models) ? ( -
+
{team.models.length === 0 ? ( All Proxy Models ) : ( - team.models.map((model: string, index: number) => ( - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - {model.length > 30 ? `${model.slice(0, 30)}...` : model} - - ) - )) + team.models.map( + (model: string, index: number) => + model === "all-proxy-models" ? ( + + All Proxy Models + + ) : ( + + + {model.length > 30 + ? `${model.slice(0, 30)}...` + : model} + + + ) + ) )}
) : null} - - + - TPM:{" "} - {team.tpm_limit ? team.tpm_limit : "Unlimited"}{" "} + TPM: {team.tpm_limit ? team.tpm_limit : "Unlimited"}{" "}

RPM:{" "} {team.rpm_limit ? team.rpm_limit : "Unlimited"} - {perTeamInfo && team.team_id && perTeamInfo[team.team_id] && perTeamInfo[team.team_id].keys && perTeamInfo[team.team_id].keys.length} Keys - {perTeamInfo && team.team_id && perTeamInfo[team.team_id] && perTeamInfo[team.team_id].team_info && perTeamInfo[team.team_id].team_info.members_with_roles && perTeamInfo[team.team_id].team_info.members_with_roles.length} Members + + {perTeamInfo && + team.team_id && + perTeamInfo[team.team_id] && + perTeamInfo[team.team_id].keys && + perTeamInfo[team.team_id].keys.length}{" "} + Keys + + + {perTeamInfo && + team.team_id && + perTeamInfo[team.team_id] && + perTeamInfo[team.team_id].team_info && + perTeamInfo[team.team_id].team_info + .members_with_roles && + perTeamInfo[team.team_id].team_info + .members_with_roles.length}{" "} + Members + - handleEditClick(team)} /> - handleDelete(team.team_id)} icon={TrashIcon} size="sm" @@ -481,7 +565,11 @@ const handleEditSubmit = async (formValues: Record) => {
- @@ -515,10 +603,12 @@ const handleEditSubmit = async (formValues: Record) => { labelAlign="left" > <> - @@ -528,7 +618,10 @@ const handleEditSubmit = async (formValues: Record) => { placeholder="Select models" style={{ width: "100%" }} > - + All Proxy Models {userModels.map((model) => ( @@ -606,8 +699,8 @@ const handleEditSubmit = async (formValues: Record) => { {member["user_email"] ? member["user_email"] : member["user_id"] - ? member["user_id"] - : null} + ? member["user_id"] + : null} {member["role"]} @@ -618,13 +711,13 @@ const handleEditSubmit = async (formValues: Record) => { {selectedTeam && ( - - )} + + )}