From a020563149177dd59578442cfa2e916b57e9906f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 14 Aug 2024 19:07:10 -0700 Subject: [PATCH 1/4] feat(proxy_server.py): support returning available fields for pass_through_endpoints via `/config/field/list --- litellm/proxy/_new_secret_config.yaml | 16 ++++---- litellm/proxy/_types.py | 24 +++++++++++- litellm/proxy/proxy_server.py | 53 ++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index bc3e0680f8..e4e180727d 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -3,11 +3,11 @@ model_list: litellm_params: model: "*" -general_settings: - master_key: sk-1234 - pass_through_endpoints: - - path: "/api/public/ingestion" # route you want to add to LiteLLM Proxy Server - target: "https://us.cloud.langfuse.com/api/public/ingestion" # URL this route should forward - headers: - LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_PUBLIC_KEY" # your langfuse account public key - LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_SECRET_KEY" # your langfuse account secret key \ No newline at end of file +# general_settings: +# master_key: sk-1234 +# pass_through_endpoints: +# - path: "/api/public/ingestion" # route you want to add to LiteLLM Proxy Server +# target: "https://us.cloud.langfuse.com/api/public/ingestion" # URL this route should forward +# headers: +# LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_PUBLIC_KEY" # your langfuse account public key +# LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_SECRET_KEY" # your langfuse account secret key \ No newline at end of file diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index deb496f2e9..cb60fa5f04 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5,10 +5,10 @@ import sys import uuid from dataclasses import fields from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, TypedDict, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Extra, Field, Json, model_validator -from typing_extensions import Annotated +from typing_extensions import Annotated, TypedDict from litellm.types.router import UpdateRouterConfig from litellm.types.utils import ProviderField @@ -1082,6 +1082,12 @@ class DynamoDBArgs(LiteLLMBase): assume_role_aws_session_name: Optional[str] = None +class PassThroughEndpointTypedDict(TypedDict): + path: str + target: str + headers: dict + + class ConfigFieldUpdate(LiteLLMBase): field_name: str field_value: Any @@ -1093,6 +1099,13 @@ class ConfigFieldDelete(LiteLLMBase): field_name: str +class FieldDetail(BaseModel): + field_name: str + field_type: str + field_description: str + field_default_value: Any = None + + class ConfigList(LiteLLMBase): field_name: str field_type: str @@ -1101,6 +1114,9 @@ class ConfigList(LiteLLMBase): stored_in_db: Optional[bool] field_default_value: Any premium_field: bool = False + nested_fields: Optional[List[FieldDetail]] = ( + None # For nested dictionary or Pydantic fields + ) class ConfigGeneralSettings(LiteLLMBase): @@ -1203,6 +1219,10 @@ class ConfigGeneralSettings(LiteLLMBase): default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", ) + pass_through_endpoints: Optional[PassThroughEndpointTypedDict] = Field( + default=None, + description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through", + ) class ConfigYAML(LiteLLMBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c79a18a5cc..2213e348d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13,7 +13,15 @@ import traceback import uuid import warnings from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Any, List, Optional +from typing import ( + TYPE_CHECKING, + Any, + List, + Optional, + get_args, + get_origin, + get_type_hints, +) import requests @@ -548,6 +556,20 @@ async def check_request_disconnection(request: Request, llm_api_call_task): ) +def _resolve_typed_dict_type(typ): + """Resolve the actual TypedDict class from a potentially wrapped type.""" + from typing_extensions import _TypedDictMeta # type: ignore + + origin = get_origin(typ) + if origin is Union: # Check if it's a Union (like Optional) + for arg in get_args(typ): + if isinstance(arg, _TypedDictMeta): + return arg + elif isinstance(typ, type) and isinstance(typ, dict): + return typ + return None + + def prisma_setup(database_url: Optional[str]): global prisma_client, proxy_logging_obj, user_api_key_cache @@ -9409,6 +9431,7 @@ async def get_config_list( "global_max_parallel_requests": {"type": "Integer"}, "max_request_size_mb": {"type": "Integer"}, "max_response_size_mb": {"type": "Integer"}, + "pass_through_endpoints": {"type": "TypedDictionary"}, } return_val = [] @@ -9416,6 +9439,33 @@ async def get_config_list( for field_name, field_info in ConfigGeneralSettings.model_fields.items(): if field_name in allowed_args: + ## HANDLE TYPED DICT + + typed_dict_type = allowed_args[field_name]["type"] + + if typed_dict_type == "TypedDictionary": + typed_dict_class: Optional[Any] = _resolve_typed_dict_type( + field_info.annotation + ) + + if typed_dict_class is None: + nested_fields = None + else: + # Get type hints from the TypedDict to create FieldDetail objects + nested_fields = [ + FieldDetail( + field_name=sub_field, + field_type=type_hint.__name__, + field_description="", # Add custom logic if descriptions are available + field_default_value=general_settings.get(sub_field, None), + ) + for sub_field, type_hint in get_type_hints( + typed_dict_class + ).items() + ] + else: + nested_fields = None + _stored_in_db = None if field_name in db_general_settings_dict: _stored_in_db = True @@ -9429,6 +9479,7 @@ async def get_config_list( field_value=general_settings.get(field_name, None), stored_in_db=_stored_in_db, field_default_value=field_info.default, + nested_fields=nested_fields, ) return_val.append(_response_obj) From 28faafadb11e5d9668fa525b1d232dfadcf07edf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 14 Aug 2024 21:36:07 -0700 Subject: [PATCH 2/4] feat(pass_through_endpoints.py): initial commit of crud endpoints for pass through endpoints --- .../pass_through_endpoints.py | 40 +++++++++++++++++++ litellm/proxy/proxy_server.py | 4 ++ 2 files changed, 44 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 15129854a3..1d9f691177 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -23,6 +23,8 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +router = APIRouter() + async def set_env_variables_in_header(custom_headers: dict): """ @@ -476,3 +478,41 @@ async def initialize_pass_through_endpoints(pass_through_endpoints: list): ) verbose_proxy_logger.debug("Added new pass through endpoint: %s", _path) + + +@router.get( + "/config/pass_through_endpoint/{endpoint_id}", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_pass_through_endpoints(request: Request, endpoint_id: str): + """ + GET configured pass through endpoint. + + If no endpoint_id given, return all configured endpoints. + """ + pass + + +@router.post( + "/config/pass_through_endpoint", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], +) +async def create_pass_through_endpoints(request: Request): + """ + Create new pass-through endpoint + """ + pass + + +@router.delete( + "/config/pass_through_endpoint", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], +) +async def delete_pass_through_endpoints(request: Request): + """ + Create new pass-through endpoint + """ + pass diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2213e348d1..6483a61b53 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -194,6 +194,9 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_confi from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( initialize_pass_through_endpoints, ) +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + router as pass_through_router, +) from litellm.proxy.secret_managers.aws_secret_manager import ( load_aws_kms, load_aws_secret_manager, @@ -9895,6 +9898,7 @@ def cleanup_router_config_variables(): app.include_router(router) app.include_router(fine_tuning_router) app.include_router(vertex_router) +app.include_router(pass_through_router) app.include_router(health_router) app.include_router(key_management_router) app.include_router(internal_user_router) From 589da45c24548fcd9f1c280f55fa17ca640a1eb7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 15 Aug 2024 21:23:26 -0700 Subject: [PATCH 3/4] feat(pass_through_endpoints.py): initial working CRUD endpoints for /pass_through_endoints --- litellm/proxy/_types.py | 25 ++- .../pass_through_endpoints.py | 170 +++++++++++++++++- litellm/proxy/proxy_server.py | 107 ++++++++--- 3 files changed, 264 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index cb60fa5f04..94428c7feb 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1082,10 +1082,18 @@ class DynamoDBArgs(LiteLLMBase): assume_role_aws_session_name: Optional[str] = None -class PassThroughEndpointTypedDict(TypedDict): - path: str - target: str - headers: dict +class PassThroughGenericEndpoint(LiteLLMBase): + path: str = Field(description="The route to be added to the LiteLLM Proxy Server.") + target: str = Field( + description="The URL to which requests for this path should be forwarded." + ) + headers: dict = Field( + description="Key-value pairs of headers to be forwarded with the request. You can set any key value pair here and it will be forwarded to your target endpoint" + ) + + +class PassThroughEndpointResponse(LiteLLMBase): + endpoints: List[PassThroughGenericEndpoint] class ConfigFieldUpdate(LiteLLMBase): @@ -1104,6 +1112,7 @@ class FieldDetail(BaseModel): field_type: str field_description: str field_default_value: Any = None + stored_in_db: Optional[bool] class ConfigList(LiteLLMBase): @@ -1219,7 +1228,7 @@ class ConfigGeneralSettings(LiteLLMBase): default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", ) - pass_through_endpoints: Optional[PassThroughEndpointTypedDict] = Field( + pass_through_endpoints: Optional[List[PassThroughGenericEndpoint]] = Field( default=None, description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through", ) @@ -1781,3 +1790,9 @@ class VirtualKeyEvent(LiteLLMBase): created_by_user_role: str created_by_key_alias: Optional[str] request_kwargs: dict + + +class CreatePassThroughEndpoint(LiteLLMBase): + path: str + target: str + headers: dict diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 1d9f691177..61893a3dc4 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -20,7 +20,14 @@ from fastapi.responses import StreamingResponse import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + PassThroughEndpointResponse, + PassThroughGenericEndpoint, + ProxyException, + UserAPIKeyAuth, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router = APIRouter() @@ -481,16 +488,64 @@ async def initialize_pass_through_endpoints(pass_through_endpoints: list): @router.get( - "/config/pass_through_endpoint/{endpoint_id}", + "/config/pass_through_endpoint", tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], + response_model=PassThroughEndpointResponse, ) -async def get_pass_through_endpoints(request: Request, endpoint_id: str): +async def get_pass_through_endpoints( + endpoint_id: Optional[str] = None, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ GET configured pass through endpoint. If no endpoint_id given, return all configured endpoints. """ + from litellm.proxy.proxy_server import get_config_general_settings + + ## Get existing pass-through endpoint field value + try: + response: ConfigFieldInfo = await get_config_general_settings( + field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict + ) + except Exception: + return PassThroughEndpointResponse(endpoints=[]) + + pass_through_endpoint_data: Optional[List] = response.field_value + if pass_through_endpoint_data is None: + return PassThroughEndpointResponse(endpoints=[]) + + returned_endpoints = [] + if endpoint_id is None: + for endpoint in pass_through_endpoint_data: + if isinstance(endpoint, dict): + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint)) + elif isinstance(endpoint, PassThroughGenericEndpoint): + returned_endpoints.append(endpoint) + elif endpoint_id is not None: + for endpoint in pass_through_endpoint_data: + _endpoint: Optional[PassThroughGenericEndpoint] = None + if isinstance(endpoint, dict): + _endpoint = PassThroughGenericEndpoint(**endpoint) + elif isinstance(endpoint, PassThroughGenericEndpoint): + _endpoint = endpoint + + if _endpoint is not None and _endpoint.path == endpoint_id: + returned_endpoints.append(_endpoint) + + return PassThroughEndpointResponse(endpoints=returned_endpoints) + + +@router.post( + "/config/pass_through_endpoint/{endpoint_id}", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], +) +async def update_pass_through_endpoints(request: Request, endpoint_id: str): + """ + Update a pass-through endpoint + """ pass @@ -499,20 +554,119 @@ async def get_pass_through_endpoints(request: Request, endpoint_id: str): tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], ) -async def create_pass_through_endpoints(request: Request): +async def create_pass_through_endpoints( + data: PassThroughGenericEndpoint, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Create new pass-through endpoint """ - pass + from litellm.proxy.proxy_server import ( + get_config_general_settings, + update_config_general_settings, + ) + + ## Get existing pass-through endpoint field value + + try: + response: ConfigFieldInfo = await get_config_general_settings( + field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict + ) + except Exception: + response = ConfigFieldInfo( + field_name="pass_through_endpoints", field_value=None + ) + + ## Update field with new endpoint + data_dict = data.model_dump() + if response.field_value is None: + response.field_value = [data_dict] + elif isinstance(response.field_value, List): + response.field_value.append(data_dict) + + ## Update db + updated_data = ConfigFieldUpdate( + field_name="pass_through_endpoints", + field_value=response.field_value, + config_type="general_settings", + ) + await update_config_general_settings( + data=updated_data, user_api_key_dict=user_api_key_dict + ) @router.delete( "/config/pass_through_endpoint", tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], + response_model=PassThroughEndpointResponse, ) -async def delete_pass_through_endpoints(request: Request): +async def delete_pass_through_endpoints( + endpoint_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ - Create new pass-through endpoint + Delete a pass-through endpoint + + Returns - the deleted endpoint """ - pass + from litellm.proxy.proxy_server import ( + get_config_general_settings, + update_config_general_settings, + ) + + ## Get existing pass-through endpoint field value + + try: + response: ConfigFieldInfo = await get_config_general_settings( + field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict + ) + except Exception: + response = ConfigFieldInfo( + field_name="pass_through_endpoints", field_value=None + ) + + ## Update field by removing endpoint + pass_through_endpoint_data: Optional[List] = response.field_value + response_obj: Optional[PassThroughGenericEndpoint] = None + if response.field_value is None or pass_through_endpoint_data is None: + raise HTTPException( + status_code=400, + detail={"error": "There are no pass-through endpoints setup."}, + ) + elif isinstance(response.field_value, List): + invalid_idx: Optional[int] = None + for idx, endpoint in enumerate(pass_through_endpoint_data): + _endpoint: Optional[PassThroughGenericEndpoint] = None + if isinstance(endpoint, dict): + _endpoint = PassThroughGenericEndpoint(**endpoint) + elif isinstance(endpoint, PassThroughGenericEndpoint): + _endpoint = endpoint + + if _endpoint is not None and _endpoint.path == endpoint_id: + invalid_idx = idx + response_obj = _endpoint + + if invalid_idx is not None: + pass_through_endpoint_data.pop(invalid_idx) + + ## Update db + updated_data = ConfigFieldUpdate( + field_name="pass_through_endpoints", + field_value=pass_through_endpoint_data, + config_type="general_settings", + ) + await update_config_general_settings( + data=updated_data, user_api_key_dict=user_api_key_dict + ) + + if response_obj is None: + raise HTTPException( + status_code=400, + detail={ + "error": "Endpoint={} was not found in pass-through endpoint list.".format( + endpoint_id + ) + }, + ) + return PassThroughEndpointResponse(endpoints=[response_obj]) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6483a61b53..a331e150ef 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -573,6 +573,23 @@ def _resolve_typed_dict_type(typ): return None +def _resolve_pydantic_type(typ) -> List: + """Resolve the actual TypedDict class from a potentially wrapped type.""" + origin = get_origin(typ) + typs = [] + if origin is Union: # Check if it's a Union (like Optional) + for arg in get_args(typ): + if ( + arg is not None + and not isinstance(arg, type(None)) + and "NoneType" not in str(arg) + ): + typs.append(arg) + elif isinstance(typ, type) and isinstance(typ, BaseModel): + return [typ] + return typs + + def prisma_setup(database_url: Optional[str]): global prisma_client, proxy_logging_obj, user_api_key_cache @@ -2204,6 +2221,15 @@ class ProxyConfig: alerting_args=general_settings["alerting_args"], ) + ## PASS-THROUGH ENDPOINTS ## + if "pass_through_endpoints" in _general_settings: + general_settings["pass_through_endpoints"] = _general_settings[ + "pass_through_endpoints" + ] + await initialize_pass_through_endpoints( + pass_through_endpoints=general_settings["pass_through_endpoints"] + ) + async def add_deployment( self, prisma_client: PrismaClient, @@ -9434,7 +9460,7 @@ async def get_config_list( "global_max_parallel_requests": {"type": "Integer"}, "max_request_size_mb": {"type": "Integer"}, "max_response_size_mb": {"type": "Integer"}, - "pass_through_endpoints": {"type": "TypedDictionary"}, + "pass_through_endpoints": {"type": "PydanticModel"}, } return_val = [] @@ -9446,45 +9472,76 @@ async def get_config_list( typed_dict_type = allowed_args[field_name]["type"] - if typed_dict_type == "TypedDictionary": - typed_dict_class: Optional[Any] = _resolve_typed_dict_type( + if typed_dict_type == "PydanticModel": + pydantic_class_list: Optional[Any] = _resolve_pydantic_type( field_info.annotation ) + if pydantic_class_list is None: + continue - if typed_dict_class is None: - nested_fields = None - else: + for pydantic_class in pydantic_class_list: # Get type hints from the TypedDict to create FieldDetail objects nested_fields = [ FieldDetail( field_name=sub_field, - field_type=type_hint.__name__, + field_type=sub_field_type.__name__, field_description="", # Add custom logic if descriptions are available field_default_value=general_settings.get(sub_field, None), + stored_in_db=None, ) - for sub_field, type_hint in get_type_hints( - typed_dict_class - ).items() + for sub_field, sub_field_type in pydantic_class.__annotations__.items() ] + + idx = 0 + for ( + sub_field, + sub_field_info, + ) in pydantic_class.model_fields.items(): + if ( + hasattr(sub_field_info, "description") + and sub_field_info.description is not None + ): + nested_fields[idx].field_description = ( + sub_field_info.description + ) + idx += 1 + + _stored_in_db = None + if field_name in db_general_settings_dict: + _stored_in_db = True + elif field_name in general_settings: + _stored_in_db = False + + _response_obj = ConfigList( + field_name=field_name, + field_type=allowed_args[field_name]["type"], + field_description=field_info.description or "", + field_value=general_settings.get(field_name, None), + stored_in_db=_stored_in_db, + field_default_value=field_info.default, + nested_fields=nested_fields, + ) + return_val.append(_response_obj) + else: nested_fields = None - _stored_in_db = None - if field_name in db_general_settings_dict: - _stored_in_db = True - elif field_name in general_settings: - _stored_in_db = False + _stored_in_db = None + if field_name in db_general_settings_dict: + _stored_in_db = True + elif field_name in general_settings: + _stored_in_db = False - _response_obj = ConfigList( - field_name=field_name, - field_type=allowed_args[field_name]["type"], - field_description=field_info.description or "", - field_value=general_settings.get(field_name, None), - stored_in_db=_stored_in_db, - field_default_value=field_info.default, - nested_fields=nested_fields, - ) - return_val.append(_response_obj) + _response_obj = ConfigList( + field_name=field_name, + field_type=allowed_args[field_name]["type"], + field_description=field_info.description or "", + field_value=general_settings.get(field_name, None), + stored_in_db=_stored_in_db, + field_default_value=field_info.default, + nested_fields=nested_fields, + ) + return_val.append(_response_obj) return return_val From 6fc6df134f7477830975ed8fb367ef7bbc9cf2ed Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 15 Aug 2024 21:58:11 -0700 Subject: [PATCH 4/4] feat(ui): for adding pass-through endpoints --- .../pass_through_endpoints.py | 47 +++++ litellm/proxy/proxy_server.py | 9 +- ui/litellm-dashboard/src/app/page.tsx | 8 + .../src/components/add_pass_through.tsx | 149 +++++++++++++ .../src/components/general_settings.tsx | 2 +- .../src/components/key_value_input.tsx | 56 +++++ .../src/components/leftnav.tsx | 12 +- .../src/components/networking.tsx | 143 +++++++++++++ .../src/components/pass_through_settings.tsx | 196 ++++++++++++++++++ 9 files changed, 613 insertions(+), 9 deletions(-) create mode 100644 litellm/proxy/config_management_endpoints/pass_through_endpoints.py create mode 100644 ui/litellm-dashboard/src/components/add_pass_through.tsx create mode 100644 ui/litellm-dashboard/src/components/key_value_input.tsx create mode 100644 ui/litellm-dashboard/src/components/pass_through_settings.tsx diff --git a/litellm/proxy/config_management_endpoints/pass_through_endpoints.py b/litellm/proxy/config_management_endpoints/pass_through_endpoints.py new file mode 100644 index 0000000000..237f1b74b2 --- /dev/null +++ b/litellm/proxy/config_management_endpoints/pass_through_endpoints.py @@ -0,0 +1,47 @@ +""" +What is this? + +CRUD endpoints for managing pass-through endpoints +""" + +import asyncio +import traceback +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import fastapi +import httpx +from fastapi import ( + APIRouter, + Depends, + File, + Form, + Header, + HTTPException, + Request, + Response, + UploadFile, + status, +) + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.batches.main import FileObject +from litellm.proxy._types import * +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() + + +@router.get( + "/config/pass_through_endpoints/settings", + dependencies=[Depends(user_api_key_auth)], + tags=["pass-through-endpoints"], + summary="Create pass-through endpoints for provider specific endpoints - https://docs.litellm.ai/docs/proxy/pass_through", +) +async def create_fine_tuning_job( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + pass diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a331e150ef..4d141955b2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9473,11 +9473,10 @@ async def get_config_list( typed_dict_type = allowed_args[field_name]["type"] if typed_dict_type == "PydanticModel": - pydantic_class_list: Optional[Any] = _resolve_pydantic_type( - field_info.annotation - ) - if pydantic_class_list is None: - continue + if field_name == "pass_through_endpoints": + pydantic_class_list = [PassThroughGenericEndpoint] + else: + pydantic_class_list = [] for pydantic_class in pydantic_class_list: # Get type hints from the TypedDict to create FieldDetail objects diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 9b7a09cbfd..02ef8ebe05 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -9,6 +9,7 @@ import Teams from "@/components/teams"; import AdminPanel from "@/components/admins"; import Settings from "@/components/settings"; import GeneralSettings from "@/components/general_settings"; +import PassThroughSettings from "@/components/pass_through_settings"; import BudgetPanel from "@/components/budgets/budget_panel"; import ModelHub from "@/components/model_hub"; import APIRef from "@/components/api_ref"; @@ -263,6 +264,13 @@ const CreateKeyPage = () => { accessToken={accessToken} premiumUser={premiumUser} /> + ) : page == "pass-through-settings" ? ( + ) : ( >; +} + +const AddPassThroughEndpoint: React.FC = ({ + accessToken, setPassThroughItems, passThroughItems +}) => { + const [form] = Form.useForm(); + const [isModalVisible, setIsModalVisible] = useState(false); + const [selectedModel, setSelectedModel] = useState(""); + const handleOk = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const handleCancel = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const addPassThrough = (formValues: Record) => { + // Print the received value + console.log(formValues); + + // // Extract model_name and models from formValues + // const { model_name, models } = formValues; + + // // Create new fallback + // const newFallback = { [model_name]: models }; + + // // Get current fallbacks, or an empty array if it's null + // const currentFallbacks = routerSettings.fallbacks || []; + + // // Add new fallback to the current fallbacks + // const updatedFallbacks = [...currentFallbacks, newFallback]; + + // // Create a new routerSettings object with updated fallbacks + // const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks }; + + const newPassThroughItem: passThroughItem = { + "headers": formValues["headers"], + "path": formValues["path"], + "target": formValues["target"] + } + const updatedPassThroughSettings = [...passThroughItems, newPassThroughItem] + + + try { + createPassThroughEndpoint(accessToken, formValues); + setPassThroughItems(updatedPassThroughSettings) + } catch (error) { + message.error("Failed to update router settings: " + error, 20); + } + + message.success("Pass through endpoint successfully added"); + + setIsModalVisible(false) + form.resetFields(); + }; + + + return ( +
+ + +
+ <> + + + + + + + + + + + + +
+ Add Pass-Through Endpoint +
+
+ + +
+ ); +}; + +export default AddPassThroughEndpoint; diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index f80ca203ba..87750b6070 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -597,7 +597,7 @@ const GeneralSettings: React.FC = ({ - {generalSettings.map((value, index) => ( + {generalSettings.filter((value) => value.field_type !== "TypedDictionary").map((value, index) => ( {value.field_name} diff --git a/ui/litellm-dashboard/src/components/key_value_input.tsx b/ui/litellm-dashboard/src/components/key_value_input.tsx new file mode 100644 index 0000000000..90f58eede6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/key_value_input.tsx @@ -0,0 +1,56 @@ +import React, { useState } from 'react'; +import { Form, Input, Button, Space } from 'antd'; +import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons'; +import { TextInput, Grid, Col } from "@tremor/react"; +import { TrashIcon } from "@heroicons/react/outline"; + +interface KeyValueInputProps { + value?: Record; + onChange?: (value: Record) => void; +} + +const KeyValueInput: React.FC = ({ value = {}, onChange }) => { + const [pairs, setPairs] = useState<[string, string][]>(Object.entries(value)); + + const handleAdd = () => { + setPairs([...pairs, ['', '']]); + }; + + const handleRemove = (index: number) => { + const newPairs = pairs.filter((_, i) => i !== index); + setPairs(newPairs); + onChange?.(Object.fromEntries(newPairs)); + }; + + const handleChange = (index: number, key: string, val: string) => { + const newPairs = [...pairs]; + newPairs[index] = [key, val]; + setPairs(newPairs); + onChange?.(Object.fromEntries(newPairs)); + }; + + return ( +
+ {pairs.map(([key, val], index) => ( + + handleChange(index, e.target.value, val)} + /> + handleChange(index, key, e.target.value)} + /> + handleRemove(index)} /> + + ))} + +
+ ); +}; + +export default KeyValueInput; diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index b33dda982e..c8f5745ed4 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -102,15 +102,21 @@ const Sidebar: React.FC = ({ Router Settings ) : null} + {userRole == "Admin" ? ( - setPage("admin-panel")}> + setPage("pass-through-settings")}> + Pass-Through + + ) : null} + {userRole == "Admin" ? ( + setPage("admin-panel")}> Admin Settings ) : null} - setPage("api_ref")}> + setPage("api_ref")}> API Reference - setPage("model-hub")}> + setPage("model-hub")}> Model Hub diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 263616c917..f550764789 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2388,6 +2388,38 @@ export const getGeneralSettingsCall = async (accessToken: String) => { } }; + +export const getPassThroughEndpointsCall = async (accessToken: String) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/config/pass_through_endpoint` + : `/config/pass_through_endpoint`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to get callbacks:", error); + throw error; + } +}; + export const getConfigFieldSetting = async ( accessToken: String, fieldName: string @@ -2420,6 +2452,85 @@ export const getConfigFieldSetting = async ( } }; +export const updatePassThroughFieldSetting = async ( + accessToken: String, + fieldName: string, + fieldValue: any +) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/config/pass_through_endpoint` + : `/config/pass_through_endpoint`; + + let formData = { + field_name: fieldName, + field_value: fieldValue, + }; + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(formData), + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + message.success("Successfully updated value!"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to set callbacks:", error); + throw error; + } +}; + +export const createPassThroughEndpoint = async ( + accessToken: String, + formValues: Record +) => { + /** + * Set callbacks on proxy + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/config/pass_through_endpoint` : `/config/pass_through_endpoint`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + ...formValues, // Include formValues in the request body + }), + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to set callbacks:", error); + throw error; + } +}; + export const updateConfigFieldSetting = async ( accessToken: String, fieldName: string, @@ -2500,6 +2611,38 @@ export const deleteConfigFieldSetting = async ( throw error; } }; + +export const deletePassThroughEndpointsCall = async (accessToken: String, endpointId: string) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/config/pass_through_endpoint?endpoint_id=${endpointId}` + : `/config/pass_through_endpoint${endpointId}`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "DELETE", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to get callbacks:", error); + throw error; + } +}; + export const setCallbacksCall = async ( accessToken: String, formValues: Record diff --git a/ui/litellm-dashboard/src/components/pass_through_settings.tsx b/ui/litellm-dashboard/src/components/pass_through_settings.tsx new file mode 100644 index 0000000000..c979076a2a --- /dev/null +++ b/ui/litellm-dashboard/src/components/pass_through_settings.tsx @@ -0,0 +1,196 @@ +import React, { useState, useEffect } from "react"; +import { + Card, + Title, + Subtitle, + Table, + TableHead, + TableRow, + Badge, + TableHeaderCell, + TableCell, + TableBody, + Metric, + Text, + Grid, + Button, + TextInput, + Select as Select2, + SelectItem, + Col, + Accordion, + AccordionBody, + AccordionHeader, + AccordionList, +} from "@tremor/react"; +import { + TabPanel, + TabPanels, + TabGroup, + TabList, + Tab, + Icon, +} from "@tremor/react"; +import { + getCallbacksCall, + setCallbacksCall, + getGeneralSettingsCall, + deletePassThroughEndpointsCall, + getPassThroughEndpointsCall, + serviceHealthCheck, + updateConfigFieldSetting, + deleteConfigFieldSetting, +} from "./networking"; +import { + Modal, + Form, + Input, + Select, + Button as Button2, + message, + InputNumber, +} from "antd"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, + RefreshIcon, + CheckCircleIcon, + XCircleIcon, + QuestionMarkCircleIcon, +} from "@heroicons/react/outline"; +import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; +import AddFallbacks from "./add_fallbacks"; +import AddPassThroughEndpoint from "./add_pass_through"; +import openai from "openai"; +import Paragraph from "antd/es/skeleton/Paragraph"; +interface GeneralSettingsPageProps { + accessToken: string | null; + userRole: string | null; + userID: string | null; + modelData: any; +} + + +interface routingStrategyArgs { + ttl?: number; + lowest_latency_buffer?: number; +} + +interface nestedFieldItem { + field_name: string; + field_type: string; + field_value: any; + field_description: string; + stored_in_db: boolean | null; +} + +export interface passThroughItem { + path: string + target: string + headers: object +} + + + + +const PassThroughSettings: React.FC = ({ + accessToken, + userRole, + userID, + modelData, +}) => { + const [generalSettings, setGeneralSettings] = useState( + [] + ); + useEffect(() => { + if (!accessToken || !userRole || !userID) { + return; + } + getPassThroughEndpointsCall(accessToken).then((data) => { + let general_settings = data["endpoints"]; + setGeneralSettings(general_settings); + }); + }, [accessToken, userRole, userID]); + + + const handleResetField = (fieldName: string, idx: number) => { + if (!accessToken) { + return; + } + + try { + deletePassThroughEndpointsCall(accessToken, fieldName); + // update value in state + + const updatedSettings = generalSettings.filter((setting) => setting.path !== fieldName); + setGeneralSettings(updatedSettings); + + message.success("Endpoint deleted successfully."); + + } catch (error) { + // do something + } + }; + + + if (!accessToken) { + return null; + } + + + + return ( +
+ + + + + + Path + Target + Headers + Action + + + + {generalSettings.map((value, index) => ( + + + {value.path} + + + { + value.target + } + + + { + JSON.stringify(value.headers) + } + + + + handleResetField(value.path, index) + } + > + Reset + + + + ))} + +
+ + + +
+ ); +}; + +export default PassThroughSettings;