From f52cc18adb0dd68b4484d9643eb208cc789acfde Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 28 Jun 2024 15:03:21 -0700 Subject: [PATCH 1/4] feat - support pass through endpoints --- .../pass_through_endpoints.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 litellm/proxy/pass_through_endpoints/pass_through_endpoints.py diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py new file mode 100644 index 0000000000..1d20066712 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -0,0 +1,101 @@ +import ast +import traceback + +import httpx +from fastapi import APIRouter, FastAPI, HTTPException, Request, Response, status +from fastapi.responses import StreamingResponse + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ProxyException + +async_client = httpx.AsyncClient() + + +async def pass_through_request(request: Request, target: str, custom_headers: dict): + try: + + url = httpx.URL(target) + + # Start with the original request headers + headers = custom_headers + # headers = dict(request.headers) + + request_body = await request.body() + _parsed_body = ast.literal_eval(request_body.decode("utf-8")) + + verbose_proxy_logger.debug( + "Pass through endpoint sending request to \nURL {}\nheaders: {}\nbody: {}\n".format( + url, headers, _parsed_body + ) + ) + + response = await async_client.request( + method=request.method, + url=url, + headers=headers, + params=request.query_params, + json=_parsed_body, + ) + + if response.status_code != 200: + raise HTTPException(status_code=response.status_code, detail=response.text) + + content = await response.aread() + return Response( + content=content, + status_code=response.status_code, + headers=dict(response.headers), + ) + except Exception as e: + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.pass through endpoint(): Exception occured - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e.detail)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) + + +def create_pass_through_route(endpoint, target, custom_headers=None): + async def endpoint_func(request: Request): + return await pass_through_request(request, target, custom_headers) + + return endpoint_func + + +async def initialize_pass_through_endpoints(pass_through_endpoints: list): + + verbose_proxy_logger.debug("initializing pass through endpoints") + from litellm.proxy.proxy_server import app + + for endpoint in pass_through_endpoints: + _target = endpoint.get("target", None) + _path = endpoint.get("path", None) + _custom_headers = endpoint.get("headers", None) + + if _target is None: + continue + + verbose_proxy_logger.debug("adding pass through endpoint: %s", _path) + + app.add_api_route( + path=_path, + endpoint=create_pass_through_route(_path, _target, _custom_headers), + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], + ) + + verbose_proxy_logger.debug("Added new pass through endpoint: %s", _path) From 954c6ec9eda9ce72e1883c1aaba7af72e4dbad6d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 28 Jun 2024 15:06:51 -0700 Subject: [PATCH 2/4] fix support pass through endpoints --- litellm/proxy/proxy_server.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 553c8a48cb..ff4b1e6633 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -161,6 +161,9 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( router as key_management_router, ) from litellm.proxy.management_endpoints.team_endpoints import router as team_router +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + initialize_pass_through_endpoints, +) from litellm.proxy.secret_managers.aws_secret_manager import ( load_aws_kms, load_aws_secret_manager, @@ -1856,6 +1859,11 @@ class ProxyConfig: user_custom_key_generate = get_instance_fn( value=custom_key_generate, config_file_path=config_file_path ) + ## pass through endpoints + if general_settings.get("pass_through_endpoints", None) is not None: + await initialize_pass_through_endpoints( + pass_through_endpoints=general_settings["pass_through_endpoints"] + ) ## dynamodb database_type = general_settings.get("database_type", None) if database_type is not None and ( From 8f2931937a9970d225f2c5b215432f08927c570a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 28 Jun 2024 15:30:31 -0700 Subject: [PATCH 3/4] fix use os.environ/ vars for pass through endpoints --- .../pass_through_endpoints.py | 38 +++++++++++++++++-- litellm/proxy/proxy_config.yaml | 8 +++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 1d20066712..802c84d84b 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -5,20 +5,49 @@ import httpx from fastapi import APIRouter, FastAPI, HTTPException, Request, Response, status from fastapi.responses import StreamingResponse +import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException async_client = httpx.AsyncClient() +async def set_env_variables_in_header(custom_headers: dict): + """ + checks if nay headers on config.yaml are defined as os.environ/COHERE_API_KEY etc + + only runs for headers defined on config.yaml + + example header can be + + {"Authorization": "bearer os.environ/COHERE_API_KEY"} + """ + headers = {} + for key, value in custom_headers.items(): + headers[key] = value + if isinstance(value, str) and "os.environ/" in value: + verbose_proxy_logger.debug( + "pass through endpoint - looking up 'os.environ/' variable" + ) + # get string section that is os.environ/ + start_index = value.find("os.environ/") + _variable_name = value[start_index:] + + verbose_proxy_logger.debug( + "pass through endpoint - getting secret for variable name: %s", + _variable_name, + ) + _secret_value = litellm.get_secret(_variable_name) + new_value = value.replace(_variable_name, _secret_value) + headers[key] = new_value + return headers + + async def pass_through_request(request: Request, target: str, custom_headers: dict): try: url = httpx.URL(target) - - # Start with the original request headers headers = custom_headers - # headers = dict(request.headers) request_body = await request.body() _parsed_body = ast.literal_eval(request_body.decode("utf-8")) @@ -86,6 +115,9 @@ async def initialize_pass_through_endpoints(pass_through_endpoints: list): _target = endpoint.get("target", None) _path = endpoint.get("path", None) _custom_headers = endpoint.get("headers", None) + _custom_headers = await set_env_variables_in_header( + custom_headers=_custom_headers + ) if _target is None: continue diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 0c0365f43d..04f12a8440 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -22,6 +22,13 @@ general_settings: master_key: sk-1234 alerting: ["slack", "email"] public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] + pass_through_endpoints: + - path: "/v1/rerank" + target: "https://api.cohere.com/v1/rerank" + headers: + Authorization: "bearer os.environ/COHERE_API_KEY" + content-type: application/json + accept: application/json litellm_settings: @@ -34,6 +41,5 @@ litellm_settings: - user - metadata - metadata.generation_name - cache: True From ac066462df29ba21d36d0e42c036d8c2cd166536 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 28 Jun 2024 15:55:29 -0700 Subject: [PATCH 4/4] docs - add pass through routes on litelm proxy --- docs/my-website/docs/proxy/pass_through.md | 97 ++++++++++++++++++++++ docs/my-website/sidebars.js | 3 +- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/proxy/pass_through.md diff --git a/docs/my-website/docs/proxy/pass_through.md b/docs/my-website/docs/proxy/pass_through.md new file mode 100644 index 0000000000..c19c93cec8 --- /dev/null +++ b/docs/my-website/docs/proxy/pass_through.md @@ -0,0 +1,97 @@ +# ➡️ Create Pass Through Endpoints + +Add pass through routes to LiteLLM Proxy + +**Example:** Add a route `/v1/rerank` that forwards requests to `https://api.cohere.com/v1/rerank` through LiteLLM Proxy + + +💡 This allows making the following Request to LiteLLM Proxy +```shell +curl --request POST \ + --url http://localhost:4000/v1/rerank \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data '{ + "model": "rerank-english-v3.0", + "query": "What is the capital of the United States?", + "top_n": 3, + "documents": ["Carson City is the capital city of the American state of Nevada."] + }' +``` + +## Tutorial - Setup Cohere Re-Rank Endpoint on LiteLLM Proxy + +**Step 1** Define pass through routes on [litellm config.yaml](configs.md) + +```yaml +general_settings: + master_key: sk-1234 + pass_through_endpoints: + - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server + target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to + headers: # headers to forward to this URL + Authorization: "bearer os.environ/COHERE_API_KEY" # (Optional) Auth Header to forward to your Endpoint + content-type: application/json # (Optional) Extra Headers to pass to this endpoint + accept: application/json +``` + +**Step 2** Start Proxy Server in detailed_debug mode + +```shell +litellm --config config.yaml +``` +**Step 3** Make Request to pass through endpoint + +```shell +curl --request POST \ + --url http://localhost:4000/v1/rerank \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data '{ + "model": "rerank-english-v3.0", + "query": "What is the capital of the United States?", + "top_n": 3, + "documents": ["Carson City is the capital city of the American state of Nevada.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", + "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", + "Capitalization or capitalisation in English grammar is the use of a capital letter at the start of a word. English usage varies from capitalization in other languages.", + "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states."] + }' +``` + + +🎉 **Expected Response** + +This request got forwarded from LiteLLM Proxy -> Defined Target URL (with headers) + +```shell +{ + "id": "37103a5b-8cfb-48d3-87c7-da288bedd429", + "results": [ + { + "index": 2, + "relevance_score": 0.999071 + }, + { + "index": 4, + "relevance_score": 0.7867867 + }, + { + "index": 0, + "relevance_score": 0.32713068 + } + ], + "meta": { + "api_version": { + "version": "1" + }, + "billed_units": { + "search_units": 1 + } + } +} +``` + + + + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 31bc6abcb7..82f4bd2600 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -48,6 +48,7 @@ const sidebars = { "proxy/billing", "proxy/user_keys", "proxy/virtual_keys", + "proxy/token_auth", "proxy/alerting", { type: "category", @@ -56,11 +57,11 @@ const sidebars = { }, "proxy/ui", "proxy/prometheus", + "proxy/pass_through", "proxy/email", "proxy/multiple_admins", "proxy/team_based_routing", "proxy/customer_routing", - "proxy/token_auth", { type: "category", label: "Extra Load Balancing",