Merge pull request #4463 from BerriAI/docs_create_pass_through_routes_litellm_proxy

Docs create pass through routes litellm proxy (tutorial setup cohere Re-Rank Endpoint)
2026-08-20 06:23:46 +00:00 · 2024-06-28 16:55:44 -07:00
parent 1980a07f32 ac066462df
commit d0d1efcbdf
5 changed files with 247 additions and 2 deletions
@@ -0,0 +1,97 @@
+# ➡️ Create Pass Through Endpoints 
+
+Add pass through routes to LiteLLM Proxy
+
+**Example:** Add a route `/v1/rerank` that forwards requests to `https://api.cohere.com/v1/rerank` through LiteLLM Proxy
+
+
+💡 This allows making the following Request to LiteLLM Proxy
+```shell
+curl --request POST \
+  --url http://localhost:4000/v1/rerank \
+  --header 'accept: application/json' \
+  --header 'content-type: application/json' \
+  --data '{
+    "model": "rerank-english-v3.0",
+    "query": "What is the capital of the United States?",
+    "top_n": 3,
+    "documents": ["Carson City is the capital city of the American state of Nevada."]
+  }'
+```
+
+## Tutorial - Setup Cohere Re-Rank Endpoint on LiteLLM Proxy
+
+**Step 1** Define pass through routes on [litellm config.yaml](configs.md)
+
+```yaml
+general_settings:
+  master_key: sk-1234
+  pass_through_endpoints:
+    - path: "/v1/rerank"                                  # route you want to add to LiteLLM Proxy Server
+      target: "https://api.cohere.com/v1/rerank"          # URL this route should forward requests to
+      headers:                                            # headers to forward to this URL
+        Authorization: "bearer os.environ/COHERE_API_KEY" # (Optional) Auth Header to forward to your Endpoint
+        content-type: application/json                    # (Optional) Extra Headers to pass to this endpoint 
+        accept: application/json
+```
+
+**Step 2** Start Proxy Server in detailed_debug mode
+
+```shell
+litellm --config config.yaml
+```
+**Step 3** Make Request to pass through endpoint
+
+```shell
+curl --request POST \
+  --url http://localhost:4000/v1/rerank \
+  --header 'accept: application/json' \
+  --header 'content-type: application/json' \
+  --data '{
+    "model": "rerank-english-v3.0",
+    "query": "What is the capital of the United States?",
+    "top_n": 3,
+    "documents": ["Carson City is the capital city of the American state of Nevada.",
+                  "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+                  "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.",
+                  "Capitalization or capitalisation in English grammar is the use of a capital letter at the start of a word. English usage varies from capitalization in other languages.",
+                  "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states."]
+  }'
+```
+
+
+🎉 **Expected Response**
+
+This request got forwarded from LiteLLM Proxy -> Defined Target URL (with headers)
+
+```shell
+{
+  "id": "37103a5b-8cfb-48d3-87c7-da288bedd429",
+  "results": [
+    {
+      "index": 2,
+      "relevance_score": 0.999071
+    },
+    {
+      "index": 4,
+      "relevance_score": 0.7867867
+    },
+    {
+      "index": 0,
+      "relevance_score": 0.32713068
+    }
+  ],
+  "meta": {
+    "api_version": {
+      "version": "1"
+    },
+    "billed_units": {
+      "search_units": 1
+    }
+  }
+}
+```
+
+
+
+
@@ -48,6 +48,7 @@ const sidebars = {
        "proxy/billing",
        "proxy/user_keys",
        "proxy/virtual_keys",
+        "proxy/token_auth",
        "proxy/alerting",
        {
          type: "category",
@@ -56,11 +57,11 @@ const sidebars = {
        },
        "proxy/ui",
        "proxy/prometheus",
+        "proxy/pass_through",
        "proxy/email",
        "proxy/multiple_admins",
        "proxy/team_based_routing",
        "proxy/customer_routing",
-        "proxy/token_auth",
        {
          type: "category",
          label: "Extra Load Balancing",
@@ -0,0 +1,133 @@
+import ast
+import traceback
+
+import httpx
+from fastapi import APIRouter, FastAPI, HTTPException, Request, Response, status
+from fastapi.responses import StreamingResponse
+
+import litellm
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import ProxyException
+
+async_client = httpx.AsyncClient()
+
+
+async def set_env_variables_in_header(custom_headers: dict):
+    """
+    checks if nay headers on config.yaml are defined as os.environ/COHERE_API_KEY etc
+
+    only runs for headers defined on config.yaml
+
+    example header can be
+
+    {"Authorization": "bearer os.environ/COHERE_API_KEY"}
+    """
+    headers = {}
+    for key, value in custom_headers.items():
+        headers[key] = value
+        if isinstance(value, str) and "os.environ/" in value:
+            verbose_proxy_logger.debug(
+                "pass through endpoint - looking up 'os.environ/' variable"
+            )
+            # get string section that is os.environ/
+            start_index = value.find("os.environ/")
+            _variable_name = value[start_index:]
+
+            verbose_proxy_logger.debug(
+                "pass through endpoint - getting secret for variable name: %s",
+                _variable_name,
+            )
+            _secret_value = litellm.get_secret(_variable_name)
+            new_value = value.replace(_variable_name, _secret_value)
+            headers[key] = new_value
+    return headers
+
+
+async def pass_through_request(request: Request, target: str, custom_headers: dict):
+    try:
+
+        url = httpx.URL(target)
+        headers = custom_headers
+
+        request_body = await request.body()
+        _parsed_body = ast.literal_eval(request_body.decode("utf-8"))
+
+        verbose_proxy_logger.debug(
+            "Pass through endpoint sending request to \nURL {}\nheaders: {}\nbody: {}\n".format(
+                url, headers, _parsed_body
+            )
+        )
+
+        response = await async_client.request(
+            method=request.method,
+            url=url,
+            headers=headers,
+            params=request.query_params,
+            json=_parsed_body,
+        )
+
+        if response.status_code != 200:
+            raise HTTPException(status_code=response.status_code, detail=response.text)
+
+        content = await response.aread()
+        return Response(
+            content=content,
+            status_code=response.status_code,
+            headers=dict(response.headers),
+        )
+    except Exception as e:
+        verbose_proxy_logger.error(
+            "litellm.proxy.proxy_server.pass through endpoint(): Exception occured - {}".format(
+                str(e)
+            )
+        )
+        verbose_proxy_logger.debug(traceback.format_exc())
+        if isinstance(e, HTTPException):
+            raise ProxyException(
+                message=getattr(e, "message", str(e.detail)),
+                type=getattr(e, "type", "None"),
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
+            )
+        else:
+            error_msg = f"{str(e)}"
+            raise ProxyException(
+                message=getattr(e, "message", error_msg),
+                type=getattr(e, "type", "None"),
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", 500),
+            )
+
+
+def create_pass_through_route(endpoint, target, custom_headers=None):
+    async def endpoint_func(request: Request):
+        return await pass_through_request(request, target, custom_headers)
+
+    return endpoint_func
+
+
+async def initialize_pass_through_endpoints(pass_through_endpoints: list):
+
+    verbose_proxy_logger.debug("initializing pass through endpoints")
+    from litellm.proxy.proxy_server import app
+
+    for endpoint in pass_through_endpoints:
+        _target = endpoint.get("target", None)
+        _path = endpoint.get("path", None)
+        _custom_headers = endpoint.get("headers", None)
+        _custom_headers = await set_env_variables_in_header(
+            custom_headers=_custom_headers
+        )
+
+        if _target is None:
+            continue
+
+        verbose_proxy_logger.debug("adding pass through endpoint: %s", _path)
+
+        app.add_api_route(
+            path=_path,
+            endpoint=create_pass_through_route(_path, _target, _custom_headers),
+            methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+        )
+
+        verbose_proxy_logger.debug("Added new pass through endpoint: %s", _path)
@@ -22,6 +22,13 @@ general_settings:
  master_key: sk-1234
  alerting: ["slack", "email"]
  public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
+  pass_through_endpoints:
+    - path: "/v1/rerank"
+      target: "https://api.cohere.com/v1/rerank"
+      headers:
+        Authorization: "bearer os.environ/COHERE_API_KEY"
+        content-type: application/json
+        accept: application/json


 litellm_settings:
@@ -34,6 +41,5 @@ litellm_settings:
    - user
    - metadata
    - metadata.generation_name
-  cache: True


@@ -161,6 +161,9 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
    router as key_management_router,
 )
 from litellm.proxy.management_endpoints.team_endpoints import router as team_router
+from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+    initialize_pass_through_endpoints,
+)
 from litellm.proxy.secret_managers.aws_secret_manager import (
    load_aws_kms,
    load_aws_secret_manager,
@@ -1856,6 +1859,11 @@ class ProxyConfig:
                user_custom_key_generate = get_instance_fn(
                    value=custom_key_generate, config_file_path=config_file_path
                )
+            ## pass through endpoints
+            if general_settings.get("pass_through_endpoints", None) is not None:
+                await initialize_pass_through_endpoints(
+                    pass_through_endpoints=general_settings["pass_through_endpoints"]
+                )
            ## dynamodb
            database_type = general_settings.get("database_type", None)
            if database_type is not None and (