Merge pull request #4465 from BerriAI/litellm_pass_through_langfuse_requests

[Feat] - Proxy support Passing through Langfuse requests

docs/my-website/docs/proxy/pass_through.md

@@ -1,3 +1,5 @@
+import Image from '@theme/IdealImage';
+
 # ➡️ Create Pass Through Endpoints 
 
 Add pass through routes to LiteLLM Proxy
@@ -19,7 +21,7 @@ curl --request POST \
   }'
 ```
 
-## Tutorial - Setup Cohere Re-Rank Endpoint on LiteLLM Proxy
+## Tutorial - Pass through Cohere Re-Rank Endpoint
 
 **Step 1** Define pass through routes on [litellm config.yaml](configs.md)
 
@@ -38,10 +40,12 @@ general_settings:
 **Step 2** Start Proxy Server in detailed_debug mode
 
 ```shell
-litellm --config config.yaml
+litellm --config config.yaml --detailed_debug
 ```
 **Step 3** Make Request to pass through endpoint
 
+Here `http://localhost:4000` is your litellm proxy endpoint
+
 ```shell
 curl --request POST \
   --url http://localhost:4000/v1/rerank \
@@ -92,6 +96,59 @@ This request got forwarded from LiteLLM Proxy -> Defined Target URL (with header
 }
 ```
 
+## Tutorial - Pass Through Langfuse Requests
 
 
+**Step 1** Define pass through routes on [litellm config.yaml](configs.md)
+
+```yaml
+general_settings:
+  master_key: sk-1234
+  pass_through_endpoints:
+    - path: "/api/public/ingestion"                                # route you want to add to LiteLLM Proxy Server
+      target: "https://us.cloud.langfuse.com/api/public/ingestion" # URL this route should forward 
+      headers:
+        LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_DEV_PUBLIC_KEY" # your langfuse account public key
+        LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_DEV_SK_KEY"     # your langfuse account secret key
+```
+
+**Step 2** Start Proxy Server in detailed_debug mode
+
+```shell
+litellm --config config.yaml --detailed_debug
+```
+**Step 3** Make Request to pass through endpoint
+
+Run this code to make a sample trace 
+```python
+from langfuse import Langfuse
+
+langfuse = Langfuse(
+    host="http://localhost:4000", # your litellm proxy endpoint
+    public_key="anything",        # no key required since this is a pass through
+    secret_key="anything",        # no key required since this is a pass through
+)
+
+print("sending langfuse trace request")
+trace = langfuse.trace(name="test-trace-litellm-proxy-passthrough")
+print("flushing langfuse request")
+langfuse.flush()
+
+print("flushed langfuse request")
+```
+
+
+🎉 **Expected Response**
+
+On success
+Expect to see the following Trace Generated on your Langfuse Dashboard
+
+<Image img={require('../../img/proxy_langfuse.png')} />
+
+You will see the following endpoint called on your litellm proxy server logs
+
+```shell
+POST /api/public/ingestion HTTP/1.1" 207 Multi-Status
+```
+
 

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -1,5 +1,6 @@
 import ast
 import traceback
+from base64 import b64encode
 
 import httpx
 from fastapi import APIRouter, FastAPI, HTTPException, Request, Response, status
@@ -24,22 +25,41 @@ async def set_env_variables_in_header(custom_headers: dict):
     """
     headers = {}
     for key, value in custom_headers.items():
-        headers[key] = value
-        if isinstance(value, str) and "os.environ/" in value:
-            verbose_proxy_logger.debug(
-                "pass through endpoint - looking up 'os.environ/' variable"
-            )
-            # get string section that is os.environ/
-            start_index = value.find("os.environ/")
-            _variable_name = value[start_index:]
+        # langfuse Api requires base64 encoded headers - it's simpleer to just ask litellm users to set their langfuse public and secret keys
+        # we can then get the b64 encoded keys here
+        if key == "LANGFUSE_PUBLIC_KEY" or key == "LANGFUSE_SECRET_KEY":
+            # langfuse requires b64 encoded headers - we construct that here
+            _langfuse_public_key = custom_headers["LANGFUSE_PUBLIC_KEY"]
+            _langfuse_secret_key = custom_headers["LANGFUSE_SECRET_KEY"]
+            if isinstance(
+                _langfuse_public_key, str
+            ) and _langfuse_public_key.startswith("os.environ/"):
+                _langfuse_public_key = litellm.get_secret(_langfuse_public_key)
+            if isinstance(
+                _langfuse_secret_key, str
+            ) and _langfuse_secret_key.startswith("os.environ/"):
+                _langfuse_secret_key = litellm.get_secret(_langfuse_secret_key)
+            headers["Authorization"] = "Basic " + b64encode(
+                f"{_langfuse_public_key}:{_langfuse_secret_key}".encode("utf-8")
+            ).decode("ascii")
+        else:
+            # for all other headers
+            headers[key] = value
+            if isinstance(value, str) and "os.environ/" in value:
+                verbose_proxy_logger.debug(
+                    "pass through endpoint - looking up 'os.environ/' variable"
+                )
+                # get string section that is os.environ/
+                start_index = value.find("os.environ/")
+                _variable_name = value[start_index:]
 
-            verbose_proxy_logger.debug(
-                "pass through endpoint - getting secret for variable name: %s",
-                _variable_name,
-            )
-            _secret_value = litellm.get_secret(_variable_name)
-            new_value = value.replace(_variable_name, _secret_value)
-            headers[key] = new_value
+                verbose_proxy_logger.debug(
+                    "pass through endpoint - getting secret for variable name: %s",
+                    _variable_name,
+                )
+                _secret_value = litellm.get_secret(_variable_name)
+                new_value = value.replace(_variable_name, _secret_value)
+                headers[key] = new_value
     return headers
 
 

litellm/proxy/proxy_config.yaml

@@ -20,8 +20,6 @@ model_list:
 
 general_settings:
   master_key: sk-1234
-  alerting: ["slack", "email"]
-  public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"]
   pass_through_endpoints:
     - path: "/v1/rerank"
       target: "https://api.cohere.com/v1/rerank"
@@ -29,7 +27,11 @@ general_settings:
         Authorization: "bearer os.environ/COHERE_API_KEY"
         content-type: application/json
         accept: application/json
-
+    - path: "/api/public/ingestion"
+      target: "https://us.cloud.langfuse.com/api/public/ingestion"
+      headers:
+        LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_DEV_PUBLIC_KEY"
+        LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_DEV_SK_KEY"
 
 litellm_settings:
   success_callback: ["prometheus"]

litellm/proxy/tests/test_pass_through_langfuse.py

@@ -0,0 +1,14 @@
+from langfuse import Langfuse
+
+langfuse = Langfuse(
+    host="http://localhost:4000",
+    public_key="anything",
+    secret_key="anything",
+)
+
+print("sending langfuse trace request")
+trace = langfuse.trace(name="test-trace-litellm-proxy-passthrough")
+print("flushing langfuse request")
+langfuse.flush()
+
+print("flushed langfuse request")