diff --git a/docs/my-website/docs/proxy/pass_through.md b/docs/my-website/docs/proxy/pass_through.md index c19c93cec8..c479d36cc5 100644 --- a/docs/my-website/docs/proxy/pass_through.md +++ b/docs/my-website/docs/proxy/pass_through.md @@ -1,3 +1,5 @@ +import Image from '@theme/IdealImage'; + # ➡️ Create Pass Through Endpoints Add pass through routes to LiteLLM Proxy @@ -19,7 +21,7 @@ curl --request POST \ }' ``` -## Tutorial - Setup Cohere Re-Rank Endpoint on LiteLLM Proxy +## Tutorial - Pass through Cohere Re-Rank Endpoint **Step 1** Define pass through routes on [litellm config.yaml](configs.md) @@ -38,10 +40,12 @@ general_settings: **Step 2** Start Proxy Server in detailed_debug mode ```shell -litellm --config config.yaml +litellm --config config.yaml --detailed_debug ``` **Step 3** Make Request to pass through endpoint +Here `http://localhost:4000` is your litellm proxy endpoint + ```shell curl --request POST \ --url http://localhost:4000/v1/rerank \ @@ -92,6 +96,59 @@ This request got forwarded from LiteLLM Proxy -> Defined Target URL (with header } ``` +## Tutorial - Pass Through Langfuse Requests +**Step 1** Define pass through routes on [litellm config.yaml](configs.md) + +```yaml +general_settings: + master_key: sk-1234 + pass_through_endpoints: + - path: "/api/public/ingestion" # route you want to add to LiteLLM Proxy Server + target: "https://us.cloud.langfuse.com/api/public/ingestion" # URL this route should forward + headers: + LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_DEV_PUBLIC_KEY" # your langfuse account public key + LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_DEV_SK_KEY" # your langfuse account secret key +``` + +**Step 2** Start Proxy Server in detailed_debug mode + +```shell +litellm --config config.yaml --detailed_debug +``` +**Step 3** Make Request to pass through endpoint + +Run this code to make a sample trace +```python +from langfuse import Langfuse + +langfuse = Langfuse( + host="http://localhost:4000", # your litellm proxy endpoint + public_key="anything", # no key required since this is a pass through + secret_key="anything", # no key required since this is a pass through +) + +print("sending langfuse trace request") +trace = langfuse.trace(name="test-trace-litellm-proxy-passthrough") +print("flushing langfuse request") +langfuse.flush() + +print("flushed langfuse request") +``` + + +🎉 **Expected Response** + +On success +Expect to see the following Trace Generated on your Langfuse Dashboard + + + +You will see the following endpoint called on your litellm proxy server logs + +```shell +POST /api/public/ingestion HTTP/1.1" 207 Multi-Status +``` + diff --git a/docs/my-website/img/proxy_langfuse.png b/docs/my-website/img/proxy_langfuse.png new file mode 100644 index 0000000000..4a3ca28eef Binary files /dev/null and b/docs/my-website/img/proxy_langfuse.png differ diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 802c84d84b..d4b4484965 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -1,5 +1,6 @@ import ast import traceback +from base64 import b64encode import httpx from fastapi import APIRouter, FastAPI, HTTPException, Request, Response, status @@ -24,22 +25,41 @@ async def set_env_variables_in_header(custom_headers: dict): """ headers = {} for key, value in custom_headers.items(): - headers[key] = value - if isinstance(value, str) and "os.environ/" in value: - verbose_proxy_logger.debug( - "pass through endpoint - looking up 'os.environ/' variable" - ) - # get string section that is os.environ/ - start_index = value.find("os.environ/") - _variable_name = value[start_index:] + # langfuse Api requires base64 encoded headers - it's simpleer to just ask litellm users to set their langfuse public and secret keys + # we can then get the b64 encoded keys here + if key == "LANGFUSE_PUBLIC_KEY" or key == "LANGFUSE_SECRET_KEY": + # langfuse requires b64 encoded headers - we construct that here + _langfuse_public_key = custom_headers["LANGFUSE_PUBLIC_KEY"] + _langfuse_secret_key = custom_headers["LANGFUSE_SECRET_KEY"] + if isinstance( + _langfuse_public_key, str + ) and _langfuse_public_key.startswith("os.environ/"): + _langfuse_public_key = litellm.get_secret(_langfuse_public_key) + if isinstance( + _langfuse_secret_key, str + ) and _langfuse_secret_key.startswith("os.environ/"): + _langfuse_secret_key = litellm.get_secret(_langfuse_secret_key) + headers["Authorization"] = "Basic " + b64encode( + f"{_langfuse_public_key}:{_langfuse_secret_key}".encode("utf-8") + ).decode("ascii") + else: + # for all other headers + headers[key] = value + if isinstance(value, str) and "os.environ/" in value: + verbose_proxy_logger.debug( + "pass through endpoint - looking up 'os.environ/' variable" + ) + # get string section that is os.environ/ + start_index = value.find("os.environ/") + _variable_name = value[start_index:] - verbose_proxy_logger.debug( - "pass through endpoint - getting secret for variable name: %s", - _variable_name, - ) - _secret_value = litellm.get_secret(_variable_name) - new_value = value.replace(_variable_name, _secret_value) - headers[key] = new_value + verbose_proxy_logger.debug( + "pass through endpoint - getting secret for variable name: %s", + _variable_name, + ) + _secret_value = litellm.get_secret(_variable_name) + new_value = value.replace(_variable_name, _secret_value) + headers[key] = new_value return headers diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 04f12a8440..40e0386f45 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -20,8 +20,6 @@ model_list: general_settings: master_key: sk-1234 - alerting: ["slack", "email"] - public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate"] pass_through_endpoints: - path: "/v1/rerank" target: "https://api.cohere.com/v1/rerank" @@ -29,7 +27,11 @@ general_settings: Authorization: "bearer os.environ/COHERE_API_KEY" content-type: application/json accept: application/json - + - path: "/api/public/ingestion" + target: "https://us.cloud.langfuse.com/api/public/ingestion" + headers: + LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_DEV_PUBLIC_KEY" + LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_DEV_SK_KEY" litellm_settings: success_callback: ["prometheus"] diff --git a/litellm/proxy/tests/test_pass_through_langfuse.py b/litellm/proxy/tests/test_pass_through_langfuse.py new file mode 100644 index 0000000000..dfc91ee1b1 --- /dev/null +++ b/litellm/proxy/tests/test_pass_through_langfuse.py @@ -0,0 +1,14 @@ +from langfuse import Langfuse + +langfuse = Langfuse( + host="http://localhost:4000", + public_key="anything", + secret_key="anything", +) + +print("sending langfuse trace request") +trace = langfuse.trace(name="test-trace-litellm-proxy-passthrough") +print("flushing langfuse request") +langfuse.flush() + +print("flushed langfuse request")