From 7b78f4fc2f65f4dbf951b780a6c099fe6fc80890 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 21:53:07 -0700 Subject: [PATCH] docs - spend tracking with litellm --- docs/my-website/docs/proxy/cost_tracking.md | 63 +++++++++++++-------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index f2d456febb..de1a63a4c3 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -8,16 +8,12 @@ Track spend for keys, users, and teams across 100+ LLMs. ### How to Track Spend with LiteLLM +**Step 1** + 👉 [Setup LiteLLM with a Database](https://docs.litellm.ai/docs/proxy/deploy) -That's it, that's all you need to do for basic LLM spend tracking -### What spend gets tracked - -For the Request below LiteLLM will **automatically** track spend for -- Virtual Key = `sk-1234` -- Customer = `palantir` -- Tags = `jobID:214590dsff09fds`, `taskName:run_page_classification` +**Step2** Send `/chat/completions` request @@ -31,9 +27,8 @@ client = openai.OpenAI( base_url="http://0.0.0.0:4000" ) -# request sent to model set on litellm proxy, `litellm --model` response = client.chat.completions.create( - model="gpt-3.5-turbo", + model="llama3", messages = [ { "role": "user", @@ -61,7 +56,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer sk-1234' \ --data '{ - "model": "gpt-3.5-turbo", + "model": "llama3", "messages": [ { "role": "user", @@ -91,7 +86,7 @@ os.environ["OPENAI_API_KEY"] = "sk-1234" chat = ChatOpenAI( openai_api_base="http://0.0.0.0:4000", - model = "gpt-3.5-turbo", + model = "llama3", user="palantir", extra_body={ "metadata": { @@ -116,25 +111,45 @@ print(response) -## View Spend on LiteLLM Admin UI +**Step3 - Verify Spend Tracked** +That's IT. Now Verify your spend was tracked -Navigate to the `Usage` Tab on the LiteLLM UI (found on `https://your-proxy-endpoint/ui`) +The following spend gets tracked in Table `LiteLLM_SpendLogs` + +```json +{ + "api_key": "fe6b0cab4ff5a5a8df823196cc8a450*****", # Hash of API Key used + "user": "default_user", # Internal User (LiteLLM_UserTable) that owns `api_key=sk-1234`. + "team_id": "e8d1460f-846c-45d7-9b43-55f3cc52ac32", # Team (LiteLLM_TeamTable) that owns `api_key=sk-1234` + "request_tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"],# Tags sent in request + "end_user": "palantir", # Customer - the `user` sent in the request + "model_group": "llama3", # "model" passed to LiteLLM + "api_base": "https://api.groq.com/openai/v1/", # "api_base" of model used by LiteLLM + "spend": 0.000002, # Spend in $ + "total_tokens": 100, + "completion_tokens": 80, + "prompt_tokens": 20, + +} +``` + +Navigate to the Usage Tab on the LiteLLM UI (found on https://your-proxy-endpoint/ui) and verify you see spend tracked under `Usage` ## API Endpoints to get Spend -### Getting Spend Reports - To Charge Other Teams, API Keys +#### Getting Spend Reports - To Charge Other Teams, API Keys Use the `/global/spend/report` endpoint to get daily spend per team, with a breakdown of spend per API Key, Model -#### Example Request +##### Example Request ```shell curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30' \ -H 'Authorization: Bearer sk-1234' ``` -#### Example Response +##### Example Response @@ -243,7 +258,7 @@ Output from script -### Allowing Non-Proxy Admins to access `/spend` endpoints +#### Allowing Non-Proxy Admins to access `/spend` endpoints Use this when you want non-proxy admins to access `/spend` endpoints @@ -253,7 +268,7 @@ Schedule a [meeting with us to get your Enterprise License](https://calendly.com ::: -#### Create Key +##### Create Key Create Key with with `permissions={"get_spend_routes": true}` ```shell curl --location 'http://0.0.0.0:4000/key/generate' \ @@ -264,7 +279,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \ }' ``` -#### Use generated key on `/spend` endpoints +##### Use generated key on `/spend` endpoints Access spend Routes with newly generate keys ```shell @@ -274,14 +289,14 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end -### Reset Team, API Key Spend - MASTER KEY ONLY +#### Reset Team, API Key Spend - MASTER KEY ONLY Use `/global/spend/reset` if you want to: - Reset the Spend for all API Keys, Teams. The `spend` for ALL Teams and Keys in `LiteLLM_TeamTable` and `LiteLLM_VerificationToken` will be set to `spend=0` - LiteLLM will maintain all the logs in `LiteLLMSpendLogs` for Auditing Purposes -#### Request +##### Request Only the `LITELLM_MASTER_KEY` you set can access this route ```shell curl -X POST \ @@ -290,7 +305,7 @@ curl -X POST \ -H 'Content-Type: application/json' ``` -#### Expected Responses +##### Expected Responses ```shell {"message":"Spend for all API Keys and Teams reset successfully","status":"success"} @@ -303,7 +318,7 @@ curl -X POST \ Set base model for cost tracking azure image-gen call -### Image Generation +#### Image Generation ```yaml model_list: @@ -318,7 +333,7 @@ model_list: mode: image_generation ``` -### Chat Completions / Embeddings +#### Chat Completions / Embeddings **Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking