diff --git a/docs/my-website/docs/completion/usage.md b/docs/my-website/docs/completion/usage.md index c388e5bfee..d610afeae5 100644 --- a/docs/my-website/docs/completion/usage.md +++ b/docs/my-website/docs/completion/usage.md @@ -50,3 +50,51 @@ for chunk in completion: print(chunk.choices[0].delta) ``` + +### Proxy: Always Include Streaming Usage + +When using the LiteLLM Proxy, you can configure it to automatically include usage information in all streaming responses, even if the client doesn't send `stream_options={"include_usage": True}`. + +#### Configuration + +Add the following to your config.yaml: + +```yaml +general_settings: + always_include_stream_usage: true +``` + +Alternatively, configure it through the UI: + +1. Navigate to the LiteLLM Proxy UI +2. Go to `Settings` > `Router Settings` > `General` +3. Find the `always_include_stream_usage` setting +4. Toggle it to `true` +5. Click `Update` to save + +#### How it works + +When `always_include_stream_usage` is enabled: +- All streaming requests will automatically have `stream_options={"include_usage": True}` added +- Clients will receive usage information in the final chunk, even if they didn't explicitly request it +- If a client already provides `stream_options`, `include_usage: True` will be added without overwriting other options +- Non-streaming requests are not affected + +#### Example + +With this setting enabled, a simple streaming request like: + +```bash +curl -X POST http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello!"}], + "stream": true + }' +``` + +Will automatically receive usage information in the response, without needing to explicitly include `stream_options`. + +``` diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7dfa3bb239..b8866fb88a 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -619,6 +619,23 @@ class ProxyBaseLLMRequestProcessing: self.data["litellm_call_id"] = request.headers.get( "x-litellm-call-id", str(uuid.uuid4()) ) + + ### AUTO STREAM USAGE TRACKING ### + # If always_include_stream_usage is enabled and this is a streaming request + # automatically add stream_options={'include_usage': True} if not already set + if ( + general_settings.get("always_include_stream_usage", False) is True + and self.data.get("stream", False) is True + ): + # Only set if stream_options is not already provided by the client + if "stream_options" not in self.data: + self.data["stream_options"] = {"include_usage": True} + elif ( + isinstance(self.data["stream_options"], dict) + and "include_usage" not in self.data["stream_options"] + ): + self.data["stream_options"]["include_usage"] = True + ### CALL HOOKS ### - modify/reject incoming data before calling the model ## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 216e6b3cca..0e702abfc6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11364,6 +11364,7 @@ async def get_config_list( "maximum_spend_logs_retention_period": {"type": "String"}, "mcp_internal_ip_ranges": {"type": "List"}, "mcp_trusted_proxy_ranges": {"type": "List"}, + "always_include_stream_usage": {"type": "Boolean"}, } return_val = [] diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index 18f891705e..22c8d38a37 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -11,6 +11,7 @@ import { Text, Button, Icon, + Switch, } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; import { @@ -163,7 +164,12 @@ const GeneralSettings: React.FC = ({ accessToken, user handleInputChange(value.field_name, newValue)} // Handle value change + onChange={(newValue) => handleInputChange(value.field_name, newValue)} + /> + ) : value.field_type == "Boolean" ? ( + handleInputChange(value.field_name, checked)} /> ) : null}