diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index 817d70b87c..43ab82b8e6 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -18,13 +18,17 @@ model_list: ### 1 Instance LiteLLM Proxy -In these tests the median latency of directly calling the fake-openai-endpoint is 60ms. +In these tests the baseline latency characteristics are measured against a fake-openai-endpoint. -| Metric | Litellm Proxy (1 Instance) | -|--------|------------------------| -| RPS | 475 | -| Median Latency (ms) | 100 | -| Latency overhead added by LiteLLM Proxy | 40ms | +#### Performance Metrics + +| Metric | Value | +|--------|-------| +| **Requests per Second (RPS)** | 475 | +| **End-to-End Latency P50 (ms)** | 100 | +| **LiteLLM Overhead P50 (ms)** | 3 | +| **LiteLLM Overhead P90 (ms)** | 17 | +| **LiteLLM Overhead P99 (ms)** | 31 | @@ -33,7 +37,8 @@ In these tests the median latency of directly calling the fake-openai-endpoint i --> #### Key Findings -- Single instance: 475 RPS @ 100ms latency +- Single instance: 475 RPS @ 100ms median latency +- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead - 2 LiteLLM instances: 950 RPS @ 100ms latency - 4 LiteLLM instances: 1900 RPS @ 100ms latency @@ -54,6 +59,62 @@ Each machine deploying LiteLLM had the following specs: - 2 CPU - 4GB RAM +## How to measure LiteLLM Overhead + +All responses from litellm will include the `x-litellm-overhead-duration-ms` header, this is the latency overhead in milliseconds added by LiteLLM Proxy. + + +If you want to measure this on locust you can use the following code: + +```python showLineNumbers title="Locust Code for measuring LiteLLM Overhead" +import os +import uuid +from locust import HttpUser, task, between, events + +# Custom metric to track LiteLLM overhead duration +overhead_durations = [] + +@events.request.add_listener +def on_request(request_type, name, response_time, response_length, response, context, exception, start_time, url, **kwargs): + if response and hasattr(response, 'headers'): + overhead_duration = response.headers.get('x-litellm-overhead-duration-ms') + if overhead_duration: + try: + duration_ms = float(overhead_duration) + overhead_durations.append(duration_ms) + # Report as custom metric + events.request.fire( + request_type="Custom", + name="LiteLLM Overhead Duration (ms)", + response_time=duration_ms, + response_length=0, + ) + except (ValueError, TypeError): + pass + +class MyUser(HttpUser): + wait_time = between(0.5, 1) # Random wait time between requests + + def on_start(self): + self.api_key = os.getenv('API_KEY', 'sk-1234567890') + self.client.headers.update({'Authorization': f'Bearer {self.api_key}'}) + + @task + def litellm_completion(self): + # no cache hits with this + payload = { + "model": "db-openai-endpoint", + "messages": [{"role": "user", "content": f"{uuid.uuid4()} This is a test there will be no cache hits and we'll fill up the context" * 150}], + "user": "my-new-end-user-1" + } + response = self.client.post("chat/completions", json=payload) + + if response.status_code != 200: + # log the errors in error.txt + with open("error.txt", "a") as error_log: + error_log.write(response.text + "\n") +``` + ## Logging Callbacks diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a4df33642a..a28f3ab6ef 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10907,7 +10907,7 @@ "litellm_provider": "qwen", "mode": "chat", "supports_tool_choice": false - }, + }, "openrouter/switchpoint/router": { "max_tokens": 131072, "max_input_tokens": 131072,