From 48b9250a3d022df7ded71fc732337cf3fc51b6d1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 22 Mar 2024 19:44:16 -0700 Subject: [PATCH 1/3] (fix) update load test used --- litellm/proxy/proxy_load_test/locustfile.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/proxy_load_test/locustfile.py b/litellm/proxy/proxy_load_test/locustfile.py index f439f72746..263c871efd 100644 --- a/litellm/proxy/proxy_load_test/locustfile.py +++ b/litellm/proxy/proxy_load_test/locustfile.py @@ -6,11 +6,11 @@ import time class MyUser(HttpUser): wait_time = between(1, 5) - @task(3) + @task def chat_completion(self): headers = { "Content-Type": "application/json", - "Authorization": f"Bearer sk-mh3YNUDs1d_f6fMXfvEqBA", + "Authorization": f"Bearer sk-1234", # Include any additional headers you may need for authentication, etc. } @@ -28,11 +28,3 @@ class MyUser(HttpUser): response = self.client.post("chat/completions", json=payload, headers=headers) # Print or log the response if needed - - @task(10) - def health_readiness(self): - response = self.client.get("health/readiness") - - @task(10) - def health_liveliness(self): - response = self.client.get("health/liveliness") From 311918b99c083e20bb30b99fe7699932180424c3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 22 Mar 2024 19:45:24 -0700 Subject: [PATCH 2/3] (fix) add some better load testing --- .../litellm_router_proxy/Dockerfile | 20 +++++++ .../litellm_router_proxy/main.py | 59 +++++++++++++++++++ .../proxy_load_test/simple_litellm_proxy.py | 54 +++++++++++++++++ .../simple_litellm_router_proxy.py | 59 +++++++++++++++++++ litellm/proxy/proxy_load_test/simple_proxy.py | 52 ++++++++++++++++ litellm/proxy/tests/load_test_completion.py | 40 ++++++++----- 6 files changed, 270 insertions(+), 14 deletions(-) create mode 100644 litellm/proxy/proxy_load_test/litellm_router_proxy/Dockerfile create mode 100644 litellm/proxy/proxy_load_test/litellm_router_proxy/main.py create mode 100644 litellm/proxy/proxy_load_test/simple_litellm_proxy.py create mode 100644 litellm/proxy/proxy_load_test/simple_litellm_router_proxy.py create mode 100644 litellm/proxy/proxy_load_test/simple_proxy.py diff --git a/litellm/proxy/proxy_load_test/litellm_router_proxy/Dockerfile b/litellm/proxy/proxy_load_test/litellm_router_proxy/Dockerfile new file mode 100644 index 0000000000..f5787f0da3 --- /dev/null +++ b/litellm/proxy/proxy_load_test/litellm_router_proxy/Dockerfile @@ -0,0 +1,20 @@ +# Use the official Python image as the base image +FROM python:3.9-slim + +# Set the working directory in the container +WORKDIR /app + +# Copy the Python requirements file +COPY requirements.txt . + +# Install the Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the application code +COPY . . + +# Expose the port the app will run on +EXPOSE 8090 + +# Start the application +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8090"] \ No newline at end of file diff --git a/litellm/proxy/proxy_load_test/litellm_router_proxy/main.py b/litellm/proxy/proxy_load_test/litellm_router_proxy/main.py new file mode 100644 index 0000000000..95e2abc15a --- /dev/null +++ b/litellm/proxy/proxy_load_test/litellm_router_proxy/main.py @@ -0,0 +1,59 @@ +# import sys, os +# sys.path.insert( +# 0, os.path.abspath("../") +# ) # Adds the parent directory to the system path +from fastapi import FastAPI, Request, status, HTTPException, Depends +from fastapi.responses import StreamingResponse +from fastapi.security import OAuth2PasswordBearer +from fastapi.middleware.cors import CORSMiddleware +import uuid +import litellm + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +litellm_router = litellm.Router( + model_list=[ + { + "model_name": "anything", # model alias -> loadbalance between models with same `model_name` + "litellm_params": { # params for litellm completion/embedding call + "model": "openai/anything", # actual model name + "api_key": "sk-1234", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + }, + } + ] +) + + +# for completion +@app.post("/chat/completions") +@app.post("/v1/chat/completions") +async def completion(request: Request): + # this proxy uses the OpenAI SDK to call a fixed endpoint + + response = await litellm_router.acompletion( + model="anything", + messages=[ + { + "role": "user", + "content": "hello who are you", + } + ], + ) + + return response + + +if __name__ == "__main__": + import uvicorn + + # run this on 8090, 8091, 8092 and 8093 + uvicorn.run(app, host="0.0.0.0", port=8090) diff --git a/litellm/proxy/proxy_load_test/simple_litellm_proxy.py b/litellm/proxy/proxy_load_test/simple_litellm_proxy.py new file mode 100644 index 0000000000..003c89c777 --- /dev/null +++ b/litellm/proxy/proxy_load_test/simple_litellm_proxy.py @@ -0,0 +1,54 @@ +# import sys, os +# sys.path.insert( +# 0, os.path.abspath("../") +# ) # Adds the parent directory to the system path +from fastapi import FastAPI, Request, status, HTTPException, Depends +from fastapi.responses import StreamingResponse +from fastapi.security import OAuth2PasswordBearer +from fastapi.middleware.cors import CORSMiddleware +import uuid +import litellm +import openai +from openai import AsyncOpenAI + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +litellm_client = AsyncOpenAI( + base_url="https://exampleopenaiendpoint-production.up.railway.app/", + api_key="sk-1234", +) + + +# for completion +@app.post("/chat/completions") +@app.post("/v1/chat/completions") +async def completion(request: Request): + # this proxy uses the OpenAI SDK to call a fixed endpoint + + response = await litellm.acompletion( + model="openai/anything", + messages=[ + { + "role": "user", + "content": "hello who are you", + } + ], + client=litellm_client, + ) + + return response + + +if __name__ == "__main__": + import uvicorn + + # run this on 8090, 8091, 8092 and 8093 + uvicorn.run(app, host="0.0.0.0", port=8090) diff --git a/litellm/proxy/proxy_load_test/simple_litellm_router_proxy.py b/litellm/proxy/proxy_load_test/simple_litellm_router_proxy.py new file mode 100644 index 0000000000..95e2abc15a --- /dev/null +++ b/litellm/proxy/proxy_load_test/simple_litellm_router_proxy.py @@ -0,0 +1,59 @@ +# import sys, os +# sys.path.insert( +# 0, os.path.abspath("../") +# ) # Adds the parent directory to the system path +from fastapi import FastAPI, Request, status, HTTPException, Depends +from fastapi.responses import StreamingResponse +from fastapi.security import OAuth2PasswordBearer +from fastapi.middleware.cors import CORSMiddleware +import uuid +import litellm + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +litellm_router = litellm.Router( + model_list=[ + { + "model_name": "anything", # model alias -> loadbalance between models with same `model_name` + "litellm_params": { # params for litellm completion/embedding call + "model": "openai/anything", # actual model name + "api_key": "sk-1234", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + }, + } + ] +) + + +# for completion +@app.post("/chat/completions") +@app.post("/v1/chat/completions") +async def completion(request: Request): + # this proxy uses the OpenAI SDK to call a fixed endpoint + + response = await litellm_router.acompletion( + model="anything", + messages=[ + { + "role": "user", + "content": "hello who are you", + } + ], + ) + + return response + + +if __name__ == "__main__": + import uvicorn + + # run this on 8090, 8091, 8092 and 8093 + uvicorn.run(app, host="0.0.0.0", port=8090) diff --git a/litellm/proxy/proxy_load_test/simple_proxy.py b/litellm/proxy/proxy_load_test/simple_proxy.py new file mode 100644 index 0000000000..12fb6cffb4 --- /dev/null +++ b/litellm/proxy/proxy_load_test/simple_proxy.py @@ -0,0 +1,52 @@ +# import sys, os +# sys.path.insert( +# 0, os.path.abspath("../") +# ) # Adds the parent directory to the system path +from fastapi import FastAPI, Request, status, HTTPException, Depends +from fastapi.responses import StreamingResponse +from fastapi.security import OAuth2PasswordBearer +from fastapi.middleware.cors import CORSMiddleware +import uuid +import openai +from openai import AsyncOpenAI + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +litellm_client = AsyncOpenAI( + base_url="https://exampleopenaiendpoint-production.up.railway.app/", + api_key="sk-1234", +) + + +# for completion +@app.post("/chat/completions") +@app.post("/v1/chat/completions") +async def completion(request: Request): + # this proxy uses the OpenAI SDK to call a fixed endpoint + + response = await litellm_client.chat.completions.create( + model="anything", + messages=[ + { + "role": "user", + "content": "hello who are you", + } + ], + ) + + return response + + +if __name__ == "__main__": + import uvicorn + + # run this on 8090, 8091, 8092 and 8093 + uvicorn.run(app, host="0.0.0.0", port=8090) diff --git a/litellm/proxy/tests/load_test_completion.py b/litellm/proxy/tests/load_test_completion.py index 9450c1cb5b..29d8924ab5 100644 --- a/litellm/proxy/tests/load_test_completion.py +++ b/litellm/proxy/tests/load_test_completion.py @@ -1,56 +1,68 @@ -import time, asyncio, os +import time +import asyncio +import os from openai import AsyncOpenAI, AsyncAzureOpenAI import uuid import traceback from large_text import text from dotenv import load_dotenv +from statistics import mean, median -litellm_client = AsyncOpenAI(base_url="http://0.0.0.0:4000", api_key="sk-1234") +litellm_client = AsyncOpenAI(base_url="http://0.0.0.0:4000/", api_key="sk-1234") async def litellm_completion(): - # Your existing code for litellm_completion goes here try: + start_time = time.time() response = await litellm_client.chat.completions.create( - model="fake_openai", + model="fake-openai-endpoint", messages=[ { "role": "user", - "content": f"{text}. Who was alexander the great? {uuid.uuid4()}", + "content": f"This is a test{uuid.uuid4()}", } ], user="my-new-end-user-1", ) - return response + end_time = time.time() + latency = end_time - start_time + print("response time=", latency) + return response, latency except Exception as e: - # If there's an exception, log the error message with open("error_log.txt", "a") as error_log: error_log.write(f"Error during completion: {str(e)}\n") - pass + return None, 0 async def main(): - for i in range(3): + latencies = [] + for i in range(5): start = time.time() - n = 10 # Number of concurrent tasks + n = 100 # Number of concurrent tasks tasks = [litellm_completion() for _ in range(n)] chat_completions = await asyncio.gather(*tasks) - successful_completions = [c for c in chat_completions if c is not None] + successful_completions = [c for c, l in chat_completions if c is not None] + completion_latencies = [l for c, l in chat_completions if c is not None] + latencies.extend(completion_latencies) - # Write errors to error_log.txt with open("error_log.txt", "a") as error_log: - for completion in chat_completions: + for completion, latency in chat_completions: if isinstance(completion, str): error_log.write(completion + "\n") print(n, time.time() - start, len(successful_completions)) + if latencies: + average_latency = mean(latencies) + median_latency = median(latencies) + print(f"Average Latency per Response: {average_latency} seconds") + print(f"Median Latency per Response: {median_latency} seconds") + if __name__ == "__main__": - # Blank out contents of error_log.txt open("error_log.txt", "w").close() asyncio.run(main()) From 9c483dbae48d37bcdcddf8e77161e0bcab353475 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 22 Mar 2024 19:47:44 -0700 Subject: [PATCH 3/3] (feat) add sample kubernetes for litellm --- deploy/kubernetes/kub.yaml | 55 ++++++++++++++++++++++++++++++++++ deploy/kubernetes/service.yaml | 12 ++++++++ 2 files changed, 67 insertions(+) create mode 100644 deploy/kubernetes/kub.yaml create mode 100644 deploy/kubernetes/service.yaml diff --git a/deploy/kubernetes/kub.yaml b/deploy/kubernetes/kub.yaml new file mode 100644 index 0000000000..1ddc0e95f1 --- /dev/null +++ b/deploy/kubernetes/kub.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-deployment +spec: + replicas: 5 + selector: + matchLabels: + app: litellm + template: + metadata: + labels: + app: litellm + spec: + containers: + - name: litellm-container + image: ghcr.io/berriai/litellm:main-latest + env: + - name: AZURE_API_KEY + value: "d699s" + - name: AZURE_API_BASE + value: "https://openai/" + - name: LITELLM_MASTER_KEY + value: "sk-1234" + ports: + - containerPort: 4000 + args: + - "--config" + - "/app/proxy_config.yaml" # Update the path to mount the config file + volumeMounts: # Define volume mount for proxy_config.yaml + - name: config-volume + mountPath: /app + readOnly: true + livenessProbe: + httpGet: + path: /health/liveliness + port: 4000 + initialDelaySeconds: 120 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /health/readiness + port: 4000 + initialDelaySeconds: 120 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + volumes: # Define volume to mount proxy_config.yaml + - name: config-volume + configMap: + name: litellm-config diff --git a/deploy/kubernetes/service.yaml b/deploy/kubernetes/service.yaml new file mode 100644 index 0000000000..4751c83725 --- /dev/null +++ b/deploy/kubernetes/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: litellm-service +spec: + selector: + app: litellm + ports: + - protocol: TCP + port: 4000 + targetPort: 4000 + type: LoadBalancer \ No newline at end of file