diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index bb0410f333..acba1fb0e4 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -56,6 +56,8 @@ print(response) ## Redis Queue +In production, we use Redis to track usage across multiple Azure deployments. + ```python router = Router(model_list=model_list, redis_host=os.getenv("REDIS_HOST"), @@ -63,4 +65,52 @@ router = Router(model_list=model_list, redis_port=os.getenv("REDIS_PORT")) print(response) -``` \ No newline at end of file +``` + +## Handle Multiple Azure Deployments via OpenAI Proxy Server + +#### 1. Clone repo +```shell +git clone https://github.com/BerriAI/litellm.git +``` + +#### 2. Add Azure/OpenAI deployments to `secrets_template.toml` +```python +[model."gpt-3.5-turbo"] +model_list = [{ # list of model deployments + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-v-2", + "api_key": "my-azure-api-key-1", + "api_version": "my-azure-api-version-1", + "api_base": "my-azure-api-base-1" + }, + "tpm": 240000, + "rpm": 1800 +}, { # list of model deployments + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-function-calling", + "api_key": "my-azure-api-key-2", + "api_version": "my-azure-api-version-2", + "api_base": "my-azure-api-base-2" + }, + "tpm": 240000, + "rpm": 1800 +}, { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "gpt-3.5-turbo", + "api_key": "sk-...", + }, + "tpm": 1000000, + "rpm": 9000 +}] +``` + +#### 3. Run with Docker Image +```shell +docker build -t litellm . && docker run -p 8000:8000 litellm +``` + +