mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-12 03:06:22 +00:00
docs(routing.md): add proxy with multiple azure deployments to docs
This commit is contained in:
@@ -56,6 +56,8 @@ print(response)
|
||||
|
||||
## Redis Queue
|
||||
|
||||
In production, we use Redis to track usage across multiple Azure deployments.
|
||||
|
||||
```python
|
||||
router = Router(model_list=model_list,
|
||||
redis_host=os.getenv("REDIS_HOST"),
|
||||
@@ -63,4 +65,52 @@ router = Router(model_list=model_list,
|
||||
redis_port=os.getenv("REDIS_PORT"))
|
||||
|
||||
print(response)
|
||||
```
|
||||
```
|
||||
|
||||
## Handle Multiple Azure Deployments via OpenAI Proxy Server
|
||||
|
||||
#### 1. Clone repo
|
||||
```shell
|
||||
git clone https://github.com/BerriAI/litellm.git
|
||||
```
|
||||
|
||||
#### 2. Add Azure/OpenAI deployments to `secrets_template.toml`
|
||||
```python
|
||||
[model."gpt-3.5-turbo"]
|
||||
model_list = [{ # list of model deployments
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/chatgpt-v-2",
|
||||
"api_key": "my-azure-api-key-1",
|
||||
"api_version": "my-azure-api-version-1",
|
||||
"api_base": "my-azure-api-base-1"
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800
|
||||
}, { # list of model deployments
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/chatgpt-function-calling",
|
||||
"api_key": "my-azure-api-key-2",
|
||||
"api_version": "my-azure-api-version-2",
|
||||
"api_base": "my-azure-api-base-2"
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800
|
||||
}, {
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "gpt-3.5-turbo",
|
||||
"api_key": "sk-...",
|
||||
},
|
||||
"tpm": 1000000,
|
||||
"rpm": 9000
|
||||
}]
|
||||
```
|
||||
|
||||
#### 3. Run with Docker Image
|
||||
```shell
|
||||
docker build -t litellm . && docker run -p 8000:8000 litellm
|
||||
```
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user