mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-24 10:27:13 +00:00
docs(routing.md): add calling via proxy tutorial to router docs
This commit is contained in:
@@ -274,7 +274,7 @@ print(response)
|
||||
</TabItem>
|
||||
<TabItem value="usage-based-v2" label="Rate-Limit Aware v2 (ASYNC)">
|
||||
|
||||
**NEW**
|
||||
**🎉 NEW**
|
||||
|
||||
This is an async implementation of usage-based-routing.
|
||||
|
||||
@@ -286,6 +286,8 @@ If you pass in the deployment's tpm/rpm limits, this will also check against tha
|
||||
|
||||
For Azure, your RPM = TPM/6.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="sdk">
|
||||
|
||||
```python
|
||||
from litellm import Router
|
||||
@@ -333,6 +335,59 @@ response = await router.acompletion(model="gpt-3.5-turbo",
|
||||
|
||||
print(response)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="proxy">
|
||||
|
||||
**1. Set strategy in config**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo # model alias
|
||||
litellm_params: # params for litellm completion/embedding call
|
||||
model: azure/chatgpt-v-2 # actual model name
|
||||
api_key: os.environ/AZURE_API_KEY
|
||||
api_version: os.environ/AZURE_API_VERSION
|
||||
api_base: os.environ/AZURE_API_BASE
|
||||
tpm: 100000
|
||||
rpm: 10000
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params: # params for litellm completion/embedding call
|
||||
model: gpt-3.5-turbo
|
||||
api_key: os.getenv(OPENAI_API_KEY)
|
||||
tpm: 100000
|
||||
rpm: 1000
|
||||
|
||||
router_settings:
|
||||
routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE
|
||||
redis_host: <your-redis-host>
|
||||
redis_password: <your-redis-password>
|
||||
redis_port: <your-redis-port>
|
||||
enable_pre_call_check: true
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
```
|
||||
|
||||
**2. Start proxy**
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it!**
|
||||
|
||||
```bash
|
||||
curl --location 'http://localhost:4000/v1/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--data '{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
</TabItem>
|
||||
|
||||
Reference in New Issue
Block a user