mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-24 02:25:29 +00:00
docs(configs.md): add wildcard model name to docs
This commit is contained in:
@@ -277,6 +277,54 @@ curl --location 'http://0.0.0.0:4000/v1/model/info' \
|
||||
--data ''
|
||||
```
|
||||
|
||||
## Wildcard Model Name (Add ALL MODELS from env)
|
||||
|
||||
Dynamically call any model from any given provider without the need to predefine it in the config YAML file. As long as the relevant keys are in the environment (see [providers list](../providers/)), LiteLLM will make the call correctly.
|
||||
|
||||
|
||||
|
||||
1. Setup config.yaml
|
||||
```
|
||||
model_list:
|
||||
- model_name: "*" # all requests where model not in your config go to this deployment
|
||||
litellm_params:
|
||||
model: "openai/*" # passes our validation check that a real provider is given
|
||||
```
|
||||
|
||||
2. Start LiteLLM proxy
|
||||
|
||||
```
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Try claude 3-5 sonnet from anthropic
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-D '{
|
||||
"model": "claude-3-5-sonnet-20240620",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hey, how'\''s it going?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "I'\''m doing well. Would like to hear the rest of the story?"
|
||||
},
|
||||
{"role": "user", "content": "Na"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "No problem, is there anything else i can help you with today?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I think you'\''re getting cut off sometimes"
|
||||
}
|
||||
]
|
||||
}
|
||||
'
|
||||
```
|
||||
|
||||
## Load Balancing
|
||||
|
||||
:::info
|
||||
|
||||
@@ -1,54 +1,4 @@
|
||||
# model_list:
|
||||
# - model_name: my-fake-model
|
||||
# litellm_params:
|
||||
# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
|
||||
# api_key: my-fake-key
|
||||
# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
|
||||
# mock_response: "Hello world 1"
|
||||
# model_info:
|
||||
# max_input_tokens: 0 # trigger context window fallback
|
||||
# - model_name: my-fake-model
|
||||
# litellm_params:
|
||||
# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
|
||||
# api_key: my-fake-key
|
||||
# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000
|
||||
# mock_response: "Hello world 2"
|
||||
# model_info:
|
||||
# max_input_tokens: 0
|
||||
|
||||
# router_settings:
|
||||
# enable_pre_call_checks: True
|
||||
|
||||
|
||||
# litellm_settings:
|
||||
# failure_callback: ["langfuse"]
|
||||
|
||||
model_list:
|
||||
- model_name: summarize
|
||||
- model_name: "*" # all requests where model not in your config go to this deployment
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
rpm: 10000
|
||||
tpm: 12000000
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
mock_response: Hello world 1
|
||||
|
||||
- model_name: summarize-l
|
||||
litellm_params:
|
||||
model: claude-3-5-sonnet-20240620
|
||||
rpm: 4000
|
||||
tpm: 400000
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
mock_response: Hello world 2
|
||||
|
||||
litellm_settings:
|
||||
num_retries: 3
|
||||
request_timeout: 120
|
||||
allowed_fails: 3
|
||||
# fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
|
||||
# context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
|
||||
|
||||
|
||||
|
||||
router_settings:
|
||||
routing_strategy: simple-shuffle
|
||||
enable_pre_call_checks: true.
|
||||
model: "openai/*"
|
||||
|
||||
Reference in New Issue
Block a user