diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 3ab644855b..00457dbc4d 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -277,6 +277,54 @@ curl --location 'http://0.0.0.0:4000/v1/model/info' \ --data '' ``` +## Wildcard Model Name (Add ALL MODELS from env) + +Dynamically call any model from any given provider without the need to predefine it in the config YAML file. As long as the relevant keys are in the environment (see [providers list](../providers/)), LiteLLM will make the call correctly. + + + +1. Setup config.yaml +``` +model_list: + - model_name: "*" # all requests where model not in your config go to this deployment + litellm_params: + model: "openai/*" # passes our validation check that a real provider is given +``` + +2. Start LiteLLM proxy + +``` +litellm --config /path/to/config.yaml +``` + +3. Try claude 3-5 sonnet from anthropic + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "claude-3-5-sonnet-20240620", + "messages": [ + {"role": "user", "content": "Hey, how'\''s it going?"}, + { + "role": "assistant", + "content": "I'\''m doing well. Would like to hear the rest of the story?" + }, + {"role": "user", "content": "Na"}, + { + "role": "assistant", + "content": "No problem, is there anything else i can help you with today?" + }, + { + "role": "user", + "content": "I think you'\''re getting cut off sometimes" + } + ] +} +' +``` + ## Load Balancing :::info diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 938e74b5e7..2693ce3d30 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,54 +1,4 @@ -# model_list: -# - model_name: my-fake-model -# litellm_params: -# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 -# api_key: my-fake-key -# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000 -# mock_response: "Hello world 1" -# model_info: -# max_input_tokens: 0 # trigger context window fallback -# - model_name: my-fake-model -# litellm_params: -# model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 -# api_key: my-fake-key -# aws_bedrock_runtime_endpoint: http://127.0.0.1:8000 -# mock_response: "Hello world 2" -# model_info: -# max_input_tokens: 0 - -# router_settings: -# enable_pre_call_checks: True - - -# litellm_settings: -# failure_callback: ["langfuse"] - model_list: - - model_name: summarize + - model_name: "*" # all requests where model not in your config go to this deployment litellm_params: - model: openai/gpt-4o - rpm: 10000 - tpm: 12000000 - api_key: os.environ/OPENAI_API_KEY - mock_response: Hello world 1 - - - model_name: summarize-l - litellm_params: - model: claude-3-5-sonnet-20240620 - rpm: 4000 - tpm: 400000 - api_key: os.environ/ANTHROPIC_API_KEY - mock_response: Hello world 2 - -litellm_settings: - num_retries: 3 - request_timeout: 120 - allowed_fails: 3 - # fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}] - # context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}] - - - -router_settings: - routing_strategy: simple-shuffle - enable_pre_call_checks: true. + model: "openai/*"