From 5c03109b6fe8d59602aa4616fffc67e6705eef98 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 5 Mar 2024 07:39:06 -0800 Subject: [PATCH] docs(configs.md): add load balancing to proxy config docs --- docs/my-website/docs/proxy/configs.md | 65 ++++++++++++++++----------- litellm/llms/aleph_alpha.py | 11 +++-- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 0a155828b0..2b3edfadb9 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -202,7 +202,7 @@ print(response) -## Save Model-specific params (API Base, API Keys, Temperature, Max Tokens, Seed, Organization, Headers etc.) +## Save Model-specific params (API Base, Keys, Temperature, Max Tokens, Organization, Headers etc.) You can use the config to save model-specific information like api_base, api_key, temperature, max_tokens, etc. [**All input params**](https://docs.litellm.ai/docs/completion/input#input-params-1) @@ -244,6 +244,45 @@ $ litellm --config /path/to/config.yaml ``` +## Load Balancing + +Use this to call multiple instances of the same model and configure things like [routing strategy](../routing.md#advanced). + +```yaml +router_settings: + routing_strategy: "latency-based-routing" # routes to the fastest deployment in the group + +model_list: + - model_name: zephyr-beta + litellm_params: + model: huggingface/HuggingFaceH4/zephyr-7b-beta + api_base: http://0.0.0.0:8001 + - model_name: zephyr-beta + litellm_params: + model: huggingface/HuggingFaceH4/zephyr-7b-beta + api_base: http://0.0.0.0:8002 + - model_name: zephyr-beta + litellm_params: + model: huggingface/HuggingFaceH4/zephyr-7b-beta + api_base: http://0.0.0.0:8003 + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: + - model_name: gpt-3.5-turbo-16k + litellm_params: + model: gpt-3.5-turbo-16k + api_key: + +litellm_settings: + num_retries: 3 # retry call 3 times on each model_name (e.g. zephyr-beta) + request_timeout: 10 # raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout + fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo"]}] # fallback to gpt-3.5-turbo if call fails num_retries + context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error + allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. +``` + + ## Set Azure `base_model` for cost tracking **Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking @@ -512,30 +551,6 @@ curl --location 'http://0.0.0.0:8000/chat/completions' \ ``` -## Router Settings - -Use this to configure things like routing strategy. - -```yaml -router_settings: - routing_strategy: "least-busy" - -model_list: # will route requests to the least busy ollama model - - model_name: ollama-models - litellm_params: - model: "ollama/mistral" - api_base: "http://127.0.0.1:8001" - - model_name: ollama-models - litellm_params: - model: "ollama/codellama" - api_base: "http://127.0.0.1:8002" - - model_name: ollama-models - litellm_params: - model: "ollama/llama2" - api_base: "http://127.0.0.1:8003" -``` - - ## Configure DB Pool Limits + Connection Timeouts ```yaml diff --git a/litellm/llms/aleph_alpha.py b/litellm/llms/aleph_alpha.py index 7168e7369c..3c1bd5dde0 100644 --- a/litellm/llms/aleph_alpha.py +++ b/litellm/llms/aleph_alpha.py @@ -77,9 +77,9 @@ class AlephAlphaConfig: - `control_log_additive` (boolean; default value: true): Method of applying control to attention scores. """ - maximum_tokens: Optional[ - int - ] = litellm.max_tokens # aleph alpha requires max tokens + maximum_tokens: Optional[int] = ( + litellm.max_tokens + ) # aleph alpha requires max tokens minimum_tokens: Optional[int] = None echo: Optional[bool] = None temperature: Optional[int] = None @@ -285,7 +285,10 @@ def completion( ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here. prompt_tokens = len(encoding.encode(prompt)) completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"]["content"]) + encoding.encode( + model_response["choices"][0]["message"]["content"], + disallowed_special=(), + ) ) model_response["created"] = int(time.time())