From fc4e900a232d206429da61d55b2fc4a09275243a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 22 Jun 2024 15:26:58 -0700 Subject: [PATCH] docs(team_budgets.md): update docs with script for testing dynamic rate limiting --- docs/my-website/docs/proxy/team_budgets.md | 57 ++++++++++++++++- litellm/proxy/_new_secret_config.yaml | 71 +++------------------- 2 files changed, 65 insertions(+), 63 deletions(-) diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md index d7f6d7fdc9..f6b7c22824 100644 --- a/docs/my-website/docs/proxy/team_budgets.md +++ b/docs/my-website/docs/proxy/team_budgets.md @@ -167,8 +167,12 @@ model_list: mock_response: hello-world tpm: 60 -general_settings: - callbacks: ["dynamic_rate_limiting"] +litellm_settings: + callbacks: ["dynamic_rate_limiter"] + +general_settings: + master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env + database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env ``` 2. Start proxy @@ -186,6 +190,55 @@ litellm --config /path/to/config.yaml - Mock response returns 30 total tokens / request - Each team will only be able to make 1 request per minute """ +import requests +from openai import OpenAI, RateLimitError + +def create_key(api_key: str, base_url: str): + response = requests.post( + url="{}/key/generate".format(base_url), + json={}, + headers={ + "Authorization": "Bearer {}".format(api_key) + } + ) + + _response = response.json() + + print(f"_response: {_response}") + return _response["key"] + +key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000") +key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# call proxy with key 1 - works +openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000") + +response = openai_client_1.chat.completions.with_raw_response.create( + model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}], +) + +print("Headers for call - {}".format(response.headers)) +_response = response.parse() +print("Total tokens for call - {}".format(_response.usage.total_tokens)) + + +# call proxy with key 2 - works +openai_client_2 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000") + +response = openai_client_2.chat.completions.with_raw_response.create( + model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}], +) + +print("Headers for call - {}".format(response.headers)) +_response = response.parse() +print("Total tokens for call - {}".format(_response.usage.total_tokens)) +# call proxy with key 2 - fails +try: + openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}]) + raise Exception("This should have failed!") +except RateLimitError as e: + print("This was rate limited b/c - {}".format(str(e))) + ``` \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 866ca0ab0a..01f09ca02b 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,61 +1,10 @@ -environment_variables: - LANGFUSE_PUBLIC_KEY: Q6K8MQN6L7sPYSJiFKM9eNrETOx6V/FxVPup4FqdKsZK1hyR4gyanlQ2KHLg5D5afng99uIt0JCEQ2jiKF9UxFvtnb4BbJ4qpeceH+iK8v/bdg== - LANGFUSE_SECRET_KEY: 5xQ7KMa6YMLsm+H/Pf1VmlqWq1NON5IoCxABhkUBeSck7ftsj2CmpkL2ZwrxwrktgiTUBH+3gJYBX+XBk7lqOOUpvmiLjol/E5lCqq0M1CqLWA== - SLACK_WEBHOOK_URL: RJjhS0Hhz0/s07sCIf1OTXmTGodpK9L2K9p953Z+fOX0l2SkPFT6mB9+yIrLufmlwEaku5NNEBKy//+AG01yOd+7wV1GhK65vfj3B/gTN8t5cuVnR4vFxKY5Rx4eSGLtzyAs+aIBTp4GoNXDIjroCqfCjPkItEZWCg== -general_settings: - alerting: - - slack - alerting_threshold: 300 - database_connection_pool_limit: 100 - database_connection_timeout: 60 - disable_master_key_return: true - health_check_interval: 300 - proxy_batch_write_at: 60 - ui_access_mode: all - # master_key: sk-1234 -litellm_settings: - allowed_fails: 3 - failure_callback: - - prometheus - num_retries: 3 - service_callback: - - prometheus_system - success_callback: - - langfuse - - prometheus - - langsmith -model_list: -- litellm_params: - model: gpt-3.5-turbo - model_name: gpt-3.5-turbo -- litellm_params: - api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ - api_key: my-fake-key - model: openai/my-fake-model - stream_timeout: 0.001 - model_name: fake-openai-endpoint -- litellm_params: - api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ - api_key: my-fake-key - model: openai/my-fake-model-2 - stream_timeout: 0.001 - model_name: fake-openai-endpoint -- litellm_params: - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: 2023-07-01-preview - model: azure/chatgpt-v-2 - stream_timeout: 0.001 - model_name: azure-gpt-3.5 -- litellm_params: - api_key: os.environ/OPENAI_API_KEY - model: text-embedding-ada-002 - model_name: text-embedding-ada-002 -- litellm_params: - model: text-completion-openai/gpt-3.5-turbo-instruct - model_name: gpt-instruct -router_settings: - enable_pre_call_checks: true - redis_host: os.environ/REDIS_HOST - redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT +model_list: + - model_name: my-fake-model + litellm_params: + model: gpt-3.5-turbo + api_key: my-fake-key + mock_response: hello-world + tpm: 60 + +litellm_settings: + callbacks: ["dynamic_rate_limiter"] \ No newline at end of file