From 1475ae59f3ad4fd472a3e0ff259ffd82b8cd0f5a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 13 Aug 2024 20:16:54 -0700 Subject: [PATCH 1/6] add helper to load config from s3 --- .../proxy/common_utils/load_config_utils.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 litellm/proxy/common_utils/load_config_utils.py diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py new file mode 100644 index 0000000000..0c7f5047e2 --- /dev/null +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -0,0 +1,40 @@ +import tempfile + +import boto3 +import yaml + +from litellm._logging import verbose_proxy_logger + + +def get_file_contents_from_s3(bucket_name, object_key): + s3_client = boto3.client("s3") + try: + verbose_proxy_logger.debug( + f"Retrieving {object_key} from S3 bucket: {bucket_name}" + ) + response = s3_client.get_object(Bucket=bucket_name, Key=object_key) + verbose_proxy_logger.debug(f"Response: {response}") + + # Read the file contents + file_contents = response["Body"].read().decode("utf-8") + verbose_proxy_logger.debug(f"File contents retrieved from S3") + + # Create a temporary file with YAML extension + with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as temp_file: + temp_file.write(file_contents.encode("utf-8")) + temp_file_path = temp_file.name + verbose_proxy_logger.debug(f"File stored temporarily at: {temp_file_path}") + + # Load the YAML file content + with open(temp_file_path, "r") as yaml_file: + config = yaml.safe_load(yaml_file) + + return config + except Exception as e: + verbose_proxy_logger.error(f"Error retrieving file contents: {str(e)}") + return None + + +# # Example usage +# bucket_name = 'litellm-proxy' +# object_key = 'litellm_proxy_config.yaml' From 4e7b0ce76e2d87ff98bd4c79ae4b052fa225891c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 13 Aug 2024 20:18:59 -0700 Subject: [PATCH 2/6] feat read config from s3 --- litellm/proxy/proxy_server.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c79a18a5cc..b637bee21b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -151,6 +151,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( check_file_size_under_limit, ) from litellm.proxy.common_utils.init_callbacks import initialize_callbacks_on_proxy +from litellm.proxy.common_utils.load_config_utils import get_file_contents_from_s3 from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) @@ -1402,7 +1403,18 @@ class ProxyConfig: global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, use_background_health_checks, health_check_interval, use_queue, custom_db_client, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details # Load existing config - config = await self.get_config(config_file_path=config_file_path) + if os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is not None: + bucket_name = os.environ.get("LITELLM_CONFIG_BUCKET_NAME") + object_key = os.environ.get("LITELLM_CONFIG_BUCKET_OBJECT_KEY") + verbose_proxy_logger.debug( + "bucket_name: %s, object_key: %s", bucket_name, object_key + ) + config = get_file_contents_from_s3( + bucket_name=bucket_name, object_key=object_key + ) + else: + # default to file + config = await self.get_config(config_file_path=config_file_path) ## PRINT YAML FOR CONFIRMING IT WORKS printed_yaml = copy.deepcopy(config) printed_yaml.pop("environment_variables", None) @@ -2601,6 +2613,15 @@ async def startup_event(): ) else: await initialize(**worker_config) + elif os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is not None: + ( + llm_router, + llm_model_list, + general_settings, + ) = await proxy_config.load_config( + router=llm_router, config_file_path=worker_config + ) + else: # if not, assume it's a json string worker_config = json.loads(os.getenv("WORKER_CONFIG")) From 6f7b20429491fe7a4d8ff308457e024a4e1fa6f5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 13 Aug 2024 20:26:29 -0700 Subject: [PATCH 3/6] docs - set litellm config as s3 object --- docs/my-website/docs/proxy/deploy.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 7c254ed35d..9f21068e03 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -705,6 +705,29 @@ docker run ghcr.io/berriai/litellm:main-latest \ Provide an ssl certificate when starting litellm proxy server +### 3. Providing LiteLLM config.yaml file as a s3 Object/url + +Use this if you cannot mount a config file on your deployment service (example - AWS Fargate, Railway etc) + +LiteLLM Proxy will read your config.yaml from an s3 Bucket + +Set the following .env vars +```shell +LITELLM_CONFIG_BUCKET_NAME = "litellm-proxy" # your bucket name on s3 +LITELLM_CONFIG_BUCKET_OBJECT_KEY = "litellm_proxy_config.yaml" # object key on s3 +``` + +Start litellm proxy with these env vars - litellm will read your config from s3 + +```shell +docker run --name litellm-proxy \ + -e DATABASE_URL= \ + -e LITELLM_CONFIG_BUCKET_NAME= \ + -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ + -p 4000:4000 \ + ghcr.io/berriai/litellm-database:main-latest +``` + ## Platform-specific Guide From 86818ddffc362a17f8149eb72bfa60092dabbed6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 13 Aug 2024 20:28:40 -0700 Subject: [PATCH 4/6] comment on using boto3 --- litellm/proxy/common_utils/load_config_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 0c7f5047e2..acafb6416b 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -7,6 +7,7 @@ from litellm._logging import verbose_proxy_logger def get_file_contents_from_s3(bucket_name, object_key): + # v0 rely on boto3 for authentication - allowing boto3 to handle IAM credentials etc s3_client = boto3.client("s3") try: verbose_proxy_logger.debug( From a37f087b624c7aebd23948fae255db47b4df0028 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 13 Aug 2024 20:33:33 -0700 Subject: [PATCH 5/6] fix ci/cd pipeline --- .circleci/config.yml | 2 ++ litellm/tests/test_completion.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26a2ae356b..b43a8aa64c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -125,6 +125,7 @@ jobs: pip install tiktoken pip install aiohttp pip install click + pip install "boto3==1.34.34" pip install jinja2 pip install tokenizers pip install openai @@ -287,6 +288,7 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" + pip install "boto3==1.34.34" pip install mypy pip install pyarrow pip install numpydoc diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index db0239ca33..4ea9ee3b0f 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.prompt_templates.factory import anthropic_messages_pt -# litellm.num_retries = 3 +# litellm.num_retries =3 litellm.cache = None litellm.success_callback = [] user_message = "Write a short poem about the sky" From 09535b25f4e3c7d7ce5a8cbe438b189e6e7060e8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 13 Aug 2024 21:18:06 -0700 Subject: [PATCH 6/6] fix use s3 get_credentials to get boto3 creds --- litellm/proxy/common_utils/load_config_utils.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index acafb6416b..bded2e3470 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -8,7 +8,19 @@ from litellm._logging import verbose_proxy_logger def get_file_contents_from_s3(bucket_name, object_key): # v0 rely on boto3 for authentication - allowing boto3 to handle IAM credentials etc - s3_client = boto3.client("s3") + from botocore.config import Config + from botocore.credentials import Credentials + + from litellm.main import bedrock_converse_chat_completion + + credentials: Credentials = bedrock_converse_chat_completion.get_credentials() + s3_client = boto3.client( + "s3", + aws_access_key_id=credentials.access_key, + aws_secret_access_key=credentials.secret_key, + aws_session_token=credentials.token, # Optional, if using temporary credentials + ) + try: verbose_proxy_logger.debug( f"Retrieving {object_key} from S3 bucket: {bucket_name}"