Merge pull request #5191 from BerriAI/litellm_load_config_from_s3

[Feat] Allow loading LiteLLM config from s3 buckets

.circleci/config.yml

@@ -125,6 +125,7 @@ jobs:
             pip install tiktoken
             pip install aiohttp
             pip install click
+            pip install "boto3==1.34.34"
             pip install jinja2
             pip install tokenizers
             pip install openai
@@ -287,6 +288,7 @@ jobs:
             pip install "pytest==7.3.1"
             pip install "pytest-mock==3.12.0"
             pip install "pytest-asyncio==0.21.1"
+            pip install "boto3==1.34.34"
             pip install mypy
             pip install pyarrow
             pip install numpydoc

docs/my-website/docs/proxy/deploy.md

@@ -705,6 +705,29 @@ docker run ghcr.io/berriai/litellm:main-latest \
 
 Provide an ssl certificate when starting litellm proxy server 
 
+### 3. Providing LiteLLM config.yaml file as a s3 Object/url
+
+Use this if you cannot mount a config file on your deployment service (example - AWS Fargate, Railway etc)
+
+LiteLLM Proxy will read your config.yaml from an s3 Bucket
+
+Set the following .env vars 
+```shell
+LITELLM_CONFIG_BUCKET_NAME = "litellm-proxy"                    # your bucket name on s3 
+LITELLM_CONFIG_BUCKET_OBJECT_KEY = "litellm_proxy_config.yaml"  # object key on s3
+```
+
+Start litellm proxy with these env vars - litellm will read your config from s3 
+
+```shell
+docker run --name litellm-proxy \
+   -e DATABASE_URL=<database_url> \
+   -e LITELLM_CONFIG_BUCKET_NAME=<bucket_name> \
+   -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
+   -p 4000:4000 \
+   ghcr.io/berriai/litellm-database:main-latest
+```
+
 ## Platform-specific Guide
 
 <Tabs>

litellm/proxy/common_utils/load_config_utils.py

@@ -0,0 +1,53 @@
+import tempfile
+
+import boto3
+import yaml
+
+from litellm._logging import verbose_proxy_logger
+
+
+def get_file_contents_from_s3(bucket_name, object_key):
+    # v0 rely on boto3 for authentication - allowing boto3 to handle IAM credentials etc
+    from botocore.config import Config
+    from botocore.credentials import Credentials
+
+    from litellm.main import bedrock_converse_chat_completion
+
+    credentials: Credentials = bedrock_converse_chat_completion.get_credentials()
+    s3_client = boto3.client(
+        "s3",
+        aws_access_key_id=credentials.access_key,
+        aws_secret_access_key=credentials.secret_key,
+        aws_session_token=credentials.token,  # Optional, if using temporary credentials
+    )
+
+    try:
+        verbose_proxy_logger.debug(
+            f"Retrieving {object_key} from S3 bucket: {bucket_name}"
+        )
+        response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
+        verbose_proxy_logger.debug(f"Response: {response}")
+
+        # Read the file contents
+        file_contents = response["Body"].read().decode("utf-8")
+        verbose_proxy_logger.debug(f"File contents retrieved from S3")
+
+        # Create a temporary file with YAML extension
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as temp_file:
+            temp_file.write(file_contents.encode("utf-8"))
+            temp_file_path = temp_file.name
+            verbose_proxy_logger.debug(f"File stored temporarily at: {temp_file_path}")
+
+        # Load the YAML file content
+        with open(temp_file_path, "r") as yaml_file:
+            config = yaml.safe_load(yaml_file)
+
+        return config
+    except Exception as e:
+        verbose_proxy_logger.error(f"Error retrieving file contents: {str(e)}")
+        return None
+
+
+# # Example usage
+# bucket_name = 'litellm-proxy'
+# object_key = 'litellm_proxy_config.yaml'

litellm/proxy/proxy_server.py

@@ -151,6 +151,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
     check_file_size_under_limit,
 )
 from litellm.proxy.common_utils.init_callbacks import initialize_callbacks_on_proxy
+from litellm.proxy.common_utils.load_config_utils import get_file_contents_from_s3
 from litellm.proxy.common_utils.openai_endpoint_utils import (
     remove_sensitive_info_from_deployment,
 )
@@ -1402,7 +1403,18 @@ class ProxyConfig:
         global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, use_background_health_checks, health_check_interval, use_queue, custom_db_client, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details
 
         # Load existing config
-        config = await self.get_config(config_file_path=config_file_path)
+        if os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is not None:
+            bucket_name = os.environ.get("LITELLM_CONFIG_BUCKET_NAME")
+            object_key = os.environ.get("LITELLM_CONFIG_BUCKET_OBJECT_KEY")
+            verbose_proxy_logger.debug(
+                "bucket_name: %s, object_key: %s", bucket_name, object_key
+            )
+            config = get_file_contents_from_s3(
+                bucket_name=bucket_name, object_key=object_key
+            )
+        else:
+            # default to file
+            config = await self.get_config(config_file_path=config_file_path)
         ## PRINT YAML FOR CONFIRMING IT WORKS
         printed_yaml = copy.deepcopy(config)
         printed_yaml.pop("environment_variables", None)
@@ -2601,6 +2613,15 @@ async def startup_event():
             )
         else:
             await initialize(**worker_config)
+    elif os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is not None:
+        (
+            llm_router,
+            llm_model_list,
+            general_settings,
+        ) = await proxy_config.load_config(
+            router=llm_router, config_file_path=worker_config
+        )
+
     else:
         # if not, assume it's a json string
         worker_config = json.loads(os.getenv("WORKER_CONFIG"))

litellm/tests/test_completion.py

@@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.prompt_templates.factory import anthropic_messages_pt
 
-# litellm.num_retries = 3
+# litellm.num_retries =3
 litellm.cache = None
 litellm.success_callback = []
 user_message = "Write a short poem about the sky"