diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 4878d15a3f..0a91a179ff 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,63 +1,15 @@ model_list: - - model_name: bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0 + - model_name: db-openai-endpoint litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - ######################################################### - ########## batch specific params ######################## - s3_bucket_name: litellm-proxy - s3_region_name: us-west-2 - s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID - s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV - model_info: - mode: batch - - model_name: anthropic/* - litellm_params: - model: anthropic/* - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: openai/* - litellm_params: - model: openai/* - api_key: os.environ/OPENAI_API_KEY - - model_name: gemini/* - litellm_params: - model: gemini/* - api_key: os.environ/GEMINI_API_KEY - - model_name: vertex_ai/* - litellm_params: - model: vertex_ai/* - - model_name: "grok-4" - model_info: - mode: completion - litellm_params: - model: oci/xai.grok-4 - oci_key: ocid1.api_key.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk - oci_region: us-phoenix-1 - oci_user: ocid1.user.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk - oci_fingerprint: aa:bb:cc:dd:ee:ff:11:22:33:44:55:66:77:88:99:00 - oci_tenancy: ocid1.tenancy.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk - oci_key_file: /path/to/oci_api_key.pem - oci_compartment_id: ocid1.compartment.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk - drop_params: True + model: openai/gm + api_key: hi + api_base: https://exampleopenaiendpoint-production.up.railway.app/ -guardrails: - - guardrail_name: "bedrock-pre-guard" - litellm_params: - guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" - mode: "during_call" - guardrailIdentifier: ff6ujrregl1q - guardrailVersion: "DRAFT" - litellm_settings: - callbacks: ["datadog"] - include_cost_in_streaming_usage: true - datadog_params: - turn_off_message_logging: true - datadog_llm_observability_params: - turn_off_message_logging: true - - cache: True - cache_params: - type: redis - redis_startup_nodes: [{"host": "127.0.0.1", "port": "7000"}, {"host": "127.0.0.1", "port": "7001"}, {"host": "127.0.0.1", "port": "7002"}, {"host": "127.0.0.1", "port": "7003"}] \ No newline at end of file + callbacks: ["dynamic_rate_limiter_v3"] + priority_reservation: + "prod": 0.9 # 90% reserved for production (9 RPM) + "dev": 0.1 # 10% reserved for development (1 RPM) + priority_reservation_settings: + default_priority: 0.2 # Weight (0%) assigned to keys without explicit priority metadata \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e60482bc8d..7a1e8f1e73 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1880,6 +1880,10 @@ class ProxyConfig: verbose_proxy_logger.info( f"{blue_color_code}Set Global Gitlab Config on LiteLLM Proxy{reset_color_code}" ) + elif key == "priority_reservation_settings": + from litellm.types.utils import PriorityReservationSettings + + litellm.priority_reservation_settings = PriorityReservationSettings(**value) elif key == "callbacks": initialize_callbacks_on_proxy( value=value, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0da0ba4e93..cac3bdfb01 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2717,10 +2717,5 @@ class PriorityReservationSettings(BaseModel): default=0.50, description="Saturation threshold (0.0-1.0) at which strict priority enforcement begins. Below this threshold, generous mode allows priority borrowing. Above this threshold, strict mode enforces normalized priority limits." ) - - tracking_multiplier: int = Field( - default=10, - description="Multiplier for model-wide tracking limits in strict mode. Set to 10x because v3_limiter.should_rate_limit() both increments counters AND enforces limits - we need the counter increment (for saturation checks) but not the enforcement (priority limits handle that). High multiplier ensures tracking never blocks." - ) model_config = ConfigDict(protected_namespaces=())