diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index a77beb01e1..01ff24da69 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -331,19 +331,14 @@ router_settings: | AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow | AZURE_FEDERATED_TOKEN_FILE | File path to Azure federated token | AZURE_KEY_VAULT_URI | URI for Azure Key Vault -| AZURE_OPERATION_POLLING_TIMEOUT | Timeout in seconds for Azure operation polling | AZURE_STORAGE_ACCOUNT_KEY | The Azure Storage Account Key to use for Authentication to Azure Blob Storage logging | AZURE_STORAGE_ACCOUNT_NAME | Name of the Azure Storage Account to use for logging to Azure Blob Storage | AZURE_STORAGE_FILE_SYSTEM | Name of the Azure Storage File System to use for logging to Azure Blob Storage. (Typically the Container name) | AZURE_STORAGE_TENANT_ID | The Application Tenant ID to use for Authentication to Azure Blob Storage logging | AZURE_STORAGE_CLIENT_ID | The Application Client ID to use for Authentication to Azure Blob Storage logging | AZURE_STORAGE_CLIENT_SECRET | The Application Client Secret to use for Authentication to Azure Blob Storage logging -| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour) -| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours) -| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75 | BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service | BRAINTRUST_API_KEY | API key for Braintrust integration -| CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02 | CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI | CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI | CONFIG_FILE_PATH | File path for configuration file @@ -357,9 +352,6 @@ router_settings: | DATABASE_USER | Username for database connection | DATABASE_USERNAME | Alias for database user | DATABRICKS_API_BASE | Base URL for Databricks API -| DAYS_IN_A_MONTH | Days in a month for calculation purposes. Default is 28 -| DAYS_IN_A_WEEK | Days in a week for calculation purposes. Default is 7 -| DAYS_IN_A_YEAR | Days in a year for calculation purposes. Default is 365 | DD_BASE_URL | Base URL for Datadog integration | DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration | _DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration @@ -370,38 +362,6 @@ router_settings: | DD_SERVICE | Service identifier for Datadog logs. Defaults to "litellm-server" | DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown" | DEBUG_OTEL | Enable debug mode for OpenTelemetry -| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3 -| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096 -| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512 -| DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5 -| DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute) -| DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%) -| DEFAULT_FLUSH_INTERVAL_SECONDS | Default interval in seconds for flushing operations. Default is 5 -| DEFAULT_HEALTH_CHECK_INTERVAL | Default interval in seconds for health checks. Default is 300 (5 minutes) -| DEFAULT_IMAGE_HEIGHT | Default height for images. Default is 300 -| DEFAULT_IMAGE_TOKEN_COUNT | Default token count for images. Default is 250 -| DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300 -| DEFAULT_IN_MEMORY_TTL | Default time-to-live for in-memory cache in seconds. Default is 5 -| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 16 -| DEFAULT_MAX_RECURSE_DEPTH | Default maximum recursion depth. Default is 100 -| DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2 -| DEFAULT_MAX_TOKENS | Default maximum tokens for LLM calls. Default is 4096 -| DEFAULT_MAX_TOKENS_FOR_TRITON | Default maximum tokens for Triton models. Default is 2000 -| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 -| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 -| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 -| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7 -| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03 -| DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET | Default high reasoning effort thinking budget. Default is 4096 -| DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET | Default low reasoning effort thinking budget. Default is 1024 -| DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET | Default medium reasoning effort thinking budget. Default is 2048 -| DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1 -| DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400 -| DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1 -| DEFAULT_REPLICATE_POLLING_RETRIES | Default number of retries for Replicate polling. Default is 5 -| DEFAULT_SLACK_ALERTING_THRESHOLD | Default threshold for Slack alerting. Default is 300 -| DEFAULT_SOFT_BUDGET | Default soft budget for LiteLLM proxy keys. Default is 50.0 -| DEFAULT_TRIM_RATIO | Default ratio of tokens to trim from prompt end. Default is 0.75 | DIRECT_URL | Direct URL for service endpoint | DISABLE_ADMIN_UI | Toggle to disable the admin UI | DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates @@ -409,19 +369,8 @@ router_settings: | DOCS_FILTERED | Flag indicating filtered documentation | DOCS_TITLE | Title of the documentation pages | DOCS_URL | The path to the Swagger API documentation. **By default this is "/"** -| EMAIL_LOGO_URL | URL for the logo used in emails | EMAIL_SUPPORT_CONTACT | Support contact email address | EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False** -| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4 -| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16 -| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56 -| FIREWORKS_AI_80_B | Size parameter for Fireworks AI 80B model. Default is 80 -| FIREWORKS_AI_176_B_MOE | Size parameter for Fireworks AI 176B MOE model. Default is 176 -| FUNCTION_DEFINITION_TOKEN_COUNT | Token count for function definitions. Default is 9 -| GALILEO_BASE_URL | Base URL for Galileo platform -| GALILEO_PASSWORD | Password for Galileo authentication -| GALILEO_PROJECT_ID | Project ID for Galileo usage -| GALILEO_USERNAME | Username for Galileo authentication | GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket | GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file | GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds** @@ -453,7 +402,6 @@ router_settings: | GOOGLE_CLIENT_ID | Client ID for Google OAuth | GOOGLE_CLIENT_SECRET | Client secret for Google OAuth | GOOGLE_KMS_RESOURCE_NAME | Name of the resource in Google KMS -| HEALTH_CHECK_TIMEOUT_SECONDS | Timeout in seconds for health checks. Default is 60 | HF_API_BASE | Base URL for Hugging Face API | HCP_VAULT_ADDR | Address for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) | HCP_VAULT_CLIENT_CERT | Path to client certificate for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) @@ -463,13 +411,9 @@ router_settings: | HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault) | HELICONE_API_KEY | API key for Helicone service | HOSTNAME | Hostname for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) -| HOURS_IN_A_DAY | Hours in a day for calculation purposes. Default is 24 | HUGGINGFACE_API_BASE | Base URL for Hugging Face API | HUGGINGFACE_API_KEY | API key for Hugging Face API -| HUMANLOOP_PROMPT_CACHE_TTL_SECONDS | Time-to-live in seconds for cached prompts in Humanloop. Default is 60 | IAM_TOKEN_DB_AUTH | IAM token for database authentication -| INITIAL_RETRY_DELAY | Initial delay in seconds for retrying requests. Default is 0.5 -| JITTER | Jitter factor for retry delay calculations. Default is 0.75 | JSON_LOGS | Enable JSON formatted logging | JWT_AUDIENCE | Expected audience for JWT tokens | JWT_PUBLIC_KEY_URL | URL to fetch public key for JWT verification @@ -490,7 +434,6 @@ router_settings: | LANGSMITH_PROJECT | Project name for Langsmith integration | LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging | LANGTRACE_API_KEY | API key for Langtrace service -| LENGTH_OF_LITELLM_GENERATED_KEY | Length of keys generated by LiteLLM. Default is 16 | LITERAL_API_KEY | API key for Literal integration | LITERAL_API_URL | API URL for Literal service | LITERAL_BATCH_SIZE | Batch size for Literal operations @@ -511,21 +454,6 @@ router_settings: | LITELLM_TOKEN | Access token for LiteLLM integration | LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging | LOGFIRE_TOKEN | Token for Logfire logging service -| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000 -| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000 -| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000 -| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100 -| MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the short side of high-resolution images. Default is 768 -| MAX_SIZE_IN_MEMORY_QUEUE | Maximum size for in-memory queue. Default is 10000 -| MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB | Maximum size in KB for each item in memory cache. Default is 512 or 1024 -| MAX_SPENDLOG_ROWS_TO_QUERY | Maximum number of spend log rows to query. Default is 1,000,000 -| MAX_TEAM_LIST_LIMIT | Maximum number of teams to list. Default is 20 -| MAX_TILE_HEIGHT | Maximum height for image tiles. Default is 512 -| MAX_TILE_WIDTH | Maximum width for image tiles. Default is 512 -| MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10 -| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0 -| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001 -| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024 | MISTRAL_API_BASE | Base URL for Mistral API | MISTRAL_API_KEY | API key for Mistral API | MICROSOFT_CLIENT_ID | Client ID for Microsoft services @@ -534,12 +462,10 @@ router_settings: | MICROSOFT_SERVICE_PRINCIPAL_ID | Service Principal ID for Microsoft Enterprise Application. (This is an advanced feature if you want litellm to auto-assign members to Litellm Teams based on their Microsoft Entra ID Groups) | NO_DOCS | Flag to disable documentation generation | NO_PROXY | List of addresses to bypass proxy -| NON_LLM_CONNECTION_TIMEOUT | Timeout in seconds for non-LLM service connections. Default is 15 | OAUTH_TOKEN_INFO_ENDPOINT | Endpoint for OAuth token info retrieval | OPENAI_BASE_URL | Base URL for OpenAI API | OPENAI_API_BASE | Base URL for OpenAI API | OPENAI_API_KEY | API key for OpenAI services -| OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025 | OPENAI_ORGANIZATION | Organization identifier for OpenAI | OPENID_BASE_URL | Base URL for OpenID Connect services | OPENID_CLIENT_ID | Client ID for OpenID Connect authentication @@ -561,37 +487,21 @@ router_settings: | PREDIBASE_API_BASE | Base URL for Predibase API | PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service | PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service -| PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES | Refresh interval in minutes for Prometheus budget metrics. Default is 5 -| PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS | Fallback time in hours for sending stats to Prometheus. Default is 9 | PROMETHEUS_URL | URL for Prometheus service | PROMPTLAYER_API_KEY | API key for PromptLayer integration | PROXY_ADMIN_ID | Admin identifier for proxy server | PROXY_BASE_URL | Base URL for proxy service -| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10 -| PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605 -| PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597 | PROXY_LOGOUT_URL | URL for logging out of the proxy service | LITELLM_MASTER_KEY | Master key for proxy authentication | QDRANT_API_BASE | Base URL for Qdrant API | QDRANT_API_KEY | API key for Qdrant service -| QDRANT_SCALAR_QUANTILE | Scalar quantile for Qdrant operations. Default is 0.99 | QDRANT_URL | Connection URL for Qdrant database -| QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536 -| REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5 | REDIS_HOST | Hostname for Redis server | REDIS_PASSWORD | Password for Redis service | REDIS_PORT | Port number for Redis server -| REDIS_SOCKET_TIMEOUT | Timeout in seconds for Redis socket operations. Default is 0.1 | REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"** -| REPEATED_STREAMING_CHUNK_LIMIT | Limit for repeated streaming chunks to detect looping. Default is 100 -| REPLICATE_MODEL_NAME_WITH_ID_LENGTH | Length of Replicate model names with ID. Default is 64 -| REPLICATE_POLLING_DELAY_SECONDS | Delay in seconds for Replicate polling operations. Default is 0.5 -| REQUEST_TIMEOUT | Timeout in seconds for requests. Default is 6000 -| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5 -| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours) | SERVER_ROOT_PATH | Root path for the server application | SET_VERBOSE | Flag to enable verbose logging -| SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000 | SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly) | SLACK_WEBHOOK_URL | Webhook URL for Slack integration | SMTP_HOST | Hostname for the SMTP server @@ -608,17 +518,7 @@ router_settings: | SUPABASE_KEY | API key for Supabase service | SUPABASE_URL | Base URL for Supabase instance | STORE_MODEL_IN_DB | If true, enables storing model + credential information in the DB. -| SYSTEM_MESSAGE_TOKEN_COUNT | Token count for system messages. Default is 4 | TEST_EMAIL_ADDRESS | Email address used for testing purposes -| TOGETHER_AI_4_B | Size parameter for Together AI 4B model. Default is 4 -| TOGETHER_AI_8_B | Size parameter for Together AI 8B model. Default is 8 -| TOGETHER_AI_21_B | Size parameter for Together AI 21B model. Default is 21 -| TOGETHER_AI_41_B | Size parameter for Together AI 41B model. Default is 41 -| TOGETHER_AI_80_B | Size parameter for Together AI 80B model. Default is 80 -| TOGETHER_AI_110_B | Size parameter for Together AI 110B model. Default is 110 -| TOGETHER_AI_EMBEDDING_150_M | Size parameter for Together AI 150M embedding model. Default is 150 -| TOGETHER_AI_EMBEDDING_350_M | Size parameter for Together AI 350M embedding model. Default is 350 -| TOOL_CHOICE_OBJECT_TOKEN_COUNT | Token count for tool choice objects. Default is 4 | UI_LOGO_PATH | Path to the logo image used in the UI | UI_PASSWORD | Password for accessing the UI | UI_USERNAME | Username for accessing the UI diff --git a/litellm/constants.py b/litellm/constants.py index a3ce465d4c..8d49f2bef2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1,48 +1,30 @@ -import os from typing import List, Literal -ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) -DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) -DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) -DEFAULT_MAX_RETRIES = int(os.getenv("DEFAULT_MAX_RETRIES", 2)) -DEFAULT_MAX_RECURSE_DEPTH = int(os.getenv("DEFAULT_MAX_RECURSE_DEPTH", 100)) -DEFAULT_FAILURE_THRESHOLD_PERCENT = float( - os.getenv("DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5) -) # default cooldown a deployment if 50% of requests fail in a given minute -DEFAULT_MAX_TOKENS = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) -DEFAULT_ALLOWED_FAILS = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) -DEFAULT_REDIS_SYNC_INTERVAL = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) -DEFAULT_COOLDOWN_TIME_SECONDS = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) -DEFAULT_REPLICATE_POLLING_RETRIES = int( - os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5) +ROUTER_MAX_FALLBACKS = 5 +DEFAULT_BATCH_SIZE = 512 +DEFAULT_FLUSH_INTERVAL_SECONDS = 5 +DEFAULT_MAX_RETRIES = 2 +DEFAULT_MAX_RECURSE_DEPTH = 100 +DEFAULT_FAILURE_THRESHOLD_PERCENT = ( + 0.5 # default cooldown a deployment if 50% of requests fail in a given minute ) -DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int( - os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) -) -DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) -DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300)) -DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300)) -MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( - os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024) -) # 1MB = 1024KB -SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = int( - os.getenv("SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD", 1000) -) # Minimum number of requests to consider "reasonable traffic". Used for single-deployment cooldown logic. - -DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int( - os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024) -) -DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET = int( - os.getenv("DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET", 2048) -) -DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = int( - os.getenv("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096) -) -MAX_TOKEN_TRIMMING_ATTEMPTS = int( - os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10) -) # Maximum number of attempts to trim the message - +DEFAULT_MAX_TOKENS = 4096 +DEFAULT_ALLOWED_FAILS = 3 +DEFAULT_REDIS_SYNC_INTERVAL = 1 +DEFAULT_COOLDOWN_TIME_SECONDS = 5 +DEFAULT_REPLICATE_POLLING_RETRIES = 5 +DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = 1 +DEFAULT_IMAGE_TOKEN_COUNT = 250 +DEFAULT_IMAGE_WIDTH = 300 +DEFAULT_IMAGE_HEIGHT = 300 +DEFAULT_MAX_TOKENS = 256 # used when providers need a default +MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = 1024 # 1MB = 1024KB +SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = 1000 # Minimum number of requests to consider "reasonable traffic". Used for single-deployment cooldown logic. +DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = 1024 +DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET = 2048 +DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = 4096 +MAX_TOKEN_TRIMMING_ATTEMPTS = 10 # Maximum number of attempts to trim the message ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour @@ -51,102 +33,70 @@ REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer" REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer" REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer" -MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)) -MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 10000)) -MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int( - os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000) -) +MAX_REDIS_BUFFER_DEQUEUE_COUNT = 100 +MAX_SIZE_IN_MEMORY_QUEUE = 10000 +MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = 1000 ############################################################################################### -MINIMUM_PROMPT_CACHE_TOKEN_COUNT = int( - os.getenv("MINIMUM_PROMPT_CACHE_TOKEN_COUNT", 1024) -) # minimum number of tokens to cache a prompt by Anthropic -DEFAULT_TRIM_RATIO = float( - os.getenv("DEFAULT_TRIM_RATIO", 0.75) -) # default ratio of tokens to trim from the end of a prompt -HOURS_IN_A_DAY = int(os.getenv("HOURS_IN_A_DAY", 24)) -DAYS_IN_A_WEEK = int(os.getenv("DAYS_IN_A_WEEK", 7)) -DAYS_IN_A_MONTH = int(os.getenv("DAYS_IN_A_MONTH", 28)) -DAYS_IN_A_YEAR = int(os.getenv("DAYS_IN_A_YEAR", 365)) -REPLICATE_MODEL_NAME_WITH_ID_LENGTH = int( - os.getenv("REPLICATE_MODEL_NAME_WITH_ID_LENGTH", 64) +MINIMUM_PROMPT_CACHE_TOKEN_COUNT = ( + 1024 # minimum number of tokens to cache a prompt by Anthropic ) +DEFAULT_TRIM_RATIO = 0.75 # default ratio of tokens to trim from the end of a prompt +HOURS_IN_A_DAY = 24 +DAYS_IN_A_WEEK = 7 +DAYS_IN_A_MONTH = 28 +DAYS_IN_A_YEAR = 365 +REPLICATE_MODEL_NAME_WITH_ID_LENGTH = 64 #### TOKEN COUNTING #### -FUNCTION_DEFINITION_TOKEN_COUNT = int(os.getenv("FUNCTION_DEFINITION_TOKEN_COUNT", 9)) -SYSTEM_MESSAGE_TOKEN_COUNT = int(os.getenv("SYSTEM_MESSAGE_TOKEN_COUNT", 4)) -TOOL_CHOICE_OBJECT_TOKEN_COUNT = int(os.getenv("TOOL_CHOICE_OBJECT_TOKEN_COUNT", 4)) -DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT = int( - os.getenv("DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT", 10) -) -DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT = int( - os.getenv("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20) -) -MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES = int( - os.getenv("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768) -) -MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES = int( - os.getenv("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000) -) -MAX_TILE_WIDTH = int(os.getenv("MAX_TILE_WIDTH", 512)) -MAX_TILE_HEIGHT = int(os.getenv("MAX_TILE_HEIGHT", 512)) -OPENAI_FILE_SEARCH_COST_PER_1K_CALLS = float( - os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000) -) -MIN_NON_ZERO_TEMPERATURE = float(os.getenv("MIN_NON_ZERO_TEMPERATURE", 0.0001)) +FUNCTION_DEFINITION_TOKEN_COUNT = 9 +SYSTEM_MESSAGE_TOKEN_COUNT = 4 +TOOL_CHOICE_OBJECT_TOKEN_COUNT = 4 +DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT = 10 +DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT = 20 +MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES = 768 +MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES = 2000 +MAX_TILE_WIDTH = 512 +MAX_TILE_HEIGHT = 512 +OPENAI_FILE_SEARCH_COST_PER_1K_CALLS = 2.5 / 1000 +MIN_NON_ZERO_TEMPERATURE = 0.0001 #### RELIABILITY #### -REPEATED_STREAMING_CHUNK_LIMIT = int( - os.getenv("REPEATED_STREAMING_CHUNK_LIMIT", 100) -) # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. -DEFAULT_MAX_LRU_CACHE_SIZE = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 16)) -INITIAL_RETRY_DELAY = float(os.getenv("INITIAL_RETRY_DELAY", 0.5)) -MAX_RETRY_DELAY = float(os.getenv("MAX_RETRY_DELAY", 8.0)) -JITTER = float(os.getenv("JITTER", 0.75)) -DEFAULT_IN_MEMORY_TTL = int( - os.getenv("DEFAULT_IN_MEMORY_TTL", 5) -) # default time to live for the in-memory cache -DEFAULT_POLLING_INTERVAL = float( - os.getenv("DEFAULT_POLLING_INTERVAL", 0.03) -) # default polling interval for the scheduler -AZURE_OPERATION_POLLING_TIMEOUT = int(os.getenv("AZURE_OPERATION_POLLING_TIMEOUT", 120)) -REDIS_SOCKET_TIMEOUT = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1)) -REDIS_CONNECTION_POOL_TIMEOUT = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5)) -NON_LLM_CONNECTION_TIMEOUT = int( - os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15) -) # timeout for adjacent services (e.g. jwt auth) -MAX_EXCEPTION_MESSAGE_LENGTH = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)) -BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75)) -REPLICATE_POLLING_DELAY_SECONDS = float( - os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5) -) -DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS = int( - os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096) -) -TOGETHER_AI_4_B = int(os.getenv("TOGETHER_AI_4_B", 4)) -TOGETHER_AI_8_B = int(os.getenv("TOGETHER_AI_8_B", 8)) -TOGETHER_AI_21_B = int(os.getenv("TOGETHER_AI_21_B", 21)) -TOGETHER_AI_41_B = int(os.getenv("TOGETHER_AI_41_B", 41)) -TOGETHER_AI_80_B = int(os.getenv("TOGETHER_AI_80_B", 80)) -TOGETHER_AI_110_B = int(os.getenv("TOGETHER_AI_110_B", 110)) -TOGETHER_AI_EMBEDDING_150_M = int(os.getenv("TOGETHER_AI_EMBEDDING_150_M", 150)) -TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350)) -QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99)) -QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536)) -CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02)) -MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( - os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512) -) -DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000)) +REPEATED_STREAMING_CHUNK_LIMIT = 100 # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. +DEFAULT_MAX_LRU_CACHE_SIZE = 16 +INITIAL_RETRY_DELAY = 0.5 +MAX_RETRY_DELAY = 8.0 +JITTER = 0.75 +DEFAULT_IN_MEMORY_TTL = 5 # default time to live for the in-memory cache +DEFAULT_POLLING_INTERVAL = 0.03 # default polling interval for the scheduler +AZURE_OPERATION_POLLING_TIMEOUT = 120 +REDIS_SOCKET_TIMEOUT = 0.1 +REDIS_CONNECTION_POOL_TIMEOUT = 5 +NON_LLM_CONNECTION_TIMEOUT = 15 # timeout for adjacent services (e.g. jwt auth) +MAX_EXCEPTION_MESSAGE_LENGTH = 2000 +BEDROCK_MAX_POLICY_SIZE = 75 +REPLICATE_POLLING_DELAY_SECONDS = 0.5 +DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS = 4096 +TOGETHER_AI_4_B = 4 +TOGETHER_AI_8_B = 8 +TOGETHER_AI_21_B = 21 +TOGETHER_AI_41_B = 41 +TOGETHER_AI_80_B = 80 +TOGETHER_AI_110_B = 110 +TOGETHER_AI_EMBEDDING_150_M = 150 +TOGETHER_AI_EMBEDDING_350_M = 350 +QDRANT_SCALAR_QUANTILE = 0.99 +QDRANT_VECTOR_SIZE = 1536 +CACHED_STREAMING_CHUNK_DELAY = 0.02 +MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = 512 +DEFAULT_MAX_TOKENS_FOR_TRITON = 2000 #### Networking settings #### -request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds +request_timeout: float = 6000 # time in seconds STREAM_SSE_DONE_STRING: str = "[DONE]" ### SPEND TRACKING ### -DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = float( - os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400) -) # price per second for a100 80GB -FIREWORKS_AI_56_B_MOE = int(os.getenv("FIREWORKS_AI_56_B_MOE", 56)) -FIREWORKS_AI_176_B_MOE = int(os.getenv("FIREWORKS_AI_176_B_MOE", 176)) -FIREWORKS_AI_4_B = int(os.getenv("FIREWORKS_AI_4_B", 4)) -FIREWORKS_AI_16_B = int(os.getenv("FIREWORKS_AI_16_B", 16)) -FIREWORKS_AI_80_B = int(os.getenv("FIREWORKS_AI_80_B", 80)) +DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = 0.001400 # price per second for a100 80GB +FIREWORKS_AI_56_B_MOE = 56 +FIREWORKS_AI_176_B_MOE = 176 +FIREWORKS_AI_4_B = 4 +FIREWORKS_AI_16_B = 16 +FIREWORKS_AI_80_B = 80 #### Logging callback constants #### REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM" @@ -549,26 +499,22 @@ known_tokenizer_config = { OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null"] -HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = int( - os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) -) # 1 minute +HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = 60 # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION = "2019-07-07" -PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES = int( - os.getenv("PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5) -) +PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES = 5 MCP_TOOL_NAME_PREFIX = "mcp_tool" ########################### LiteLLM Proxy Specific Constants ########################### ######################################################################################## -MAX_SPENDLOG_ROWS_TO_QUERY = int( - os.getenv("MAX_SPENDLOG_ROWS_TO_QUERY", 1_000_000) -) # if spendLogs has more than 1M rows, do not query the DB -DEFAULT_SOFT_BUDGET = float( - os.getenv("DEFAULT_SOFT_BUDGET", 50.0) -) # by default all litellm proxy keys have a soft budget of 50.0 +MAX_SPENDLOG_ROWS_TO_QUERY = ( + 1_000_000 # if spendLogs has more than 1M rows, do not query the DB +) +DEFAULT_SOFT_BUDGET = ( + 50.0 # by default all litellm proxy keys have a soft budget of 50.0 +) # makes it clear this is a rate limit error for a litellm virtual key RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY = "LiteLLM Virtual Key user_api_key_hash" @@ -583,16 +529,10 @@ BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES = [ "optimize-prompt/", ] -BATCH_STATUS_POLL_INTERVAL_SECONDS = int( - os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600) -) # 1 hour -BATCH_STATUS_POLL_MAX_ATTEMPTS = int( - os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24) -) # for 24 hours +BATCH_STATUS_POLL_INTERVAL_SECONDS = 3600 # 1 hour +BATCH_STATUS_POLL_MAX_ATTEMPTS = 24 # for 24 hours -HEALTH_CHECK_TIMEOUT_SECONDS = int( - os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60) -) # 60 seconds +HEALTH_CHECK_TIMEOUT_SECONDS = 60 # 60 seconds UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard" LITELLM_PROXY_ADMIN_NAME = "default_user_id" @@ -600,33 +540,15 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id" ########################### DB CRON JOB NAMES ########################### DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job" PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics_job" -DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = int( - os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) -) # 1 minute -PROXY_BUDGET_RESCHEDULER_MIN_TIME = int( - os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597) -) -PROXY_BUDGET_RESCHEDULER_MAX_TIME = int( - os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605) -) -PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds -DEFAULT_HEALTH_CHECK_INTERVAL = int( - os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300) -) # 5 minutes -PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS = int( - os.getenv("PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS", 9) -) -DEFAULT_MODEL_CREATED_AT_TIME = int( - os.getenv("DEFAULT_MODEL_CREATED_AT_TIME", 1677610602) -) # returns on `/models` endpoint -DEFAULT_SLACK_ALERTING_THRESHOLD = int( - os.getenv("DEFAULT_SLACK_ALERTING_THRESHOLD", 300) -) -MAX_TEAM_LIST_LIMIT = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20)) -DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float( - os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7) -) -LENGTH_OF_LITELLM_GENERATED_KEY = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16)) -SECRET_MANAGER_REFRESH_INTERVAL = int( - os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400) -) +DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = 60 # 1 minute +PROXY_BUDGET_RESCHEDULER_MIN_TIME = 597 +PROXY_BUDGET_RESCHEDULER_MAX_TIME = 605 +PROXY_BATCH_WRITE_AT = 10 # in seconds +DEFAULT_HEALTH_CHECK_INTERVAL = 300 # 5 minutes +PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS = 9 +DEFAULT_MODEL_CREATED_AT_TIME = 1677610602 # returns on `/models` endpoint +DEFAULT_SLACK_ALERTING_THRESHOLD = 300 +MAX_TEAM_LIST_LIMIT = 20 +DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = 0.7 +LENGTH_OF_LITELLM_GENERATED_KEY = 16 +SECRET_MANAGER_REFRESH_INTERVAL = 86400 diff --git a/tests/litellm/test_constants.py b/tests/litellm/test_constants.py deleted file mode 100644 index 8fea734230..0000000000 --- a/tests/litellm/test_constants.py +++ /dev/null @@ -1,60 +0,0 @@ -import inspect -import json -import os -import sys -from unittest import mock - -import httpx -import pytest -import respx -from fastapi.testclient import TestClient - -sys.path.insert(0, os.path.abspath("../..")) # - -import importlib - -import litellm -from litellm import constants - - -def test_all_numeric_constants_can_be_overridden(): - """ - Test that all integer and float constants in constants.py can be overridden with environment variables. - This ensures that any new constants added in the future will be configurable via environment variables. - """ - # Get all attributes from the constants module - constants_attributes = inspect.getmembers(constants) - - # Filter for uppercase constants (by convention) that are integers or floats - numeric_constants = [ - (name, value) - for name, value in constants_attributes - if name.isupper() and isinstance(value, (int, float)) - ] - - # Ensure we found some constants to test - assert len(numeric_constants) > 0, "No numeric constants found to test" - - print("all numeric constants", json.dumps(numeric_constants, indent=4)) - - # Verify all numeric constants have environment variable support - for name, value in numeric_constants: - # Skip constants that are not meant to be overridden (if any) - if name.startswith("_"): - continue - - # Create a test value that's different from the default - test_value = value + 1 if isinstance(value, int) else value + 0.1 - - # Set the environment variable - with mock.patch.dict(os.environ, {name: str(test_value)}): - print("overriding", name, "with", test_value) - importlib.reload(constants) - - # Get the new value after reload - new_value = getattr(constants, name) - - # Verify the value was overridden - assert ( - new_value == test_value - ), f"Failed to override {name} with environment variable. Expected {test_value}, got {new_value}"