mirror of
https://github.com/tiennm99/litellm.git
synced 2026-06-17 22:48:35 +00:00
Yassin Kortam
b5d3a5fc85
- Introduce RoutingPrismaWrapper that transparently routes read operations (find_*, count, group_by, query_raw, query_first) to a reader endpoint while writes remain on the writer, enabling Aurora-style reader/writer endpoint splits - Add IAMEndpoint dataclass and parse_iam_endpoint_from_url() to capture static connection fields from a reader URL so only the IAM token needs to rotate, avoiding the need for separate DATABASE_HOST_READ_REPLICA/etc. env vars - Enhance PrismaWrapper with per-instance knobs (db_url_env_var, iam_endpoint, recreate_uses_datasource, log_prefix) so writer and reader wrappers are independent: the reader writes its fresh URL to DATABASE_URL_READ_REPLICA and passes datasource override to Prisma since Prisma only auto-reads DATABASE_URL - Fix deadlock in PrismaWrapper.__getattr__: when called from inside a running event loop, schedule the token refresh as a background task instead of blocking with run_coroutine_threadsafe + future.result(), which would deadlock the loop thread waiting for a coroutine that needs the loop to run - Fix botocore crash when DATABASE_PORT is unset by defaulting to "5432" in both proxy_cli.py and PrismaWrapper.get_rds_iam_token(); passing None caused botocore to embed the literal string "None" in the presigned URL - Implement graceful reader degradation: reader connect/recreate failures are non-fatal; wrapper sets _reader_unavailable=True and silently routes reads to the writer to keep the proxy serving traffic during transient reader outages - Add PrismaClient.writer_db property so the reconnect smoke-test always validates the writer engine specifically; query_raw on the routing wrapper would route to the reader and not verify the newly-recreated writer - Expose DATABASE_URL_READ_REPLICA in Helm chart (values.yaml + deployment.yaml) via both plain value and secret key reference, and document the field in docker-compose.yml - Add 887-line test suite covering routing logic, IAM token refresh paths, reader degradation scenarios, datasource override behavior, and the deadlock regression Co-authored-by: Yassin Kortam <yassinkortam@g.ucla.edu>
74 lines
2.7 KiB
YAML
74 lines
2.7 KiB
YAML
services:
|
|
litellm:
|
|
build:
|
|
context: .
|
|
args:
|
|
target: runtime
|
|
image: docker.litellm.ai/berriai/litellm:main-stable
|
|
#########################################
|
|
## Uncomment these lines to start proxy with a config.yaml file ##
|
|
# volumes:
|
|
# - ./config.yaml:/app/config.yaml
|
|
# command:
|
|
# - "--config=/app/config.yaml"
|
|
##############################################
|
|
ports:
|
|
- "4000:4000" # Map the container port to the host, change the host port if necessary
|
|
environment:
|
|
DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
|
|
# Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
|
|
# to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
|
|
# single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
|
|
# is auto-refreshed alongside the writer.
|
|
# DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
|
|
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
|
|
env_file:
|
|
- .env # Load local .env file
|
|
depends_on:
|
|
- db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
|
|
healthcheck: # Defines the health check configuration for the container
|
|
test:
|
|
- CMD-SHELL
|
|
- python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check
|
|
interval: 30s # Perform health check every 30 seconds
|
|
timeout: 10s # Health check command times out after 10 seconds
|
|
retries: 3 # Retry up to 3 times if health check fails
|
|
start_period: 40s # Wait 40 seconds after container start before beginning health checks
|
|
|
|
db:
|
|
image: postgres:16
|
|
restart: always
|
|
container_name: litellm_db
|
|
environment:
|
|
POSTGRES_DB: litellm
|
|
POSTGRES_USER: llmproxy
|
|
POSTGRES_PASSWORD: dbpassword9090
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
|
|
interval: 1s
|
|
timeout: 5s
|
|
retries: 10
|
|
|
|
prometheus:
|
|
image: prom/prometheus
|
|
volumes:
|
|
- prometheus_data:/prometheus
|
|
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
|
ports:
|
|
- "9090:9090"
|
|
command:
|
|
- "--config.file=/etc/prometheus/prometheus.yml"
|
|
- "--storage.tsdb.path=/prometheus"
|
|
- "--storage.tsdb.retention.time=15d"
|
|
restart: always
|
|
|
|
volumes:
|
|
prometheus_data:
|
|
driver: local
|
|
postgres_data:
|
|
name: litellm_postgres_data # Named volume for Postgres data persistence
|