From e09ef4afc72a0a0e2bc34d24984bdd3835d10cc9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 17:39:48 -0700 Subject: [PATCH 01/24] use service logger for tracking pod lock status --- litellm/types/services.py | 68 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/litellm/types/services.py b/litellm/types/services.py index 3eb283dbe9..05944772d7 100644 --- a/litellm/types/services.py +++ b/litellm/types/services.py @@ -1,8 +1,15 @@ import enum import uuid -from typing import Optional +from typing import List, Optional from pydantic import BaseModel, Field +from typing_extensions import TypedDict + + +class ServiceMetrics(enum.Enum): + COUNTER = "counter" + HISTOGRAM = "histogram" + GAUGE = "gauge" class ServiceTypes(str, enum.Enum): @@ -18,6 +25,62 @@ class ServiceTypes(str, enum.Enum): ROUTER = "router" AUTH = "auth" PROXY_PRE_CALL = "proxy_pre_call" + POD_LOCK_MANAGER = "pod_lock_manager" + + +class ServiceConfig(TypedDict): + """ + Configuration for services and their metrics + """ + + metrics: List[ServiceMetrics] # What metrics this service should support + + +""" +Metric types to use for each service + +- REDIS only needs Counter, Histogram +- Pod Lock Manager only needs a gauge metric +""" +DEFAULT_SERVICE_CONFIGS = { + ServiceTypes.REDIS.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.DB.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.BATCH_WRITE_TO_DB.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.RESET_BUDGET_JOB.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.LITELLM.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.ROUTER.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.AUTH.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.PROXY_PRE_CALL.value: { + "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] + }, + ServiceTypes.POD_LOCK_MANAGER.value: {"metrics": [ServiceMetrics.GAUGE]}, +} + + +class ServiceEventMetadata(TypedDict, total=False): + """ + The metadata logged during service success/failure + + Add any extra fields you expect to access in the service_success_hook/service_failure_hook + """ + + # Dynamically control gauge labels and values + gauge_labels: Optional[str] + gauge_value: Optional[float] class ServiceLoggerPayload(BaseModel): @@ -30,6 +93,9 @@ class ServiceLoggerPayload(BaseModel): service: ServiceTypes = Field(description="who is this for? - postgres/redis") duration: float = Field(description="How long did the request take?") call_type: str = Field(description="The call of the service, being made") + event_metadata: Optional[dict] = Field( + description="The metadata logged during service success/failure" + ) def to_json(self, **kwargs): try: From 73bbd0a4460e808ba1e385293ec23bad16114fe8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 17:40:25 -0700 Subject: [PATCH 02/24] emit lock acquired and released events --- .../db_transaction_queue/pod_lock_manager.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 5b640033a0..3f63afe62a 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -1,15 +1,20 @@ +import asyncio import uuid from typing import TYPE_CHECKING, Any, Optional from litellm._logging import verbose_proxy_logger +from litellm._service_logger import ServiceLogging from litellm.caching.redis_cache import RedisCache from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS +from litellm.types.services import ServiceTypes if TYPE_CHECKING: ProxyLogging = Any else: ProxyLogging = Any +service_logger_obj = ServiceLogging() # used for tracking current pod lock status + class PodLockManager: """ @@ -57,6 +62,7 @@ class PodLockManager: self.pod_id, self.cronjob_id, ) + return True else: # Check if the current pod already holds the lock @@ -70,6 +76,7 @@ class PodLockManager: self.pod_id, self.cronjob_id, ) + self._emit_acquired_lock_event(self.cronjob_id, self.pod_id) return True return False except Exception as e: @@ -104,6 +111,7 @@ class PodLockManager: self.pod_id, self.cronjob_id, ) + self._emit_released_lock_event(self.cronjob_id, self.pod_id) else: verbose_proxy_logger.debug( "Pod %s failed to release Redis lock for cronjob_id=%s", @@ -127,3 +135,31 @@ class PodLockManager: verbose_proxy_logger.error( f"Error releasing Redis lock for {self.cronjob_id}: {e}" ) + + @staticmethod + def _emit_acquired_lock_event(cronjob_id: str, pod_id: str): + asyncio.create_task( + service_logger_obj.async_service_success_hook( + service=ServiceTypes.POD_LOCK_MANAGER, + duration=DEFAULT_CRON_JOB_LOCK_TTL_SECONDS, + call_type="_emit_acquired_lock_event", + event_metadata={ + "gauge_labels": f"{cronjob_id}:{pod_id}", + "gauge_value": 1, + }, + ) + ) + + @staticmethod + def _emit_released_lock_event(cronjob_id: str, pod_id: str): + asyncio.create_task( + service_logger_obj.async_service_success_hook( + service=ServiceTypes.POD_LOCK_MANAGER, + duration=DEFAULT_CRON_JOB_LOCK_TTL_SECONDS, + call_type="_emit_released_lock_event", + event_metadata={ + "gauge_labels": f"{cronjob_id}:{pod_id}", + "gauge_value": 0, + }, + ) + ) From 05b30e28db38a5e56e08bf8af9f7516fd22d9c61 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 17:50:41 -0700 Subject: [PATCH 03/24] clean up service metrics --- litellm/_service_logger.py | 2 + litellm/integrations/prometheus_services.py | 106 +++++++++++++++----- litellm/proxy/proxy_config.yaml | 4 +- 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index 8f835bea83..7a60359d54 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -124,6 +124,7 @@ class ServiceLogging(CustomLogger): service=service, duration=duration, call_type=call_type, + event_metadata=event_metadata, ) for callback in litellm.service_callback: @@ -229,6 +230,7 @@ class ServiceLogging(CustomLogger): service=service, duration=duration, call_type=call_type, + event_metadata=event_metadata, ) for callback in litellm.service_callback: diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 4bf293fb01..d14cbd7469 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -7,7 +7,12 @@ from typing import List, Optional, Union from litellm._logging import print_verbose, verbose_logger from litellm.types.integrations.prometheus import LATENCY_BUCKETS -from litellm.types.services import ServiceLoggerPayload, ServiceTypes +from litellm.types.services import ( + DEFAULT_SERVICE_CONFIGS, + ServiceLoggerPayload, + ServiceMetrics, + ServiceTypes, +) FAILED_REQUESTS_LABELS = ["error_class", "function_name"] @@ -23,7 +28,8 @@ class PrometheusServicesLogger: ): try: try: - from prometheus_client import REGISTRY, Counter, Histogram + from prometheus_client import REGISTRY, Counter, Gauge, Histogram + from prometheus_client.gc_collector import Collector except ImportError: raise Exception( "Missing prometheus_client. Run `pip install prometheus-client`" @@ -31,36 +37,52 @@ class PrometheusServicesLogger: self.Histogram = Histogram self.Counter = Counter + self.Gauge = Gauge self.REGISTRY = REGISTRY verbose_logger.debug("in init prometheus services metrics") - self.services = [item.value for item in ServiceTypes] - - self.payload_to_prometheus_map = ( - {} - ) # store the prometheus histogram/counter we need to call for each field in payload + self.services = [item for item in ServiceTypes] + self.payload_to_prometheus_map = {} for service in self.services: - histogram = self.create_histogram(service, type_of_request="latency") - counter_failed_request = self.create_counter( - service, - type_of_request="failed_requests", - additional_labels=FAILED_REQUESTS_LABELS, - ) - counter_total_requests = self.create_counter( - service, type_of_request="total_requests" - ) - self.payload_to_prometheus_map[service] = [ - histogram, - counter_failed_request, - counter_total_requests, - ] + service_metrics: List[Union[Histogram, Counter, Gauge, Collector]] = [] - self.prometheus_to_amount_map: dict = ( - {} - ) # the field / value in ServiceLoggerPayload the object needs to be incremented by + metrics_to_initialize = self._get_service_metrics_initialize(service) + # Initialize only the configured metrics for each service + if ServiceMetrics.HISTOGRAM in metrics_to_initialize: + histogram = self.create_histogram( + service, type_of_request="latency" + ) + if histogram: + service_metrics.append(histogram) + + if ServiceMetrics.COUNTER in metrics_to_initialize: + counter_failed_request = self.create_counter( + service, + type_of_request="failed_requests", + additional_labels=FAILED_REQUESTS_LABELS, + ) + if counter_failed_request: + service_metrics.append(counter_failed_request) + counter_total_requests = self.create_counter( + service, type_of_request="total_requests" + ) + if counter_total_requests: + service_metrics.append(counter_total_requests) + + if ServiceMetrics.GAUGE in metrics_to_initialize: + gauge = self.create_gauge( + service, type_of_request="pod_lock_manager" + ) + if gauge: + service_metrics.append(gauge) + + if service_metrics: + self.payload_to_prometheus_map[service] = service_metrics + + self.prometheus_to_amount_map: dict = {} ### MOCK TESTING ### self.mock_testing = mock_testing self.mock_testing_success_calls = 0 @@ -70,6 +92,17 @@ class PrometheusServicesLogger: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e + def _get_service_metrics_initialize( + self, service: ServiceTypes + ) -> List[ServiceMetrics]: + if service not in DEFAULT_SERVICE_CONFIGS: + raise ValueError(f"Service {service} not found in DEFAULT_SERVICE_CONFIGS") + + metrics = DEFAULT_SERVICE_CONFIGS.get(service, {}).get("metrics", []) + if not metrics: + raise ValueError(f"No metrics found for service {service}") + return metrics + def is_metric_registered(self, metric_name) -> bool: for metric in self.REGISTRY.collect(): if metric_name == metric.name: @@ -94,6 +127,15 @@ class PrometheusServicesLogger: buckets=LATENCY_BUCKETS, ) + def create_gauge(self, service: str, type_of_request: str): + metric_name = "litellm_{}_{}".format(service, type_of_request) + is_registered = self.is_metric_registered(metric_name) + if is_registered: + return self._get_metric(metric_name) + return self.Gauge( + metric_name, "Gauge for {} service".format(service), labelnames=[service] + ) + def create_counter( self, service: str, @@ -120,6 +162,15 @@ class PrometheusServicesLogger: histogram.labels(labels).observe(amount) + def update_gauge( + self, + gauge, + labels: str, + amount: float, + ): + assert isinstance(gauge, self.Gauge) + gauge.labels(labels).set(amount) + def increment_counter( self, counter, @@ -190,6 +241,13 @@ class PrometheusServicesLogger: labels=payload.service.value, amount=1, # LOG TOTAL REQUESTS TO PROMETHEUS ) + elif isinstance(obj, self.Gauge): + if payload.event_metadata: + self.update_gauge( + gauge=obj, + labels=payload.event_metadata.get("gauge_labels") or "", + amount=payload.event_metadata.get("gauge_value") or 0, + ) async def async_service_failure_hook( self, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index fe8d73d26a..2ee830bca4 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -12,4 +12,6 @@ litellm_settings: cache: True cache_params: type: redis - supported_call_types: [] \ No newline at end of file + supported_call_types: [] + callbacks: ["prometheus"] + service_callback: ["prometheus_system"] \ No newline at end of file From 3256b6af6c0432f5ce310623f61c5e57eec91a08 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 18:03:09 -0700 Subject: [PATCH 04/24] track service types on prom services --- litellm/types/services.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/types/services.py b/litellm/types/services.py index 05944772d7..e7b3c91ed3 100644 --- a/litellm/types/services.py +++ b/litellm/types/services.py @@ -27,6 +27,17 @@ class ServiceTypes(str, enum.Enum): PROXY_PRE_CALL = "proxy_pre_call" POD_LOCK_MANAGER = "pod_lock_manager" + """ + Operational metrics for DB Transaction Queues + """ + # daily spend update queue - actual transaction events + IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE = "in_memory_daily_spend_update_queue" + REDIS_DAILY_SPEND_UPDATE_QUEUE = "redis_daily_spend_update_queue" + + # spend update queue - current spend of key, user, team + IN_MEMORY_SPEND_UPDATE_QUEUE = "in_memory_spend_update_queue" + REDIS_SPEND_UPDATE_QUEUE = "redis_spend_update_queue" + class ServiceConfig(TypedDict): """ From 7b768ed909a357193f8b8b74f622d6d6e62165be Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 18:38:33 -0700 Subject: [PATCH 05/24] doc fix sso login url --- docs/my-website/docs/proxy/admin_ui_sso.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/admin_ui_sso.md b/docs/my-website/docs/proxy/admin_ui_sso.md index 882e3df0b2..0bbba57fd9 100644 --- a/docs/my-website/docs/proxy/admin_ui_sso.md +++ b/docs/my-website/docs/proxy/admin_ui_sso.md @@ -156,7 +156,7 @@ PROXY_LOGOUT_URL="https://www.google.com" Set this in your .env (so the proxy can set the correct redirect url) ```shell -PROXY_BASE_URL=https://litellm-api.up.railway.app/ +PROXY_BASE_URL=https://litellm-api.up.railway.app ``` #### Step 4. Test flow From 80fb4ece9770e26d33b11bfd63d8bc146591eb74 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 18:39:29 -0700 Subject: [PATCH 06/24] prom emit size of DB TX queues for observability --- litellm/integrations/prometheus_services.py | 10 +++--- .../db_transaction_queue/base_update_queue.py | 16 +++++++++ .../daily_spend_update_queue.py | 24 ++++++++++++-- .../db_transaction_queue/pod_lock_manager.py | 4 +-- .../redis_update_buffer.py | 33 +++++++++++++++++-- .../spend_update_queue.py | 24 ++++++++++++-- 6 files changed, 97 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index d14cbd7469..dddaa4d064 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -73,9 +73,7 @@ class PrometheusServicesLogger: service_metrics.append(counter_total_requests) if ServiceMetrics.GAUGE in metrics_to_initialize: - gauge = self.create_gauge( - service, type_of_request="pod_lock_manager" - ) + gauge = self.create_gauge(service, type_of_request="size") if gauge: service_metrics.append(gauge) @@ -95,12 +93,14 @@ class PrometheusServicesLogger: def _get_service_metrics_initialize( self, service: ServiceTypes ) -> List[ServiceMetrics]: + DEFAULT_METRICS = [ServiceMetrics.COUNTER, ServiceMetrics.GAUGE] if service not in DEFAULT_SERVICE_CONFIGS: - raise ValueError(f"Service {service} not found in DEFAULT_SERVICE_CONFIGS") + return DEFAULT_METRICS metrics = DEFAULT_SERVICE_CONFIGS.get(service, {}).get("metrics", []) if not metrics: - raise ValueError(f"No metrics found for service {service}") + verbose_logger.debug(f"No metrics found for service {service}") + return DEFAULT_METRICS return metrics def is_metric_registered(self, metric_name) -> bool: diff --git a/litellm/proxy/db/db_transaction_queue/base_update_queue.py b/litellm/proxy/db/db_transaction_queue/base_update_queue.py index b3c3c26c84..2bf2393127 100644 --- a/litellm/proxy/db/db_transaction_queue/base_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/base_update_queue.py @@ -2,8 +2,14 @@ Base class for in memory buffer for database transactions """ import asyncio +from typing import Optional from litellm._logging import verbose_proxy_logger +from litellm._service_logger import ServiceLogging + +service_logger_obj = ( + ServiceLogging() +) # used for tracking metrics for In memory buffer, redis buffer, pod lock manager class BaseUpdateQueue: @@ -16,6 +22,9 @@ class BaseUpdateQueue: """Enqueue an update.""" verbose_proxy_logger.debug("Adding update to queue: %s", update) await self.update_queue.put(update) + await self._emit_new_item_added_to_queue_event( + queue_size=self.update_queue.qsize() + ) async def flush_all_updates_from_in_memory_queue(self): """Get all updates from the queue.""" @@ -23,3 +32,10 @@ class BaseUpdateQueue: while not self.update_queue.empty(): updates.append(await self.update_queue.get()) return updates + + async def _emit_new_item_added_to_queue_event( + self, + queue_size: Optional[int] = None, + ): + """placeholder, emit event when a new item is added to the queue""" + pass diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index dedb8c8f8f..afae431370 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -1,9 +1,13 @@ import asyncio -from typing import Dict, List +from typing import Dict, List, Optional from litellm._logging import verbose_proxy_logger from litellm.proxy._types import DailyUserSpendTransaction -from litellm.proxy.db.db_transaction_queue.base_update_queue import BaseUpdateQueue +from litellm.proxy.db.db_transaction_queue.base_update_queue import ( + BaseUpdateQueue, + service_logger_obj, +) +from litellm.types.services import ServiceTypes class DailySpendUpdateQueue(BaseUpdateQueue): @@ -93,3 +97,19 @@ class DailySpendUpdateQueue(BaseUpdateQueue): else: aggregated_daily_spend_update_transactions[_key] = payload return aggregated_daily_spend_update_transactions + + async def _emit_new_item_added_to_queue_event( + self, + queue_size: Optional[int] = None, + ): + asyncio.create_task( + service_logger_obj.async_service_success_hook( + service=ServiceTypes.IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE, + duration=0, + call_type="_emit_new_item_added_to_queue_event", + event_metadata={ + "gauge_labels": ServiceTypes.IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE, + "gauge_value": queue_size, + }, + ) + ) diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 3f63afe62a..cb4a43a802 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -3,9 +3,9 @@ import uuid from typing import TYPE_CHECKING, Any, Optional from litellm._logging import verbose_proxy_logger -from litellm._service_logger import ServiceLogging from litellm.caching.redis_cache import RedisCache from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS +from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj from litellm.types.services import ServiceTypes if TYPE_CHECKING: @@ -13,8 +13,6 @@ if TYPE_CHECKING: else: ProxyLogging = Any -service_logger_obj = ServiceLogging() # used for tracking current pod lock status - class PodLockManager: """ diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index ea1356159a..88741fbb18 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -4,6 +4,7 @@ Handles buffering database `UPDATE` transactions in Redis before committing them This is to prevent deadlocks and improve reliability """ +import asyncio import json from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -16,11 +17,13 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import DailyUserSpendTransaction, DBSpendUpdateTransactions +from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( DailySpendUpdateQueue, ) from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue from litellm.secret_managers.main import str_to_bool +from litellm.types.services import ServiceTypes if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -136,18 +139,27 @@ class RedisUpdateBuffer: return list_of_transactions = [safe_dumps(db_spend_update_transactions)] - await self.redis_cache.async_rpush( + current_redis_buffer_size = await self.redis_cache.async_rpush( key=REDIS_UPDATE_BUFFER_KEY, values=list_of_transactions, ) + await self._emit_new_item_added_to_redis_buffer_event( + queue_size=current_redis_buffer_size, + service=ServiceTypes.REDIS_SPEND_UPDATE_QUEUE, + ) list_of_daily_spend_update_transactions = [ safe_dumps(daily_spend_update_transactions) ] - await self.redis_cache.async_rpush( + + current_redis_buffer_size = await self.redis_cache.async_rpush( key=REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY, values=list_of_daily_spend_update_transactions, ) + await self._emit_new_item_added_to_redis_buffer_event( + queue_size=current_redis_buffer_size, + service=ServiceTypes.REDIS_DAILY_SPEND_UPDATE_QUEUE, + ) @staticmethod def _number_of_transactions_to_store_in_redis( @@ -300,3 +312,20 @@ class RedisUpdateBuffer: ) return combined_transaction + + async def _emit_new_item_added_to_redis_buffer_event( + self, + service: ServiceTypes, + queue_size: int, + ): + asyncio.create_task( + service_logger_obj.async_service_success_hook( + service=service, + duration=0, + call_type="_emit_new_item_added_to_queue_event", + event_metadata={ + "gauge_labels": service, + "gauge_value": queue_size, + }, + ) + ) diff --git a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py index ce181d1478..60e9379751 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py @@ -1,5 +1,5 @@ import asyncio -from typing import List +from typing import List, Optional from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( @@ -7,7 +7,11 @@ from litellm.proxy._types import ( Litellm_EntityType, SpendUpdateQueueItem, ) -from litellm.proxy.db.db_transaction_queue.base_update_queue import BaseUpdateQueue +from litellm.proxy.db.db_transaction_queue.base_update_queue import ( + BaseUpdateQueue, + service_logger_obj, +) +from litellm.types.services import ServiceTypes class SpendUpdateQueue(BaseUpdateQueue): @@ -111,3 +115,19 @@ class SpendUpdateQueue(BaseUpdateQueue): transactions_dict[entity_id] += response_cost or 0 return db_spend_update_transactions + + async def _emit_new_item_added_to_queue_event( + self, + queue_size: Optional[int] = None, + ): + asyncio.create_task( + service_logger_obj.async_service_success_hook( + service=ServiceTypes.IN_MEMORY_SPEND_UPDATE_QUEUE, + duration=0, + call_type="_emit_new_item_added_to_queue_event", + event_metadata={ + "gauge_labels": ServiceTypes.IN_MEMORY_SPEND_UPDATE_QUEUE, + "gauge_value": queue_size, + }, + ) + ) From c4e8b9607d9323c8910321d2c736f4d72aac1425 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 18:51:41 -0700 Subject: [PATCH 07/24] fix async_set_cache --- litellm/caching/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 1d553c9c80..31e11abf97 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -303,7 +303,7 @@ class RedisCache(BaseCache): raise e key = self.check_and_fix_namespace(key=key) - ttl = self.get_ttl(**kwargs) or kwargs.get("ex", None) + ttl = self.get_ttl(**kwargs) nx = kwargs.get("nx", False) print_verbose(f"Set ASYNC Redis Cache: key: {key}\nValue {value}\nttl={ttl}") From bcf42fd82d8fb5806cced0287f65b05bf779117b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 21:19:05 -0700 Subject: [PATCH 08/24] linting fix prometheus services --- litellm/integrations/prometheus_services.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index dddaa4d064..37f0d696fb 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -3,7 +3,7 @@ # On success + failure, log events to Prometheus for litellm / adjacent services (litellm, redis, postgres, llm api providers) -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union from litellm._logging import print_verbose, verbose_logger from litellm.types.integrations.prometheus import LATENCY_BUCKETS @@ -43,7 +43,9 @@ class PrometheusServicesLogger: verbose_logger.debug("in init prometheus services metrics") self.services = [item for item in ServiceTypes] - self.payload_to_prometheus_map = {} + self.payload_to_prometheus_map: Dict[ + str, List[Union[Histogram, Counter, Gauge, Collector]] + ] = {} for service in self.services: service_metrics: List[Union[Histogram, Counter, Gauge, Collector]] = [] @@ -78,7 +80,7 @@ class PrometheusServicesLogger: service_metrics.append(gauge) if service_metrics: - self.payload_to_prometheus_map[service] = service_metrics + self.payload_to_prometheus_map[service.value] = service_metrics self.prometheus_to_amount_map: dict = {} ### MOCK TESTING ### From e68603e176efe8075232c8e50004d17a84e694ab Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 21:31:19 -0700 Subject: [PATCH 09/24] test create and update gauge --- .../integrations/test_prometheus_services.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/litellm/integrations/test_prometheus_services.py diff --git a/tests/litellm/integrations/test_prometheus_services.py b/tests/litellm/integrations/test_prometheus_services.py new file mode 100644 index 0000000000..b627d31fda --- /dev/null +++ b/tests/litellm/integrations/test_prometheus_services.py @@ -0,0 +1,48 @@ +import json +import os +import sys +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi.testclient import TestClient + +from litellm.integrations.prometheus_services import ( + PrometheusServicesLogger, + ServiceMetrics, + ServiceTypes, +) + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + + +def test_create_gauge_new(): + """Test creating a new gauge""" + pl = PrometheusServicesLogger() + + # Create new gauge + gauge = pl.create_gauge(service="test_service", type_of_request="size") + + assert gauge is not None + assert pl._get_metric("litellm_test_service_size") is gauge + + +def test_update_gauge(): + """Test updating a gauge's value""" + pl = PrometheusServicesLogger() + + # Create a gauge to test with + gauge = pl.create_gauge(service="test_service", type_of_request="size") + + # Mock the labels method to verify it's called correctly + with patch.object(gauge, "labels") as mock_labels: + mock_gauge = AsyncMock() + mock_labels.return_value = mock_gauge + + # Call update_gauge + pl.update_gauge(gauge=gauge, labels="test_label", amount=42.5) + + # Verify correct methods were called + mock_labels.assert_called_once_with("test_label") + mock_gauge.set.assert_called_once_with(42.5) From e3b788ea29dceac589ff08e24cab42d83206e72b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Apr 2025 21:58:35 -0700 Subject: [PATCH 10/24] fix test --- litellm/integrations/prometheus_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 37f0d696fb..c060026e15 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -95,7 +95,7 @@ class PrometheusServicesLogger: def _get_service_metrics_initialize( self, service: ServiceTypes ) -> List[ServiceMetrics]: - DEFAULT_METRICS = [ServiceMetrics.COUNTER, ServiceMetrics.GAUGE] + DEFAULT_METRICS = [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] if service not in DEFAULT_SERVICE_CONFIGS: return DEFAULT_METRICS From bde88b3ba61df1f9f64e086e6b154099f08e4700 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 4 Apr 2025 16:34:43 -0700 Subject: [PATCH 11/24] fix type error --- litellm/integrations/prometheus_services.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index c060026e15..2bd38c2ae9 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -42,7 +42,7 @@ class PrometheusServicesLogger: verbose_logger.debug("in init prometheus services metrics") - self.services = [item for item in ServiceTypes] + self.services: List[ServiceTypes] = [item for item in ServiceTypes] self.payload_to_prometheus_map: Dict[ str, List[Union[Histogram, Counter, Gauge, Collector]] ] = {} @@ -55,27 +55,27 @@ class PrometheusServicesLogger: # Initialize only the configured metrics for each service if ServiceMetrics.HISTOGRAM in metrics_to_initialize: histogram = self.create_histogram( - service, type_of_request="latency" + service.value, type_of_request="latency" ) if histogram: service_metrics.append(histogram) if ServiceMetrics.COUNTER in metrics_to_initialize: counter_failed_request = self.create_counter( - service, + service.value, type_of_request="failed_requests", additional_labels=FAILED_REQUESTS_LABELS, ) if counter_failed_request: service_metrics.append(counter_failed_request) counter_total_requests = self.create_counter( - service, type_of_request="total_requests" + service.value, type_of_request="total_requests" ) if counter_total_requests: service_metrics.append(counter_total_requests) if ServiceMetrics.GAUGE in metrics_to_initialize: - gauge = self.create_gauge(service, type_of_request="size") + gauge = self.create_gauge(service.value, type_of_request="size") if gauge: service_metrics.append(gauge) From 901d6fe7b79c159afe95bbb33a7aa01850072030 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 4 Apr 2025 16:41:07 -0700 Subject: [PATCH 12/24] add operational metrics for pod lock manager v2 arch --- litellm/integrations/prometheus_services.py | 3 +-- litellm/proxy/proxy_config.yaml | 9 ++------- litellm/types/services.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 2bd38c2ae9..a5f2f0b5c7 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -42,12 +42,11 @@ class PrometheusServicesLogger: verbose_logger.debug("in init prometheus services metrics") - self.services: List[ServiceTypes] = [item for item in ServiceTypes] self.payload_to_prometheus_map: Dict[ str, List[Union[Histogram, Counter, Gauge, Collector]] ] = {} - for service in self.services: + for service in ServiceTypes: service_metrics: List[Union[Histogram, Counter, Gauge, Collector]] = [] metrics_to_initialize = self._get_service_metrics_initialize(service) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 52948c927e..56eb2ca39f 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -5,11 +5,6 @@ model_list: api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ -general_settings: - use_redis_transaction_buffer: true - litellm_settings: - cache: True - cache_params: - type: redis - supported_call_types: [] + callbacks: ["prometheus"] + service_callback: ["prometheus_system"] \ No newline at end of file diff --git a/litellm/types/services.py b/litellm/types/services.py index e7b3c91ed3..865827f0f8 100644 --- a/litellm/types/services.py +++ b/litellm/types/services.py @@ -78,7 +78,18 @@ DEFAULT_SERVICE_CONFIGS = { ServiceTypes.PROXY_PRE_CALL.value: { "metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM] }, + # Operational metrics for DB Transaction Queues ServiceTypes.POD_LOCK_MANAGER.value: {"metrics": [ServiceMetrics.GAUGE]}, + ServiceTypes.IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE.value: { + "metrics": [ServiceMetrics.GAUGE] + }, + ServiceTypes.REDIS_DAILY_SPEND_UPDATE_QUEUE.value: { + "metrics": [ServiceMetrics.GAUGE] + }, + ServiceTypes.IN_MEMORY_SPEND_UPDATE_QUEUE.value: { + "metrics": [ServiceMetrics.GAUGE] + }, + ServiceTypes.REDIS_SPEND_UPDATE_QUEUE.value: {"metrics": [ServiceMetrics.GAUGE]}, } From c555c15ad7d65850b3348436d91b910afc1c1907 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Fri, 4 Apr 2025 18:40:14 -0700 Subject: [PATCH 13/24] fix(router.py): support reusable credentials via passthrough router (#9758) * fix(router.py): support reusable credentials via passthrough router enables reusable vertex credentials to be used in passthrough * test: fix test * test(test_router_adding_deployments.py): add unit testing --- .../litellm_core_utils/credential_accessor.py | 1 + litellm/router.py | 45 ++++++++++++++---- litellm/types/router.py | 1 + .../test_router_adding_deployments.py | 47 ++++++++++++++----- 4 files changed, 75 insertions(+), 19 deletions(-) diff --git a/litellm/litellm_core_utils/credential_accessor.py b/litellm/litellm_core_utils/credential_accessor.py index d87dcc116b..45e1ea2c49 100644 --- a/litellm/litellm_core_utils/credential_accessor.py +++ b/litellm/litellm_core_utils/credential_accessor.py @@ -10,6 +10,7 @@ class CredentialAccessor: @staticmethod def get_credential_values(credential_name: str) -> dict: """Safe accessor for credentials.""" + if not litellm.credential_list: return {} for credential in litellm.credential_list: diff --git a/litellm/router.py b/litellm/router.py index 3c1e441582..456e8641e0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -54,6 +54,7 @@ from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.router_strategy.budget_limiter import RouterBudgetLimiting @@ -4506,25 +4507,53 @@ class Router: passthrough_endpoint_router, ) + if deployment.litellm_params.litellm_credential_name is not None: + credential_values = CredentialAccessor.get_credential_values( + deployment.litellm_params.litellm_credential_name + ) + else: + credential_values = {} + if custom_llm_provider == "vertex_ai": + vertex_project = ( + credential_values.get("vertex_project") + or deployment.litellm_params.vertex_project + ) + vertex_location = ( + credential_values.get("vertex_location") + or deployment.litellm_params.vertex_location + ) + vertex_credentials = ( + credential_values.get("vertex_credentials") + or deployment.litellm_params.vertex_credentials + ) + if ( - deployment.litellm_params.vertex_project is None - or deployment.litellm_params.vertex_location is None - or deployment.litellm_params.vertex_credentials is None + vertex_project is None + or vertex_location is None + or vertex_credentials is None ): raise ValueError( "vertex_project, vertex_location, and vertex_credentials must be set in litellm_params for pass-through endpoints" ) passthrough_endpoint_router.add_vertex_credentials( - project_id=deployment.litellm_params.vertex_project, - location=deployment.litellm_params.vertex_location, - vertex_credentials=deployment.litellm_params.vertex_credentials, + project_id=vertex_project, + location=vertex_location, + vertex_credentials=vertex_credentials, ) else: + api_base = ( + credential_values.get("api_base") + or deployment.litellm_params.api_base + ) + api_key = ( + credential_values.get("api_key") + or deployment.litellm_params.api_key + ) passthrough_endpoint_router.set_pass_through_credentials( custom_llm_provider=custom_llm_provider, - api_base=deployment.litellm_params.api_base, - api_key=deployment.litellm_params.api_key, + api_base=api_base, + api_key=api_key, ) pass pass diff --git a/litellm/types/router.py b/litellm/types/router.py index fde7b67b8d..5609c3f67f 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -179,6 +179,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams): max_retries: Optional[int] = None organization: Optional[str] = None # for openai orgs configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None + litellm_credential_name: Optional[str] = None ## LOGGING PARAMS ## litellm_trace_id: Optional[str] = None diff --git a/tests/router_unit_tests/test_router_adding_deployments.py b/tests/router_unit_tests/test_router_adding_deployments.py index 53fe7347d3..55481394bb 100644 --- a/tests/router_unit_tests/test_router_adding_deployments.py +++ b/tests/router_unit_tests/test_router_adding_deployments.py @@ -9,23 +9,48 @@ from litellm.router import Deployment, LiteLLM_Params from unittest.mock import patch import json - -def test_initialize_deployment_for_pass_through_success(): +@pytest.mark.parametrize("reusable_credentials", [True, False]) +def test_initialize_deployment_for_pass_through_success(reusable_credentials): """ Test successful initialization of a Vertex AI pass-through deployment """ + from litellm.litellm_core_utils.credential_accessor import CredentialAccessor + from litellm.types.utils import CredentialItem + + vertex_project="test-project" + vertex_location="us-central1" + vertex_credentials=json.dumps({"type": "service_account", "project_id": "test"}) + + if not reusable_credentials: + litellm_params = LiteLLM_Params( + model="vertex_ai/test-model", + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_credentials=vertex_credentials, + use_in_pass_through=True, + ) + else: + # add credentials to the credential accessor + CredentialAccessor.upsert_credentials([ + CredentialItem( + credential_name="vertex_credentials", + credential_values={ + "vertex_project": vertex_project, + "vertex_location": vertex_location, + "vertex_credentials": vertex_credentials, + }, + credential_info={} + ) + ]) + litellm_params = LiteLLM_Params( + model="vertex_ai/test-model", + litellm_credential_name="vertex_credentials", + use_in_pass_through=True, + ) router = Router(model_list=[]) deployment = Deployment( model_name="vertex-test", - litellm_params=LiteLLM_Params( - model="vertex_ai/test-model", - vertex_project="test-project", - vertex_location="us-central1", - vertex_credentials=json.dumps( - {"type": "service_account", "project_id": "test"} - ), - use_in_pass_through=True, - ), + litellm_params=litellm_params, ) # Test the initialization From 6395bd8d652878cb1f503ae3c6fd85faaf7bf950 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 4 Apr 2025 20:25:05 -0700 Subject: [PATCH 14/24] test: mark flaky test --- tests/litellm/litellm_core_utils/test_streaming_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/litellm/litellm_core_utils/test_streaming_handler.py b/tests/litellm/litellm_core_utils/test_streaming_handler.py index d79be260d8..7ee667d819 100644 --- a/tests/litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/litellm/litellm_core_utils/test_streaming_handler.py @@ -502,6 +502,7 @@ async def test_streaming_handler_with_usage( @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio +@pytest.mark.flaky(reruns=3) async def test_streaming_with_usage_and_logging(sync_mode: bool): import time From b5851769fca5f6bc70a00a5582aa899c058e5e03 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 4 Apr 2025 20:26:11 -0700 Subject: [PATCH 15/24] fix: fix import --- tests/llm_translation/test_max_completion_tokens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm_translation/test_max_completion_tokens.py b/tests/llm_translation/test_max_completion_tokens.py index a8f3dd50a8..f1374a22a2 100644 --- a/tests/llm_translation/test_max_completion_tokens.py +++ b/tests/llm_translation/test_max_completion_tokens.py @@ -330,7 +330,7 @@ def test_all_model_configs(): drop_params=False, ) == {"max_tokens_to_sample": 10} - from litellm.llms.databricks.chat.handler import DatabricksConfig + from litellm.llms.databricks.chat.transformation import DatabricksConfig assert "max_completion_tokens" in DatabricksConfig().get_supported_openai_params() From d66db2207b48458645fd861e60a269ff047fa38b Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Fri, 4 Apr 2025 20:36:48 -0700 Subject: [PATCH 16/24] Allow team members to see team models (#9742) * fix(proxy_server.py): allow team member to see team models * fix(model_dashboard.tsx): show edit + delete icons to be disabled if user is not admin and did not create models * fix(proxy_server.py): fix ruff function size error * fix(proxy_server.py): fix user model filter check --- litellm/proxy/proxy_server.py | 75 ++++++++++++++++--- .../src/components/model_dashboard.tsx | 4 +- .../components/model_dashboard/columns.tsx | 17 ++++- 3 files changed, 81 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 100b0bf6db..acc6b6175e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5333,6 +5333,67 @@ async def _check_if_model_is_user_added( return filtered_models +def _check_if_model_is_team_model( + models: List[DeploymentTypedDict], user_row: LiteLLM_UserTable +) -> List[Dict]: + """ + Check if model is a team model + + Check if user is a member of the team that the model belongs to + """ + + user_team_models: List[Dict] = [] + for model in models: + model_team_id = model.get("model_info", {}).get("team_id", None) + + if model_team_id is not None: + if model_team_id in user_row.teams: + user_team_models.append(cast(Dict, model)) + + return user_team_models + + +async def non_admin_all_models( + all_models: List[Dict], + llm_router: Router, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Optional[PrismaClient], +): + """ + Check if model is in db + + Check if db model is 'created_by' == user_api_key_dict.user_id + + Only return models that match + """ + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + all_models = await _check_if_model_is_user_added( + models=all_models, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + ) + + if user_api_key_dict.user_id: + try: + user_row = await prisma_client.db.litellm_usertable.find_unique( + where={"user_id": user_api_key_dict.user_id} + ) + except Exception: + raise HTTPException(status_code=400, detail={"error": "User not found"}) + + all_models += _check_if_model_is_team_model( + models=llm_router.get_model_list() or [], + user_row=user_row, + ) + + return all_models + + @router.get( "/v2/model/info", description="v2 - returns models available to the user based on their API key permissions. Shows model info from config.yaml (except api key and api base). Filter to just user-added models with ?user_models_only=true", @@ -5378,16 +5439,10 @@ async def model_info_v2( if model is not None: all_models = [m for m in all_models if m["model_name"] == model] - if user_models_only is True: - """ - Check if model is in db - - Check if db model is 'created_by' == user_api_key_dict.user_id - - Only return models that match - """ - all_models = await _check_if_model_is_user_added( - models=all_models, + if user_models_only: + all_models = await non_admin_all_models( + all_models=all_models, + llm_router=llm_router, user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, ) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index e87da27c24..d3dda3a96d 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1126,13 +1126,15 @@ const ModelDashboard: React.FC = ({ diff --git a/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx index b1acf9ea5d..2602761a5b 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx @@ -7,6 +7,8 @@ import { TrashIcon, PencilIcon, PencilAltIcon } from "@heroicons/react/outline"; import DeleteModelButton from "../delete_model_button"; export const columns = ( + userRole: string, + userID: string, premiumUser: boolean, setSelectedModelId: (id: string) => void, setSelectedTeamId: (id: string) => void, @@ -226,23 +228,30 @@ export const columns = ( header: "", cell: ({ row }) => { const model = row.original; + const canEditModel = userRole === "Admin" || model.model_info?.created_by === userID; return (
{ - setSelectedModelId(model.model_info.id); - setEditModel(true); + if (canEditModel) { + setSelectedModelId(model.model_info.id); + setEditModel(true); + } }} + className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer"} /> { - setSelectedModelId(model.model_info.id); - setEditModel(false); + if (canEditModel) { + setSelectedModelId(model.model_info.id); + setEditModel(false); + } }} + className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer"} />
); From 90a4dfab3c044a7795642ad8cc28bbc04fcfc276 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Fri, 4 Apr 2025 20:37:08 -0700 Subject: [PATCH 17/24] =?UTF-8?q?fix(xai/chat/transformation.py):=20filter?= =?UTF-8?q?=20out=20'name'=20param=20for=20xai=20non-=E2=80=A6=20(#9761)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(xai/chat/transformation.py): filter out 'name' param for xai non-user roles Fixes https://github.com/BerriAI/litellm/issues/9720 * test fix test_hf_chat_template --------- Co-authored-by: Ishaan Jaff --- .../prompt_templates/common_utils.py | 4 ++-- litellm/llms/xai/chat/transformation.py | 24 ++++++++++++++++++- tests/llm_translation/test_xai.py | 18 ++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 9ba1153c08..8d3845969a 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -35,7 +35,7 @@ def handle_messages_with_content_list_to_str_conversion( def strip_name_from_messages( - messages: List[AllMessageValues], + messages: List[AllMessageValues], allowed_name_roles: List[str] = ["user"] ) -> List[AllMessageValues]: """ Removes 'name' from messages @@ -44,7 +44,7 @@ def strip_name_from_messages( for message in messages: msg_role = message.get("role") msg_copy = message.copy() - if msg_role == "user": + if msg_role not in allowed_name_roles: msg_copy.pop("name", None) # type: ignore new_messages.append(msg_copy) return new_messages diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 734c6eb2e0..614509020e 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -1,6 +1,10 @@ -from typing import Optional, Tuple +from typing import List, Optional, Tuple +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + strip_name_from_messages, +) from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -51,3 +55,21 @@ class XAIChatConfig(OpenAIGPTConfig): if value is not None: optional_params[param] = value return optional_params + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Handle https://github.com/BerriAI/litellm/issues/9720 + + Filter out 'name' from messages + """ + messages = strip_name_from_messages(messages) + return super().transform_request( + model, messages, optional_params, litellm_params, headers + ) diff --git a/tests/llm_translation/test_xai.py b/tests/llm_translation/test_xai.py index de4bfc907d..3846a4f1f0 100644 --- a/tests/llm_translation/test_xai.py +++ b/tests/llm_translation/test_xai.py @@ -142,3 +142,21 @@ def test_completion_xai(stream): assert response.choices[0].message.content is not None except Exception as e: pytest.fail(f"Error occurred: {e}") + + +def test_xai_message_name_filtering(): + messages = [ + { + "role": "system", + "content": "*I press the green button*", + "name": "example_user" + }, + {"role": "user", "content": "Hello", "name": "John"}, + {"role": "assistant", "content": "Hello", "name": "Jane"}, + ] + response = completion( + model="xai/grok-beta", + messages=messages, + ) + assert response is not None + assert response.choices[0].message.content is not None From af42e5855ff6a4e740c4c5b9af08ca84ffc13292 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Fri, 4 Apr 2025 20:37:48 -0700 Subject: [PATCH 18/24] Gemini image generation output support (#9646) * fix(gemini/transformation.py): make GET request to get uri details, if cannot be inferred * fix: fix linting errors * Revert "fix: fix linting errors" This reverts commit 926a5a527ff27a107b39da8f5a26b0ee8e2d9884. * fix(gemini/transformation.py): modalities param support Partially resolves https://github.com/BerriAI/litellm/issues/9237 * feat(google_ai_studio/): add image generation support Closes https://github.com/BerriAI/litellm/issues/9237 * fix: fix types * fix: fix ruff check --- litellm/llms/gemini/chat/transformation.py | 1 + .../llms/vertex_ai/gemini/transformation.py | 13 +- .../vertex_and_google_ai_studio_gemini.py | 135 +++++++++++------- litellm/types/llms/vertex_ai.py | 12 +- tests/llm_translation/test_gemini.py | 13 +- tests/llm_translation/test_optional_params.py | 12 +- 6 files changed, 123 insertions(+), 63 deletions(-) diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 0d5956122e..795333d598 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -81,6 +81,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): "stop", "logprobs", "frequency_penalty", + "modalities", ] def map_openai_params( diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 8067d51c87..d70fa1a089 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -224,17 +224,12 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 if not file_id: continue - mime_type = format or _get_image_mime_type_from_url(file_id) - - if mime_type is not None: - _part = PartType( - file_data=FileDataType( - file_uri=file_id, - mime_type=mime_type, - ) + try: + _part = _process_gemini_image( + image_url=file_id, format=format ) _parts.append(_part) - else: + except Exception: raise Exception( "Unable to determine mime type for file_id: {}, set this explicitly using message[{}].content[{}].file.format".format( file_id, msg_i, element_idx diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 36382831c6..d38c24bb2e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -208,6 +208,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "seed", "logprobs", "top_logprobs", # Added this to list of supported openAI params + "modalities", ] def map_tool_choice_values( @@ -312,6 +313,30 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): old_schema = _build_vertex_schema(parameters=old_schema) return old_schema + def apply_response_schema_transformation(self, value: dict, optional_params: dict): + # remove 'additionalProperties' from json schema + value = _remove_additional_properties(value) + # remove 'strict' from json schema + value = _remove_strict_from_schema(value) + if value["type"] == "json_object": + optional_params["response_mime_type"] = "application/json" + elif value["type"] == "text": + optional_params["response_mime_type"] = "text/plain" + if "response_schema" in value: + optional_params["response_mime_type"] = "application/json" + optional_params["response_schema"] = value["response_schema"] + elif value["type"] == "json_schema": # type: ignore + if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore + optional_params["response_mime_type"] = "application/json" + optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore + + if "response_schema" in optional_params and isinstance( + optional_params["response_schema"], dict + ): + optional_params["response_schema"] = self._map_response_schema( + value=optional_params["response_schema"] + ) + def map_openai_params( self, non_default_params: Dict, @@ -322,58 +347,39 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): for param, value in non_default_params.items(): if param == "temperature": optional_params["temperature"] = value - if param == "top_p": + elif param == "top_p": optional_params["top_p"] = value - if ( + elif ( param == "stream" and value is True ): # sending stream = False, can cause it to get passed unchecked and raise issues optional_params["stream"] = value - if param == "n": + elif param == "n": optional_params["candidate_count"] = value - if param == "stop": + elif param == "stop": if isinstance(value, str): optional_params["stop_sequences"] = [value] elif isinstance(value, list): optional_params["stop_sequences"] = value - if param == "max_tokens" or param == "max_completion_tokens": + elif param == "max_tokens" or param == "max_completion_tokens": optional_params["max_output_tokens"] = value - if param == "response_format" and isinstance(value, dict): # type: ignore - # remove 'additionalProperties' from json schema - value = _remove_additional_properties(value) - # remove 'strict' from json schema - value = _remove_strict_from_schema(value) - if value["type"] == "json_object": - optional_params["response_mime_type"] = "application/json" - elif value["type"] == "text": - optional_params["response_mime_type"] = "text/plain" - if "response_schema" in value: - optional_params["response_mime_type"] = "application/json" - optional_params["response_schema"] = value["response_schema"] - elif value["type"] == "json_schema": # type: ignore - if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore - optional_params["response_mime_type"] = "application/json" - optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore - - if "response_schema" in optional_params and isinstance( - optional_params["response_schema"], dict - ): - optional_params["response_schema"] = self._map_response_schema( - value=optional_params["response_schema"] - ) - if param == "frequency_penalty": + elif param == "response_format" and isinstance(value, dict): # type: ignore + self.apply_response_schema_transformation( + value=value, optional_params=optional_params + ) + elif param == "frequency_penalty": optional_params["frequency_penalty"] = value - if param == "presence_penalty": + elif param == "presence_penalty": optional_params["presence_penalty"] = value - if param == "logprobs": + elif param == "logprobs": optional_params["responseLogprobs"] = value - if param == "top_logprobs": + elif param == "top_logprobs": optional_params["logprobs"] = value - if (param == "tools" or param == "functions") and isinstance(value, list): + elif (param == "tools" or param == "functions") and isinstance(value, list): optional_params["tools"] = self._map_function(value=value) optional_params["litellm_param_is_function_call"] = ( True if param == "functions" else False ) - if param == "tool_choice" and ( + elif param == "tool_choice" and ( isinstance(value, str) or isinstance(value, dict) ): _tool_choice_value = self.map_tool_choice_values( @@ -381,8 +387,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value - if param == "seed": + elif param == "seed": optional_params["seed"] = value + elif param == "modalities" and isinstance(value, list): + response_modalities = [] + for modality in value: + if modality == "text": + response_modalities.append("TEXT") + elif modality == "image": + response_modalities.append("IMAGE") + else: + response_modalities.append("MODALITY_UNSPECIFIED") + optional_params["responseModalities"] = response_modalities if litellm.vertex_ai_safety_settings is not None: optional_params["safety_settings"] = litellm.vertex_ai_safety_settings @@ -493,6 +509,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): for part in parts: if "text" in part: _content_str += part["text"] + elif "inlineData" in part: # base64 encoded image + _content_str += "data:{};base64,{}".format( + part["inlineData"]["mimeType"], part["inlineData"]["data"] + ) + if _content_str: return _content_str return None @@ -685,7 +706,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_logprobs: Optional[ChoiceLogprobs] = None tools: Optional[List[ChatCompletionToolCallChunk]] = [] functions: Optional[ChatCompletionToolCallFunctionChunk] = None - + for idx, candidate in enumerate(_candidates): if "content" not in candidate: continue @@ -698,16 +719,20 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "citationMetadata" in candidate: citation_metadata.append(candidate["citationMetadata"]) - + if "parts" in candidate["content"]: - chat_completion_message["content"] = VertexGeminiConfig().get_assistant_content_message( + chat_completion_message[ + "content" + ] = VertexGeminiConfig().get_assistant_content_message( parts=candidate["content"]["parts"] ) functions, tools = self._transform_parts( parts=candidate["content"]["parts"], index=candidate.get("index", idx), - is_function_call=litellm_params.get("litellm_param_is_function_call"), + is_function_call=litellm_params.get( + "litellm_param_is_function_call" + ), ) if "logprobsResult" in candidate: @@ -723,7 +748,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if functions is not None: chat_completion_message["function_call"] = functions - + choice = litellm.Choices( finish_reason=candidate.get("finishReason", "stop"), index=candidate.get("index", idx), @@ -733,7 +758,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) model_response.choices.append(choice) - + return grounding_metadata, safety_ratings, citation_metadata def transform_response( @@ -785,7 +810,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _candidates = completion_response.get("candidates") if _candidates and len(_candidates) > 0: - content_policy_violations = VertexGeminiConfig().get_flagged_finish_reasons() + content_policy_violations = ( + VertexGeminiConfig().get_flagged_finish_reasons() + ) if ( "finishReason" in _candidates[0] and _candidates[0]["finishReason"] in content_policy_violations.keys() @@ -795,12 +822,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): completion_response=completion_response, ) - model_response.choices = [] # type: ignore + model_response.choices = [] try: grounding_metadata, safety_ratings, citation_metadata = [], [], [] if _candidates: - grounding_metadata, safety_ratings, citation_metadata = self._process_candidates( + ( + grounding_metadata, + safety_ratings, + citation_metadata, + ) = self._process_candidates( _candidates, model_response, litellm_params ) @@ -809,14 +840,20 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ## ADD METADATA TO RESPONSE ## setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) - model_response._hidden_params["vertex_ai_grounding_metadata"] = grounding_metadata - + model_response._hidden_params[ + "vertex_ai_grounding_metadata" + ] = grounding_metadata + setattr(model_response, "vertex_ai_safety_results", safety_ratings) - model_response._hidden_params["vertex_ai_safety_results"] = safety_ratings # older approach - maintaining to prevent regressions - + model_response._hidden_params[ + "vertex_ai_safety_results" + ] = safety_ratings # older approach - maintaining to prevent regressions + ## ADD CITATION METADATA ## setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) - model_response._hidden_params["vertex_ai_citation_metadata"] = citation_metadata # older approach - maintaining to prevent regressions + model_response._hidden_params[ + "vertex_ai_citation_metadata" + ] = citation_metadata # older approach - maintaining to prevent regressions except Exception as e: raise VertexAIError( diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index fef034ce80..27d79ec992 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -56,12 +56,17 @@ class HttpxCodeExecutionResult(TypedDict): output: str +class HttpxBlobType(TypedDict): + mimeType: str + data: str + + class HttpxPartType(TypedDict, total=False): text: str - inline_data: BlobType - file_data: FileDataType + inlineData: HttpxBlobType + fileData: FileDataType functionCall: HttpxFunctionCall - function_response: FunctionResponse + functionResponse: FunctionResponse executableCode: HttpxExecutableCode codeExecutionResult: HttpxCodeExecutionResult @@ -160,6 +165,7 @@ class GenerationConfig(TypedDict, total=False): seed: int responseLogprobs: bool logprobs: int + responseModalities: List[Literal["TEXT", "IMAGE", "AUDIO", "VIDEO"]] class Tools(TypedDict, total=False): diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 7c7c10daee..2763f451f6 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -11,7 +11,8 @@ from base_llm_unit_tests import BaseLLMChatTest from litellm.llms.vertex_ai.context_caching.transformation import ( separate_cached_messages, ) - +import litellm +from litellm import completion class TestGoogleAIStudioGemini(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: @@ -72,3 +73,13 @@ def test_gemini_context_caching_separate_messages(): print(non_cached_messages) assert len(cached_messages) > 0, "Cached messages should be present" assert len(non_cached_messages) > 0, "Non-cached messages should be present" + + +def test_gemini_image_generation(): + # litellm._turn_on_debug() + response = completion( + model="gemini/gemini-2.0-flash-exp-image-generation", + messages=[{"role": "user", "content": "Generate an image of a cat"}], + modalities=["image", "text"], + ) + assert response.choices[0].message.content is not None \ No newline at end of file diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 5e792d46e9..8180cc7279 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -1405,6 +1405,17 @@ def test_azure_modalities_param(): assert optional_params["audio"] == {"type": "audio_input", "input": "test.wav"} +def test_gemini_modalities_param(): + optional_params = get_optional_params( + model="gemini-1.5-pro", + custom_llm_provider="gemini", + modalities=["text", "image"], + ) + + assert optional_params["responseModalities"] == ["TEXT", "IMAGE"] + + + def test_azure_response_format_param(): optional_params = litellm.get_optional_params( @@ -1430,4 +1441,3 @@ def test_anthropic_unified_reasoning_content(model, provider): reasoning_effort="high", ) assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096} - From 001043ba0592392efa6f67e4355778deb72366d9 Mon Sep 17 00:00:00 2001 From: Chaos Yu Date: Sat, 5 Apr 2025 11:39:12 +0800 Subject: [PATCH 19/24] make sure metadata available and have a value (#9764) --- litellm/proxy/litellm_pre_call_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index dade6c933e..785ad8dc29 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -528,7 +528,7 @@ async def add_litellm_data_to_request( # noqa: PLR0915 _metadata_variable_name = _get_metadata_variable_name(request) - if _metadata_variable_name not in data: + if data.get(_metadata_variable_name, None) is None: data[_metadata_variable_name] = {} # We want to log the "metadata" from the client side request. Avoid circular reference by not directly assigning metadata to itself. From 08f9e1447be964f63e545be494e5bfa4d65a2f97 Mon Sep 17 00:00:00 2001 From: Hugo Liu Date: Sat, 5 Apr 2025 11:43:46 +0800 Subject: [PATCH 20/24] fix(asr-groq): add groq whisper models to model cost map (#9648) Co-authored-by: liuhu --- model_prices_and_context_window.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4c56210625..c1b738ca8c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3303,6 +3303,24 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "groq/whisper-large-v3": { + "mode": "audio_transcription", + "input_cost_per_second": 0.00003083, + "output_cost_per_second": 0, + "litellm_provider": "groq" + }, + "groq/whisper-large-v3-turbo": { + "mode": "audio_transcription", + "input_cost_per_second": 0.00001111, + "output_cost_per_second": 0, + "litellm_provider": "groq" + }, + "groq/distil-whisper-large-v3-en": { + "mode": "audio_transcription", + "input_cost_per_second": 0.00000556, + "output_cost_per_second": 0, + "litellm_provider": "groq" + }, "cerebras/llama3.1-8b": { "max_tokens": 128000, "max_input_tokens": 128000, From 3e9066e91d00b56ebba2f3b09f66c89e85a2880f Mon Sep 17 00:00:00 2001 From: caramulrooney <30801834+caramulrooney@users.noreply.github.com> Date: Fri, 4 Apr 2025 23:44:06 -0400 Subject: [PATCH 21/24] Update model_prices_and_context_window.json (#9620) Add watsonx/ibm/granite-3-8b-instruct --- model_prices_and_context_window.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c1b738ca8c..6ab7501a27 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -88,6 +88,24 @@ "search_context_size_high": 0.050 } }, + "watsonx/ibm/granite-3-8b-instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 1024, + "input_cost_per_token": 0.0002, + "output_cost_per_token": 0.0002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_vision": false, + "supports_audio_input": false, + "supports_audio_output": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "deprecation_date": null + } "gpt-4o-search-preview-2025-03-11": { "max_tokens": 16384, "max_input_tokens": 128000, From 5826108c9ab0e7d16fbd44bc64a8b5950a41d516 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 4 Apr 2025 20:45:27 -0700 Subject: [PATCH 22/24] build: bump --- ...odel_prices_and_context_window_backup.json | 36 +++++++++++++++++++ pyproject.toml | 4 +-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4c56210625..6ab7501a27 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -88,6 +88,24 @@ "search_context_size_high": 0.050 } }, + "watsonx/ibm/granite-3-8b-instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 1024, + "input_cost_per_token": 0.0002, + "output_cost_per_token": 0.0002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_vision": false, + "supports_audio_input": false, + "supports_audio_output": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "deprecation_date": null + } "gpt-4o-search-preview-2025-03-11": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -3303,6 +3321,24 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "groq/whisper-large-v3": { + "mode": "audio_transcription", + "input_cost_per_second": 0.00003083, + "output_cost_per_second": 0, + "litellm_provider": "groq" + }, + "groq/whisper-large-v3-turbo": { + "mode": "audio_transcription", + "input_cost_per_second": 0.00001111, + "output_cost_per_second": 0, + "litellm_provider": "groq" + }, + "groq/distil-whisper-large-v3-en": { + "mode": "audio_transcription", + "input_cost_per_second": 0.00000556, + "output_cost_per_second": 0, + "litellm_provider": "groq" + }, "cerebras/llama3.1-8b": { "max_tokens": 128000, "max_input_tokens": 128000, diff --git a/pyproject.toml b/pyproject.toml index ac14a9af51..8cb15bb14e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.65.3" +version = "1.65.4" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -117,7 +117,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.65.3" +version = "1.65.4" version_files = [ "pyproject.toml:^version" ] From 7cd7bdbd0fc8b9adea843d34e1530c4c0af0dab8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 4 Apr 2025 20:48:29 -0700 Subject: [PATCH 23/24] build: fix model cost map --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ab7501a27..96a63b87f2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -105,7 +105,7 @@ "supports_response_schema": true, "supports_system_messages": true, "deprecation_date": null - } + }, "gpt-4o-search-preview-2025-03-11": { "max_tokens": 16384, "max_input_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6ab7501a27..96a63b87f2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -105,7 +105,7 @@ "supports_response_schema": true, "supports_system_messages": true, "deprecation_date": null - } + }, "gpt-4o-search-preview-2025-03-11": { "max_tokens": 16384, "max_input_tokens": 128000, From 8559bcc2525d119d4bd4a365384a501e80f7eefc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 4 Apr 2025 21:16:12 -0700 Subject: [PATCH 24/24] DB Transaction Queue Health Metrics --- docs/my-website/docs/proxy/db_deadlocks.md | 12 ++++++++++++ docs/my-website/docs/proxy/prometheus.md | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/docs/my-website/docs/proxy/db_deadlocks.md b/docs/my-website/docs/proxy/db_deadlocks.md index e649bdccc0..332374995d 100644 --- a/docs/my-website/docs/proxy/db_deadlocks.md +++ b/docs/my-website/docs/proxy/db_deadlocks.md @@ -71,4 +71,16 @@ litellm_settings: supported_call_types: [] # Optional: Set cache for proxy, but not on the actual llm api call ``` +## Monitoring + +LiteLLM emits the following prometheus metrics to monitor the health/status of the in memory buffer and redis buffer. + + +| Metric Name | Description | Storage Type | +|-----------------------------------------------------|-----------------------------------------------------------------------------|--------------| +| `litellm_pod_lock_manager_size` | Indicates which pod has the lock to write updates to the database. | Redis | +| `litellm_in_memory_daily_spend_update_queue_size` | Number of items in the in-memory daily spend update queue. These are the aggregate spend logs for each user. | In-Memory | +| `litellm_redis_daily_spend_update_queue_size` | Number of items in the Redis daily spend update queue. These are the aggregate spend logs for each user. | Redis | +| `litellm_in_memory_spend_update_queue_size` | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory | +| `litellm_redis_spend_update_queue_size` | Redis aggregate spend values for keys, users, teams, etc. | Redis | diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 8dff527ae5..220a3c2c12 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -242,6 +242,19 @@ litellm_settings: | `litellm_redis_fails` | Number of failed redis calls | | `litellm_self_latency` | Histogram latency for successful litellm api call | +#### DB Transaction Queue Health Metrics + +Use these metrics to monitor the health of the DB Transaction Queue. Eg. Monitoring the size of the in-memory and redis buffers. + +| Metric Name | Description | Storage Type | +|-----------------------------------------------------|-----------------------------------------------------------------------------|--------------| +| `litellm_pod_lock_manager_size` | Indicates which pod has the lock to write updates to the database. | Redis | +| `litellm_in_memory_daily_spend_update_queue_size` | Number of items in the in-memory daily spend update queue. These are the aggregate spend logs for each user. | In-Memory | +| `litellm_redis_daily_spend_update_queue_size` | Number of items in the Redis daily spend update queue. These are the aggregate spend logs for each user. | Redis | +| `litellm_in_memory_spend_update_queue_size` | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory | +| `litellm_redis_spend_update_queue_size` | Redis aggregate spend values for keys, users, teams, etc. | Redis | + + ## **🔥 LiteLLM Maintained Grafana Dashboards **