tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-20 08:23:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into litellm_add_auth_metrics_endpoint

Browse Source
This commit is contained in:
Ishaan Jaff
2025-04-04 21:28:06 -07:00
committed by GitHub
parent fc4c453cb9 8559bcc252
commit b89ed69257
35 changed files with 774 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/my-website/docs/proxy/admin_ui_sso.md
+1 -1
View File
@@ -156,7 +156,7 @@ PROXY_LOGOUT_URL="https://www.google.com"
Set this in your .env (so the proxy can set the correct redirect url)
```shell
PROXY_BASE_URL=https://litellm-api.up.railway.app/
PROXY_BASE_URL=https://litellm-api.up.railway.app
```
#### Step 4. Test flow
docs/my-website/docs/proxy/db_deadlocks.md
+12
View File
@@ -71,4 +71,16 @@ litellm_settings:
supported_call_types: [] # Optional: Set cache for proxy, but not on the actual llm api call
```
## Monitoring
LiteLLM emits the following prometheus metrics to monitor the health/status of the in memory buffer and redis buffer.
| Metric Name | Description | Storage Type |
|-----------------------------------------------------|-----------------------------------------------------------------------------|--------------|
| `litellm_pod_lock_manager_size` | Indicates which pod has the lock to write updates to the database. | Redis |
| `litellm_in_memory_daily_spend_update_queue_size` | Number of items in the in-memory daily spend update queue. These are the aggregate spend logs for each user. | In-Memory |
| `litellm_redis_daily_spend_update_queue_size` | Number of items in the Redis daily spend update queue. These are the aggregate spend logs for each user. | Redis |
| `litellm_in_memory_spend_update_queue_size` | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory |
| `litellm_redis_spend_update_queue_size` | Redis aggregate spend values for keys, users, teams, etc. | Redis |
docs/my-website/docs/proxy/prometheus.md
+13
View File
@@ -242,6 +242,19 @@ litellm_settings:
| `litellm_redis_fails` | Number of failed redis calls |
| `litellm_self_latency` | Histogram latency for successful litellm api call |
#### DB Transaction Queue Health Metrics
Use these metrics to monitor the health of the DB Transaction Queue. Eg. Monitoring the size of the in-memory and redis buffers.
| Metric Name | Description | Storage Type |
|-----------------------------------------------------|-----------------------------------------------------------------------------|--------------|
| `litellm_pod_lock_manager_size` | Indicates which pod has the lock to write updates to the database. | Redis |
| `litellm_in_memory_daily_spend_update_queue_size` | Number of items in the in-memory daily spend update queue. These are the aggregate spend logs for each user. | In-Memory |
| `litellm_redis_daily_spend_update_queue_size` | Number of items in the Redis daily spend update queue. These are the aggregate spend logs for each user. | Redis |
| `litellm_in_memory_spend_update_queue_size` | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory |
| `litellm_redis_spend_update_queue_size` | Redis aggregate spend values for keys, users, teams, etc. | Redis |
## **🔥 LiteLLM Maintained Grafana Dashboards **
litellm/_service_logger.py
+2
View File
@@ -124,6 +124,7 @@ class ServiceLogging(CustomLogger):
service=service,
duration=duration,
call_type=call_type,
event_metadata=event_metadata,
)
for callback in litellm.service_callback:
@@ -229,6 +230,7 @@ class ServiceLogging(CustomLogger):
service=service,
duration=duration,
call_type=call_type,
event_metadata=event_metadata,
)
for callback in litellm.service_callback:
litellm/integrations/prometheus_services.py
+84 -25
View File
@@ -3,11 +3,16 @@
# On success + failure, log events to Prometheus for litellm / adjacent services (litellm, redis, postgres, llm api providers)
from typing import List, Optional, Union
from typing import Dict, List, Optional, Union
from litellm._logging import print_verbose, verbose_logger
from litellm.types.integrations.prometheus import LATENCY_BUCKETS
from litellm.types.services import ServiceLoggerPayload, ServiceTypes
from litellm.types.services import (
DEFAULT_SERVICE_CONFIGS,
ServiceLoggerPayload,
ServiceMetrics,
ServiceTypes,
)
FAILED_REQUESTS_LABELS = ["error_class", "function_name"]
@@ -23,7 +28,8 @@ class PrometheusServicesLogger:
):
try:
try:
from prometheus_client import REGISTRY, Counter, Histogram
from prometheus_client import REGISTRY, Counter, Gauge, Histogram
from prometheus_client.gc_collector import Collector
except ImportError:
raise Exception(
"Missing prometheus_client. Run `pip install prometheus-client`"
@@ -31,36 +37,51 @@ class PrometheusServicesLogger:
self.Histogram = Histogram
self.Counter = Counter
self.Gauge = Gauge
self.REGISTRY = REGISTRY
verbose_logger.debug("in init prometheus services metrics")
self.services = [item.value for item in ServiceTypes]
self.payload_to_prometheus_map: Dict[
str, List[Union[Histogram, Counter, Gauge, Collector]]
] = {}
self.payload_to_prometheus_map = (
{}
) # store the prometheus histogram/counter we need to call for each field in payload
for service in ServiceTypes:
service_metrics: List[Union[Histogram, Counter, Gauge, Collector]] = []
for service in self.services:
histogram = self.create_histogram(service, type_of_request="latency")
counter_failed_request = self.create_counter(
service,
type_of_request="failed_requests",
additional_labels=FAILED_REQUESTS_LABELS,
)
counter_total_requests = self.create_counter(
service, type_of_request="total_requests"
)
self.payload_to_prometheus_map[service] = [
histogram,
counter_failed_request,
counter_total_requests,
]
metrics_to_initialize = self._get_service_metrics_initialize(service)
self.prometheus_to_amount_map: dict = (
{}
) # the field / value in ServiceLoggerPayload the object needs to be incremented by
# Initialize only the configured metrics for each service
if ServiceMetrics.HISTOGRAM in metrics_to_initialize:
histogram = self.create_histogram(
service.value, type_of_request="latency"
)
if histogram:
service_metrics.append(histogram)
if ServiceMetrics.COUNTER in metrics_to_initialize:
counter_failed_request = self.create_counter(
service.value,
type_of_request="failed_requests",
additional_labels=FAILED_REQUESTS_LABELS,
)
if counter_failed_request:
service_metrics.append(counter_failed_request)
counter_total_requests = self.create_counter(
service.value, type_of_request="total_requests"
)
if counter_total_requests:
service_metrics.append(counter_total_requests)
if ServiceMetrics.GAUGE in metrics_to_initialize:
gauge = self.create_gauge(service.value, type_of_request="size")
if gauge:
service_metrics.append(gauge)
if service_metrics:
self.payload_to_prometheus_map[service.value] = service_metrics
self.prometheus_to_amount_map: dict = {}
### MOCK TESTING ###
self.mock_testing = mock_testing
self.mock_testing_success_calls = 0
@@ -70,6 +91,19 @@ class PrometheusServicesLogger:
print_verbose(f"Got exception on init prometheus client {str(e)}")
raise e
def _get_service_metrics_initialize(
self, service: ServiceTypes
) -> List[ServiceMetrics]:
DEFAULT_METRICS = [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
if service not in DEFAULT_SERVICE_CONFIGS:
return DEFAULT_METRICS
metrics = DEFAULT_SERVICE_CONFIGS.get(service, {}).get("metrics", [])
if not metrics:
verbose_logger.debug(f"No metrics found for service {service}")
return DEFAULT_METRICS
return metrics
def is_metric_registered(self, metric_name) -> bool:
for metric in self.REGISTRY.collect():
if metric_name == metric.name:
@@ -94,6 +128,15 @@ class PrometheusServicesLogger:
buckets=LATENCY_BUCKETS,
)
def create_gauge(self, service: str, type_of_request: str):
metric_name = "litellm_{}_{}".format(service, type_of_request)
is_registered = self.is_metric_registered(metric_name)
if is_registered:
return self._get_metric(metric_name)
return self.Gauge(
metric_name, "Gauge for {} service".format(service), labelnames=[service]
)
def create_counter(
self,
service: str,
@@ -120,6 +163,15 @@ class PrometheusServicesLogger:
histogram.labels(labels).observe(amount)
def update_gauge(
self,
gauge,
labels: str,
amount: float,
):
assert isinstance(gauge, self.Gauge)
gauge.labels(labels).set(amount)
def increment_counter(
self,
counter,
@@ -190,6 +242,13 @@ class PrometheusServicesLogger:
labels=payload.service.value,
amount=1, # LOG TOTAL REQUESTS TO PROMETHEUS
)
elif isinstance(obj, self.Gauge):
if payload.event_metadata:
self.update_gauge(
gauge=obj,
labels=payload.event_metadata.get("gauge_labels") or "",
amount=payload.event_metadata.get("gauge_value") or 0,
)
async def async_service_failure_hook(
self,
litellm/litellm_core_utils/credential_accessor.py
+1
View File
@@ -10,6 +10,7 @@ class CredentialAccessor:
@staticmethod
def get_credential_values(credential_name: str) -> dict:
"""Safe accessor for credentials."""
if not litellm.credential_list:
return {}
for credential in litellm.credential_list:
litellm/litellm_core_utils/prompt_templates/common_utils.py
+2 -2
View File
@@ -35,7 +35,7 @@ def handle_messages_with_content_list_to_str_conversion(
def strip_name_from_messages(
messages: List[AllMessageValues],
messages: List[AllMessageValues], allowed_name_roles: List[str] = ["user"]
) -> List[AllMessageValues]:
"""
Removes 'name' from messages
@@ -44,7 +44,7 @@ def strip_name_from_messages(
for message in messages:
msg_role = message.get("role")
msg_copy = message.copy()
if msg_role == "user":
if msg_role not in allowed_name_roles:
msg_copy.pop("name", None) # type: ignore
new_messages.append(msg_copy)
return new_messages
litellm/llms/gemini/chat/transformation.py
+1
View File
@@ -81,6 +81,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
"stop",
"logprobs",
"frequency_penalty",
"modalities",
]
def map_openai_params(
litellm/llms/vertex_ai/gemini/transformation.py
+4 -9
View File
@@ -224,17 +224,12 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
if not file_id:
continue
mime_type = format or _get_image_mime_type_from_url(file_id)
if mime_type is not None:
_part = PartType(
file_data=FileDataType(
file_uri=file_id,
mime_type=mime_type,
)
try:
_part = _process_gemini_image(
image_url=file_id, format=format
)
_parts.append(_part)
else:
except Exception:
raise Exception(
"Unable to determine mime type for file_id: {}, set this explicitly using message[{}].content[{}].file.format".format(
file_id, msg_i, element_idx
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+86 -49
View File
@@ -208,6 +208,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"seed",
"logprobs",
"top_logprobs", # Added this to list of supported openAI params
"modalities",
]
def map_tool_choice_values(
@@ -312,6 +313,30 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
old_schema = _build_vertex_schema(parameters=old_schema)
return old_schema
def apply_response_schema_transformation(self, value: dict, optional_params: dict):
# remove 'additionalProperties' from json schema
value = _remove_additional_properties(value)
# remove 'strict' from json schema
value = _remove_strict_from_schema(value)
if value["type"] == "json_object":
optional_params["response_mime_type"] = "application/json"
elif value["type"] == "text":
optional_params["response_mime_type"] = "text/plain"
if "response_schema" in value:
optional_params["response_mime_type"] = "application/json"
optional_params["response_schema"] = value["response_schema"]
elif value["type"] == "json_schema": # type: ignore
if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore
optional_params["response_mime_type"] = "application/json"
optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore
if "response_schema" in optional_params and isinstance(
optional_params["response_schema"], dict
):
optional_params["response_schema"] = self._map_response_schema(
value=optional_params["response_schema"]
)
def map_openai_params(
self,
non_default_params: Dict,
@@ -322,58 +347,39 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
for param, value in non_default_params.items():
if param == "temperature":
optional_params["temperature"] = value
if param == "top_p":
elif param == "top_p":
optional_params["top_p"] = value
if (
elif (
param == "stream" and value is True
): # sending stream = False, can cause it to get passed unchecked and raise issues
optional_params["stream"] = value
if param == "n":
elif param == "n":
optional_params["candidate_count"] = value
if param == "stop":
elif param == "stop":
if isinstance(value, str):
optional_params["stop_sequences"] = [value]
elif isinstance(value, list):
optional_params["stop_sequences"] = value
if param == "max_tokens" or param == "max_completion_tokens":
elif param == "max_tokens" or param == "max_completion_tokens":
optional_params["max_output_tokens"] = value
if param == "response_format" and isinstance(value, dict): # type: ignore
# remove 'additionalProperties' from json schema
value = _remove_additional_properties(value)
# remove 'strict' from json schema
value = _remove_strict_from_schema(value)
if value["type"] == "json_object":
optional_params["response_mime_type"] = "application/json"
elif value["type"] == "text":
optional_params["response_mime_type"] = "text/plain"
if "response_schema" in value:
optional_params["response_mime_type"] = "application/json"
optional_params["response_schema"] = value["response_schema"]
elif value["type"] == "json_schema": # type: ignore
if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore
optional_params["response_mime_type"] = "application/json"
optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore
if "response_schema" in optional_params and isinstance(
optional_params["response_schema"], dict
):
optional_params["response_schema"] = self._map_response_schema(
value=optional_params["response_schema"]
)
if param == "frequency_penalty":
elif param == "response_format" and isinstance(value, dict): # type: ignore
self.apply_response_schema_transformation(
value=value, optional_params=optional_params
)
elif param == "frequency_penalty":
optional_params["frequency_penalty"] = value
if param == "presence_penalty":
elif param == "presence_penalty":
optional_params["presence_penalty"] = value
if param == "logprobs":
elif param == "logprobs":
optional_params["responseLogprobs"] = value
if param == "top_logprobs":
elif param == "top_logprobs":
optional_params["logprobs"] = value
if (param == "tools" or param == "functions") and isinstance(value, list):
elif (param == "tools" or param == "functions") and isinstance(value, list):
optional_params["tools"] = self._map_function(value=value)
optional_params["litellm_param_is_function_call"] = (
True if param == "functions" else False
)
if param == "tool_choice" and (
elif param == "tool_choice" and (
isinstance(value, str) or isinstance(value, dict)
):
_tool_choice_value = self.map_tool_choice_values(
@@ -381,8 +387,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
)
if _tool_choice_value is not None:
optional_params["tool_choice"] = _tool_choice_value
if param == "seed":
elif param == "seed":
optional_params["seed"] = value
elif param == "modalities" and isinstance(value, list):
response_modalities = []
for modality in value:
if modality == "text":
response_modalities.append("TEXT")
elif modality == "image":
response_modalities.append("IMAGE")
else:
response_modalities.append("MODALITY_UNSPECIFIED")
optional_params["responseModalities"] = response_modalities
if litellm.vertex_ai_safety_settings is not None:
optional_params["safety_settings"] = litellm.vertex_ai_safety_settings
@@ -493,6 +509,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
for part in parts:
if "text" in part:
_content_str += part["text"]
elif "inlineData" in part: # base64 encoded image
_content_str += "data:{};base64,{}".format(
part["inlineData"]["mimeType"], part["inlineData"]["data"]
)
if _content_str:
return _content_str
return None
@@ -685,7 +706,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
chat_completion_logprobs: Optional[ChoiceLogprobs] = None
tools: Optional[List[ChatCompletionToolCallChunk]] = []
functions: Optional[ChatCompletionToolCallFunctionChunk] = None
for idx, candidate in enumerate(_candidates):
if "content" not in candidate:
continue
@@ -698,16 +719,20 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if "citationMetadata" in candidate:
citation_metadata.append(candidate["citationMetadata"])
if "parts" in candidate["content"]:
chat_completion_message["content"] = VertexGeminiConfig().get_assistant_content_message(
chat_completion_message[
"content"
] = VertexGeminiConfig().get_assistant_content_message(
parts=candidate["content"]["parts"]
)
functions, tools = self._transform_parts(
parts=candidate["content"]["parts"],
index=candidate.get("index", idx),
is_function_call=litellm_params.get("litellm_param_is_function_call"),
is_function_call=litellm_params.get(
"litellm_param_is_function_call"
),
)
if "logprobsResult" in candidate:
@@ -723,7 +748,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if functions is not None:
chat_completion_message["function_call"] = functions
choice = litellm.Choices(
finish_reason=candidate.get("finishReason", "stop"),
index=candidate.get("index", idx),
@@ -733,7 +758,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
)
model_response.choices.append(choice)
return grounding_metadata, safety_ratings, citation_metadata
def transform_response(
@@ -785,7 +810,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
_candidates = completion_response.get("candidates")
if _candidates and len(_candidates) > 0:
content_policy_violations = VertexGeminiConfig().get_flagged_finish_reasons()
content_policy_violations = (
VertexGeminiConfig().get_flagged_finish_reasons()
)
if (
"finishReason" in _candidates[0]
and _candidates[0]["finishReason"] in content_policy_violations.keys()
@@ -795,12 +822,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
completion_response=completion_response,
)
model_response.choices = [] # type: ignore
model_response.choices = []
try:
grounding_metadata, safety_ratings, citation_metadata = [], [], []
if _candidates:
grounding_metadata, safety_ratings, citation_metadata = self._process_candidates(
(
grounding_metadata,
safety_ratings,
citation_metadata,
) = self._process_candidates(
_candidates, model_response, litellm_params
)
@@ -809,14 +840,20 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
## ADD METADATA TO RESPONSE ##
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
model_response._hidden_params["vertex_ai_grounding_metadata"] = grounding_metadata
model_response._hidden_params[
"vertex_ai_grounding_metadata"
] = grounding_metadata
setattr(model_response, "vertex_ai_safety_results", safety_ratings)
model_response._hidden_params["vertex_ai_safety_results"] = safety_ratings # older approach - maintaining to prevent regressions
model_response._hidden_params[
"vertex_ai_safety_results"
] = safety_ratings # older approach - maintaining to prevent regressions
## ADD CITATION METADATA ##
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata)
model_response._hidden_params["vertex_ai_citation_metadata"] = citation_metadata # older approach - maintaining to prevent regressions
model_response._hidden_params[
"vertex_ai_citation_metadata"
] = citation_metadata # older approach - maintaining to prevent regressions
except Exception as e:
raise VertexAIError(
litellm/llms/xai/chat/transformation.py
+23 -1
View File
@@ -1,6 +1,10 @@
from typing import Optional, Tuple
from typing import List, Optional, Tuple
from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_name_from_messages,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@@ -51,3 +55,21 @@ class XAIChatConfig(OpenAIGPTConfig):
if value is not None:
optional_params[param] = value
return optional_params
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Handle https://github.com/BerriAI/litellm/issues/9720
Filter out 'name' from messages
"""
messages = strip_name_from_messages(messages)
return super().transform_request(
model, messages, optional_params, litellm_params, headers
)
litellm/model_prices_and_context_window_backup.json
+36
View File
@@ -88,6 +88,24 @@
"search_context_size_high": 0.050
}
},
"watsonx/ibm/granite-3-8b-instruct": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 1024,
"input_cost_per_token": 0.0002,
"output_cost_per_token": 0.0002,
"litellm_provider": "watsonx",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_vision": false,
"supports_audio_input": false,
"supports_audio_output": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"deprecation_date": null
},
"gpt-4o-search-preview-2025-03-11": {
"max_tokens": 16384,
"max_input_tokens": 128000,
@@ -3303,6 +3321,24 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"groq/whisper-large-v3": {
"mode": "audio_transcription",
"input_cost_per_second": 0.00003083,
"output_cost_per_second": 0,
"litellm_provider": "groq"
},
"groq/whisper-large-v3-turbo": {
"mode": "audio_transcription",
"input_cost_per_second": 0.00001111,
"output_cost_per_second": 0,
"litellm_provider": "groq"
},
"groq/distil-whisper-large-v3-en": {
"mode": "audio_transcription",
"input_cost_per_second": 0.00000556,
"output_cost_per_second": 0,
"litellm_provider": "groq"
},
"cerebras/llama3.1-8b": {
"max_tokens": 128000,
"max_input_tokens": 128000,
litellm/proxy/db/db_transaction_queue/base_update_queue.py
+16
View File
@@ -2,8 +2,14 @@
Base class for in memory buffer for database transactions
"""
import asyncio
from typing import Optional
from litellm._logging import verbose_proxy_logger
from litellm._service_logger import ServiceLogging
service_logger_obj = (
ServiceLogging()
) # used for tracking metrics for In memory buffer, redis buffer, pod lock manager
from litellm.constants import MAX_IN_MEMORY_QUEUE_FLUSH_COUNT, MAX_SIZE_IN_MEMORY_QUEUE
@@ -18,6 +24,9 @@ class BaseUpdateQueue:
"""Enqueue an update."""
verbose_proxy_logger.debug("Adding update to queue: %s", update)
await self.update_queue.put(update)
await self._emit_new_item_added_to_queue_event(
queue_size=self.update_queue.qsize()
)
async def flush_all_updates_from_in_memory_queue(self):
"""Get all updates from the queue."""
@@ -31,3 +40,10 @@ class BaseUpdateQueue:
break
updates.append(await self.update_queue.get())
return updates
async def _emit_new_item_added_to_queue_event(
self,
queue_size: Optional[int] = None,
):
"""placeholder, emit event when a new item is added to the queue"""
pass
litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
+22 -2
View File
@@ -1,10 +1,14 @@
import asyncio
from copy import deepcopy
from typing import Dict, List
from typing import Dict, List, Optional
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import DailyUserSpendTransaction
from litellm.proxy.db.db_transaction_queue.base_update_queue import BaseUpdateQueue
from litellm.proxy.db.db_transaction_queue.base_update_queue import (
BaseUpdateQueue,
service_logger_obj,
)
from litellm.types.services import ServiceTypes
class DailySpendUpdateQueue(BaseUpdateQueue):
@@ -117,3 +121,19 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
return aggregated_daily_spend_update_transactions
async def _emit_new_item_added_to_queue_event(
self,
queue_size: Optional[int] = None,
):
asyncio.create_task(
service_logger_obj.async_service_success_hook(
service=ServiceTypes.IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE,
duration=0,
call_type="_emit_new_item_added_to_queue_event",
event_metadata={
"gauge_labels": ServiceTypes.IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE,
"gauge_value": queue_size,
},
)
)
litellm/proxy/db/db_transaction_queue/pod_lock_manager.py
+34
View File
@@ -1,9 +1,12 @@
import asyncio
import uuid
from typing import TYPE_CHECKING, Any, Optional
from litellm._logging import verbose_proxy_logger
from litellm.caching.redis_cache import RedisCache
from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS
from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj
from litellm.types.services import ServiceTypes
if TYPE_CHECKING:
ProxyLogging = Any
@@ -57,6 +60,7 @@ class PodLockManager:
self.pod_id,
self.cronjob_id,
)
return True
else:
# Check if the current pod already holds the lock
@@ -70,6 +74,7 @@ class PodLockManager:
self.pod_id,
self.cronjob_id,
)
self._emit_acquired_lock_event(self.cronjob_id, self.pod_id)
return True
return False
except Exception as e:
@@ -104,6 +109,7 @@ class PodLockManager:
self.pod_id,
self.cronjob_id,
)
self._emit_released_lock_event(self.cronjob_id, self.pod_id)
else:
verbose_proxy_logger.debug(
"Pod %s failed to release Redis lock for cronjob_id=%s",
@@ -127,3 +133,31 @@ class PodLockManager:
verbose_proxy_logger.error(
f"Error releasing Redis lock for {self.cronjob_id}: {e}"
)
@staticmethod
def _emit_acquired_lock_event(cronjob_id: str, pod_id: str):
asyncio.create_task(
service_logger_obj.async_service_success_hook(
service=ServiceTypes.POD_LOCK_MANAGER,
duration=DEFAULT_CRON_JOB_LOCK_TTL_SECONDS,
call_type="_emit_acquired_lock_event",
event_metadata={
"gauge_labels": f"{cronjob_id}:{pod_id}",
"gauge_value": 1,
},
)
)
@staticmethod
def _emit_released_lock_event(cronjob_id: str, pod_id: str):
asyncio.create_task(
service_logger_obj.async_service_success_hook(
service=ServiceTypes.POD_LOCK_MANAGER,
duration=DEFAULT_CRON_JOB_LOCK_TTL_SECONDS,
call_type="_emit_released_lock_event",
event_metadata={
"gauge_labels": f"{cronjob_id}:{pod_id}",
"gauge_value": 0,
},
)
)
litellm/proxy/db/db_transaction_queue/redis_update_buffer.py
+31 -2
View File
@@ -4,6 +4,7 @@ Handles buffering database `UPDATE` transactions in Redis before committing them
This is to prevent deadlocks and improve reliability
"""
import asyncio
import json
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
@@ -16,11 +17,13 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import DailyUserSpendTransaction, DBSpendUpdateTransactions
from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj
from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
DailySpendUpdateQueue,
)
from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue
from litellm.secret_managers.main import str_to_bool
from litellm.types.services import ServiceTypes
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
@@ -136,18 +139,27 @@ class RedisUpdateBuffer:
return
list_of_transactions = [safe_dumps(db_spend_update_transactions)]
await self.redis_cache.async_rpush(
current_redis_buffer_size = await self.redis_cache.async_rpush(
key=REDIS_UPDATE_BUFFER_KEY,
values=list_of_transactions,
)
await self._emit_new_item_added_to_redis_buffer_event(
queue_size=current_redis_buffer_size,
service=ServiceTypes.REDIS_SPEND_UPDATE_QUEUE,
)
list_of_daily_spend_update_transactions = [
safe_dumps(daily_spend_update_transactions)
]
await self.redis_cache.async_rpush(
current_redis_buffer_size = await self.redis_cache.async_rpush(
key=REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY,
values=list_of_daily_spend_update_transactions,
)
await self._emit_new_item_added_to_redis_buffer_event(
queue_size=current_redis_buffer_size,
service=ServiceTypes.REDIS_DAILY_SPEND_UPDATE_QUEUE,
)
@staticmethod
def _number_of_transactions_to_store_in_redis(
@@ -300,3 +312,20 @@ class RedisUpdateBuffer:
)
return combined_transaction
async def _emit_new_item_added_to_redis_buffer_event(
self,
service: ServiceTypes,
queue_size: int,
):
asyncio.create_task(
service_logger_obj.async_service_success_hook(
service=service,
duration=0,
call_type="_emit_new_item_added_to_queue_event",
event_metadata={
"gauge_labels": service,
"gauge_value": queue_size,
},
)
)
litellm/proxy/db/db_transaction_queue/spend_update_queue.py
+22 -2
View File
@@ -1,5 +1,5 @@
import asyncio
from typing import Dict, List
from typing import Dict, List, Optional
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import (
@@ -7,7 +7,11 @@ from litellm.proxy._types import (
Litellm_EntityType,
SpendUpdateQueueItem,
)
from litellm.proxy.db.db_transaction_queue.base_update_queue import BaseUpdateQueue
from litellm.proxy.db.db_transaction_queue.base_update_queue import (
BaseUpdateQueue,
service_logger_obj,
)
from litellm.types.services import ServiceTypes
class SpendUpdateQueue(BaseUpdateQueue):
@@ -203,3 +207,19 @@ class SpendUpdateQueue(BaseUpdateQueue):
transactions_dict[entity_id] += response_cost or 0
return db_spend_update_transactions
async def _emit_new_item_added_to_queue_event(
self,
queue_size: Optional[int] = None,
):
asyncio.create_task(
service_logger_obj.async_service_success_hook(
service=ServiceTypes.IN_MEMORY_SPEND_UPDATE_QUEUE,
duration=0,
call_type="_emit_new_item_added_to_queue_event",
event_metadata={
"gauge_labels": ServiceTypes.IN_MEMORY_SPEND_UPDATE_QUEUE,
"gauge_value": queue_size,
},
)
)
litellm/proxy/litellm_pre_call_utils.py
+1 -1
View File
@@ -528,7 +528,7 @@ async def add_litellm_data_to_request( # noqa: PLR0915
_metadata_variable_name = _get_metadata_variable_name(request)
if _metadata_variable_name not in data:
if data.get(_metadata_variable_name, None) is None:
data[_metadata_variable_name] = {}
# We want to log the "metadata" from the client side request. Avoid circular reference by not directly assigning metadata to itself.
litellm/proxy/proxy_config.yaml
+1
View File
@@ -11,5 +11,6 @@ model_list:
litellm_settings:
require_auth_for_metrics_endpoint: true
callbacks: ["prometheus"]
service_callback: ["prometheus_system"]
litellm/proxy/proxy_server.py
+65 -10
View File
@@ -5332,6 +5332,67 @@ async def _check_if_model_is_user_added(
return filtered_models
def _check_if_model_is_team_model(
models: List[DeploymentTypedDict], user_row: LiteLLM_UserTable
) -> List[Dict]:
"""
Check if model is a team model
Check if user is a member of the team that the model belongs to
"""
user_team_models: List[Dict] = []
for model in models:
model_team_id = model.get("model_info", {}).get("team_id", None)
if model_team_id is not None:
if model_team_id in user_row.teams:
user_team_models.append(cast(Dict, model))
return user_team_models
async def non_admin_all_models(
all_models: List[Dict],
llm_router: Router,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: Optional[PrismaClient],
):
"""
Check if model is in db
Check if db model is 'created_by' == user_api_key_dict.user_id
Only return models that match
"""
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
all_models = await _check_if_model_is_user_added(
models=all_models,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
if user_api_key_dict.user_id:
try:
user_row = await prisma_client.db.litellm_usertable.find_unique(
where={"user_id": user_api_key_dict.user_id}
)
except Exception:
raise HTTPException(status_code=400, detail={"error": "User not found"})
all_models += _check_if_model_is_team_model(
models=llm_router.get_model_list() or [],
user_row=user_row,
)
return all_models
@router.get(
"/v2/model/info",
description="v2 - returns models available to the user based on their API key permissions. Shows model info from config.yaml (except api key and api base). Filter to just user-added models with ?user_models_only=true",
@@ -5377,16 +5438,10 @@ async def model_info_v2(
if model is not None:
all_models = [m for m in all_models if m["model_name"] == model]
if user_models_only is True:
"""
Check if model is in db
Check if db model is 'created_by' == user_api_key_dict.user_id
Only return models that match
"""
all_models = await _check_if_model_is_user_added(
models=all_models,
if user_models_only:
all_models = await non_admin_all_models(
all_models=all_models,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
litellm/router.py
+37 -8
View File
@@ -54,6 +54,7 @@ from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
@@ -4506,25 +4507,53 @@ class Router:
passthrough_endpoint_router,
)
if deployment.litellm_params.litellm_credential_name is not None:
credential_values = CredentialAccessor.get_credential_values(
deployment.litellm_params.litellm_credential_name
)
else:
credential_values = {}
if custom_llm_provider == "vertex_ai":
vertex_project = (
credential_values.get("vertex_project")
or deployment.litellm_params.vertex_project
)
vertex_location = (
credential_values.get("vertex_location")
or deployment.litellm_params.vertex_location
)
vertex_credentials = (
credential_values.get("vertex_credentials")
or deployment.litellm_params.vertex_credentials
)
if (
deployment.litellm_params.vertex_project is None
or deployment.litellm_params.vertex_location is None
or deployment.litellm_params.vertex_credentials is None
vertex_project is None
or vertex_location is None
or vertex_credentials is None
):
raise ValueError(
"vertex_project, vertex_location, and vertex_credentials must be set in litellm_params for pass-through endpoints"
)
passthrough_endpoint_router.add_vertex_credentials(
project_id=deployment.litellm_params.vertex_project,
location=deployment.litellm_params.vertex_location,
vertex_credentials=deployment.litellm_params.vertex_credentials,
project_id=vertex_project,
location=vertex_location,
vertex_credentials=vertex_credentials,
)
else:
api_base = (
credential_values.get("api_base")
or deployment.litellm_params.api_base
)
api_key = (
credential_values.get("api_key")
or deployment.litellm_params.api_key
)
passthrough_endpoint_router.set_pass_through_credentials(
custom_llm_provider=custom_llm_provider,
api_base=deployment.litellm_params.api_base,
api_key=deployment.litellm_params.api_key,
api_base=api_base,
api_key=api_key,
)
pass
pass
litellm/types/llms/vertex_ai.py
+9 -3
View File
@@ -56,12 +56,17 @@ class HttpxCodeExecutionResult(TypedDict):
output: str
class HttpxBlobType(TypedDict):
mimeType: str
data: str
class HttpxPartType(TypedDict, total=False):
text: str
inline_data: BlobType
file_data: FileDataType
inlineData: HttpxBlobType
fileData: FileDataType
functionCall: HttpxFunctionCall
function_response: FunctionResponse
functionResponse: FunctionResponse
executableCode: HttpxExecutableCode
codeExecutionResult: HttpxCodeExecutionResult
@@ -160,6 +165,7 @@ class GenerationConfig(TypedDict, total=False):
seed: int
responseLogprobs: bool
logprobs: int
responseModalities: List[Literal["TEXT", "IMAGE", "AUDIO", "VIDEO"]]
class Tools(TypedDict, total=False):
litellm/types/router.py
+1
View File
@@ -179,6 +179,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams):
max_retries: Optional[int] = None
organization: Optional[str] = None # for openai orgs
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
litellm_credential_name: Optional[str] = None
## LOGGING PARAMS ##
litellm_trace_id: Optional[str] = None
litellm/types/services.py
+89 -1
View File
@@ -1,8 +1,15 @@
import enum
import uuid
from typing import Optional
from typing import List, Optional
from pydantic import BaseModel, Field
from typing_extensions import TypedDict
class ServiceMetrics(enum.Enum):
COUNTER = "counter"
HISTOGRAM = "histogram"
GAUGE = "gauge"
class ServiceTypes(str, enum.Enum):
@@ -18,6 +25,84 @@ class ServiceTypes(str, enum.Enum):
ROUTER = "router"
AUTH = "auth"
PROXY_PRE_CALL = "proxy_pre_call"
POD_LOCK_MANAGER = "pod_lock_manager"
"""
Operational metrics for DB Transaction Queues
"""
# daily spend update queue - actual transaction events
IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE = "in_memory_daily_spend_update_queue"
REDIS_DAILY_SPEND_UPDATE_QUEUE = "redis_daily_spend_update_queue"
# spend update queue - current spend of key, user, team
IN_MEMORY_SPEND_UPDATE_QUEUE = "in_memory_spend_update_queue"
REDIS_SPEND_UPDATE_QUEUE = "redis_spend_update_queue"
class ServiceConfig(TypedDict):
"""
Configuration for services and their metrics
"""
metrics: List[ServiceMetrics] # What metrics this service should support
"""
Metric types to use for each service
- REDIS only needs Counter, Histogram
- Pod Lock Manager only needs a gauge metric
"""
DEFAULT_SERVICE_CONFIGS = {
ServiceTypes.REDIS.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
ServiceTypes.DB.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
ServiceTypes.BATCH_WRITE_TO_DB.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
ServiceTypes.RESET_BUDGET_JOB.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
ServiceTypes.LITELLM.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
ServiceTypes.ROUTER.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
ServiceTypes.AUTH.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
ServiceTypes.PROXY_PRE_CALL.value: {
"metrics": [ServiceMetrics.COUNTER, ServiceMetrics.HISTOGRAM]
},
# Operational metrics for DB Transaction Queues
ServiceTypes.POD_LOCK_MANAGER.value: {"metrics": [ServiceMetrics.GAUGE]},
ServiceTypes.IN_MEMORY_DAILY_SPEND_UPDATE_QUEUE.value: {
"metrics": [ServiceMetrics.GAUGE]
},
ServiceTypes.REDIS_DAILY_SPEND_UPDATE_QUEUE.value: {
"metrics": [ServiceMetrics.GAUGE]
},
ServiceTypes.IN_MEMORY_SPEND_UPDATE_QUEUE.value: {
"metrics": [ServiceMetrics.GAUGE]
},
ServiceTypes.REDIS_SPEND_UPDATE_QUEUE.value: {"metrics": [ServiceMetrics.GAUGE]},
}
class ServiceEventMetadata(TypedDict, total=False):
"""
The metadata logged during service success/failure
Add any extra fields you expect to access in the service_success_hook/service_failure_hook
"""
# Dynamically control gauge labels and values
gauge_labels: Optional[str]
gauge_value: Optional[float]
class ServiceLoggerPayload(BaseModel):
@@ -30,6 +115,9 @@ class ServiceLoggerPayload(BaseModel):
service: ServiceTypes = Field(description="who is this for? - postgres/redis")
duration: float = Field(description="How long did the request take?")
call_type: str = Field(description="The call of the service, being made")
event_metadata: Optional[dict] = Field(
description="The metadata logged during service success/failure"
)
def to_json(self, **kwargs):
try:
model_prices_and_context_window.json
+36
View File
@@ -88,6 +88,24 @@
"search_context_size_high": 0.050
}
},
"watsonx/ibm/granite-3-8b-instruct": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 1024,
"input_cost_per_token": 0.0002,
"output_cost_per_token": 0.0002,
"litellm_provider": "watsonx",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_vision": false,
"supports_audio_input": false,
"supports_audio_output": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"deprecation_date": null
},
"gpt-4o-search-preview-2025-03-11": {
"max_tokens": 16384,
"max_input_tokens": 128000,
@@ -3303,6 +3321,24 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"groq/whisper-large-v3": {
"mode": "audio_transcription",
"input_cost_per_second": 0.00003083,
"output_cost_per_second": 0,
"litellm_provider": "groq"
},
"groq/whisper-large-v3-turbo": {
"mode": "audio_transcription",
"input_cost_per_second": 0.00001111,
"output_cost_per_second": 0,
"litellm_provider": "groq"
},
"groq/distil-whisper-large-v3-en": {
"mode": "audio_transcription",
"input_cost_per_second": 0.00000556,
"output_cost_per_second": 0,
"litellm_provider": "groq"
},
"cerebras/llama3.1-8b": {
"max_tokens": 128000,
"max_input_tokens": 128000,
pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.65.3"
version = "1.65.4"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@@ -117,7 +117,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.65.3"
version = "1.65.4"
version_files = [
"pyproject.toml:^version"
]
tests/litellm/integrations/test_prometheus_services.py
+48
View File
@@ -0,0 +1,48 @@
import json
import os
import sys
from unittest.mock import AsyncMock, patch
import pytest
from fastapi.testclient import TestClient
from litellm.integrations.prometheus_services import (
PrometheusServicesLogger,
ServiceMetrics,
ServiceTypes,
)
sys.path.insert(
0, os.path.abspath("../../..")
) # Adds the parent directory to the system path
def test_create_gauge_new():
"""Test creating a new gauge"""
pl = PrometheusServicesLogger()
# Create new gauge
gauge = pl.create_gauge(service="test_service", type_of_request="size")
assert gauge is not None
assert pl._get_metric("litellm_test_service_size") is gauge
def test_update_gauge():
"""Test updating a gauge's value"""
pl = PrometheusServicesLogger()
# Create a gauge to test with
gauge = pl.create_gauge(service="test_service", type_of_request="size")
# Mock the labels method to verify it's called correctly
with patch.object(gauge, "labels") as mock_labels:
mock_gauge = AsyncMock()
mock_labels.return_value = mock_gauge
# Call update_gauge
pl.update_gauge(gauge=gauge, labels="test_label", amount=42.5)
# Verify correct methods were called
mock_labels.assert_called_once_with("test_label")
mock_gauge.set.assert_called_once_with(42.5)
tests/litellm/litellm_core_utils/test_streaming_handler.py
+1
View File
@@ -502,6 +502,7 @@ async def test_streaming_handler_with_usage(
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
@pytest.mark.flaky(reruns=3)
async def test_streaming_with_usage_and_logging(sync_mode: bool):
import time
tests/llm_translation/test_gemini.py
+12 -1
View File
@@ -11,7 +11,8 @@ from base_llm_unit_tests import BaseLLMChatTest
from litellm.llms.vertex_ai.context_caching.transformation import (
separate_cached_messages,
)
import litellm
from litellm import completion
class TestGoogleAIStudioGemini(BaseLLMChatTest):
def get_base_completion_call_args(self) -> dict:
@@ -72,3 +73,13 @@ def test_gemini_context_caching_separate_messages():
print(non_cached_messages)
assert len(cached_messages) > 0, "Cached messages should be present"
assert len(non_cached_messages) > 0, "Non-cached messages should be present"
def test_gemini_image_generation():
# litellm._turn_on_debug()
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
assert response.choices[0].message.content is not None
tests/llm_translation/test_max_completion_tokens.py
+1 -1
View File
@@ -330,7 +330,7 @@ def test_all_model_configs():
drop_params=False,
) == {"max_tokens_to_sample": 10}
from litellm.llms.databricks.chat.handler import DatabricksConfig
from litellm.llms.databricks.chat.transformation import DatabricksConfig
assert "max_completion_tokens" in DatabricksConfig().get_supported_openai_params()
tests/llm_translation/test_optional_params.py
+11 -1
View File
@@ -1405,6 +1405,17 @@ def test_azure_modalities_param():
assert optional_params["audio"] == {"type": "audio_input", "input": "test.wav"}
def test_gemini_modalities_param():
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="gemini",
modalities=["text", "image"],
)
assert optional_params["responseModalities"] == ["TEXT", "IMAGE"]
def test_azure_response_format_param():
optional_params = litellm.get_optional_params(
@@ -1430,4 +1441,3 @@ def test_anthropic_unified_reasoning_content(model, provider):
reasoning_effort="high",
)
assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096}
tests/llm_translation/test_xai.py
+18
View File
@@ -142,3 +142,21 @@ def test_completion_xai(stream):
assert response.choices[0].message.content is not None
except Exception as e:
pytest.fail(f"Error occurred: {e}")
def test_xai_message_name_filtering():
messages = [
{
"role": "system",
"content": "*I press the green button*",
"name": "example_user"
},
{"role": "user", "content": "Hello", "name": "John"},
{"role": "assistant", "content": "Hello", "name": "Jane"},
]
response = completion(
model="xai/grok-beta",
messages=messages,
)
assert response is not None
assert response.choices[0].message.content is not None
tests/router_unit_tests/test_router_adding_deployments.py
+36 -11
View File
@@ -9,23 +9,48 @@ from litellm.router import Deployment, LiteLLM_Params
from unittest.mock import patch
import json
def test_initialize_deployment_for_pass_through_success():
@pytest.mark.parametrize("reusable_credentials", [True, False])
def test_initialize_deployment_for_pass_through_success(reusable_credentials):
"""
Test successful initialization of a Vertex AI pass-through deployment
"""
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.types.utils import CredentialItem
vertex_project="test-project"
vertex_location="us-central1"
vertex_credentials=json.dumps({"type": "service_account", "project_id": "test"})
if not reusable_credentials:
litellm_params = LiteLLM_Params(
model="vertex_ai/test-model",
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_credentials=vertex_credentials,
use_in_pass_through=True,
)
else:
# add credentials to the credential accessor
CredentialAccessor.upsert_credentials([
CredentialItem(
credential_name="vertex_credentials",
credential_values={
"vertex_project": vertex_project,
"vertex_location": vertex_location,
"vertex_credentials": vertex_credentials,
},
credential_info={}
)
])
litellm_params = LiteLLM_Params(
model="vertex_ai/test-model",
litellm_credential_name="vertex_credentials",
use_in_pass_through=True,
)
router = Router(model_list=[])
deployment = Deployment(
model_name="vertex-test",
litellm_params=LiteLLM_Params(
model="vertex_ai/test-model",
vertex_project="test-project",
vertex_location="us-central1",
vertex_credentials=json.dumps(
{"type": "service_account", "project_id": "test"}
),
use_in_pass_through=True,
),
litellm_params=litellm_params,
)
# Test the initialization
ui/litellm-dashboard/src/components/model_dashboard.tsx
+3 -1
View File
@@ -1126,13 +1126,15 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
</div>
<ModelDataTable
columns={columns(
userRole,
userID,
premiumUser,
setSelectedModelId,
setSelectedTeamId,
getDisplayModelName,
handleEditClick,
handleRefreshClick,
setEditModel
setEditModel,
)}
data={modelData.data.filter(
(model: any) =>
ui/litellm-dashboard/src/components/model_dashboard/columns.tsx
+13 -4
View File
@@ -7,6 +7,8 @@ import { TrashIcon, PencilIcon, PencilAltIcon } from "@heroicons/react/outline";
import DeleteModelButton from "../delete_model_button";
export const columns = (
userRole: string,
userID: string,
premiumUser: boolean,
setSelectedModelId: (id: string) => void,
setSelectedTeamId: (id: string) => void,
@@ -226,23 +228,30 @@ export const columns = (
header: "",
cell: ({ row }) => {
const model = row.original;
const canEditModel = userRole === "Admin" || model.model_info?.created_by === userID;
return (
<div className="flex items-center justify-end gap-2 pr-4">
<Icon
icon={PencilAltIcon}
size="sm"
onClick={() => {
setSelectedModelId(model.model_info.id);
setEditModel(true);
if (canEditModel) {
setSelectedModelId(model.model_info.id);
setEditModel(true);
}
}}
className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer"}
/>
<Icon
icon={TrashIcon}
size="sm"
onClick={() => {
setSelectedModelId(model.model_info.id);
setEditModel(false);
if (canEditModel) {
setSelectedModelId(model.model_info.id);
setEditModel(false);
}
}}
className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer"}
/>
</div>
);