Merge branch 'BerriAI:main' into main

.github/scripts/scan_keywords.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import urllib.request
+import urllib.error
+
+
+def read_event_payload() -> dict:
+    event_path = os.environ.get("GITHUB_EVENT_PATH")
+    if not event_path or not os.path.exists(event_path):
+        return {}
+    with open(event_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
+    issue = event.get("issue") or {}
+    title = (issue.get("title") or "").strip()
+    body = (issue.get("body") or "").strip()
+    number = issue.get("number") or 0
+    html_url = issue.get("html_url") or ""
+    author = ((issue.get("user") or {}).get("login") or "").strip()
+    return title, body, number, html_url, author
+
+
+def detect_keywords(text: str, keywords: list[str]) -> list[str]:
+    lowered = text.lower()
+    matches = []
+    for keyword in keywords:
+        k = keyword.strip().lower()
+        if not k:
+            continue
+        if k in lowered:
+            matches.append(keyword.strip())
+    # Deduplicate while preserving order
+    seen = set()
+    unique_matches = []
+    for m in matches:
+        if m not in seen:
+            unique_matches.append(m)
+            seen.add(m)
+    return unique_matches
+
+
+def send_webhook(webhook_url: str, payload: dict) -> None:
+    if not webhook_url:
+        return
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(
+        webhook_url,
+        data=data,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            resp.read()
+    except urllib.error.HTTPError as e:
+        print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
+    except urllib.error.URLError as e:
+        print(f"Webhook URL error: {e.reason}", file=sys.stderr)
+    except Exception as e:
+        print(f"Webhook unexpected error: {e}", file=sys.stderr)
+
+
+def _excerpt(text: str, max_len: int = 400) -> str:
+    if not text:
+        return ""
+    
+    # Keep original formatting
+    if len(text) <= max_len:
+        return text
+    return text[: max_len - 1] + "…"
+
+
+
+def main() -> int:
+    event = read_event_payload()
+    if not event:
+        print("::warning::No event payload found; exiting without labeling.")
+        return 0
+
+    # Read issue details
+    title, body, number, html_url, author = get_issue_text(event)
+    combined_text = f"{title}\n\n{body}".strip()
+
+    # Keywords from env or defaults
+    keywords_env = os.environ.get("KEYWORDS", "")
+    default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
+    keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
+
+    matches = detect_keywords(combined_text, keywords)
+    found = bool(matches)
+
+    # Emit outputs
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as fh:
+            fh.write(f"found={'true' if found else 'false'}\n")
+            fh.write(f"matches={','.join(matches)}\n")
+
+    # Optional webhook notification
+    webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
+    if found and webhook_url:
+        repo_full = (event.get("repository") or {}).get("full_name", "")
+        title_part = f"*{title}*" if title else "New issue"
+        author_part = f" by @{author}" if author else ""
+        body_preview = _excerpt(body)
+        preview_block = f"\n{body_preview}" if body_preview else ""
+        payload = {
+            "text": (
+                f"New issue 🚨\n"
+                f"{title_part}\n\n{preview_block}\n"
+                f"<{html_url}|View issue>\n"
+                f"Author: {author}"
+            )
+        }
+        send_webhook(webhook_url, payload)
+
+    # Print a short log line for Actions UI
+    if found:
+        print(f"Detected provider keywords: {', '.join(matches)}")
+    else:
+        print("No provider keywords detected.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

.github/workflows/issue-keyword-labeler.yml

@@ -0,0 +1,64 @@
+name: Issue Keyword Labeler
+
+on:
+  issues:
+    types: 
+        - opened
+
+jobs:
+  scan-and-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Scan for provider keywords
+        id: scan
+        env:
+          PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
+          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
+        run: python3 .github/scripts/scan_keywords.py
+
+      - name: Ensure label exists
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const labelName = 'llm translation';
+            try {
+              await github.rest.issues.getLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: labelName
+              });
+            } catch (error) {
+              if (error.status === 404) {
+                await github.rest.issues.createLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: labelName,
+                  color: 'c1ff72',
+                  description: 'Issues related to LLM provider translation/mapping'
+                });
+              } else {
+                throw error;
+              }
+            }
+
+      - name: Add label to the issue
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: ['llm translation']
+            });
+

docs/my-website/docs/proxy/prod.md

@@ -199,7 +199,7 @@ USE_PRISMA_MIGRATE="True"
 <TabItem value="cli" label="CLI">
 
 ```bash
-litellm --use_prisma_migrate
+litellm
 ```
 
 </TabItem>

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -243,7 +243,6 @@ class ProxyExtrasDBManager:
             bool: True if setup was successful, False otherwise
         """
         schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
-        use_migrate = str_to_bool(os.getenv("USE_PRISMA_MIGRATE")) or use_migrate
         for attempt in range(4):
             original_dir = os.getcwd()
             migrations_dir = ProxyExtrasDBManager._get_prisma_dir()

litellm/litellm_core_utils/model_response_utils.py

@@ -0,0 +1,213 @@
+"""
+Utility functions for ModelResponse and ModelResponseStream objects.
+"""
+
+from typing import Any
+
+from litellm.types.utils import Delta, ModelResponseBase, ModelResponseStream
+
+
+def is_model_response_stream_empty(model_response: ModelResponseStream) -> bool:
+    """
+    Check if a ModelResponseStream is empty based on:
+    - If finish_reason is set -> it's non empty
+    - If any field in choices is set (e.g. content, tool calls, etc.) it's non empty
+    - If usage exists -> it's non empty
+
+    This function is robust and ignores fields that are always set (from ModelResponseBase)
+    and checks for any meaningful content in other fields.
+
+    Args:
+        model_response: The ModelResponseStream to check
+
+    Returns:
+        bool: True if the stream is empty, False if it contains meaningful data
+    """
+    # Fields that are always set in ModelResponseBase and should be ignored
+    # These are structural fields that don't indicate content
+    BASE_FIELDS = ModelResponseBase.model_fields.keys()
+
+    # Check if usage exists - this indicates meaningful data
+    if getattr(model_response, "usage", None) is not None:
+        return False
+
+    # Check provider_specific_fields at the top level
+    if (
+        hasattr(model_response, "provider_specific_fields")
+        and model_response.provider_specific_fields is not None
+        and model_response.provider_specific_fields != {}
+    ):
+        return False
+
+    # Check model_extra for dynamically added fields (this is where Pydantic stores them)
+    if hasattr(model_response, "model_extra") and model_response.model_extra:
+        for extra_field_name, extra_field_value in model_response.model_extra.items():
+            if _has_meaningful_content(extra_field_value):
+                return False
+
+    # Check for any non-base fields that are set
+    for model_response_field in model_response.model_fields.keys():
+        # Skip base fields that are always set
+        if model_response_field in BASE_FIELDS:
+            continue
+
+        # Skip choices - we'll handle them separately with deep inspection
+        if model_response_field == "choices":
+            continue
+
+        # Check if any other field has meaningful content
+        model_response_value = getattr(model_response, model_response_field, None)
+        if _has_meaningful_content(model_response_value):
+            return False
+
+    # Deep check of choices for any meaningful content
+    if hasattr(model_response, "choices") and model_response.choices:
+        for choice in model_response.choices:
+            if _is_choice_non_empty(choice):
+                return False
+
+    # If we get here, the stream is empty
+    return True
+
+
+def _has_meaningful_content(value: Any) -> bool:
+    """
+    Check if a value contains meaningful content.
+
+    Args:
+        value: The value to check
+
+    Returns:
+        bool: True if the value has meaningful content, False otherwise
+    """
+    if value is None:
+        return False
+
+    if isinstance(value, str):
+        return len(value.strip()) > 0
+
+    if isinstance(value, (list, dict)):
+        return len(value) > 0
+
+    if isinstance(value, bool):
+        return True  # Any boolean value is meaningful
+
+    if isinstance(value, (int, float)):
+        return True  # Any numeric value is meaningful
+
+    # For other types (objects), consider them meaningful if they exist
+    return True
+
+
+def _is_choice_non_empty(choice: Any) -> bool:
+    """
+    Deep check if a choice contains any meaningful content.
+
+    Args:
+        choice: The choice object to check
+
+    Returns:
+        bool: True if the choice has meaningful content, False otherwise
+    """
+    # Check finish_reason
+    if hasattr(choice, "finish_reason") and choice.finish_reason is not None:
+
+        return True
+
+    # Check logprobs
+    if hasattr(choice, "logprobs") and choice.logprobs is not None:
+
+        return True
+
+    # Check enhancements (if present)
+    if hasattr(choice, "enhancements") and choice.enhancements is not None:
+
+        return True
+
+    # Deep check delta object
+    if hasattr(choice, "delta") and choice.delta is not None:
+        if _is_delta_non_empty(choice.delta):
+
+            return True
+
+    # Check model_extra for dynamically added fields on the choice
+    if hasattr(choice, "model_extra") and choice.model_extra:
+        for extra_field_name, extra_field_value in choice.model_extra.items():
+            # Skip certain structural fields that are just default/None placeholders
+            if extra_field_name == "index" and extra_field_value == 0:
+
+                continue
+            if (
+                extra_field_name in {"finish_reason", "logprobs"}
+                and extra_field_value is None
+            ):
+
+                continue
+            if extra_field_name == "delta":
+
+                continue
+            if _has_meaningful_content(extra_field_value):
+
+                return True
+
+    # Check for any other non-standard fields on the choice
+    for attr_name in dir(choice):
+        # Skip private attributes, methods, and known empty fields
+        if (
+            attr_name.startswith("_")
+            or callable(getattr(choice, attr_name))
+            or attr_name.startswith("model_")
+            or attr_name
+            in {
+                "finish_reason",
+                "index",
+                "delta",
+                "logprobs",
+                "enhancements",
+            }
+        ):
+
+            continue
+
+        attr_value = getattr(choice, attr_name, None)
+        if _has_meaningful_content(attr_value):
+
+            return True
+
+    return False
+
+
+def _is_delta_non_empty(delta: Delta) -> bool:
+    """
+    Deep check if a delta object contains any meaningful content.
+
+    Args:
+        delta: The delta object to check
+
+    Returns:
+        bool: True if the delta has meaningful content, False otherwise
+    """
+    # Check model_extra for dynamically added fields (this is where Pydantic stores them)
+    if hasattr(delta, "model_extra") and delta.model_extra:
+        for extra_field_name, extra_field_value in delta.model_extra.items():
+            # Even structural fields are meaningful if they have actual content
+            if _has_meaningful_content(extra_field_value):
+
+                return True
+
+    # Check all regular attributes of the delta object
+    for attr_name in dir(delta):
+        # Skip private attributes, methods, and Pydantic-specific fields
+        if (
+            attr_name.startswith("_")
+            or callable(getattr(delta, attr_name))
+            or attr_name.startswith("model_")
+        ):
+            continue
+
+        attr_value = getattr(delta, attr_name, None)
+        if _has_meaningful_content(attr_value):
+
+            return True
+
+    return False

litellm/litellm_core_utils/streaming_handler.py

@@ -13,6 +13,9 @@ from pydantic import BaseModel
 
 import litellm
 from litellm import verbose_logger
+from litellm.litellm_core_utils.model_response_utils import (
+    is_model_response_stream_empty,
+)
 from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
 from litellm.litellm_core_utils.thread_pool_executor import executor
 from litellm.types.llms.openai import ChatCompletionChunk
@@ -1574,6 +1577,13 @@ class CustomStreamWrapper:
                         response = self.model_response_creator(
                             chunk=obj_dict, hidden_params=response._hidden_params
                         )
+                        ## check if empty
+                        is_empty = is_model_response_stream_empty(
+                            model_response=cast(ModelResponseStream, response)
+                        )
+
+                        if is_empty:
+                            continue
                     # add usage as hidden param
                     if self.sent_last_chunk is True and self.stream_options is None:
                         usage = calculate_total_usage(chunks=self.chunks)
@@ -1730,6 +1740,12 @@ class CustomStreamWrapper:
 
                         # Create a new object without the removed attribute
                         processed_chunk = self.model_response_creator(chunk=obj_dict)
+                        is_empty = is_model_response_stream_empty(
+                            model_response=cast(ModelResponseStream, processed_chunk)
+                        )
+
+                        if is_empty:
+                            continue
                     print_verbose(f"final returned processed chunk: {processed_chunk}")
                     return processed_chunk
                 raise StopAsyncIteration

litellm/llms/gemini/count_tokens/handler.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
 
 import httpx
 
@@ -12,21 +12,36 @@ else:
     GenerateContentContentListUnionDict = Any
 
 class GoogleAIStudioTokenCounter:
-    def validate_environment(
+
+    def _construct_url(self, model: str, api_base: Optional[str] = None) -> str:
+        """
+        Construct the URL for the Google Gen AI Studio countTokens endpoint.
+        """
+        base_url = api_base or "https://generativelanguage.googleapis.com"
+        return f"{base_url}/v1beta/models/{model}:countTokens"
+
+        
+    async def validate_environment(
         self,
+        api_base: Optional[str] = None,
         api_key: Optional[str] = None,
         headers: Optional[Dict[str, Any]] = None,
         model: str = "",
         litellm_params: Optional[Dict[str, Any]] = None,
-    ):
+    ) -> Tuple[Dict[str, Any], str]:
+        """
+        Returns a Tuple of headers and url for the Google Gen AI Studio countTokens endpoint.
+        """
         from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
-        return GoogleGenAIConfig().validate_environment(
+        headers =  GoogleGenAIConfig().validate_environment(
             api_key=api_key,
             headers=headers,
             model=model,
             litellm_params=litellm_params,
         )
 
+        url = self._construct_url(model=model, api_base=api_base)
+        return headers, url
 
     async def acount_tokens(
         self,
@@ -70,12 +85,11 @@ class GoogleAIStudioTokenCounter:
             Exception: For any other unexpected errors
         """
         # Set up API base URL
-        base_url = api_base or "https://generativelanguage.googleapis.com"
-        url = f"{base_url}/v1beta/models/{model}:countTokens"
-        
+
         # Prepare headers
-        headers = self.validate_environment(
+        headers, url = await self.validate_environment(
             api_key=api_key,
+            api_base=api_base,
             headers={},
             model=model,
             litellm_params=kwargs,
@@ -89,7 +103,7 @@ class GoogleAIStudioTokenCounter:
         async_httpx_client = get_async_httpx_client(
             llm_provider=LlmProviders.GEMINI,
         )
-        
+
         try:
             response = await async_httpx_client.post(
                 url=url,

litellm/llms/litellm_proxy/chat/transformation.py

@@ -4,6 +4,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
 
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
+from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
 from litellm.secret_managers.main import get_secret_bool, get_secret_str
 from litellm.types.router import LiteLLM_Params
 
@@ -16,8 +17,7 @@ if TYPE_CHECKING:
 class LiteLLMProxyChatConfig(OpenAIGPTConfig):
     def get_supported_openai_params(self, model: str) -> List:
         params_list = super().get_supported_openai_params(model)
-        params_list.append("thinking")
-        params_list.append("reasoning_effort")
+        params_list.extend(OPENAI_CHAT_COMPLETION_PARAMS)
         return params_list
 
     def _map_openai_params(

litellm/llms/vertex_ai/common_utils.py

@@ -7,8 +7,11 @@ import litellm
 from litellm import supports_response_schema, supports_system_messages, verbose_logger
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
 from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
+from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.openai import AllMessageValues
 from litellm.types.llms.vertex_ai import PartType, Schema
+from litellm.types.utils import TokenCountResponse
 
 
 class VertexAIError(BaseLLMException):
@@ -63,7 +66,7 @@ def get_supports_response_schema(
 from typing import Literal, Optional
 
 all_gemini_url_modes = Literal[
-    "chat", "embedding", "batch_embedding", "image_generation"
+    "chat", "embedding", "batch_embedding", "image_generation", "count_tokens"
 ]
 
 
@@ -113,6 +116,12 @@ def _get_vertex_url(
         url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
         if model.isdigit():
             url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
+    elif mode == "count_tokens":
+        endpoint = "countTokens"
+        if vertex_location == "global":
+            url = f"https://aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/global/publishers/google/models/{model}:{endpoint}"
+        else:
+            url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
     if not url or not endpoint:
         raise ValueError(f"Unable to get vertex url/endpoint for mode: {mode}")
     return url, endpoint
@@ -148,10 +157,17 @@ def _get_gemini_url(
         url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format(
             _gemini_model_name, endpoint, gemini_api_key
         )
+    elif mode == "count_tokens":
+        endpoint = "countTokens"
+        url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format(
+            _gemini_model_name, endpoint, gemini_api_key
+        )
     elif mode == "image_generation":
         raise ValueError(
             "LiteLLM's `gemini/` route does not support image generation yet. Let us know if you need this feature by opening an issue at https://github.com/BerriAI/litellm/issues"
         )
+    else:
+        raise ValueError(f"Unsupported mode: {mode}")
 
     return url, endpoint
 
@@ -522,3 +538,99 @@ def is_global_only_vertex_model(model: str) -> bool:
     if supported_regions is None:
         return False
     return "global" in supported_regions
+
+class VertexAIModelInfo(BaseLLMModelInfo):    
+    def get_token_counter(self) -> Optional[BaseTokenCounter]:
+        """
+        Factory method to create a token counter for this provider.
+        
+        Returns:
+            Optional TokenCounterInterface implementation for this provider,
+            or None if token counting is not supported.
+        """
+        return VertexAITokenCounter()
+    
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        raise NotImplementedError("Vertex AI models are not supported yet")
+    
+    def get_models(
+        self, api_key: Optional[str] = None, api_base: Optional[str] = None
+    ) -> List[str]:
+        """
+        Returns a list of models supported by this provider.
+        """
+        raise NotImplementedError("Vertex AI models are not supported yet")
+
+    @staticmethod
+    def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+        raise NotImplementedError("Vertex AI models are not supported yet")
+
+    @staticmethod
+    def get_api_base(
+        api_base: Optional[str] = None,
+    ) -> Optional[str]:
+        raise NotImplementedError("Vertex AI models are not supported yet")
+
+
+
+    @staticmethod
+    def get_base_model(model: str) -> Optional[str]:
+        """
+        Returns the base model name from the given model name.
+
+        Some providers like bedrock - can receive model=`invoke/anthropic.claude-3-opus-20240229-v1:0` or `converse/anthropic.claude-3-opus-20240229-v1:0`
+            This function will return `anthropic.claude-3-opus-20240229-v1:0`
+        """
+        raise NotImplementedError("Vertex AI models are not supported yet")
+
+
+class VertexAITokenCounter(BaseTokenCounter):
+    """Token counter implementation for Google AI Studio provider."""
+    def should_use_token_counting_api(
+        self, 
+        custom_llm_provider: Optional[str] = None,
+    ) -> bool:
+        from litellm.types.utils import LlmProviders
+        return custom_llm_provider == LlmProviders.VERTEX_AI.value
+    
+    async def count_tokens(
+        self,
+        model_to_use: str,
+        messages: Optional[List[Dict[str, Any]]],
+        contents: Optional[List[Dict[str, Any]]],
+        deployment: Optional[Dict[str, Any]] = None,
+        request_model: str = "",
+    ) -> Optional[TokenCountResponse]:
+        import copy
+
+        from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
+        deployment = deployment or {}
+        count_tokens_params_request = copy.deepcopy(deployment.get("litellm_params", {}))
+        count_tokens_params = {
+            "model": model_to_use,
+            "contents": contents,
+        }
+        count_tokens_params_request.update(count_tokens_params)
+        result = await VertexAITokenCounter().acount_tokens(
+            **count_tokens_params_request,
+        )
+        
+        if result is not None:
+            return TokenCountResponse(
+                total_tokens=result.get("totalTokens", 0),
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type=result.get("tokenizer_used", ""),
+                original_response=result,
+            )
+        
+        return None

litellm/llms/vertex_ai/count_tokens/handler.py

@@ -0,0 +1,46 @@
+from typing import Any, Dict, Optional, Tuple
+
+from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+
+
+class VertexAITokenCounter(GoogleAIStudioTokenCounter, VertexBase):
+    async def validate_environment(
+        self,
+        api_base: Optional[str] = None,
+        api_key: Optional[str] = None,
+        headers: Optional[Dict[str, Any]] = None,
+        model: str = "",
+        litellm_params: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict[str, Any], str]:
+        """
+        Returns a Tuple of headers and url for the Vertex AI countTokens endpoint.
+        """
+        litellm_params = litellm_params or {}
+        vertex_credentials = self.get_vertex_ai_credentials(litellm_params=litellm_params)
+        vertex_project = self.get_vertex_ai_project(litellm_params=litellm_params)
+        vertex_location = self.get_vertex_ai_location(litellm_params=litellm_params)
+        should_use_v1beta1_features = self.is_using_v1beta1_features(litellm_params)
+        _auth_header, vertex_project = await self._ensure_access_token_async(
+            credentials=vertex_credentials,
+            project_id=vertex_project,
+            custom_llm_provider="vertex_ai",
+        )
+
+        auth_header, api_base = self._get_token_and_url(
+            model=model,
+            gemini_api_key=None,
+            auth_header=_auth_header,
+            vertex_project=vertex_project,
+            vertex_location=vertex_location,
+            vertex_credentials=vertex_credentials,
+            stream=False,
+            custom_llm_provider="vertex_ai",
+            api_base=None,
+            should_use_v1beta1_features=should_use_v1beta1_features,
+            mode="count_tokens",
+        )
+        headers = {
+            "Authorization": f"Bearer {auth_header}",
+        }
+        return headers, api_base

litellm/proxy/_new_secret_config.yaml

@@ -6,6 +6,7 @@ model_list:
     api_base: https://exampleopenaiendpoint-production.up.railway.app/
 
 litellm_settings:
+  callbacks: ["otel"]
   cache: true
   cache_params:
     type: redis

litellm/proxy/db/prisma_client.py

@@ -87,12 +87,11 @@ class PrismaWrapper:
         self, new_db_url: str, http_client: Optional[Any] = None
     ):
         from prisma import Prisma  # type: ignore
+
         try:
             await self._original_prisma.disconnect()
         except Exception as e:
-            verbose_proxy_logger.warning(
-                f"Failed to disconnect Prisma client: {e}"
-            )
+            verbose_proxy_logger.warning(f"Failed to disconnect Prisma client: {e}")
 
         if http_client is not None:
             self._original_prisma = Prisma(http=http_client)
@@ -139,7 +138,6 @@ class PrismaManager:
             bool: True if setup was successful, False otherwise
         """
 
-        use_migrate = str_to_bool(os.getenv("USE_PRISMA_MIGRATE")) or use_migrate
         for attempt in range(4):
             original_dir = os.getcwd()
             prisma_dir = PrismaManager._get_prisma_dir()
@@ -185,7 +183,7 @@ class PrismaManager:
 
 
 def should_update_prisma_schema(
-    disable_updates: Optional[Union[bool, str]] = None
+    disable_updates: Optional[Union[bool, str]] = None,
 ) -> bool:
     """
     Determines if Prisma Schema updates should be applied during startup.

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -901,12 +901,14 @@ async def update_group(
 
         # Update team in database
         existing_metadata = existing_team.metadata if existing_team.metadata else {}
+        updated_metadata = {**existing_metadata, "scim_data": group.model_dump()}
+        
         updated_team = await prisma_client.db.litellm_teamtable.update(
             where={"team_id": group_id},
             data={
                 "team_alias": group.displayName,
                 "members": member_ids,
-                "metadata": {**existing_metadata, "scim_data": group.model_dump()},
+                "metadata": safe_dumps(updated_metadata),
             },
         )
 

litellm/proxy/prisma_migration.py

@@ -13,7 +13,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.proxy.proxy_cli import run_server
 
 # Call the Click command with standalone_mode=False
-run_server(["--use_prisma_migrate", "--skip_server_startup"], standalone_mode=False)
+run_server(["--skip_server_startup"], standalone_mode=False)
 
 # run prisma generate
 verbose_proxy_logger.info("Running 'prisma generate'...")

litellm/proxy/proxy_cli.py

@@ -465,10 +465,10 @@ class ProxyInitializationHelpers:
     help="Ciphers to use for the SSL setup.",
 )
 @click.option(
-    "--use_prisma_migrate",
+    "--use_prisma_db_push",
     is_flag=True,
-    default=True,
-    help="Use prisma migrate instead of prisma db push for database schema updates",
+    default=False,
+    help="Use prisma db push instead of prisma migrate for database schema updates",
 )
 @click.option("--local", is_flag=True, default=False, help="for local debugging")
 @click.option(
@@ -519,7 +519,7 @@ def run_server(  # noqa: PLR0915
     ssl_certfile_path,
     ciphers,
     log_config,
-    use_prisma_migrate,
+    use_prisma_db_push: bool,
     skip_server_startup,
     keepalive_timeout,
 ):
@@ -777,7 +777,7 @@ def run_server(  # noqa: PLR0915
                 ):
                     check_prisma_schema_diff(db_url=None)
                 else:
-                    PrismaManager.setup_database(use_migrate=use_prisma_migrate)
+                    PrismaManager.setup_database(use_migrate=not use_prisma_db_push)
             else:
                 print(  # noqa
                     f"Unable to connect to DB. DATABASE_URL found in environment, but prisma package not found."  # noqa

litellm/proxy/proxy_config.yaml

@@ -1,4 +1,4 @@
 model_list:
   - model_name: vertex_ai/*
     litellm_params:
-      model: gemini/*
+      model: vertex_ai/*

litellm/router.py

@@ -4351,16 +4351,22 @@ class Router:
                     tpm_model_info = deployment_model_info.get("tpm", None)
                     rpm_model_info = deployment_model_info.get("rpm", None)
 
-                    ## if all are none, return - no need to track current tpm/rpm usage for models with no tpm/rpm set
-                    if (
-                        tpm is None
-                        and rpm is None
-                        and tpm_litellm_params is None
-                        and rpm_litellm_params is None
-                        and tpm_model_info is None
-                        and rpm_model_info is None
-                    ):
-                        return
+                # Always track deployment successes for cooldown logic, regardless of TPM/RPM limits
+                increment_deployment_successes_for_current_minute(
+                    litellm_router_instance=self,
+                    deployment_id=id,
+                )
+
+                ## if all are none, return - no need to track current tpm/rpm usage for models with no tpm/rpm set
+                if (
+                    tpm is None
+                    and rpm is None
+                    and tpm_litellm_params is None
+                    and rpm_litellm_params is None
+                    and tpm_model_info is None
+                    and rpm_model_info is None
+                ):
+                    return
 
                 parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs)
                 total_tokens: float = standard_logging_object.get("total_tokens", 0)
@@ -4409,11 +4415,6 @@ class Router:
                     parent_otel_span=parent_otel_span,
                 )
 
-                increment_deployment_successes_for_current_minute(
-                    litellm_router_instance=self,
-                    deployment_id=id,
-                )
-
                 return tpm_key
 
         except Exception as e:

litellm/utils.py

@@ -7140,6 +7140,9 @@ class ProviderConfigManager:
             return litellm.OpenAIGPTConfig()
         elif LlmProviders.GEMINI == provider:
             return litellm.GeminiModelInfo()
+        elif LlmProviders.VERTEX_AI == provider:
+            from litellm.llms.vertex_ai.common_utils import VertexAIModelInfo
+            return VertexAIModelInfo()
         elif LlmProviders.LITELLM_PROXY == provider:
             return litellm.LiteLLMProxyChatConfig()
         elif LlmProviders.TOPAZ == provider:

pyproject.toml

@@ -139,6 +139,7 @@ opentelemetry-api = "1.25.0"
 opentelemetry-sdk = "1.25.0"
 opentelemetry-exporter-otlp = "1.25.0"
 langfuse = "^2.45.0"
+fastapi-offline = "^1.7.3"
 
 [tool.poetry.group.proxy-dev.dependencies]
 prisma = "0.11.0"

tests/llm_translation/test_litellm_proxy_provider.py

@@ -452,7 +452,7 @@ def test_litellm_gateway_from_sdk_with_response_cost_in_additional_headers():
 
 
 def test_litellm_gateway_from_sdk_with_thinking_param():
-    try: 
+    try:
         response = litellm.completion(
             model="litellm_proxy/anthropic.claude-3-7-sonnet-20250219-v1:0",
             messages=[{"role": "user", "content": "Hello world"}],
@@ -464,4 +464,3 @@ def test_litellm_gateway_from_sdk_with_thinking_param():
         pytest.fail("Expected an error to be raised")
     except Exception as e:
         assert "Connection error." in str(e)
-    

tests/local_testing/test_router_cooldowns.py

@@ -22,7 +22,10 @@ import openai
 import litellm
 from litellm import Router
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments, _should_run_cooldown_logic
+from litellm.router_utils.cooldown_handlers import (
+    _async_get_cooldown_deployments,
+    _should_run_cooldown_logic,
+)
 from litellm.types.router import (
     DeploymentTypedDict,
     LiteLLMParamsTypedDict,
@@ -148,7 +151,9 @@ async def test_cooldown_time_zero_uses_zero_not_default():
     )
 
     # Mock the add_deployment_to_cooldown method to verify it's NOT called
-    with patch.object(router.cooldown_cache, "add_deployment_to_cooldown") as mock_add_cooldown:
+    with patch.object(
+        router.cooldown_cache, "add_deployment_to_cooldown"
+    ) as mock_add_cooldown:
         try:
             await router.acompletion(
                 model="gpt-3.5-turbo",
@@ -160,13 +165,13 @@ async def test_cooldown_time_zero_uses_zero_not_default():
 
         # Verify that add_deployment_to_cooldown was NOT called due to early exit
         mock_add_cooldown.assert_not_called()
-    
+
     # Also verify the deployment is not in cooldown
     cooldown_list = await _async_get_cooldown_deployments(
         litellm_router_instance=router, parent_otel_span=None
     )
     assert len(cooldown_list) == 0
-    
+
     # Verify the deployment is still healthy and available
     healthy_deployments, _ = await router._async_get_healthy_deployments(
         model="gpt-3.5-turbo", parent_otel_span=None
@@ -192,44 +197,54 @@ def test_should_run_cooldown_logic_early_exit_on_zero_cooldown():
         ],
         cooldown_time=300,
     )
-    
+
     # Test with time_to_cooldown = 0 - should return False (don't run cooldown logic)
     result = _should_run_cooldown_logic(
         litellm_router_instance=router,
         deployment="test-deployment-id",
         exception_status=429,
-        original_exception=litellm.RateLimitError("test error", "openai", "gpt-3.5-turbo"),
-        time_to_cooldown=0.0
+        original_exception=litellm.RateLimitError(
+            "test error", "openai", "gpt-3.5-turbo"
+        ),
+        time_to_cooldown=0.0,
     )
     assert result is False, "Should not run cooldown logic when time_to_cooldown is 0"
-    
+
     # Test with very small time_to_cooldown (effectively 0) - should return False
     result = _should_run_cooldown_logic(
         litellm_router_instance=router,
         deployment="test-deployment-id",
         exception_status=429,
-        original_exception=litellm.RateLimitError("test error", "openai", "gpt-3.5-turbo"),
-        time_to_cooldown=1e-10
+        original_exception=litellm.RateLimitError(
+            "test error", "openai", "gpt-3.5-turbo"
+        ),
+        time_to_cooldown=1e-10,
     )
-    assert result is False, "Should not run cooldown logic when time_to_cooldown is effectively 0"
-    
+    assert (
+        result is False
+    ), "Should not run cooldown logic when time_to_cooldown is effectively 0"
+
     # Test with None time_to_cooldown - should return True (use default cooldown logic)
     result = _should_run_cooldown_logic(
         litellm_router_instance=router,
-        deployment="test-deployment-id", 
+        deployment="test-deployment-id",
         exception_status=429,
-        original_exception=litellm.RateLimitError("test error", "openai", "gpt-3.5-turbo"),
-        time_to_cooldown=None
+        original_exception=litellm.RateLimitError(
+            "test error", "openai", "gpt-3.5-turbo"
+        ),
+        time_to_cooldown=None,
     )
     assert result is True, "Should run cooldown logic when time_to_cooldown is None"
-    
+
     # Test with positive time_to_cooldown - should return True
     result = _should_run_cooldown_logic(
         litellm_router_instance=router,
         deployment="test-deployment-id",
         exception_status=429,
-        original_exception=litellm.RateLimitError("test error", "openai", "gpt-3.5-turbo"),
-        time_to_cooldown=60.0
+        original_exception=litellm.RateLimitError(
+            "test error", "openai", "gpt-3.5-turbo"
+        ),
+        time_to_cooldown=60.0,
     )
     assert result is True, "Should run cooldown logic when time_to_cooldown is positive"
 

tests/local_testing/test_streaming.py

@@ -476,6 +476,7 @@ def test_completion_azure_stream():
 async def test_completion_predibase_streaming(sync_mode):
     try:
         litellm.set_verbose = True
+        litellm._turn_on_debug()
         if sync_mode:
             response = completion(
                 model="predibase/llama-3-8b-instruct",
@@ -701,7 +702,12 @@ async def test_completion_gemini_stream(sync_mode):
                 },
             }
         ]
-        messages = [{"role": "user", "content": "What is the weather like in Boston, MA?. You must provide me with a tool call in your response."}]
+        messages = [
+            {
+                "role": "user",
+                "content": "What is the weather like in Boston, MA?. You must provide me with a tool call in your response.",
+            }
+        ]
         print("testing gemini streaming")
         complete_response = ""
         # Add any assertions here to check the response
@@ -817,7 +823,12 @@ async def test_completion_gemini_stream_accumulated_json(sync_mode):
                 },
             }
         ]
-        messages = [{"role": "user", "content": "What is the weather like in Boston, MA?. You must provide me with a tool call in your response."}]
+        messages = [
+            {
+                "role": "user",
+                "content": "What is the weather like in Boston, MA?. You must provide me with a tool call in your response.",
+            }
+        ]
         print("testing gemini streaming")
         complete_response = ""
         # Add any assertions here to check the response
@@ -3990,3 +4001,48 @@ def test_streaming_with_cost_calculation():
     assert usage_object.prompt_tokens > 0
     assert usage_object.cost is not None
     assert usage_object.cost > 0
+
+
+def test_streaming_finish_reason():
+    litellm.set_verbose = False
+
+    openai_finish_reason_idx: Optional[int] = None
+    openai_last_chunk_idx: Optional[int] = None
+    anthropic_finish_reason_idx: Optional[int] = None
+    anthropic_last_chunk_idx: Optional[int] = None
+
+    ## OpenAI
+    response = litellm.completion(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": "What is the capital of France?"}],
+        stream=True,
+        stream_options={"include_usage": True},
+    )
+    for idx, chunk in enumerate(response):
+        print(f"OPENAI CHUNK: {chunk}")
+        if chunk.choices[0].finish_reason is not None:
+            openai_finish_reason_idx = idx
+        openai_last_chunk_idx = idx
+
+    assert openai_finish_reason_idx is not None
+    assert openai_finish_reason_idx > 0
+
+    ## Anthropic
+    response = litellm.completion(
+        model="anthropic/claude-3-5-sonnet-latest",
+        messages=[{"role": "user", "content": "What is the capital of France?"}],
+        stream=True,
+        stream_options={"include_usage": True},
+    )
+    for idx, chunk in enumerate(response):
+        print(f"ANTHROPIC CHUNK: {chunk}")
+        if chunk.choices[0].finish_reason is not None:
+            anthropic_finish_reason_idx = idx
+        anthropic_last_chunk_idx = idx
+
+    assert anthropic_finish_reason_idx is not None
+    assert anthropic_finish_reason_idx > 0
+
+    relative_anthropic_idx = anthropic_finish_reason_idx - anthropic_last_chunk_idx
+    relative_openai_idx = openai_finish_reason_idx - openai_last_chunk_idx
+    assert relative_anthropic_idx == relative_openai_idx

tests/proxy_unit_tests/test_proxy_token_counter.py

@@ -26,11 +26,86 @@ verbose_proxy_logger.setLevel(level=logging.DEBUG)
 
 from litellm.proxy._types import TokenCountRequest
 from litellm.types.utils import TokenCountResponse
+import json, tempfile
 
 
 from litellm import Router
 
 
+def get_vertex_ai_creds_json() -> dict:
+    # Define the path to the vertex_key.json file
+    print("loading vertex ai credentials")
+    filepath = os.path.dirname(os.path.abspath(__file__))
+    vertex_key_path = filepath + "/vertex_key.json"
+    # Read the existing content of the file or create an empty dictionary
+    try:
+        with open(vertex_key_path, "r") as file:
+            # Read the file content
+            print("Read vertexai file path")
+            content = file.read()
+
+            # If the file is empty or not valid JSON, create an empty dictionary
+            if not content or not content.strip():
+                service_account_key_data = {}
+            else:
+                # Attempt to load the existing JSON content
+                file.seek(0)
+                service_account_key_data = json.load(file)
+    except FileNotFoundError:
+        # If the file doesn't exist, create an empty dictionary
+        service_account_key_data = {}
+
+    # Update the service_account_key_data with environment variables
+    private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
+    private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
+    private_key = private_key.replace("\\n", "\n")
+    service_account_key_data["private_key_id"] = private_key_id
+    service_account_key_data["private_key"] = private_key
+
+    return service_account_key_data
+
+
+def load_vertex_ai_credentials():
+    # Define the path to the vertex_key.json file
+    print("loading vertex ai credentials")
+    filepath = os.path.dirname(os.path.abspath(__file__))
+    vertex_key_path = filepath + "/vertex_key.json"
+
+    # Read the existing content of the file or create an empty dictionary
+    try:
+        with open(vertex_key_path, "r") as file:
+            # Read the file content
+            print("Read vertexai file path")
+            content = file.read()
+
+            # If the file is empty or not valid JSON, create an empty dictionary
+            if not content or not content.strip():
+                service_account_key_data = {}
+            else:
+                # Attempt to load the existing JSON content
+                file.seek(0)
+                service_account_key_data = json.load(file)
+    except FileNotFoundError:
+        # If the file doesn't exist, create an empty dictionary
+        service_account_key_data = {}
+
+    # Update the service_account_key_data with environment variables
+    private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
+    private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
+    private_key = private_key.replace("\\n", "\n")
+    service_account_key_data["private_key_id"] = private_key_id
+    service_account_key_data["private_key"] = private_key
+
+    # Create a temporary file
+    with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
+        # Write the updated content to the temporary files
+        json.dump(service_account_key_data, temp_file, indent=2)
+
+    # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS
+    os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
+
+
+
 @pytest.mark.asyncio
 async def test_vLLM_token_counting():
     """
@@ -558,10 +633,12 @@ async def test_factory_registration():
 
 
 @pytest.mark.asyncio
-async def test_vertex_ai_gemini_token_counting_with_contents():
+@pytest.mark.parametrize("model_name", ["gemini-2.5-pro", "vertex-ai-gemini-2.5-pro"])
+async def test_vertex_ai_gemini_token_counting_with_contents(model_name):
     """
     Test token counting for Vertex AI Gemini model using contents format with call_endpoint=True
     """
+    load_vertex_ai_credentials()
     llm_router = Router(
         model_list=[
             {
@@ -569,7 +646,13 @@ async def test_vertex_ai_gemini_token_counting_with_contents():
                 "litellm_params": {
                     "model": "gemini/gemini-2.5-pro",
                 },
-            }
+            },
+            {
+                "model_name": "vertex-ai-gemini-2.5-pro",
+                "litellm_params": {
+                    "model": "vertex_ai/gemini-2.5-pro",
+                },
+            },
         ]
     )
     
@@ -578,7 +661,7 @@ async def test_vertex_ai_gemini_token_counting_with_contents():
     # Test with contents format and call_endpoint=True
     response = await token_counter(
         request=TokenCountRequest(
-            model="gemini-2.5-pro",
+            model=model_name,
             contents=[
                 {
                     "parts": [

tests/test_litellm/litellm_core_utils/test_model_response_utils.py

@@ -0,0 +1,60 @@
+from litellm.litellm_core_utils.model_response_utils import (
+    is_model_response_stream_empty,
+)
+from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
+
+
+def test_is_model_response_stream_empty():
+    chunk = ModelResponseStream(
+        id="chatcmpl-C3sWKN2RWbn6CZ1IGU2QCpRh4RhYf",
+        created=1755040596,
+        model="gpt-4o-mini",
+        object="chat.completion.chunk",
+        system_fingerprint="fp_34a54ae93c",
+        choices=[
+            StreamingChoices(
+                finish_reason=None,
+                index=0,
+                delta=Delta(
+                    provider_specific_fields=None,
+                    content=None,
+                    role=None,
+                    function_call=None,
+                    tool_calls=None,
+                    audio=None,
+                ),
+                logprobs=None,
+            )
+        ],
+        provider_specific_fields=None,
+    )
+    assert is_model_response_stream_empty(chunk) is True
+
+
+def test_is_model_response_stream_empty_with_custom_value():
+    chunk = ModelResponseStream(
+        id="chatcmpl-C3sWKN2RWbn6CZ1IGU2QCpRh4RhYf",
+        created=1755040596,
+        model="gpt-4o-mini",
+        object="chat.completion.chunk",
+        system_fingerprint="fp_34a54ae93c",
+        choices=[
+            StreamingChoices(
+                finish_reason=None,
+                index=0,
+                delta=Delta(
+                    provider_specific_fields=None,
+                    content=None,
+                    role=None,
+                    function_call=None,
+                    tool_calls=None,
+                    audio=None,
+                ),
+                logprobs=None,
+            )
+        ],
+        provider_specific_fields=None,
+    )
+
+    setattr(chunk.choices[0].delta, "custom_field", "test")
+    assert is_model_response_stream_empty(chunk) is False

tests/test_litellm/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py

@@ -30,3 +30,13 @@ def test_litellm_proxy_chat_transformation():
         litellm_params={},
         headers={},
     ) == {"model": "model", "messages": messages}
+
+
+def test_litellm_gateway_from_sdk_with_user_param():
+    from litellm.llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig
+
+    supported_params = LiteLLMProxyChatConfig().get_supported_openai_params(
+        "openai/gpt-4o"
+    )
+    print(f"supported_params: {supported_params}")
+    assert "user" in supported_params

tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py

@@ -579,3 +579,103 @@ async def test_get_service_provider_config(mocker):
     assert result.bulk.supported is False
     assert result.meta is not None
     assert result.meta["resourceType"] == "ServiceProviderConfig"
+
+
+@pytest.mark.asyncio
+async def test_update_group_metadata_serialization_issue(mocker):
+    """
+    Test that update_group properly serializes metadata to avoid Prisma DataError.
+    
+    This test reproduces the issue where metadata was passed as a dict instead of
+    a JSON string, causing: "Invalid argument type. `metadata` should be of any 
+    of the following types: `JsonNullValueInput`, `Json`"
+    """
+    from litellm.proxy.management_endpoints.scim.scim_v2 import update_group
+    from litellm.types.proxy.management_endpoints.scim_v2 import SCIMGroup, SCIMMember
+
+    # Create test data
+    group_id = "test-group-id"
+    scim_group = SCIMGroup(
+        schemas=["urn:ietf:params:scim:schemas:core:2.0:Group"],
+        id=group_id,
+        displayName="Test Group",
+        members=[SCIMMember(value="user1", display="User One")]
+    )
+    
+    # Mock existing team with metadata
+    mock_existing_team = mocker.MagicMock()
+    mock_existing_team.team_id = group_id
+    mock_existing_team.team_alias = "Old Group Name"
+    mock_existing_team.members = ["user1"]
+    mock_existing_team.metadata = {"existing_key": "existing_value"}
+    mock_existing_team.created_at = None
+    mock_existing_team.updated_at = None
+    
+    # Mock updated team response
+    mock_updated_team = mocker.MagicMock()
+    mock_updated_team.team_id = group_id
+    mock_updated_team.team_alias = "Test Group"
+    mock_updated_team.members = ["user1"]
+    mock_updated_team.created_at = None
+    mock_updated_team.updated_at = None
+    
+    # Create a properly structured mock for the prisma client
+    mock_prisma_client = mocker.MagicMock()
+    mock_prisma_client.db = mocker.MagicMock()
+    mock_prisma_client.db.litellm_teamtable = mocker.MagicMock()
+    mock_prisma_client.db.litellm_usertable = mocker.MagicMock()
+    
+    # Mock team operations
+    mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=mock_existing_team)
+    mock_prisma_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_updated_team)
+    
+    # Mock user operations
+    mock_user = mocker.MagicMock()
+    mock_user.user_id = "user1"
+    mock_user.user_email = "user1@example.com"  # Add proper string value for user_email
+    mock_user.teams = [group_id]
+    mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=mock_user)
+    mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value=mock_user)
+    
+    # Mock the _get_prisma_client_or_raise_exception to return our mock
+    mocker.patch(
+        "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception",
+        AsyncMock(return_value=mock_prisma_client),
+    )
+    
+    # Mock the transformation function
+    mock_scim_group_response = SCIMGroup(
+        schemas=["urn:ietf:params:scim:schemas:core:2.0:Group"],
+        id=group_id,
+        displayName="Test Group",
+        members=[SCIMMember(value="user1", display="User One")]
+    )
+    mocker.patch(
+        "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_team_to_scim_group",
+        AsyncMock(return_value=mock_scim_group_response),
+    )
+    
+    # Call the function that had the bug
+    result = await update_group(group_id=group_id, group=scim_group)
+    
+    # Verify the team update was called
+    mock_prisma_client.db.litellm_teamtable.update.assert_called_once()
+    
+    # Get the call arguments to verify metadata serialization
+    call_args = mock_prisma_client.db.litellm_teamtable.update.call_args
+    update_data = call_args[1]["data"]
+    
+    # Verify that metadata is properly serialized as a string, not a dict
+    # This is the critical check that would have caught the original bug
+    assert "metadata" in update_data
+    metadata = update_data["metadata"]
+    
+    # The fix should ensure metadata is serialized as a JSON string
+    assert isinstance(metadata, str), f"metadata should be a JSON string, but got {type(metadata)}"
+    
+    # Verify we can parse it back to verify it contains the expected data
+    import json
+    parsed_metadata = json.loads(metadata)
+    assert "existing_key" in parsed_metadata
+    assert "scim_data" in parsed_metadata
+    assert parsed_metadata["existing_key"] == "existing_value"

tests/test_litellm/proxy/test_proxy_cli.py

@@ -2,18 +2,19 @@ import os
 import sys
 from unittest.mock import MagicMock, patch
 
-import pytest
 import fastapi
+import pytest
 
 sys.path.insert(
     0, os.path.abspath("../../..")
 )  # Adds the parent directory to the system-path
 
-from litellm.proxy.proxy_cli import ProxyInitializationHelpers
-from litellm.proxy.health_endpoints.health_app_factory import build_health_app
 import builtins
 import types
 
+from litellm.proxy.health_endpoints.health_app_factory import build_health_app
+from litellm.proxy.proxy_cli import ProxyInitializationHelpers
+
 
 class TestProxyInitializationHelpers:
     @patch("importlib.metadata.version")
@@ -308,7 +309,7 @@ class TestProxyInitializationHelpers:
                 keepalive_timeout=30,
             )
             mock_uvicorn_run.assert_called_once()
-            
+
             # Check that the uvicorn.run was called with the timeout_keep_alive parameter
             call_args = mock_uvicorn_run.call_args
             assert call_args[1]["timeout_keep_alive"] == 30
@@ -376,10 +377,12 @@ class TestProxyInitializationHelpers:
     @patch("builtins.print")
     def test_run_server_no_config_passed(self, mock_print, mock_uvicorn_run):
         """Test that run_server properly handles the case when no config is passed"""
-        from click.testing import CliRunner
-        from litellm.proxy.proxy_cli import run_server
         import asyncio
 
+        from click.testing import CliRunner
+
+        from litellm.proxy.proxy_cli import run_server
+
         runner = CliRunner()
 
         mock_app = MagicMock()
@@ -389,11 +392,8 @@ class TestProxyInitializationHelpers:
 
         # Mock the ProxyConfig.get_config method to return a proper async config
         async def mock_get_config(config_file_path=None):
-            return {
-                "general_settings": {},
-                "litellm_settings": {}
-            }
-        
+            return {"general_settings": {}, "litellm_settings": {}}
+
         mock_proxy_config_instance = MagicMock()
         mock_proxy_config_instance.get_config = mock_get_config
         mock_proxy_config.return_value = mock_proxy_config_instance
@@ -423,7 +423,7 @@ class TestProxyInitializationHelpers:
                 result = runner.invoke(run_server, ["--local"])
 
                 assert result.exit_code == 0
-                
+
                 # Verify that uvicorn.run was called
                 mock_uvicorn_run.assert_called_once()
 
@@ -434,34 +434,34 @@ class TestProxyInitializationHelpers:
                 result = runner.invoke(run_server, ["--local", "--config", "None"])
 
                 assert result.exit_code == 0
-                
+
                 # Verify that uvicorn.run was called again
                 mock_uvicorn_run.assert_called_once()
 
 
 class TestHealthAppFactory:
     """Test cases for the health app factory module"""
-    
+
     def test_build_health_app(self):
         """Test that build_health_app creates a FastAPI app with the correct title and includes the health router"""
         # Execute
         health_app = build_health_app()
-        
+
         # Assert
         assert health_app.title == "LiteLLM Health Endpoints"
         assert isinstance(health_app, fastapi.FastAPI)
-        
+
         # Verify that the app has the expected health endpoints by checking route paths
         # When a router is included, its routes are flattened into the main app's routes
         route_paths = []
         for route in health_app.routes:
-            if hasattr(route, 'path'):
+            if hasattr(route, "path"):
                 route_paths.append(route.path)
-        
+
         # Check for some expected health endpoints
         expected_paths = [
             "/test",
-            "/health/services", 
+            "/health/services",
             "/health",
             "/health/history",
             "/health/latest",
@@ -470,24 +470,100 @@ class TestHealthAppFactory:
             "/health/readiness",
             "/health/liveliness",
             "/health/liveness",
-            "/health/test_connection"
+            "/health/test_connection",
         ]
-        
+
         # At least some of the expected health endpoints should be present
         found_paths = [path for path in expected_paths if path in route_paths]
-        assert len(found_paths) > 0, f"Expected to find health endpoints, but found: {route_paths}"
-        
+        assert (
+            len(found_paths) > 0
+        ), f"Expected to find health endpoints, but found: {route_paths}"
+
         # Verify that the app has routes (indicating the router was included)
-        assert len(health_app.routes) > 0, "Health app should have routes from the included router"
-        
+        assert (
+            len(health_app.routes) > 0
+        ), "Health app should have routes from the included router"
+
     def test_build_health_app_returns_different_instances(self):
         """Test that build_health_app returns different FastAPI instances on each call"""
         # Execute
         health_app_1 = build_health_app()
         health_app_2 = build_health_app()
-        
+
         # Assert
         assert health_app_1 is not health_app_2
         assert health_app_1.title == health_app_2.title
         assert isinstance(health_app_1, fastapi.FastAPI)
         assert isinstance(health_app_2, fastapi.FastAPI)
+
+    @patch("subprocess.run")
+    @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database")
+    @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff")
+    @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema")
+    @patch.dict(
+        os.environ, {"DATABASE_URL": "postgresql://test:test@localhost:5432/test"}
+    )
+    def test_use_prisma_db_push_flag_behavior(
+        self,
+        mock_should_update_schema,
+        mock_check_schema_diff,
+        mock_setup_database,
+        mock_subprocess_run,
+    ):
+        """Test that use_prisma_db_push flag correctly controls PrismaManager.setup_database use_migrate parameter"""
+        from click.testing import CliRunner
+
+        from litellm.proxy.proxy_cli import run_server
+
+        runner = CliRunner()
+
+        # Mock subprocess.run to simulate prisma being available
+        mock_subprocess_run.return_value = MagicMock(returncode=0)
+
+        # Mock should_update_prisma_schema to return True (so setup_database gets called)
+        mock_should_update_schema.return_value = True
+
+        mock_app = MagicMock()
+        mock_proxy_config = MagicMock()
+        mock_key_mgmt = MagicMock()
+        mock_save_worker_config = MagicMock()
+
+        with patch.dict(
+            "sys.modules",
+            {
+                "proxy_server": MagicMock(
+                    app=mock_app,
+                    ProxyConfig=mock_proxy_config,
+                    KeyManagementSettings=mock_key_mgmt,
+                    save_worker_config=mock_save_worker_config,
+                )
+            },
+        ), patch(
+            "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args"
+        ) as mock_get_args:
+            mock_get_args.return_value = {
+                "app": "litellm.proxy.proxy_server:app",
+                "host": "localhost",
+                "port": 8000,
+            }
+
+            # Test 1: Without --use_prisma_db_push flag (default behavior)
+            # use_prisma_db_push should be False (default), so use_migrate should be True
+            result = runner.invoke(run_server, ["--local", "--skip_server_startup"])
+
+            assert result.exit_code == 0
+            mock_setup_database.assert_called_with(use_migrate=True)
+
+            # Reset mocks
+            mock_setup_database.reset_mock()
+            mock_should_update_schema.reset_mock()
+            mock_should_update_schema.return_value = True
+
+            # Test 2: With --use_prisma_db_push flag set
+            # use_prisma_db_push should be True, so use_migrate should be False
+            result = runner.invoke(
+                run_server, ["--local", "--skip_server_startup", "--use_prisma_db_push"]
+            )
+
+            assert result.exit_code == 0
+            mock_setup_database.assert_called_with(use_migrate=False)

ui/litellm-dashboard/src/components/model_add/CredentialDeleteModal.tsx

@@ -0,0 +1,99 @@
+import React, { useState } from "react";
+import { Modal } from "antd";
+import { Button as TremorButton } from "@tremor/react";
+import { ExclamationIcon } from "@heroicons/react/outline";
+
+interface CredentialDeleteModalProps {
+  isVisible: boolean;
+  onCancel: () => void;
+  onConfirm: () => void;
+  credentialName: string;
+}
+
+const CredentialDeleteModal: React.FC<CredentialDeleteModalProps> = ({
+  isVisible,
+  onCancel,
+  onConfirm,
+  credentialName,
+}) => {
+  const [deleteConfirmInput, setDeleteConfirmInput] = useState("");
+  const isValid = deleteConfirmInput === credentialName;
+
+  const handleCancel = () => {
+    setDeleteConfirmInput("");
+    onCancel();
+  };
+
+  const handleConfirm = () => {
+    if (isValid) {
+      setDeleteConfirmInput("");
+      onConfirm();
+    }
+  };
+
+  return (
+    <Modal
+      title={
+        <div className="flex items-center">
+          <ExclamationIcon className="h-6 w-6 text-red-600 mr-2" />
+          Delete Credential
+        </div>
+      }
+      open={isVisible}
+      footer={null}
+      onCancel={handleCancel}
+      closable={true}
+      destroyOnClose={true}
+      maskClosable={false}
+    >
+      <div className="mt-4">
+        <div className="flex items-start gap-3 p-4 bg-red-50 border border-red-100 rounded-md mb-5">
+          <div className="text-red-500 mt-0.5">
+            <ExclamationIcon className="h-5 w-5" />
+          </div>
+          <div>
+            <p className="text-base font-medium text-red-600">
+              This action cannot be undone and may break existing integrations.
+            </p>
+          </div>
+        </div>
+        
+        <div className="mb-5">
+          <label className="block text-base font-medium text-gray-700 mb-2">
+            {`Type `}
+            <span className="underline italic">&apos;{credentialName}&apos;</span>
+            {` to confirm deletion:`}
+          </label>
+          <input
+            type="text"
+            value={deleteConfirmInput}
+            onChange={(e) => setDeleteConfirmInput(e.target.value)}
+            placeholder="Enter credential name exactly"
+            className="w-full px-4 py-3 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 text-base"
+            autoFocus
+          />
+        </div>
+        
+        <div className="flex justify-end space-x-2">
+          <TremorButton 
+            onClick={handleCancel} 
+            variant="secondary"
+            className="mr-2"
+          >
+            Cancel
+          </TremorButton>
+          <TremorButton
+            onClick={handleConfirm}
+            color="red"
+            className="focus:ring-red-500"
+            disabled={!isValid}
+          >
+            Delete Credential
+          </TremorButton>
+        </div>
+      </div>
+    </Modal>
+  );
+};
+
+export default CredentialDeleteModal;

ui/litellm-dashboard/src/components/model_add/credentials.tsx

@@ -23,6 +23,7 @@ import { UploadProps } from "antd/es/upload";
 import { PlusIcon } from "@heroicons/react/solid";
 import { credentialListCall, credentialCreateCall, credentialDeleteCall, credentialUpdateCall, CredentialItem, CredentialsResponse } from "@/components/networking"; // Assume this is your networking function
 import AddCredentialsTab from "./add_credentials_tab";
+import CredentialDeleteModal from "./CredentialDeleteModal";
 import { Form, message } from "antd";
 interface CredentialsPanelProps {
   accessToken: string | null;
@@ -37,6 +38,7 @@ const CredentialsPanel: React.FC<CredentialsPanelProps> = ({ accessToken, upload
   const [isAddModalOpen, setIsAddModalOpen] = useState(false);
   const [isUpdateModalOpen, setIsUpdateModalOpen] = useState(false);
   const [selectedCredential, setSelectedCredential] = useState<CredentialItem | null>(null);
+  const [credentialToDelete, setCredentialToDelete] = useState<string | null>(null);
   const [form] = Form.useForm();
 
   const restrictedFields = ['credential_name', 'custom_llm_provider'];
@@ -119,9 +121,18 @@ const CredentialsPanel: React.FC<CredentialsPanelProps> = ({ accessToken, upload
     }
     const response = await credentialDeleteCall(accessToken, credentialName);
     message.success('Credential deleted successfully');
+    setCredentialToDelete(null);
     fetchCredentials(accessToken);
   };
 
+  const openDeleteModal = (credentialName: string) => {
+    setCredentialToDelete(credentialName);
+  };
+
+  const closeDeleteModal = () => {
+    setCredentialToDelete(null);
+  };
+
   return (
     <div className="w-full mx-auto flex-auto overflow-y-auto m-8 p-2">
       <div className="flex justify-between items-center mb-4">
@@ -168,7 +179,7 @@ const CredentialsPanel: React.FC<CredentialsPanelProps> = ({ accessToken, upload
                       icon={TrashIcon}
                       variant="light"
                       size="sm"
-                      onClick={() => handleDeleteCredential(credential.credential_name)}
+                      onClick={() => openDeleteModal(credential.credential_name)}
                     />
                   </TableCell>
                 </TableRow>
@@ -208,6 +219,15 @@ const CredentialsPanel: React.FC<CredentialsPanelProps> = ({ accessToken, upload
           addOrEdit="edit"
         />
       )}
+      
+      {credentialToDelete && (
+        <CredentialDeleteModal
+          isVisible={true}
+          onCancel={closeDeleteModal}
+          onConfirm={() => handleDeleteCredential(credentialToDelete)}
+          credentialName={credentialToDelete}
+        />
+      )}
     </div>
   );
 };