Merge branch 'main' into litellm_metrics_pod_lock_manager

.circleci/requirements.txt

@@ -10,6 +10,6 @@ anthropic
 orjson==3.9.15
 pydantic==2.10.2
 google-cloud-aiplatform==1.43.0
-fastapi-sso==0.10.0
+fastapi-sso==0.16.0 
 uvloop==0.21.0
 mcp==1.5.0    # for MCP server  

docs/my-website/docs/completion/document_understanding.md

@@ -80,11 +80,13 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 -d '{
     "model": "bedrock-model",
     "messages": [
-        {"role": "user", "content": {"type": "text", "text": "What's this file about?"}},
-        {
-            "type": "image_url",
-            "image_url": "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
-        }
+        {"role": "user", "content": [
+            {"type": "text", "text": "What's this file about?"},
+            {
+                "type": "image_url",
+                "image_url": "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
+            }
+        ]},
     ]
 }'
 ```
@@ -135,6 +137,46 @@ response = completion(
 assert response is not None
 ```
 </TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: bedrock-model
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: os.environ/AWS_REGION_NAME
+```
+
+2. Start the proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+    "model": "bedrock-model",
+    "messages": [
+        {"role": "user", "content": [
+            {"type": "text", "text": "What's this file about?"},
+            {
+                "type": "image_url",
+                "image_url": "data:application/pdf;base64...",
+            }
+        ]},
+    ]
+}'
+```
+</TabItem>
 </Tabs>
 
 ## Checking if a model supports pdf input

docs/my-website/docs/proxy/db_deadlocks.md

@@ -31,6 +31,15 @@ Each instance writes updates to redis
 
 A single instance will acquire a lock on the DB and flush all elements in the redis queue to the DB. 
 
+- 1 instance will attempt to acquire the lock for the DB update job 
+- The status of the lock is stored in redis
+- If the instance acquires the lock to write to DB
+    - It will read all updates from redis
+    - Aggregate all updates into 1 transaction
+    - Write updates to DB
+    - Release the lock
+- Note: Only 1 instance can acquire the lock at a time, this limits the number of instances that can write to the DB at once
+
 
 <Image img={require('../../img/deadlock_fix_2.png')}  style={{ width: '900px', height: 'auto' }} />
 <p style={{textAlign: 'left', color: '#666'}}>

litellm/files/main.py

@@ -63,16 +63,17 @@ async def acreate_file(
         loop = asyncio.get_event_loop()
         kwargs["acreate_file"] = True
 
-        # Use a partial function to pass your keyword arguments
-        func = partial(
-            create_file,
-            file,
-            purpose,
-            custom_llm_provider,
-            extra_headers,
-            extra_body,
+        call_args = {
+            "file": file,
+            "purpose": purpose,
+            "custom_llm_provider": custom_llm_provider,
+            "extra_headers": extra_headers,
+            "extra_body": extra_body,
             **kwargs,
-        )
+        }
+
+        # Use a partial function to pass your keyword arguments
+        func = partial(create_file, **call_args)
 
         # Add the context to the function
         ctx = contextvars.copy_context()
@@ -92,7 +93,7 @@ async def acreate_file(
 def create_file(
     file: FileTypes,
     purpose: Literal["assistants", "batch", "fine-tune"],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai"]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -101,6 +102,8 @@ def create_file(
     Files are used to upload documents that can be used with features like Assistants, Fine-tuning, and Batch API.
 
     LiteLLM Equivalent of POST: POST https://api.openai.com/v1/files
+
+    Specify either provider_list or custom_llm_provider.
     """
     try:
         _is_async = kwargs.pop("acreate_file", False) is True
@@ -120,7 +123,7 @@ def create_file(
         if (
             timeout is not None
             and isinstance(timeout, httpx.Timeout)
-            and supports_httpx_timeout(custom_llm_provider) is False
+            and supports_httpx_timeout(cast(str, custom_llm_provider)) is False
         ):
             read_timeout = timeout.read or 600
             timeout = read_timeout  # default 10 min timeout

litellm/litellm_core_utils/litellm_logging.py

@@ -457,8 +457,12 @@ class Logging(LiteLLMLoggingBaseClass):
         non_default_params: dict,
         prompt_id: str,
         prompt_variables: Optional[dict],
+        prompt_management_logger: Optional[CustomLogger] = None,
     ) -> Tuple[str, List[AllMessageValues], dict]:
-        custom_logger = self.get_custom_logger_for_prompt_management(model)
+        custom_logger = (
+            prompt_management_logger
+            or self.get_custom_logger_for_prompt_management(model)
+        )
         if custom_logger:
             (
                 model,

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -7,6 +7,7 @@ from typing import Dict, List, Literal, Optional, Union, cast
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionAssistantMessage,
+    ChatCompletionFileObject,
     ChatCompletionUserMessage,
 )
 from litellm.types.utils import Choices, ModelResponse, StreamingChoices
@@ -292,3 +293,58 @@ def get_completion_messages(
         messages, assistant_continue_message, ensure_alternating_roles
     )
     return messages
+
+
+def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]:
+    """
+    Gets file ids from messages
+    """
+    file_ids = []
+    for message in messages:
+        if message.get("role") == "user":
+            content = message.get("content")
+            if content:
+                if isinstance(content, str):
+                    continue
+                for c in content:
+                    if c["type"] == "file":
+                        file_object = cast(ChatCompletionFileObject, c)
+                        file_object_file_field = file_object["file"]
+                        file_id = file_object_file_field.get("file_id")
+                        if file_id:
+                            file_ids.append(file_id)
+    return file_ids
+
+
+def update_messages_with_model_file_ids(
+    messages: List[AllMessageValues],
+    model_id: str,
+    model_file_id_mapping: Dict[str, Dict[str, str]],
+) -> List[AllMessageValues]:
+    """
+    Updates messages with model file ids.
+
+    model_file_id_mapping: Dict[str, Dict[str, str]] = {
+        "litellm_proxy/file_id": {
+            "model_id": "provider_file_id"
+        }
+    }
+    """
+    for message in messages:
+        if message.get("role") == "user":
+            content = message.get("content")
+            if content:
+                if isinstance(content, str):
+                    continue
+                for c in content:
+                    if c["type"] == "file":
+                        file_object = cast(ChatCompletionFileObject, c)
+                        file_object_file_field = file_object["file"]
+                        file_id = file_object_file_field.get("file_id")
+                        if file_id:
+                            provider_file_id = (
+                                model_file_id_mapping.get(file_id, {}).get(model_id)
+                                or file_id
+                            )
+                            file_object_file_field["file_id"] = provider_file_id
+    return messages

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1300,20 +1300,37 @@ def convert_to_anthropic_tool_invoke(
       ]
     }
     """
-    anthropic_tool_invoke = [
-        AnthropicMessagesToolUseParam(
+    anthropic_tool_invoke = []
+
+    for tool in tool_calls:
+        if not get_attribute_or_key(tool, "type") == "function":
+            continue
+
+        _anthropic_tool_use_param = AnthropicMessagesToolUseParam(
             type="tool_use",
-            id=get_attribute_or_key(tool, "id"),
-            name=get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"),
+            id=cast(str, get_attribute_or_key(tool, "id")),
+            name=cast(
+                str,
+                get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"),
+            ),
             input=json.loads(
                 get_attribute_or_key(
                     get_attribute_or_key(tool, "function"), "arguments"
                 )
             ),
         )
-        for tool in tool_calls
-        if get_attribute_or_key(tool, "type") == "function"
-    ]
+
+        _content_element = add_cache_control_to_content(
+            anthropic_content_element=_anthropic_tool_use_param,
+            orignal_content_element=dict(tool),
+        )
+
+        if "cache_control" in _content_element:
+            _anthropic_tool_use_param["cache_control"] = _content_element[
+                "cache_control"
+            ]
+
+        anthropic_tool_invoke.append(_anthropic_tool_use_param)
 
     return anthropic_tool_invoke
 
@@ -1324,6 +1341,7 @@ def add_cache_control_to_content(
         AnthropicMessagesImageParam,
         AnthropicMessagesTextParam,
         AnthropicMessagesDocumentParam,
+        AnthropicMessagesToolUseParam,
         ChatCompletionThinkingBlock,
     ],
     orignal_content_element: Union[dict, AllMessageValues],

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -1,6 +1,6 @@
 import base64
 import time
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, cast
 
 from litellm.types.llms.openai import (
     ChatCompletionAssistantContentValue,
@@ -9,7 +9,9 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import (
     ChatCompletionAudioResponse,
     ChatCompletionMessageToolCall,
+    Choices,
     CompletionTokensDetails,
+    CompletionTokensDetailsWrapper,
     Function,
     FunctionCall,
     ModelResponse,
@@ -203,14 +205,14 @@ class ChunkProcessor:
         )
 
     def get_combined_content(
-        self, chunks: List[Dict[str, Any]]
+        self, chunks: List[Dict[str, Any]], delta_key: str = "content"
     ) -> ChatCompletionAssistantContentValue:
         content_list: List[str] = []
         for chunk in chunks:
             choices = chunk["choices"]
             for choice in choices:
                 delta = choice.get("delta", {})
-                content = delta.get("content", "")
+                content = delta.get(delta_key, "")
                 if content is None:
                     continue  # openai v1.0.0 sets content = None for chunks
                 content_list.append(content)
@@ -221,6 +223,11 @@ class ChunkProcessor:
         # Update the "content" field within the response dictionary
         return combined_content
 
+    def get_combined_reasoning_content(
+        self, chunks: List[Dict[str, Any]]
+    ) -> ChatCompletionAssistantContentValue:
+        return self.get_combined_content(chunks, delta_key="reasoning_content")
+
     def get_combined_audio_content(
         self, chunks: List[Dict[str, Any]]
     ) -> ChatCompletionAudioResponse:
@@ -296,12 +303,27 @@ class ChunkProcessor:
             "prompt_tokens_details": prompt_tokens_details,
         }
 
+    def count_reasoning_tokens(self, response: ModelResponse) -> int:
+        reasoning_tokens = 0
+        for choice in response.choices:
+            if (
+                hasattr(cast(Choices, choice).message, "reasoning_content")
+                and cast(Choices, choice).message.reasoning_content is not None
+            ):
+                reasoning_tokens += token_counter(
+                    text=cast(Choices, choice).message.reasoning_content,
+                    count_response_tokens=True,
+                )
+
+        return reasoning_tokens
+
     def calculate_usage(
         self,
         chunks: List[Union[Dict[str, Any], ModelResponse]],
         model: str,
         completion_output: str,
         messages: Optional[List] = None,
+        reasoning_tokens: Optional[int] = None,
     ) -> Usage:
         """
         Calculate usage for the given chunks.
@@ -382,6 +404,19 @@ class ChunkProcessor:
             )  # for anthropic
         if completion_tokens_details is not None:
             returned_usage.completion_tokens_details = completion_tokens_details
+
+        if reasoning_tokens is not None:
+            if returned_usage.completion_tokens_details is None:
+                returned_usage.completion_tokens_details = (
+                    CompletionTokensDetailsWrapper(reasoning_tokens=reasoning_tokens)
+                )
+            elif (
+                returned_usage.completion_tokens_details is not None
+                and returned_usage.completion_tokens_details.reasoning_tokens is None
+            ):
+                returned_usage.completion_tokens_details.reasoning_tokens = (
+                    reasoning_tokens
+                )
         if prompt_tokens_details is not None:
             returned_usage.prompt_tokens_details = prompt_tokens_details
 

litellm/llms/anthropic/chat/handler.py

@@ -21,7 +21,6 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
 )
 from litellm.types.llms.anthropic import (
-    AnthropicChatCompletionUsageBlock,
     ContentBlockDelta,
     ContentBlockStart,
     ContentBlockStop,
@@ -32,13 +31,13 @@ from litellm.types.llms.anthropic import (
 from litellm.types.llms.openai import (
     ChatCompletionThinkingBlock,
     ChatCompletionToolCallChunk,
-    ChatCompletionUsageBlock,
 )
 from litellm.types.utils import (
     Delta,
     GenericStreamingChunk,
     ModelResponseStream,
     StreamingChoices,
+    Usage,
 )
 from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
 
@@ -487,10 +486,8 @@ class ModelResponseIterator:
             return True
         return False
 
-    def _handle_usage(
-        self, anthropic_usage_chunk: Union[dict, UsageDelta]
-    ) -> AnthropicChatCompletionUsageBlock:
-        usage_block = AnthropicChatCompletionUsageBlock(
+    def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
+        usage_block = Usage(
             prompt_tokens=anthropic_usage_chunk.get("input_tokens", 0),
             completion_tokens=anthropic_usage_chunk.get("output_tokens", 0),
             total_tokens=anthropic_usage_chunk.get("input_tokens", 0)
@@ -581,7 +578,7 @@ class ModelResponseIterator:
             text = ""
             tool_use: Optional[ChatCompletionToolCallChunk] = None
             finish_reason = ""
-            usage: Optional[ChatCompletionUsageBlock] = None
+            usage: Optional[Usage] = None
             provider_specific_fields: Dict[str, Any] = {}
             reasoning_content: Optional[str] = None
             thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None

litellm/llms/anthropic/chat/transformation.py

@@ -33,9 +33,16 @@ from litellm.types.llms.openai import (
     ChatCompletionToolCallFunctionChunk,
     ChatCompletionToolParam,
 )
+from litellm.types.utils import CompletionTokensDetailsWrapper
 from litellm.types.utils import Message as LitellmMessage
 from litellm.types.utils import PromptTokensDetailsWrapper
-from litellm.utils import ModelResponse, Usage, add_dummy_tool, has_tool_call_blocks
+from litellm.utils import (
+    ModelResponse,
+    Usage,
+    add_dummy_tool,
+    has_tool_call_blocks,
+    token_counter,
+)
 
 from ..common_utils import AnthropicError, process_anthropic_headers
 
@@ -772,6 +779,15 @@ class AnthropicConfig(BaseConfig):
         prompt_tokens_details = PromptTokensDetailsWrapper(
             cached_tokens=cache_read_input_tokens
         )
+        completion_token_details = (
+            CompletionTokensDetailsWrapper(
+                reasoning_tokens=token_counter(
+                    text=reasoning_content, count_response_tokens=True
+                )
+            )
+            if reasoning_content
+            else None
+        )
         total_tokens = prompt_tokens + completion_tokens
         usage = Usage(
             prompt_tokens=prompt_tokens,
@@ -780,6 +796,7 @@ class AnthropicConfig(BaseConfig):
             prompt_tokens_details=prompt_tokens_details,
             cache_creation_input_tokens=cache_creation_input_tokens,
             cache_read_input_tokens=cache_read_input_tokens,
+            completion_tokens_details=completion_token_details,
         )
 
         setattr(model_response, "usage", usage)  # type: ignore

litellm/llms/azure/files/handler.py

@@ -28,11 +28,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         self,
         create_file_data: CreateFileRequest,
         openai_client: AsyncAzureOpenAI,
-    ) -> FileObject:
+    ) -> OpenAIFileObject:
         verbose_logger.debug("create_file_data=%s", create_file_data)
         response = await openai_client.files.create(**create_file_data)
         verbose_logger.debug("create_file_response=%s", response)
-        return response
+        return OpenAIFileObject(**response.model_dump())
 
     def create_file(
         self,
@@ -66,7 +66,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
-            return self.acreate_file(  # type: ignore
+            return self.acreate_file(
                 create_file_data=create_file_data, openai_client=openai_client
             )
         response = cast(AzureOpenAI, openai_client).files.create(**create_file_data)

litellm/llms/openrouter/chat/transformation.py

@@ -1,17 +1,18 @@
 """
-Support for OpenAI's `/v1/chat/completions` endpoint. 
+Support for OpenAI's `/v1/chat/completions` endpoint.
 
 Calls done in OpenAI/openai.py as OpenRouter is openai-compatible.
 
 Docs: https://openrouter.ai/docs/parameters
 """
 
-from typing import Any, AsyncIterator, Iterator, Optional, Union
+from typing import Any, AsyncIterator, Iterator, List, Optional, Union
 
 import httpx
 
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.openai import AllMessageValues
 from litellm.types.llms.openrouter import OpenRouterErrorMessage
 from litellm.types.utils import ModelResponse, ModelResponseStream
 
@@ -47,6 +48,27 @@ class OpenrouterConfig(OpenAIGPTConfig):
         ] = extra_body  # openai client supports `extra_body` param
         return mapped_openai_params
 
+    def transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Transform the overall request to be sent to the API.
+
+        Returns:
+            dict: The transformed request. Sent as the body of the API call.
+        """
+        extra_body = optional_params.pop("extra_body", {})
+        response = super().transform_request(
+            model, messages, optional_params, litellm_params, headers
+        )
+        response.update(extra_body)
+        return response
+
     def get_error_class(
         self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
     ) -> BaseLLMException:

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -676,6 +676,66 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         return usage
 
+    def _process_candidates(self, _candidates, model_response, litellm_params):
+        """Helper method to process candidates and extract metadata"""
+        grounding_metadata: List[dict] = []
+        safety_ratings: List = []
+        citation_metadata: List = []
+        chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
+        chat_completion_logprobs: Optional[ChoiceLogprobs] = None
+        tools: Optional[List[ChatCompletionToolCallChunk]] = []
+        functions: Optional[ChatCompletionToolCallFunctionChunk] = None
+        
+        for idx, candidate in enumerate(_candidates):
+            if "content" not in candidate:
+                continue
+
+            if "groundingMetadata" in candidate:
+                grounding_metadata.append(candidate["groundingMetadata"])  # type: ignore
+
+            if "safetyRatings" in candidate:
+                safety_ratings.append(candidate["safetyRatings"])
+
+            if "citationMetadata" in candidate:
+                citation_metadata.append(candidate["citationMetadata"])
+                
+            if "parts" in candidate["content"]:
+                chat_completion_message["content"] = VertexGeminiConfig().get_assistant_content_message(
+                    parts=candidate["content"]["parts"]
+                )
+
+                functions, tools = self._transform_parts(
+                    parts=candidate["content"]["parts"],
+                    index=candidate.get("index", idx),
+                    is_function_call=litellm_params.get("litellm_param_is_function_call"),
+                )
+
+            if "logprobsResult" in candidate:
+                chat_completion_logprobs = self._transform_logprobs(
+                    logprobs_result=candidate["logprobsResult"]
+                )
+            # Handle avgLogprobs for Gemini Flash 2.0
+            elif "avgLogprobs" in candidate:
+                chat_completion_logprobs = candidate["avgLogprobs"]
+
+            if tools:
+                chat_completion_message["tool_calls"] = tools
+
+            if functions is not None:
+                chat_completion_message["function_call"] = functions
+                
+            choice = litellm.Choices(
+                finish_reason=candidate.get("finishReason", "stop"),
+                index=candidate.get("index", idx),
+                message=chat_completion_message,  # type: ignore
+                logprobs=chat_completion_logprobs,
+                enhancements=None,
+            )
+
+            model_response.choices.append(choice)
+            
+        return grounding_metadata, safety_ratings, citation_metadata
+
     def transform_response(
         self,
         model: str,
@@ -725,9 +785,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         _candidates = completion_response.get("candidates")
         if _candidates and len(_candidates) > 0:
-            content_policy_violations = (
-                VertexGeminiConfig().get_flagged_finish_reasons()
-            )
+            content_policy_violations = VertexGeminiConfig().get_flagged_finish_reasons()
             if (
                 "finishReason" in _candidates[0]
                 and _candidates[0]["finishReason"] in content_policy_violations.keys()
@@ -740,88 +798,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         model_response.choices = []  # type: ignore
 
         try:
-            ## CHECK IF GROUNDING METADATA IN REQUEST
-            grounding_metadata: List[dict] = []
-            safety_ratings: List = []
-            citation_metadata: List = []
-            ## GET TEXT ##
-            chat_completion_message: ChatCompletionResponseMessage = {
-                "role": "assistant"
-            }
-            chat_completion_logprobs: Optional[ChoiceLogprobs] = None
-            tools: Optional[List[ChatCompletionToolCallChunk]] = []
-            functions: Optional[ChatCompletionToolCallFunctionChunk] = None
+            grounding_metadata, safety_ratings, citation_metadata = [], [], []
             if _candidates:
-                for idx, candidate in enumerate(_candidates):
-                    if "content" not in candidate:
-                        continue
-
-                    if "groundingMetadata" in candidate:
-                        grounding_metadata.append(candidate["groundingMetadata"])  # type: ignore
-
-                    if "safetyRatings" in candidate:
-                        safety_ratings.append(candidate["safetyRatings"])
-
-                    if "citationMetadata" in candidate:
-                        citation_metadata.append(candidate["citationMetadata"])
-                    if "parts" in candidate["content"]:
-                        chat_completion_message[
-                            "content"
-                        ] = VertexGeminiConfig().get_assistant_content_message(
-                            parts=candidate["content"]["parts"]
-                        )
-
-                        functions, tools = self._transform_parts(
-                            parts=candidate["content"]["parts"],
-                            index=candidate.get("index", idx),
-                            is_function_call=litellm_params.get(
-                                "litellm_param_is_function_call"
-                            ),
-                        )
-
-                    if "logprobsResult" in candidate:
-                        chat_completion_logprobs = self._transform_logprobs(
-                            logprobs_result=candidate["logprobsResult"]
-                        )
-
-                    if tools:
-                        chat_completion_message["tool_calls"] = tools
-
-                    if functions is not None:
-                        chat_completion_message["function_call"] = functions
-                    choice = litellm.Choices(
-                        finish_reason=candidate.get("finishReason", "stop"),
-                        index=candidate.get("index", idx),
-                        message=chat_completion_message,  # type: ignore
-                        logprobs=chat_completion_logprobs,
-                        enhancements=None,
-                    )
-
-                    model_response.choices.append(choice)
+                grounding_metadata, safety_ratings, citation_metadata = self._process_candidates(
+                    _candidates, model_response, litellm_params
+                )
 
             usage = self._calculate_usage(completion_response=completion_response)
-
             setattr(model_response, "usage", usage)
 
-            ## ADD GROUNDING METADATA ##
+            ## ADD METADATA TO RESPONSE ##
             setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
-            model_response._hidden_params[
-                "vertex_ai_grounding_metadata"
-            ] = (  # older approach - maintaining to prevent regressions
-                grounding_metadata
-            )
-
-            ## ADD SAFETY RATINGS ##
+            model_response._hidden_params["vertex_ai_grounding_metadata"] = grounding_metadata
+            
             setattr(model_response, "vertex_ai_safety_results", safety_ratings)
-            model_response._hidden_params[
-                "vertex_ai_safety_results"
-            ] = safety_ratings  # older approach - maintaining to prevent regressions
-
+            model_response._hidden_params["vertex_ai_safety_results"] = safety_ratings  # older approach - maintaining to prevent regressions
+            
             ## ADD CITATION METADATA ##
             setattr(model_response, "vertex_ai_citation_metadata", citation_metadata)
-            model_response._hidden_params[
-                "vertex_ai_citation_metadata"
-            ] = citation_metadata  # older approach - maintaining to prevent regressions
+            model_response._hidden_params["vertex_ai_citation_metadata"] = citation_metadata  # older approach - maintaining to prevent regressions
 
         except Exception as e:
             raise VertexAIError(
@@ -1029,7 +1024,7 @@ class VertexLLM(VertexBase):
             input=messages,
             api_key="",
             additional_args={
-                "complete_input_dict": data,
+                "complete_input_dict": request_body,
                 "api_base": api_base,
                 "headers": headers,
             },

litellm/main.py

@@ -110,7 +110,10 @@ from .litellm_core_utils.fallback_utils import (
     async_completion_with_fallbacks,
     completion_with_fallbacks,
 )
-from .litellm_core_utils.prompt_templates.common_utils import get_completion_messages
+from .litellm_core_utils.prompt_templates.common_utils import (
+    get_completion_messages,
+    update_messages_with_model_file_ids,
+)
 from .litellm_core_utils.prompt_templates.factory import (
     custom_prompt,
     function_call_prompt,
@@ -449,7 +452,7 @@ async def acompletion(
     fallbacks = fallbacks or litellm.model_fallbacks
     if fallbacks is not None:
         response = await async_completion_with_fallbacks(
-            **completion_kwargs, kwargs={"fallbacks": fallbacks}
+            **completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs}
         )
         if response is None:
             raise Exception(
@@ -953,7 +956,6 @@ def completion(  # type: ignore # noqa: PLR0915
     non_default_params = get_non_default_completion_params(kwargs=kwargs)
     litellm_params = {}  # used to prevent unbound var errors
     ## PROMPT MANAGEMENT HOOKS ##
-
     if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and prompt_id is not None:
         (
             model,
@@ -1068,6 +1070,15 @@ def completion(  # type: ignore # noqa: PLR0915
             if eos_token:
                 custom_prompt_dict[model]["eos_token"] = eos_token
 
+        if kwargs.get("model_file_id_mapping"):
+            messages = update_messages_with_model_file_ids(
+                messages=messages,
+                model_id=kwargs.get("model_info", {}).get("id", None),
+                model_file_id_mapping=cast(
+                    Dict[str, Dict[str, str]], kwargs.get("model_file_id_mapping")
+                ),
+            )
+
         provider_config: Optional[BaseConfig] = None
         if custom_llm_provider is not None and custom_llm_provider in [
             provider.value for provider in LlmProviders
@@ -5799,6 +5810,19 @@ def stream_chunk_builder(  # noqa: PLR0915
                 "content"
             ] = processor.get_combined_content(content_chunks)
 
+        reasoning_chunks = [
+            chunk
+            for chunk in chunks
+            if len(chunk["choices"]) > 0
+            and "reasoning_content" in chunk["choices"][0]["delta"]
+            and chunk["choices"][0]["delta"]["reasoning_content"] is not None
+        ]
+
+        if len(reasoning_chunks) > 0:
+            response["choices"][0]["message"][
+                "reasoning_content"
+            ] = processor.get_combined_reasoning_content(reasoning_chunks)
+
         audio_chunks = [
             chunk
             for chunk in chunks
@@ -5813,11 +5837,14 @@ def stream_chunk_builder(  # noqa: PLR0915
 
         completion_output = get_content_from_model_response(response)
 
+        reasoning_tokens = processor.count_reasoning_tokens(response)
+
         usage = processor.calculate_usage(
             chunks=chunks,
             model=model,
             completion_output=completion_output,
             messages=messages,
+            reasoning_tokens=reasoning_tokens,
         )
 
         setattr(response, "usage", usage)

litellm/model_prices_and_context_window_backup.json

@@ -4650,6 +4650,31 @@
         "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash",
         "supports_tool_choice": true
     },
+    "gemini-2.0-flash": {
+        "max_tokens": 8192,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 8192,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 0.0000007,
+        "input_cost_per_token": 0.0000001,
+        "output_cost_per_token": 0.0000004,
+        "litellm_provider": "vertex_ai-language-models",
+        "mode": "chat",
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": true,
+        "supports_audio_input": true,
+        "supported_modalities": ["text", "image", "audio", "video"],
+        "supports_tool_choice": true,
+        "source": "https://ai.google.dev/pricing#2_0flash"
+    },
     "gemini-2.0-flash-lite": {
         "max_input_tokens": 1048576,
         "max_output_tokens": 8192,

litellm/proxy/_experimental/out/_next/static/chunks/app/layout-429ad74a94df7643.js

@@ -1 +0,0 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_cf7686', '__Inter_Fallback_cf7686'",fontStyle:"normal"},className:"__className_cf7686"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);

litellm/proxy/_experimental/out/_next/static/chunks/app/layout-af8319e6c59a08da.js

@@ -0,0 +1 @@
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{6580:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_cf7686', '__Inter_Fallback_cf7686'",fontStyle:"normal"},className:"__className_cf7686"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=6580)}),_N_E=n.O()}]);

litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-068a441595bd0fc3.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,u){Promise.resolve().then(u.bind(u,52829))},52829:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(92699);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1})}}},function(e){e.O(0,[42,261,250,699,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{11790:function(e,n,u){Promise.resolve().then(u.bind(u,52829))},52829:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(92699);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1})}}},function(e){e.O(0,[42,261,250,699,971,117,744],function(){return e(e.s=11790)}),_N_E=e.O()}]);

litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-466610167078a21c.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[461],{8672:function(e,t,n){Promise.resolve().then(n.bind(n,12011))},12011:function(e,t,n){"use strict";n.r(t),n.d(t,{default:function(){return S}});var s=n(57437),o=n(2265),a=n(99376),i=n(20831),c=n(94789),l=n(12514),r=n(49804),u=n(67101),d=n(84264),m=n(49566),h=n(96761),x=n(84566),p=n(19250),f=n(14474),k=n(13634),j=n(73002),g=n(3914);function S(){let[e]=k.Z.useForm(),t=(0,a.useSearchParams)();(0,g.e)("token");let n=t.get("invitation_id"),[S,_]=(0,o.useState)(null),[w,Z]=(0,o.useState)(""),[N,b]=(0,o.useState)(""),[T,v]=(0,o.useState)(null),[y,E]=(0,o.useState)(""),[C,U]=(0,o.useState)("");return(0,o.useEffect)(()=>{n&&(0,p.W_)(n).then(e=>{let t=e.login_url;console.log("login_url:",t),E(t);let n=e.token,s=(0,f.o)(n);U(n),console.log("decoded:",s),_(s.key),console.log("decoded user email:",s.user_email),b(s.user_email),v(s.user_id)})},[n]),(0,s.jsx)("div",{className:"mx-auto w-full max-w-md mt-10",children:(0,s.jsxs)(l.Z,{children:[(0,s.jsx)(h.Z,{className:"text-sm mb-5 text-center",children:"\uD83D\uDE85 LiteLLM"}),(0,s.jsx)(h.Z,{className:"text-xl",children:"Sign up"}),(0,s.jsx)(d.Z,{children:"Claim your user account to login to Admin UI."}),(0,s.jsx)(c.Z,{className:"mt-4",title:"SSO",icon:x.GH$,color:"sky",children:(0,s.jsxs)(u.Z,{numItems:2,className:"flex justify-between items-center",children:[(0,s.jsx)(r.Z,{children:"SSO is under the Enterprise Tirer."}),(0,s.jsx)(r.Z,{children:(0,s.jsx)(i.Z,{variant:"primary",className:"mb-2",children:(0,s.jsx)("a",{href:"https://forms.gle/W3U4PZpJGFHWtHyA9",target:"_blank",children:"Get Free Trial"})})})]})}),(0,s.jsxs)(k.Z,{className:"mt-10 mb-5 mx-auto",layout:"vertical",onFinish:e=>{console.log("in handle submit. accessToken:",S,"token:",C,"formValues:",e),S&&C&&(e.user_email=N,T&&n&&(0,p.m_)(S,n,T,e.password).then(e=>{var t;let n="/ui/";n+="?userID="+((null===(t=e.data)||void 0===t?void 0:t.user_id)||e.user_id),document.cookie="token="+C,console.log("redirecting to:",n),window.location.href=n}))},children:[(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(k.Z.Item,{label:"Email Address",name:"user_email",children:(0,s.jsx)(m.Z,{type:"email",disabled:!0,value:N,defaultValue:N,className:"max-w-md"})}),(0,s.jsx)(k.Z.Item,{label:"Password",name:"password",rules:[{required:!0,message:"password required to sign up"}],help:"Create a password for your account",children:(0,s.jsx)(m.Z,{placeholder:"",type:"password",className:"max-w-md"})})]}),(0,s.jsx)("div",{className:"mt-10",children:(0,s.jsx)(j.ZP,{htmlType:"submit",children:"Sign Up"})})]})]})})}},3914:function(e,t,n){"use strict";function s(){let e=window.location.hostname,t=["Lax","Strict","None"];["/","/ui"].forEach(n=>{document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,";"),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,";"),t.forEach(t=>{let s="None"===t?" Secure;":"";document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; SameSite=").concat(t,";").concat(s),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,"; SameSite=").concat(t,";").concat(s)})}),console.log("After clearing cookies:",document.cookie)}function o(e){let t=document.cookie.split("; ").find(t=>t.startsWith(e+"="));return t?t.split("=")[1]:null}n.d(t,{b:function(){return s},e:function(){return o}})}},function(e){e.O(0,[665,42,899,250,971,117,744],function(){return e(e.s=8672)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[461],{32922:function(e,t,n){Promise.resolve().then(n.bind(n,12011))},12011:function(e,t,n){"use strict";n.r(t),n.d(t,{default:function(){return S}});var s=n(57437),o=n(2265),a=n(99376),i=n(20831),c=n(94789),l=n(12514),r=n(49804),u=n(67101),d=n(84264),m=n(49566),h=n(96761),x=n(84566),p=n(19250),f=n(14474),k=n(13634),j=n(73002),g=n(3914);function S(){let[e]=k.Z.useForm(),t=(0,a.useSearchParams)();(0,g.e)("token");let n=t.get("invitation_id"),[S,_]=(0,o.useState)(null),[w,Z]=(0,o.useState)(""),[N,b]=(0,o.useState)(""),[T,v]=(0,o.useState)(null),[y,E]=(0,o.useState)(""),[C,U]=(0,o.useState)("");return(0,o.useEffect)(()=>{n&&(0,p.W_)(n).then(e=>{let t=e.login_url;console.log("login_url:",t),E(t);let n=e.token,s=(0,f.o)(n);U(n),console.log("decoded:",s),_(s.key),console.log("decoded user email:",s.user_email),b(s.user_email),v(s.user_id)})},[n]),(0,s.jsx)("div",{className:"mx-auto w-full max-w-md mt-10",children:(0,s.jsxs)(l.Z,{children:[(0,s.jsx)(h.Z,{className:"text-sm mb-5 text-center",children:"\uD83D\uDE85 LiteLLM"}),(0,s.jsx)(h.Z,{className:"text-xl",children:"Sign up"}),(0,s.jsx)(d.Z,{children:"Claim your user account to login to Admin UI."}),(0,s.jsx)(c.Z,{className:"mt-4",title:"SSO",icon:x.GH$,color:"sky",children:(0,s.jsxs)(u.Z,{numItems:2,className:"flex justify-between items-center",children:[(0,s.jsx)(r.Z,{children:"SSO is under the Enterprise Tier."}),(0,s.jsx)(r.Z,{children:(0,s.jsx)(i.Z,{variant:"primary",className:"mb-2",children:(0,s.jsx)("a",{href:"https://forms.gle/W3U4PZpJGFHWtHyA9",target:"_blank",children:"Get Free Trial"})})})]})}),(0,s.jsxs)(k.Z,{className:"mt-10 mb-5 mx-auto",layout:"vertical",onFinish:e=>{console.log("in handle submit. accessToken:",S,"token:",C,"formValues:",e),S&&C&&(e.user_email=N,T&&n&&(0,p.m_)(S,n,T,e.password).then(e=>{var t;let n="/ui/";n+="?userID="+((null===(t=e.data)||void 0===t?void 0:t.user_id)||e.user_id),document.cookie="token="+C,console.log("redirecting to:",n),window.location.href=n}))},children:[(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(k.Z.Item,{label:"Email Address",name:"user_email",children:(0,s.jsx)(m.Z,{type:"email",disabled:!0,value:N,defaultValue:N,className:"max-w-md"})}),(0,s.jsx)(k.Z.Item,{label:"Password",name:"password",rules:[{required:!0,message:"password required to sign up"}],help:"Create a password for your account",children:(0,s.jsx)(m.Z,{placeholder:"",type:"password",className:"max-w-md"})})]}),(0,s.jsx)("div",{className:"mt-10",children:(0,s.jsx)(j.ZP,{htmlType:"submit",children:"Sign Up"})})]})]})})}},3914:function(e,t,n){"use strict";function s(){let e=window.location.hostname,t=["Lax","Strict","None"];["/","/ui"].forEach(n=>{document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,";"),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,";"),t.forEach(t=>{let s="None"===t?" Secure;":"";document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; SameSite=").concat(t,";").concat(s),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,"; SameSite=").concat(t,";").concat(s)})}),console.log("After clearing cookies:",document.cookie)}function o(e){let t=document.cookie.split("; ").find(t=>t.startsWith(e+"="));return t?t.split("=")[1]:null}n.d(t,{b:function(){return s},e:function(){return o}})}},function(e){e.O(0,[665,42,899,250,971,117,744],function(){return e(e.s=32922)}),_N_E=e.O()}]);

litellm/proxy/_experimental/out/_next/static/chunks/main-app-475d6efe4080647d.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{10264:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(10264)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{20169:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(20169)}),_N_E=e.O()}]);

litellm/proxy/_experimental/out/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-75a5453f51d60261.js"/><script src="/ui/_next/static/chunks/fd9d1056-524b80e1a6b8bb06.js" async=""></script><script src="/ui/_next/static/chunks/117-883150efc583d711.js" async=""></script><script src="/ui/_next/static/chunks/main-app-4f7318ae681a6d94.js" async=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-75a5453f51d60261.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/a34f9d1faa5f3315-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"style\"]\n3:HL[\"/ui/_next/static/css/1f6915676624c422.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"4:I[12846,[],\"\"]\n6:I[19107,[],\"ClientPageRoot\"]\n7:I[38411,[\"665\",\"static/chunks/3014691f-0b72c78cfebbd712.js\",\"990\",\"static/chunks/13b76428-ebdf3012af0e4489.js\",\"42\",\"static/chunks/42-1cbed529ecb084e0.js\",\"261\",\"static/chunks/261-57d48f76eec1e568.js\",\"899\",\"static/chunks/899-9af4feaf6f21839c.js\",\"274\",\"static/chunks/274-bddaf0cf6c91e72f.js\",\"250\",\"static/chunks/250-dfc03a6fb4f0d254.js\",\"699\",\"static/chunks/699-87224ecba28f1f48.js\",\"931\",\"static/chunks/app/page-0f46d4a8b9bdf1c0.js\"],\"default\",1]\n8:I[4707,[],\"\"]\n9:I[36423,[],\"\"]\nb:I[61060,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L4\",null,{\"buildId\":\"Yb50LG5p7c9QpG54GIoFV\",\"assetPrefix\":\"/ui\",\"urlParts\":[\"\",\"\"],\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[[\"$L5\",[\"$\",\"$L6\",null,{\"props\":{\"params\":{},\"searchParams\":{}},\"Component\":\"$7\"}],null],null],null]},[[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/1f6915676624c422.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_cf7686\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[null,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script></body></html>
+<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-75a5453f51d60261.js"/><script src="/ui/_next/static/chunks/fd9d1056-524b80e1a6b8bb06.js" async=""></script><script src="/ui/_next/static/chunks/117-883150efc583d711.js" async=""></script><script src="/ui/_next/static/chunks/main-app-475d6efe4080647d.js" async=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-75a5453f51d60261.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/a34f9d1faa5f3315-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"style\"]\n3:HL[\"/ui/_next/static/css/6e6c0523f29030fd.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"4:I[12846,[],\"\"]\n6:I[19107,[],\"ClientPageRoot\"]\n7:I[37140,[\"665\",\"static/chunks/3014691f-0b72c78cfebbd712.js\",\"990\",\"static/chunks/13b76428-ebdf3012af0e4489.js\",\"42\",\"static/chunks/42-1cbed529ecb084e0.js\",\"261\",\"static/chunks/261-57d48f76eec1e568.js\",\"899\",\"static/chunks/899-9af4feaf6f21839c.js\",\"810\",\"static/chunks/810-493ce8d3227b491d.js\",\"250\",\"static/chunks/250-282480f9afa56ac6.js\",\"699\",\"static/chunks/699-87224ecba28f1f48.js\",\"931\",\"static/chunks/app/page-24bd7b05ba767df8.js\"],\"default\",1]\n8:I[4707,[],\"\"]\n9:I[36423,[],\"\"]\nb:I[61060,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L4\",null,{\"buildId\":\"zniqNKJW4P7vXGttXOEEQ\",\"assetPrefix\":\"/ui\",\"urlParts\":[\"\",\"\"],\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[[\"$L5\",[\"$\",\"$L6\",null,{\"props\":{\"params\":{},\"searchParams\":{}},\"Component\":\"$7\"}],null],null],null]},[[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/6e6c0523f29030fd.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_cf7686\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[null,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script></body></html>

litellm/proxy/_experimental/out/index.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[38411,["665","static/chunks/3014691f-0b72c78cfebbd712.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","899","static/chunks/899-9af4feaf6f21839c.js","274","static/chunks/274-bddaf0cf6c91e72f.js","250","static/chunks/250-dfc03a6fb4f0d254.js","699","static/chunks/699-87224ecba28f1f48.js","931","static/chunks/app/page-0f46d4a8b9bdf1c0.js"],"default",1]
+3:I[37140,["665","static/chunks/3014691f-0b72c78cfebbd712.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","899","static/chunks/899-9af4feaf6f21839c.js","810","static/chunks/810-493ce8d3227b491d.js","250","static/chunks/250-282480f9afa56ac6.js","699","static/chunks/699-87224ecba28f1f48.js","931","static/chunks/app/page-24bd7b05ba767df8.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["Yb50LG5p7c9QpG54GIoFV",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/1f6915676624c422.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zniqNKJW4P7vXGttXOEEQ",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/6e6c0523f29030fd.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_experimental/out/model_hub.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[52829,["42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","250","static/chunks/250-dfc03a6fb4f0d254.js","699","static/chunks/699-87224ecba28f1f48.js","418","static/chunks/app/model_hub/page-cde2fb783e81a6c1.js"],"default",1]
+3:I[52829,["42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","250","static/chunks/250-282480f9afa56ac6.js","699","static/chunks/699-87224ecba28f1f48.js","418","static/chunks/app/model_hub/page-068a441595bd0fc3.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["Yb50LG5p7c9QpG54GIoFV",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/1f6915676624c422.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zniqNKJW4P7vXGttXOEEQ",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/6e6c0523f29030fd.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_experimental/out/onboarding.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[12011,["665","static/chunks/3014691f-0b72c78cfebbd712.js","42","static/chunks/42-1cbed529ecb084e0.js","899","static/chunks/899-9af4feaf6f21839c.js","250","static/chunks/250-dfc03a6fb4f0d254.js","461","static/chunks/app/onboarding/page-2bf7a26db5342dbf.js"],"default",1]
+3:I[12011,["665","static/chunks/3014691f-0b72c78cfebbd712.js","42","static/chunks/42-1cbed529ecb084e0.js","899","static/chunks/899-9af4feaf6f21839c.js","250","static/chunks/250-282480f9afa56ac6.js","461","static/chunks/app/onboarding/page-466610167078a21c.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["Yb50LG5p7c9QpG54GIoFV",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/1f6915676624c422.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zniqNKJW4P7vXGttXOEEQ",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/6e6c0523f29030fd.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_new_secret_config.yaml

@@ -1,18 +1,17 @@
 model_list:
-  - model_name: "gpt-4o"
+  - model_name: "gpt-4o-azure"
     litellm_params:
-      model: azure/chatgpt-v-2
+      model: azure/gpt-4o
       api_key: os.environ/AZURE_API_KEY
-      api_base: http://0.0.0.0:8090
-      rpm: 3
+      api_base: os.environ/AZURE_API_BASE
   - model_name: "gpt-4o-mini-openai"
     litellm_params:
       model: gpt-4o-mini
       api_key: os.environ/OPENAI_API_KEY
   - model_name: "openai/*"
     litellm_params:
-      model: openai/*
-      api_key: os.environ/OPENAI_API_KEY
+        model: openai/*
+        api_key: os.environ/OPENAI_API_KEY
   - model_name: "bedrock-nova"
     litellm_params:
       model: us.amazon.nova-pro-v1:0

litellm/proxy/_types.py

@@ -2688,6 +2688,10 @@ class PrismaCompatibleUpdateDBModel(TypedDict, total=False):
     updated_by: str
 
 
+class SpecialEnums(enum.Enum):
+    LITELM_MANAGED_FILE_ID_PREFIX = "litellm_proxy/"
+
+
 class SpecialManagementEndpointEnums(enum.Enum):
     DEFAULT_ORGANIZATION = "default_organization"
 

litellm/proxy/hooks/__init__.py

@@ -1 +1,36 @@
+from typing import Literal, Union
+
 from . import *
+from .cache_control_check import _PROXY_CacheControlCheck
+from .managed_files import _PROXY_LiteLLMManagedFiles
+from .max_budget_limiter import _PROXY_MaxBudgetLimiter
+from .parallel_request_limiter import _PROXY_MaxParallelRequestsHandler
+
+# List of all available hooks that can be enabled
+PROXY_HOOKS = {
+    "max_budget_limiter": _PROXY_MaxBudgetLimiter,
+    "managed_files": _PROXY_LiteLLMManagedFiles,
+    "parallel_request_limiter": _PROXY_MaxParallelRequestsHandler,
+    "cache_control_check": _PROXY_CacheControlCheck,
+}
+
+
+def get_proxy_hook(
+    hook_name: Union[
+        Literal[
+            "max_budget_limiter",
+            "managed_files",
+            "parallel_request_limiter",
+            "cache_control_check",
+        ],
+        str,
+    ]
+):
+    """
+    Factory method to get a proxy hook instance by name
+    """
+    if hook_name not in PROXY_HOOKS:
+        raise ValueError(
+            f"Unknown hook: {hook_name}. Available hooks: {list(PROXY_HOOKS.keys())}"
+        )
+    return PROXY_HOOKS[hook_name]

litellm/proxy/hooks/managed_files.py

@@ -0,0 +1,145 @@
+# What is this?
+## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id
+
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Union, cast
+
+from litellm import verbose_logger
+from litellm.caching.caching import DualCache
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    get_file_ids_from_messages,
+)
+from litellm.proxy._types import CallTypes, SpecialEnums, UserAPIKeyAuth
+from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose
+
+if TYPE_CHECKING:
+    from opentelemetry.trace import Span as _Span
+
+    from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
+
+    Span = Union[_Span, Any]
+    InternalUsageCache = _InternalUsageCache
+else:
+    Span = Any
+    InternalUsageCache = Any
+
+
+class _PROXY_LiteLLMManagedFiles(CustomLogger):
+    # Class variables or attributes
+    def __init__(self, internal_usage_cache: InternalUsageCache):
+        self.internal_usage_cache = internal_usage_cache
+
+    async def async_pre_call_hook(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        cache: DualCache,
+        data: Dict,
+        call_type: Literal[
+            "completion",
+            "text_completion",
+            "embeddings",
+            "image_generation",
+            "moderation",
+            "audio_transcription",
+            "pass_through_endpoint",
+            "rerank",
+        ],
+    ) -> Union[Exception, str, Dict, None]:
+        """
+        - Detect litellm_proxy/ file_id
+        - add dictionary of mappings of litellm_proxy/ file_id -> provider_file_id => {litellm_proxy/file_id: {"model_id": id, "file_id": provider_file_id}}
+        """
+        if call_type == CallTypes.completion.value:
+            messages = data.get("messages")
+            if messages:
+                file_ids = get_file_ids_from_messages(messages)
+                if file_ids:
+                    model_file_id_mapping = await self.get_model_file_id_mapping(
+                        file_ids, user_api_key_dict.parent_otel_span
+                    )
+                    data["model_file_id_mapping"] = model_file_id_mapping
+
+        return data
+
+    async def get_model_file_id_mapping(
+        self, file_ids: List[str], litellm_parent_otel_span: Span
+    ) -> dict:
+        """
+        Get model-specific file IDs for a list of proxy file IDs.
+        Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
+
+        1. Get all the litellm_proxy/ file_ids from the messages
+        2. For each file_id, search for cache keys matching the pattern file_id:*
+        3. Return a dictionary of mappings of litellm_proxy/ file_id -> model_id -> model_file_id
+
+        Example:
+        {
+            "litellm_proxy/file_id": {
+                "model_id": "model_file_id"
+            }
+        }
+        """
+        file_id_mapping: Dict[str, Dict[str, str]] = {}
+        litellm_managed_file_ids = []
+
+        for file_id in file_ids:
+            ## CHECK IF FILE ID IS MANAGED BY LITELM
+            if file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value):
+                litellm_managed_file_ids.append(file_id)
+
+        if litellm_managed_file_ids:
+            # Get all cache keys matching the pattern file_id:*
+            for file_id in litellm_managed_file_ids:
+                # Search for any cache key starting with this file_id
+                cached_values = cast(
+                    Dict[str, str],
+                    await self.internal_usage_cache.async_get_cache(
+                        key=file_id, litellm_parent_otel_span=litellm_parent_otel_span
+                    ),
+                )
+                if cached_values:
+                    file_id_mapping[file_id] = cached_values
+        return file_id_mapping
+
+    @staticmethod
+    async def return_unified_file_id(
+        file_objects: List[OpenAIFileObject],
+        purpose: OpenAIFilesPurpose,
+        internal_usage_cache: InternalUsageCache,
+        litellm_parent_otel_span: Span,
+    ) -> OpenAIFileObject:
+        unified_file_id = SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value + str(
+            uuid.uuid4()
+        )
+
+        ## CREATE RESPONSE OBJECT
+        response = OpenAIFileObject(
+            id=unified_file_id,
+            object="file",
+            purpose=cast(OpenAIFilesPurpose, purpose),
+            created_at=file_objects[0].created_at,
+            bytes=1234,
+            filename=str(datetime.now().timestamp()),
+            status="uploaded",
+        )
+
+        ## STORE RESPONSE IN DB + CACHE
+        stored_values: Dict[str, str] = {}
+        for file_object in file_objects:
+            model_id = file_object._hidden_params.get("model_id")
+            if model_id is None:
+                verbose_logger.warning(
+                    f"Skipping file_object: {file_object} because model_id in hidden_params={file_object._hidden_params} is None"
+                )
+                continue
+            file_id = file_object.id
+            stored_values[model_id] = file_id
+        await internal_usage_cache.async_set_cache(
+            key=unified_file_id,
+            value=stored_values,
+            litellm_parent_otel_span=litellm_parent_otel_span,
+        )
+
+        return response

litellm/proxy/management_endpoints/ui_sso.py

@@ -485,7 +485,14 @@ async def auth_callback(request: Request):  # noqa: PLR0915
             redirect_uri=redirect_url,
             allow_insecure_http=True,
         )
-        result = await microsoft_sso.verify_and_process(request)
+        original_msft_result = await microsoft_sso.verify_and_process(
+            request=request,
+            convert_response=False,
+        )
+        result = MicrosoftSSOHandler.openid_from_response(
+            response=original_msft_result,
+            jwt_handler=jwt_handler,
+        )
     elif generic_client_id is not None:
         result = await get_generic_sso_response(
             request=request,
@@ -494,6 +501,7 @@ async def auth_callback(request: Request):  # noqa: PLR0915
             redirect_url=redirect_url,
         )
     # User is Authe'd in - generate key for the UI to access Proxy
+    verbose_proxy_logger.debug(f"SSO callback result: {result}")
     user_email: Optional[str] = getattr(result, "email", None)
     user_id: Optional[str] = getattr(result, "id", None) if result is not None else None
 
@@ -779,3 +787,27 @@ async def get_ui_settings(request: Request):
         ),
         "DISABLE_EXPENSIVE_DB_QUERIES": disable_expensive_db_queries,
     }
+
+
+class MicrosoftSSOHandler:
+    """
+    Handles Microsoft SSO callback response and returns a CustomOpenID object
+    """
+
+    @staticmethod
+    def openid_from_response(
+        response: Optional[dict], jwt_handler: JWTHandler
+    ) -> CustomOpenID:
+        response = response or {}
+        verbose_proxy_logger.debug(f"Microsoft SSO Callback Response: {response}")
+        openid_response = CustomOpenID(
+            email=response.get("mail"),
+            display_name=response.get("displayName"),
+            provider="microsoft",
+            id=response.get("id"),
+            first_name=response.get("givenName"),
+            last_name=response.get("surname"),
+            team_ids=jwt_handler.get_team_ids_from_jwt(cast(dict, response)),
+        )
+        verbose_proxy_logger.debug(f"Microsoft SSO OpenID Response: {openid_response}")
+        return openid_response

litellm/proxy/openai_files_endpoints/files_endpoints.py

@@ -7,7 +7,7 @@
 
 import asyncio
 import traceback
-from typing import Optional
+from typing import Optional, cast, get_args
 
 import httpx
 from fastapi import (
@@ -31,7 +31,10 @@ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessin
 from litellm.proxy.common_utils.openai_endpoint_utils import (
     get_custom_llm_provider_from_request_body,
 )
+from litellm.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
+from litellm.proxy.utils import ProxyLogging
 from litellm.router import Router
+from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose
 
 router = APIRouter()
 
@@ -104,6 +107,53 @@ def is_known_model(model: Optional[str], llm_router: Optional[Router]) -> bool:
     return is_in_list
 
 
+async def _deprecated_loadbalanced_create_file(
+    llm_router: Optional[Router],
+    router_model: str,
+    _create_file_request: CreateFileRequest,
+) -> OpenAIFileObject:
+    if llm_router is None:
+        raise HTTPException(
+            status_code=500,
+            detail={
+                "error": "LLM Router not initialized. Ensure models added to proxy."
+            },
+        )
+
+    response = await llm_router.acreate_file(model=router_model, **_create_file_request)
+    return response
+
+
+async def create_file_for_each_model(
+    llm_router: Optional[Router],
+    _create_file_request: CreateFileRequest,
+    target_model_names_list: List[str],
+    purpose: OpenAIFilesPurpose,
+    proxy_logging_obj: ProxyLogging,
+    user_api_key_dict: UserAPIKeyAuth,
+) -> OpenAIFileObject:
+    if llm_router is None:
+        raise HTTPException(
+            status_code=500,
+            detail={
+                "error": "LLM Router not initialized. Ensure models added to proxy."
+            },
+        )
+    responses = []
+    for model in target_model_names_list:
+        individual_response = await llm_router.acreate_file(
+            model=model, **_create_file_request
+        )
+        responses.append(individual_response)
+    response = await _PROXY_LiteLLMManagedFiles.return_unified_file_id(
+        file_objects=responses,
+        purpose=purpose,
+        internal_usage_cache=proxy_logging_obj.internal_usage_cache,
+        litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+    )
+    return response
+
+
 @router.post(
     "/{provider}/v1/files",
     dependencies=[Depends(user_api_key_auth)],
@@ -123,6 +173,7 @@ async def create_file(
     request: Request,
     fastapi_response: Response,
     purpose: str = Form(...),
+    target_model_names: str = Form(default=""),
     provider: Optional[str] = None,
     custom_llm_provider: str = Form(default="openai"),
     file: UploadFile = File(...),
@@ -162,8 +213,25 @@ async def create_file(
             or await get_custom_llm_provider_from_request_body(request=request)
             or "openai"
         )
+
+        target_model_names_list = (
+            target_model_names.split(",") if target_model_names else []
+        )
+        target_model_names_list = [model.strip() for model in target_model_names_list]
         # Prepare the data for forwarding
 
+        # Replace with:
+        valid_purposes = get_args(OpenAIFilesPurpose)
+        if purpose not in valid_purposes:
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": f"Invalid purpose: {purpose}. Must be one of: {valid_purposes}",
+                },
+            )
+        # Cast purpose to OpenAIFilesPurpose type
+        purpose = cast(OpenAIFilesPurpose, purpose)
+
         data = {"purpose": purpose}
 
         # Include original request and headers in the data
@@ -192,21 +260,25 @@ async def create_file(
 
         _create_file_request = CreateFileRequest(file=file_data, **data)
 
+        response: Optional[OpenAIFileObject] = None
         if (
             litellm.enable_loadbalancing_on_batch_endpoints is True
             and is_router_model
             and router_model is not None
         ):
-            if llm_router is None:
-                raise HTTPException(
-                    status_code=500,
-                    detail={
-                        "error": "LLM Router not initialized. Ensure models added to proxy."
-                    },
-                )
-
-            response = await llm_router.acreate_file(
-                model=router_model, **_create_file_request
+            response = await _deprecated_loadbalanced_create_file(
+                llm_router=llm_router,
+                router_model=router_model,
+                _create_file_request=_create_file_request,
+            )
+        elif target_model_names_list:
+            response = await create_file_for_each_model(
+                llm_router=llm_router,
+                _create_file_request=_create_file_request,
+                target_model_names_list=target_model_names_list,
+                purpose=purpose,
+                proxy_logging_obj=proxy_logging_obj,
+                user_api_key_dict=user_api_key_dict,
             )
         else:
             # get configs for custom_llm_provider
@@ -220,6 +292,11 @@ async def create_file(
             # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch
             response = await litellm.acreate_file(**_create_file_request, custom_llm_provider=custom_llm_provider)  # type: ignore
 
+        if response is None:
+            raise HTTPException(
+                status_code=500,
+                detail={"error": "Failed to create file. Please try again."},
+            )
         ### ALERTING ###
         asyncio.create_task(
             proxy_logging_obj.update_request_status(
@@ -248,12 +325,11 @@ async def create_file(
         await proxy_logging_obj.post_call_failure_hook(
             user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
         )
-        verbose_proxy_logger.error(
+        verbose_proxy_logger.exception(
             "litellm.proxy.proxy_server.create_file(): Exception occured - {}".format(
                 str(e)
             )
         )
-        verbose_proxy_logger.debug(traceback.format_exc())
         if isinstance(e, HTTPException):
             raise ProxyException(
                 message=getattr(e, "message", str(e.detail)),

litellm/proxy/utils.py

@@ -76,6 +76,7 @@ from litellm.proxy.db.create_views import (
 from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
 from litellm.proxy.db.log_db_metrics import log_db_metrics
 from litellm.proxy.db.prisma_client import PrismaWrapper
+from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook
 from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck
 from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter
 from litellm.proxy.hooks.parallel_request_limiter import (
@@ -352,10 +353,19 @@ class ProxyLogging:
             self.db_spend_update_writer.redis_update_buffer.redis_cache = redis_cache
             self.db_spend_update_writer.pod_lock_manager.redis_cache = redis_cache
 
+    def _add_proxy_hooks(self, llm_router: Optional[Router] = None):
+        for hook in PROXY_HOOKS:
+            proxy_hook = get_proxy_hook(hook)
+            import inspect
+
+            expected_args = inspect.getfullargspec(proxy_hook).args
+            if "internal_usage_cache" in expected_args:
+                litellm.logging_callback_manager.add_litellm_callback(proxy_hook(self.internal_usage_cache))  # type: ignore
+            else:
+                litellm.logging_callback_manager.add_litellm_callback(proxy_hook())  # type: ignore
+
     def _init_litellm_callbacks(self, llm_router: Optional[Router] = None):
-        litellm.logging_callback_manager.add_litellm_callback(self.max_parallel_request_limiter)  # type: ignore
-        litellm.logging_callback_manager.add_litellm_callback(self.max_budget_limiter)  # type: ignore
-        litellm.logging_callback_manager.add_litellm_callback(self.cache_control_check)  # type: ignore
+        self._add_proxy_hooks(llm_router)
         litellm.logging_callback_manager.add_litellm_callback(self.service_logging_obj)  # type: ignore
         for callback in litellm.callbacks:
             if isinstance(callback, str):

litellm/router.py

@@ -68,10 +68,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
     add_fallback_headers_to_response,
     add_retry_headers_to_response,
 )
-from litellm.router_utils.batch_utils import (
-    _get_router_metadata_variable_name,
-    replace_model_in_jsonl,
-)
+from litellm.router_utils.batch_utils import _get_router_metadata_variable_name
 from litellm.router_utils.client_initalization_utils import InitalizeCachedClient
 from litellm.router_utils.clientside_credential_handler import (
     get_dynamic_litellm_params,
@@ -105,7 +102,12 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import (
     increment_deployment_successes_for_current_minute,
 )
 from litellm.scheduler import FlowItem, Scheduler
-from litellm.types.llms.openai import AllMessageValues, Batch, FileObject, FileTypes
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    Batch,
+    FileTypes,
+    OpenAIFileObject,
+)
 from litellm.types.router import (
     CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
     VALID_LITELLM_ENVIRONMENTS,
@@ -2703,7 +2705,7 @@ class Router:
         self,
         model: str,
         **kwargs,
-    ) -> FileObject:
+    ) -> OpenAIFileObject:
         try:
             kwargs["model"] = model
             kwargs["original_function"] = self._acreate_file
@@ -2727,7 +2729,7 @@ class Router:
         self,
         model: str,
         **kwargs,
-    ) -> FileObject:
+    ) -> OpenAIFileObject:
         try:
             verbose_router_logger.debug(
                 f"Inside _atext_completion()- model: {model}; kwargs: {kwargs}"
@@ -2754,9 +2756,9 @@ class Router:
             stripped_model, custom_llm_provider, _, _ = get_llm_provider(
                 model=data["model"]
             )
-            kwargs["file"] = replace_model_in_jsonl(
-                file_content=kwargs["file"], new_model_name=stripped_model
-            )
+            # kwargs["file"] = replace_model_in_jsonl(
+            #     file_content=kwargs["file"], new_model_name=stripped_model
+            # )
 
             response = litellm.acreate_file(
                 **{
@@ -2796,6 +2798,7 @@ class Router:
             verbose_router_logger.info(
                 f"litellm.acreate_file(model={model_name})\033[32m 200 OK\033[0m"
             )
+
             return response  # type: ignore
         except Exception as e:
             verbose_router_logger.exception(

litellm/router_utils/common_utils.py

@@ -0,0 +1,14 @@
+import hashlib
+import json
+
+from litellm.types.router import CredentialLiteLLMParams
+
+
+def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:
+    """
+    Hash of the credential params, used for mapping the file id to the right model
+    """
+    sensitive_params = CredentialLiteLLMParams(**litellm_params)
+    return hashlib.sha256(
+        json.dumps(sensitive_params.model_dump()).encode()
+    ).hexdigest()

litellm/types/llms/anthropic.py

@@ -52,11 +52,12 @@ class AnthropicMessagesTextParam(TypedDict, total=False):
     cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
 
 
-class AnthropicMessagesToolUseParam(TypedDict):
+class AnthropicMessagesToolUseParam(TypedDict, total=False):
     type: Required[Literal["tool_use"]]
     id: str
     name: str
     input: dict
+    cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
 
 
 AnthropicMessagesAssistantMessageValues = Union[

litellm/types/llms/openai.py

@@ -234,7 +234,18 @@ class Thread(BaseModel):
     """The object type, which is always `thread`."""
 
 
-OpenAICreateFileRequestOptionalParams = Literal["purpose",]
+OpenAICreateFileRequestOptionalParams = Literal["purpose"]
+
+OpenAIFilesPurpose = Literal[
+    "assistants",
+    "assistants_output",
+    "batch",
+    "batch_output",
+    "fine-tune",
+    "fine-tune-results",
+    "vision",
+    "user_data",
+]
 
 
 class OpenAIFileObject(BaseModel):
@@ -253,16 +264,7 @@ class OpenAIFileObject(BaseModel):
     object: Literal["file"]
     """The object type, which is always `file`."""
 
-    purpose: Literal[
-        "assistants",
-        "assistants_output",
-        "batch",
-        "batch_output",
-        "fine-tune",
-        "fine-tune-results",
-        "vision",
-        "user_data",
-    ]
+    purpose: OpenAIFilesPurpose
     """The intended purpose of the file.
 
     Supported values are `assistants`, `assistants_output`, `batch`, `batch_output`,
@@ -286,6 +288,8 @@ class OpenAIFileObject(BaseModel):
     `error` field on `fine_tuning.job`.
     """
 
+    _hidden_params: dict = {}
+
 
 # OpenAI Files Types
 class CreateFileRequest(TypedDict, total=False):

litellm/types/router.py

@@ -18,6 +18,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from ..exceptions import RateLimitError
 from .completion import CompletionRequest
 from .embedding import EmbeddingRequest
+from .llms.openai import OpenAIFileObject
 from .llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
 from .utils import ModelResponse, ProviderSpecificModelInfo
 
@@ -703,3 +704,12 @@ class GenericBudgetWindowDetails(BaseModel):
 
 
 OptionalPreCallChecks = List[Literal["prompt_caching", "router_budget_limiting"]]
+
+
+class LiteLLM_RouterFileObject(TypedDict, total=False):
+    """
+    Tracking the litellm params hash, used for mapping the file id to the right model
+    """
+
+    litellm_params_sensitive_credential_hash: str
+    file_object: OpenAIFileObject

litellm/types/utils.py

@@ -1886,6 +1886,7 @@ all_litellm_params = [
     "logger_fn",
     "verbose",
     "custom_llm_provider",
+    "model_file_id_mapping",
     "litellm_logging_obj",
     "litellm_call_id",
     "use_client",

model_prices_and_context_window.json

@@ -4650,6 +4650,31 @@
         "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash",
         "supports_tool_choice": true
     },
+    "gemini-2.0-flash": {
+        "max_tokens": 8192,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 8192,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 0.0000007,
+        "input_cost_per_token": 0.0000001,
+        "output_cost_per_token": 0.0000004,
+        "litellm_provider": "vertex_ai-language-models",
+        "mode": "chat",
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": true,
+        "supports_audio_input": true,
+        "supported_modalities": ["text", "image", "audio", "video"],
+        "supports_tool_choice": true,
+        "source": "https://ai.google.dev/pricing#2_0flash"
+    },
     "gemini-2.0-flash-lite": {
         "max_input_tokens": 1048576,
         "max_output_tokens": 8192,

poetry.lock

@@ -3105,17 +3105,17 @@ requests = "2.31.0"
 
 [[package]]
 name = "respx"
-version = "0.20.2"
+version = "0.22.0"
 description = "A utility for mocking out the Python HTTPX and HTTP Core libraries."
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "respx-0.20.2-py2.py3-none-any.whl", hash = "sha256:ab8e1cf6da28a5b2dd883ea617f8130f77f676736e6e9e4a25817ad116a172c9"},
-    {file = "respx-0.20.2.tar.gz", hash = "sha256:07cf4108b1c88b82010f67d3c831dae33a375c7b436e54d87737c7f9f99be643"},
+    {file = "respx-0.22.0-py2.py3-none-any.whl", hash = "sha256:631128d4c9aba15e56903fb5f66fb1eff412ce28dd387ca3a81339e52dbd3ad0"},
+    {file = "respx-0.22.0.tar.gz", hash = "sha256:3c8924caa2a50bd71aefc07aa812f2466ff489f1848c96e954a5362d17095d91"},
 ]
 
 [package.dependencies]
-httpx = ">=0.21.0"
+httpx = ">=0.25.0"
 
 [[package]]
 name = "rpds-py"
@@ -4056,4 +4056,4 @@ proxy = ["PyJWT", "apscheduler", "backoff", "boto3", "cryptography", "fastapi",
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<4.0, !=3.9.7"
-content-hash = "524b2f8276ba057f8dc8a79dd460c1a243ef4aece7c08a8bf344e029e07b8841"
+content-hash = "27c2090e5190d8b37948419dd8dd6234dd0ab7ea81a222aa81601596382472fc"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.65.2"
+version = "1.65.3"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
@@ -101,7 +101,7 @@ mypy = "^1.0"
 pytest = "^7.4.3"
 pytest-mock = "^3.12.0"
 pytest-asyncio = "^0.21.1"
-respx = "^0.20.2"
+respx = "^0.22.0"
 ruff = "^0.1.0"
 types-requests = "*"
 types-setuptools = "*"
@@ -117,7 +117,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.65.2"
+version = "1.65.3"
 version_files = [
     "pyproject.toml:^version"
 ]

tests/litellm/litellm_core_utils/test_streaming_handler.py

@@ -17,6 +17,7 @@ import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.types.utils import (
+    CompletionTokensDetailsWrapper,
     Delta,
     ModelResponseStream,
     PromptTokensDetailsWrapper,
@@ -430,11 +431,18 @@ async def test_streaming_handler_with_usage(
         completion_tokens=392,
         prompt_tokens=1799,
         total_tokens=2191,
-        completion_tokens_details=None,
+        completion_tokens_details=CompletionTokensDetailsWrapper(  # <-- This has a value
+            accepted_prediction_tokens=None,
+            audio_tokens=None,
+            reasoning_tokens=0,
+            rejected_prediction_tokens=None,
+            text_tokens=None,
+        ),
         prompt_tokens_details=PromptTokensDetailsWrapper(
             audio_tokens=None, cached_tokens=1796, text_tokens=None, image_tokens=None
         ),
     )
+
     final_chunk = ModelResponseStream(
         id="chatcmpl-87291500-d8c5-428e-b187-36fe5a4c97ab",
         created=1742056047,
@@ -510,7 +518,13 @@ async def test_streaming_with_usage_and_logging(sync_mode: bool):
         completion_tokens=392,
         prompt_tokens=1799,
         total_tokens=2191,
-        completion_tokens_details=None,
+        completion_tokens_details=CompletionTokensDetailsWrapper(
+            accepted_prediction_tokens=None,
+            audio_tokens=None,
+            reasoning_tokens=0,
+            rejected_prediction_tokens=None,
+            text_tokens=None,
+        ),
         prompt_tokens_details=PromptTokensDetailsWrapper(
             audio_tokens=None,
             cached_tokens=1796,

tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py

@@ -1,11 +1,9 @@
-import json
 import os
 import sys
-from unittest.mock import AsyncMock, MagicMock, patch
 
-import httpx
 import pytest
 
+
 sys.path.insert(
     0, os.path.abspath("../../../../..")
 )  # Adds the parent directory to the system path
@@ -13,6 +11,7 @@ sys.path.insert(
 from litellm.llms.openrouter.chat.transformation import (
     OpenRouterChatCompletionStreamingHandler,
     OpenRouterException,
+    OpenrouterConfig,
 )
 
 
@@ -79,3 +78,20 @@ class TestOpenRouterChatCompletionStreamingHandler:
 
         assert "KeyError" in str(exc_info.value)
         assert exc_info.value.status_code == 400
+
+
+def test_openrouter_extra_body_transformation():
+
+    transformed_request = OpenrouterConfig().transform_request(
+        model="openrouter/deepseek/deepseek-chat",
+        messages=[{"role": "user", "content": "Hello, world!"}],
+        optional_params={"extra_body": {"provider": {"order": ["DeepSeek"]}}},
+        litellm_params={},
+        headers={},
+    )
+
+    # https://github.com/BerriAI/litellm/issues/8425, validate its not contained in extra_body still
+    assert transformed_request["provider"]["order"] == ["DeepSeek"]
+    assert transformed_request["messages"] == [
+        {"role": "user", "content": "Hello, world!"}
+    ]

tests/litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py

@@ -0,0 +1,66 @@
+import pytest
+import asyncio
+from unittest.mock import MagicMock
+from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
+import litellm
+from litellm import ModelResponse
+
+@pytest.mark.asyncio
+async def test_transform_response_with_avglogprobs():
+    """
+    Test that the transform_response method correctly handles the avgLogprobs key
+    from Gemini Flash 2.0 responses.
+    """
+    # Create a mock response with avgLogprobs
+    response_json = {
+        "candidates": [{
+            "content": {"parts": [{"text": "Test response"}], "role": "model"},
+            "finishReason": "STOP",
+            "avgLogprobs": -0.3445799010140555
+        }],
+        "usageMetadata": {
+            "promptTokenCount": 10,
+            "candidatesTokenCount": 5,
+            "totalTokenCount": 15
+        }
+    }
+    
+    # Create a mock HTTP response
+    mock_response = MagicMock()
+    mock_response.json.return_value = response_json
+    
+    # Create a mock logging object
+    mock_logging = MagicMock()
+    
+    # Create an instance of VertexGeminiConfig
+    config = VertexGeminiConfig()
+    
+    # Create a ModelResponse object
+    model_response = ModelResponse(
+        id="test-id",
+        choices=[],
+        created=1234567890,
+        model="gemini-2.0-flash",
+        usage={
+            "prompt_tokens": 10,
+            "completion_tokens": 5,
+            "total_tokens": 15
+        }
+    )
+    
+    # Call the transform_response method
+    transformed_response = config.transform_response(
+        model="gemini-2.0-flash",
+        raw_response=mock_response,
+        model_response=model_response,
+        logging_obj=mock_logging,
+        request_data={},
+        messages=[],
+        optional_params={},
+        litellm_params={},
+        encoding=None
+    )
+    
+    # Assert that the avgLogprobs was correctly added to the model response
+    assert len(transformed_response.choices) == 1
+    assert transformed_response.choices[0].logprobs == -0.3445799010140555

tests/litellm/proxy/management_endpoints/test_ui_sso.py

@@ -0,0 +1,81 @@
+import json
+import os
+import sys
+from typing import Optional, cast
+from unittest.mock import MagicMock, patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+sys.path.insert(
+    0, os.path.abspath("../../..")
+)  # Adds the parent directory to the system path
+
+from litellm.proxy.auth.handle_jwt import JWTHandler
+from litellm.proxy.management_endpoints.types import CustomOpenID
+from litellm.proxy.management_endpoints.ui_sso import MicrosoftSSOHandler
+
+
+def test_microsoft_sso_handler_openid_from_response():
+    # Arrange
+    # Create a mock response similar to what Microsoft SSO would return
+    mock_response = {
+        "mail": "test@example.com",
+        "displayName": "Test User",
+        "id": "user123",
+        "givenName": "Test",
+        "surname": "User",
+        "some_other_field": "value",
+    }
+
+    # Create a mock JWTHandler that returns predetermined team IDs
+    mock_jwt_handler = MagicMock(spec=JWTHandler)
+    expected_team_ids = ["team1", "team2"]
+    mock_jwt_handler.get_team_ids_from_jwt.return_value = expected_team_ids
+
+    # Act
+    # Call the method being tested
+    result = MicrosoftSSOHandler.openid_from_response(
+        response=mock_response, jwt_handler=mock_jwt_handler
+    )
+
+    # Assert
+    # Verify the JWT handler was called with the correct parameters
+    mock_jwt_handler.get_team_ids_from_jwt.assert_called_once_with(
+        cast(dict, mock_response)
+    )
+
+    # Check that the result is a CustomOpenID object with the expected values
+    assert isinstance(result, CustomOpenID)
+    assert result.email == "test@example.com"
+    assert result.display_name == "Test User"
+    assert result.provider == "microsoft"
+    assert result.id == "user123"
+    assert result.first_name == "Test"
+    assert result.last_name == "User"
+    assert result.team_ids == expected_team_ids
+
+
+def test_microsoft_sso_handler_with_empty_response():
+    # Arrange
+    # Test with None response
+    mock_jwt_handler = MagicMock(spec=JWTHandler)
+    mock_jwt_handler.get_team_ids_from_jwt.return_value = []
+
+    # Act
+    result = MicrosoftSSOHandler.openid_from_response(
+        response=None, jwt_handler=mock_jwt_handler
+    )
+
+    # Assert
+    assert isinstance(result, CustomOpenID)
+    assert result.email is None
+    assert result.display_name is None
+    assert result.provider == "microsoft"
+    assert result.id is None
+    assert result.first_name is None
+    assert result.last_name is None
+    assert result.team_ids == []
+
+    # Make sure the JWT handler was called with an empty dict
+    mock_jwt_handler.get_team_ids_from_jwt.assert_called_once_with({})

tests/litellm/proxy/openai_files_endpoint/test_files_endpoint.py

@@ -0,0 +1,306 @@
+import json
+import os
+import sys
+from unittest.mock import ANY
+
+import pytest
+import respx
+from fastapi.testclient import TestClient
+from pytest_mock import MockerFixture
+
+sys.path.insert(
+    0, os.path.abspath("../../../..")
+)  # Adds the parent directory to the system path
+
+import litellm
+from litellm import Router
+from litellm.proxy._types import LiteLLM_UserTableFiltered, UserAPIKeyAuth
+from litellm.proxy.hooks import get_proxy_hook
+from litellm.proxy.management_endpoints.internal_user_endpoints import ui_view_users
+from litellm.proxy.proxy_server import app
+
+client = TestClient(app)
+from litellm.caching.caching import DualCache
+from litellm.proxy.proxy_server import hash_token
+from litellm.proxy.utils import ProxyLogging
+
+
+@pytest.fixture
+def llm_router() -> Router:
+    llm_router = Router(
+        model_list=[
+            {
+                "model_name": "azure-gpt-3-5-turbo",
+                "litellm_params": {
+                    "model": "azure/chatgpt-v-2",
+                    "api_key": "azure_api_key",
+                    "api_base": "azure_api_base",
+                    "api_version": "azure_api_version",
+                },
+                "model_info": {
+                    "id": "azure-gpt-3-5-turbo-id",
+                },
+            },
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {
+                    "model": "openai/gpt-3.5-turbo",
+                    "api_key": "openai_api_key",
+                },
+                "model_info": {
+                    "id": "gpt-3.5-turbo-id",
+                },
+            },
+            {
+                "model_name": "gemini-2.0-flash",
+                "litellm_params": {
+                    "model": "gemini/gemini-2.0-flash",
+                },
+                "model_info": {
+                    "id": "gemini-2.0-flash-id",
+                },
+            },
+        ]
+    )
+    return llm_router
+
+
+def setup_proxy_logging_object(monkeypatch, llm_router: Router) -> ProxyLogging:
+    proxy_logging_object = ProxyLogging(
+        user_api_key_cache=DualCache(default_in_memory_ttl=1)
+    )
+    proxy_logging_object._add_proxy_hooks(llm_router)
+    monkeypatch.setattr(
+        "litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_object
+    )
+    return proxy_logging_object
+
+
+def test_invalid_purpose(mocker: MockerFixture, monkeypatch, llm_router: Router):
+    """
+    Asserts 'create_file' is called with the correct arguments
+    """
+    # Create a simple test file content
+    test_file_content = b"test audio content"
+    test_file = ("test.wav", test_file_content, "audio/wav")
+
+    response = client.post(
+        "/v1/files",
+        files={"file": test_file},
+        data={
+            "purpose": "my-bad-purpose",
+            "target_model_names": ["azure-gpt-3-5-turbo", "gpt-3.5-turbo"],
+        },
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    assert response.status_code == 400
+    print(f"response: {response.json()}")
+    assert "Invalid purpose: my-bad-purpose" in response.json()["error"]["message"]
+
+
+def test_mock_create_audio_file(mocker: MockerFixture, monkeypatch, llm_router: Router):
+    """
+    Asserts 'create_file' is called with the correct arguments
+    """
+    from litellm import Router
+
+    mock_create_file = mocker.patch("litellm.files.main.create_file")
+
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router)
+
+    # Create a simple test file content
+    test_file_content = b"test audio content"
+    test_file = ("test.wav", test_file_content, "audio/wav")
+
+    response = client.post(
+        "/v1/files",
+        files={"file": test_file},
+        data={
+            "purpose": "user_data",
+            "target_model_names": "azure-gpt-3-5-turbo, gpt-3.5-turbo",
+        },
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    print(f"response: {response.text}")
+    assert response.status_code == 200
+
+    # Get all calls made to create_file
+    calls = mock_create_file.call_args_list
+
+    # Check for Azure call
+    azure_call_found = False
+    for call in calls:
+        kwargs = call.kwargs
+        if (
+            kwargs.get("custom_llm_provider") == "azure"
+            and kwargs.get("model") == "azure/chatgpt-v-2"
+            and kwargs.get("api_key") == "azure_api_key"
+        ):
+            azure_call_found = True
+            break
+    assert (
+        azure_call_found
+    ), f"Azure call not found with expected parameters. Calls: {calls}"
+
+    # Check for OpenAI call
+    openai_call_found = False
+    for call in calls:
+        kwargs = call.kwargs
+        if (
+            kwargs.get("custom_llm_provider") == "openai"
+            and kwargs.get("model") == "openai/gpt-3.5-turbo"
+            and kwargs.get("api_key") == "openai_api_key"
+        ):
+            openai_call_found = True
+            break
+    assert openai_call_found, "OpenAI call not found with expected parameters"
+
+
+@pytest.mark.skip(reason="mock respx fails on ci/cd - unclear why")
+def test_create_file_and_call_chat_completion_e2e(
+    mocker: MockerFixture, monkeypatch, llm_router: Router
+):
+    """
+    1. Create a file
+    2. Call a chat completion with the file
+    3. Assert the file is used in the chat completion
+    """
+    # Create and enable respx mock instance
+    mock = respx.mock()
+    mock.start()
+    try:
+        from litellm.types.llms.openai import OpenAIFileObject
+
+        monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router)
+        proxy_logging_object = setup_proxy_logging_object(monkeypatch, llm_router)
+
+        # Create a simple test file content
+        test_file_content = b"test audio content"
+        test_file = ("test.wav", test_file_content, "audio/wav")
+
+        # Mock the file creation response
+        mock_file_response = OpenAIFileObject(
+            id="test-file-id",
+            object="file",
+            bytes=123,
+            created_at=1234567890,
+            filename="test.wav",
+            purpose="user_data",
+            status="uploaded",
+        )
+        mock_file_response._hidden_params = {"model_id": "gemini-2.0-flash-id"}
+        mocker.patch.object(llm_router, "acreate_file", return_value=mock_file_response)
+
+        # Mock the Gemini API call using respx
+        mock_gemini_response = {
+            "candidates": [
+                {"content": {"parts": [{"text": "This is a test audio file"}]}}
+            ]
+        }
+
+        # Mock the Gemini API endpoint with a more flexible pattern
+        gemini_route = mock.post(
+            url__regex=r".*generativelanguage\.googleapis\.com.*"
+        ).mock(
+            return_value=respx.MockResponse(status_code=200, json=mock_gemini_response),
+        )
+
+        # Print updated mock setup
+        print("\nAfter Adding Gemini Route:")
+        print("==========================")
+        print(f"Number of mocked routes: {len(mock.routes)}")
+        for route in mock.routes:
+            print(f"Mocked Route: {route}")
+            print(f"Pattern: {route.pattern}")
+
+        ## CREATE FILE
+        file = client.post(
+            "/v1/files",
+            files={"file": test_file},
+            data={
+                "purpose": "user_data",
+                "target_model_names": "gemini-2.0-flash, gpt-3.5-turbo",
+            },
+            headers={"Authorization": "Bearer test-key"},
+        )
+
+        print("\nAfter File Creation:")
+        print("====================")
+        print(f"File creation status: {file.status_code}")
+        print(f"Recorded calls so far: {len(mock.calls)}")
+        for call in mock.calls:
+            print(f"Call made to: {call.request.method} {call.request.url}")
+
+        assert file.status_code == 200
+        assert file.json()["id"] != "test-file-id"  # unified file id used
+
+        ## USE FILE IN CHAT COMPLETION
+        try:
+            completion = client.post(
+                "/v1/chat/completions",
+                json={
+                    "model": "gemini-2.0-flash",
+                    "modalities": ["text", "audio"],
+                    "audio": {"voice": "alloy", "format": "wav"},
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": "What is in this recording?"},
+                                {
+                                    "type": "file",
+                                    "file": {
+                                        "file_id": file.json()["id"],
+                                        "filename": "my-test-name",
+                                        "format": "audio/wav",
+                                    },
+                                },
+                            ],
+                        },
+                    ],
+                    "drop_params": True,
+                },
+                headers={"Authorization": "Bearer test-key"},
+            )
+        except Exception as e:
+            print(f"error: {e}")
+
+        print("\nError occurred during chat completion:")
+        print("=====================================")
+        print("\nFinal Mock State:")
+        print("=================")
+        print(f"Total mocked routes: {len(mock.routes)}")
+        for route in mock.routes:
+            print(f"\nMocked Route: {route}")
+            print(f"  Called: {route.called}")
+
+        print("\nActual Requests Made:")
+        print("=====================")
+        print(f"Total calls recorded: {len(mock.calls)}")
+        for idx, call in enumerate(mock.calls):
+            print(f"\nCall {idx + 1}:")
+            print(f"  Method: {call.request.method}")
+            print(f"  URL: {call.request.url}")
+            print(f"  Headers: {dict(call.request.headers)}")
+            try:
+                print(f"  Body: {call.request.content.decode()}")
+            except:
+                print("  Body: <could not decode>")
+
+        # Verify Gemini API was called
+        assert gemini_route.called, "Gemini API was not called"
+
+        # Print the call details
+        print("\nGemini API Call Details:")
+        print(f"URL: {gemini_route.calls.last.request.url}")
+        print(f"Method: {gemini_route.calls.last.request.method}")
+        print(f"Headers: {dict(gemini_route.calls.last.request.headers)}")
+        print(f"Content: {gemini_route.calls.last.request.content.decode()}")
+        print(f"Response: {gemini_route.calls.last.response.content.decode()}")
+
+        assert "test-file-id" in gemini_route.calls.last.request.content.decode()
+    finally:
+        # Stop the mock
+        mock.stop()

tests/litellm/proxy/test_proxy_server.py

@@ -39,7 +39,6 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
     with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), patch(
         "litellm.proxy.proxy_server.store_model_in_db", False
     ):  # set store_model_in_db to False
-
         # Test when store_model_in_db is False
         await ProxyStartupEvent.initialize_scheduled_background_jobs(
             general_settings={},
@@ -57,7 +56,6 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
     with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), patch(
         "litellm.proxy.proxy_server.store_model_in_db", True
     ), patch("litellm.proxy.proxy_server.get_secret_bool", return_value=True):
-
         await ProxyStartupEvent.initialize_scheduled_background_jobs(
             general_settings={},
             prisma_client=mock_prisma_client,

tests/litellm/test_main.py

@@ -1,8 +1,10 @@
 import json
 import os
 import sys
-
+import httpx
 import pytest
+import respx
+
 from fastapi.testclient import TestClient
 
 sys.path.insert(
@@ -259,3 +261,84 @@ def test_bedrock_latency_optimized_inference():
         mock_post.assert_called_once()
         json_data = json.loads(mock_post.call_args.kwargs["data"])
         assert json_data["performanceConfig"]["latency"] == "optimized"
+
+@pytest.fixture(autouse=True)
+def set_openrouter_api_key():
+    original_api_key = os.environ.get("OPENROUTER_API_KEY")
+    os.environ["OPENROUTER_API_KEY"] = "fake-key-for-testing"
+    yield
+    if original_api_key is not None:
+        os.environ["OPENROUTER_API_KEY"] = original_api_key
+    else:
+        del os.environ["OPENROUTER_API_KEY"]
+
+
+@pytest.mark.asyncio
+async def test_extra_body_with_fallback(respx_mock: respx.MockRouter, set_openrouter_api_key):
+    """
+    test regression for https://github.com/BerriAI/litellm/issues/8425.
+    
+    This was perhaps a wider issue with the acompletion function not passing kwargs such as extra_body correctly when fallbacks are specified.
+    """
+    # Set up test parameters
+    model = "openrouter/deepseek/deepseek-chat"
+    messages = [{"role": "user", "content": "Hello, world!"}]
+    extra_body = {
+        "provider": {
+            "order": ["DeepSeek"],
+            "allow_fallbacks": False,
+            "require_parameters": True
+        }
+    }
+    fallbacks = [
+        {
+            "model": "openrouter/google/gemini-flash-1.5-8b"
+        }
+    ]
+
+    respx_mock.post("https://openrouter.ai/api/v1/chat/completions").respond(
+        json={
+            "id": "chatcmpl-123",
+            "object": "chat.completion",
+            "created": 1677652288,
+            "model": model,
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": "Hello from mocked response!",
+                    },
+                    "finish_reason": "stop",
+                }
+            ],
+            "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21},
+        }
+    )
+
+    response = await litellm.acompletion(
+        model=model,
+        messages=messages,
+        extra_body=extra_body,
+        fallbacks=fallbacks,
+        api_key="fake-openrouter-api-key",
+    )
+
+    # Get the request from the mock
+    request: httpx.Request = respx_mock.calls[0].request
+    request_body = request.read()
+    request_body = json.loads(request_body)
+
+    # Verify basic parameters
+    assert request_body["model"] == "deepseek/deepseek-chat"
+    assert request_body["messages"] == messages
+
+    # Verify the extra_body parameters remain under the provider key
+    assert request_body["provider"]["order"] == ["DeepSeek"]
+    assert request_body["provider"]["allow_fallbacks"] is False
+    assert request_body["provider"]["require_parameters"] is True
+
+    # Verify the response
+    assert response is not None
+    assert response.choices[0].message.content == "Hello from mocked response!"
+    

tests/llm_translation/test_anthropic_completion.py

@@ -1116,3 +1116,6 @@ def test_anthropic_thinking_in_assistant_message(model):
     response = litellm.completion(**params)
 
     assert response is not None
+
+
+

tests/llm_translation/test_bedrock_completion.py

@@ -2430,63 +2430,66 @@ def test_bedrock_process_empty_text_blocks():
     assert modified_message["content"][0]["text"] == "Please continue."
 
 
-def test_nova_optional_params_tool_choice():
-    litellm.drop_params = True
-    litellm.set_verbose = True
-    litellm.completion(
-        messages=[
-            {"role": "user", "content": "A WWII competitive game for 4-8 players"}
-        ],
-        model="bedrock/us.amazon.nova-pro-v1:0",
-        temperature=0.3,
-        tools=[
-            {
-                "type": "function",
-                "function": {
-                    "name": "GameDefinition",
-                    "description": "Correctly extracted `GameDefinition` with all the required parameters with correct types",
-                    "parameters": {
-                        "$defs": {
-                            "TurnDurationEnum": {
-                                "enum": ["action", "encounter", "battle", "operation"],
-                                "title": "TurnDurationEnum",
-                                "type": "string",
-                            }
-                        },
-                        "properties": {
-                            "id": {
-                                "anyOf": [{"type": "integer"}, {"type": "null"}],
-                                "default": None,
-                                "title": "Id",
-                            },
-                            "prompt": {"title": "Prompt", "type": "string"},
-                            "name": {"title": "Name", "type": "string"},
-                            "description": {"title": "Description", "type": "string"},
-                            "competitve": {"title": "Competitve", "type": "boolean"},
-                            "players_min": {"title": "Players Min", "type": "integer"},
-                            "players_max": {"title": "Players Max", "type": "integer"},
-                            "turn_duration": {
-                                "$ref": "#/$defs/TurnDurationEnum",
-                                "description": "how long the passing of a turn should represent for a game at this scale",
-                            },
-                        },
-                        "required": [
-                            "competitve",
-                            "description",
-                            "name",
-                            "players_max",
-                            "players_min",
-                            "prompt",
-                            "turn_duration",
-                        ],
-                        "type": "object",
-                    },
-                },
-            }
-        ],
-        tool_choice={"type": "function", "function": {"name": "GameDefinition"}},
-    )
 
+def test_nova_optional_params_tool_choice():
+    try:
+        litellm.drop_params = True
+        litellm.set_verbose = True
+        litellm.completion(
+            messages=[
+                {"role": "user", "content": "A WWII competitive game for 4-8 players"}
+            ],
+            model="bedrock/us.amazon.nova-pro-v1:0",
+            temperature=0.3,
+            tools=[
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "GameDefinition",
+                        "description": "Correctly extracted `GameDefinition` with all the required parameters with correct types",
+                        "parameters": {
+                            "$defs": {
+                                "TurnDurationEnum": {
+                                    "enum": ["action", "encounter", "battle", "operation"],
+                                    "title": "TurnDurationEnum",
+                                    "type": "string",
+                                }
+                            },
+                            "properties": {
+                                "id": {
+                                    "anyOf": [{"type": "integer"}, {"type": "null"}],
+                                    "default": None,
+                                    "title": "Id",
+                                },
+                                "prompt": {"title": "Prompt", "type": "string"},
+                                "name": {"title": "Name", "type": "string"},
+                                "description": {"title": "Description", "type": "string"},
+                                "competitve": {"title": "Competitve", "type": "boolean"},
+                                "players_min": {"title": "Players Min", "type": "integer"},
+                                "players_max": {"title": "Players Max", "type": "integer"},
+                                "turn_duration": {
+                                    "$ref": "#/$defs/TurnDurationEnum",
+                                    "description": "how long the passing of a turn should represent for a game at this scale",
+                                },
+                            },
+                            "required": [
+                                "competitve",
+                                "description",
+                                "name",
+                                "players_max",
+                                "players_min",
+                                "prompt",
+                                "turn_duration",
+                            ],
+                            "type": "object",
+                        },
+                    },
+                }
+            ],
+            tool_choice={"type": "function", "function": {"name": "GameDefinition"}},
+        )
+    except litellm.APIConnectionError:
+        pass
 
 class TestBedrockEmbedding(BaseLLMEmbeddingTest):
     def get_base_embedding_call_args(self) -> dict:

tests/llm_translation/test_bedrock_nova_json.py

@@ -15,6 +15,12 @@ class TestBedrockNovaJson(BaseLLMChatTest):
         return {
             "model": "bedrock/converse/us.amazon.nova-micro-v1:0",
         }
+    
+    def test_json_response_nested_pydantic_obj(self):
+        pass
+    
+    def test_json_response_nested_json_schema(self):
+        pass
 
     def test_tool_call_no_arguments(self, tool_call_no_arguments):
         """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""

tests/llm_translation/test_prompt_factory.py

@@ -300,6 +300,60 @@ def test_anthropic_cache_controls_pt():
     print("translated_messages: ", translated_messages)
 
 
+def test_anthropic_cache_controls_tool_calls_pt():
+    """
+    Tests that cache_control is properly set in tool_calls when converting messages
+    for the Anthropic API.
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": "Can you help me get the weather?",
+        },
+        {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                {
+                    "id": "weather-tool-id-123",
+                    "function": {
+                        "arguments": '{"location": "San Francisco"}',
+                        "name": "get_weather",
+                    },
+                    "type": "function",
+                }
+            ],
+            "cache_control": {"type": "ephemeral"},
+        },
+        {
+            "role": "function",
+            "content": '{"temperature": 72, "unit": "fahrenheit", "description": "sunny"}',
+            "name": "get_weather",
+            "tool_call_id": "weather-tool-id-123",
+            "cache_control": {"type": "ephemeral"},
+        },
+    ]
+
+    translated_messages = anthropic_messages_pt(
+        messages, model="claude-3-sonnet-20240229", llm_provider="anthropic"
+    )
+
+    print("Translated tool call messages:", translated_messages)
+
+    assert translated_messages[0]["role"] == "user"
+
+    assert translated_messages[1]["role"] == "assistant"
+    for content_item in translated_messages[1]["content"]:
+        if content_item["type"] == "tool_use":
+            assert "cache_control" not in content_item
+            assert content_item["name"] == "get_weather"
+
+    assert translated_messages[2]["role"] == "user"
+    for content_item in translated_messages[2]["content"]:
+        if content_item["type"] == "tool_result":
+            assert content_item["cache_control"] == {"type": "ephemeral"}
+
+
 @pytest.mark.parametrize("provider", ["bedrock", "anthropic"])
 def test_bedrock_parallel_tool_calling_pt(provider):
     """
@@ -701,7 +755,7 @@ def test_hf_chat_template():
             "add_eos_token": False,
             "bos_token": {
                 "__type": "AddedToken",
-                "content": "<｜begin▁of▁sentence｜>",
+                "content": "",
                 "lstrip": False,
                 "normalized": True,
                 "rstrip": False,
@@ -710,7 +764,7 @@ def test_hf_chat_template():
             "clean_up_tokenization_spaces": False,
             "eos_token": {
                 "__type": "AddedToken",
-                "content": "<｜end▁of▁sentence｜>",
+                "content": "",
                 "lstrip": False,
                 "normalized": True,
                 "rstrip": False,
@@ -720,7 +774,7 @@ def test_hf_chat_template():
             "model_max_length": 16384,
             "pad_token": {
                 "__type": "AddedToken",
-                "content": "<｜end▁of▁sentence｜>",
+                "content": "",
                 "lstrip": False,
                 "normalized": True,
                 "rstrip": False,
@@ -729,7 +783,7 @@ def test_hf_chat_template():
             "sp_model_kwargs": {},
             "unk_token": None,
             "tokenizer_class": "LlamaTokenizerFast",
-            "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
+            "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{' ' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{' ' + tool['type'] + ' ' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + ' '}}{%- set ns.is_first = true -%}{%- else %}{{' ' + tool['type'] + ' ' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + ' '}}{{' '}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{' ' + message['content'] + ' '}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{' ' + content + ' '}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{' ' + message['content'] + ' '}}{%- set ns.is_output_first = false %}{%- else %}{{' ' + message['content'] + ' '}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{' '}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{' '}}{% endif %}",
         },
     )
 
@@ -741,7 +795,9 @@ def test_hf_chat_template():
     print(chat_template)
     assert (
         chat_template.rstrip()
-        == """<｜begin▁of▁sentence｜>You are a helpful assistant.<｜User｜>What is the weather in Copenhagen?<｜Assistant｜><think>"""
+        == """You are a helpful assistant.
+What is the weather in Copenhagen?
+"""
     )
 
 

tests/local_testing/test_amazing_vertex_completion.py

@@ -59,6 +59,7 @@ VERTEX_MODELS_TO_NOT_TEST = [
     "gemini-pro-experimental",
     "gemini-flash-experimental",
     "gemini-1.5-flash-exp-0827",
+    "gemini-2.0-pro-exp-02-05",
     "gemini-pro-flash",
     "gemini-1.5-flash-exp-0827",
     "gemini-2.0-flash-exp",

tests/local_testing/test_caching.py

@@ -2342,7 +2342,7 @@ async def test_redis_caching_llm_caching_ttl(sync_mode):
 
             # Verify that the set method was called on the mock Redis instance
             mock_redis_instance.set.assert_called_once_with(
-                name="test", value='"test_value"', ex=120
+                name="test", value='"test_value"', ex=120, nx=False
             )
 
     ## Increment cache

ui/litellm-dashboard/out/_next/static/chunks/app/layout-429ad74a94df7643.js

@@ -1 +0,0 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_cf7686', '__Inter_Fallback_cf7686'",fontStyle:"normal"},className:"__className_cf7686"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);

ui/litellm-dashboard/out/_next/static/chunks/app/layout-af8319e6c59a08da.js

@@ -0,0 +1 @@
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{6580:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_cf7686', '__Inter_Fallback_cf7686'",fontStyle:"normal"},className:"__className_cf7686"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=6580)}),_N_E=n.O()}]);

ui/litellm-dashboard/out/_next/static/chunks/app/model_hub/page-068a441595bd0fc3.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,u){Promise.resolve().then(u.bind(u,52829))},52829:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(92699);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1})}}},function(e){e.O(0,[42,261,250,699,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{11790:function(e,n,u){Promise.resolve().then(u.bind(u,52829))},52829:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(92699);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1})}}},function(e){e.O(0,[42,261,250,699,971,117,744],function(){return e(e.s=11790)}),_N_E=e.O()}]);

ui/litellm-dashboard/out/_next/static/chunks/app/onboarding/page-466610167078a21c.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[461],{8672:function(e,t,n){Promise.resolve().then(n.bind(n,12011))},12011:function(e,t,n){"use strict";n.r(t),n.d(t,{default:function(){return S}});var s=n(57437),o=n(2265),a=n(99376),i=n(20831),c=n(94789),l=n(12514),r=n(49804),u=n(67101),d=n(84264),m=n(49566),h=n(96761),x=n(84566),p=n(19250),f=n(14474),k=n(13634),j=n(73002),g=n(3914);function S(){let[e]=k.Z.useForm(),t=(0,a.useSearchParams)();(0,g.e)("token");let n=t.get("invitation_id"),[S,_]=(0,o.useState)(null),[w,Z]=(0,o.useState)(""),[N,b]=(0,o.useState)(""),[T,v]=(0,o.useState)(null),[y,E]=(0,o.useState)(""),[C,U]=(0,o.useState)("");return(0,o.useEffect)(()=>{n&&(0,p.W_)(n).then(e=>{let t=e.login_url;console.log("login_url:",t),E(t);let n=e.token,s=(0,f.o)(n);U(n),console.log("decoded:",s),_(s.key),console.log("decoded user email:",s.user_email),b(s.user_email),v(s.user_id)})},[n]),(0,s.jsx)("div",{className:"mx-auto w-full max-w-md mt-10",children:(0,s.jsxs)(l.Z,{children:[(0,s.jsx)(h.Z,{className:"text-sm mb-5 text-center",children:"\uD83D\uDE85 LiteLLM"}),(0,s.jsx)(h.Z,{className:"text-xl",children:"Sign up"}),(0,s.jsx)(d.Z,{children:"Claim your user account to login to Admin UI."}),(0,s.jsx)(c.Z,{className:"mt-4",title:"SSO",icon:x.GH$,color:"sky",children:(0,s.jsxs)(u.Z,{numItems:2,className:"flex justify-between items-center",children:[(0,s.jsx)(r.Z,{children:"SSO is under the Enterprise Tirer."}),(0,s.jsx)(r.Z,{children:(0,s.jsx)(i.Z,{variant:"primary",className:"mb-2",children:(0,s.jsx)("a",{href:"https://forms.gle/W3U4PZpJGFHWtHyA9",target:"_blank",children:"Get Free Trial"})})})]})}),(0,s.jsxs)(k.Z,{className:"mt-10 mb-5 mx-auto",layout:"vertical",onFinish:e=>{console.log("in handle submit. accessToken:",S,"token:",C,"formValues:",e),S&&C&&(e.user_email=N,T&&n&&(0,p.m_)(S,n,T,e.password).then(e=>{var t;let n="/ui/";n+="?userID="+((null===(t=e.data)||void 0===t?void 0:t.user_id)||e.user_id),document.cookie="token="+C,console.log("redirecting to:",n),window.location.href=n}))},children:[(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(k.Z.Item,{label:"Email Address",name:"user_email",children:(0,s.jsx)(m.Z,{type:"email",disabled:!0,value:N,defaultValue:N,className:"max-w-md"})}),(0,s.jsx)(k.Z.Item,{label:"Password",name:"password",rules:[{required:!0,message:"password required to sign up"}],help:"Create a password for your account",children:(0,s.jsx)(m.Z,{placeholder:"",type:"password",className:"max-w-md"})})]}),(0,s.jsx)("div",{className:"mt-10",children:(0,s.jsx)(j.ZP,{htmlType:"submit",children:"Sign Up"})})]})]})})}},3914:function(e,t,n){"use strict";function s(){let e=window.location.hostname,t=["Lax","Strict","None"];["/","/ui"].forEach(n=>{document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,";"),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,";"),t.forEach(t=>{let s="None"===t?" Secure;":"";document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; SameSite=").concat(t,";").concat(s),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,"; SameSite=").concat(t,";").concat(s)})}),console.log("After clearing cookies:",document.cookie)}function o(e){let t=document.cookie.split("; ").find(t=>t.startsWith(e+"="));return t?t.split("=")[1]:null}n.d(t,{b:function(){return s},e:function(){return o}})}},function(e){e.O(0,[665,42,899,250,971,117,744],function(){return e(e.s=8672)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[461],{32922:function(e,t,n){Promise.resolve().then(n.bind(n,12011))},12011:function(e,t,n){"use strict";n.r(t),n.d(t,{default:function(){return S}});var s=n(57437),o=n(2265),a=n(99376),i=n(20831),c=n(94789),l=n(12514),r=n(49804),u=n(67101),d=n(84264),m=n(49566),h=n(96761),x=n(84566),p=n(19250),f=n(14474),k=n(13634),j=n(73002),g=n(3914);function S(){let[e]=k.Z.useForm(),t=(0,a.useSearchParams)();(0,g.e)("token");let n=t.get("invitation_id"),[S,_]=(0,o.useState)(null),[w,Z]=(0,o.useState)(""),[N,b]=(0,o.useState)(""),[T,v]=(0,o.useState)(null),[y,E]=(0,o.useState)(""),[C,U]=(0,o.useState)("");return(0,o.useEffect)(()=>{n&&(0,p.W_)(n).then(e=>{let t=e.login_url;console.log("login_url:",t),E(t);let n=e.token,s=(0,f.o)(n);U(n),console.log("decoded:",s),_(s.key),console.log("decoded user email:",s.user_email),b(s.user_email),v(s.user_id)})},[n]),(0,s.jsx)("div",{className:"mx-auto w-full max-w-md mt-10",children:(0,s.jsxs)(l.Z,{children:[(0,s.jsx)(h.Z,{className:"text-sm mb-5 text-center",children:"\uD83D\uDE85 LiteLLM"}),(0,s.jsx)(h.Z,{className:"text-xl",children:"Sign up"}),(0,s.jsx)(d.Z,{children:"Claim your user account to login to Admin UI."}),(0,s.jsx)(c.Z,{className:"mt-4",title:"SSO",icon:x.GH$,color:"sky",children:(0,s.jsxs)(u.Z,{numItems:2,className:"flex justify-between items-center",children:[(0,s.jsx)(r.Z,{children:"SSO is under the Enterprise Tier."}),(0,s.jsx)(r.Z,{children:(0,s.jsx)(i.Z,{variant:"primary",className:"mb-2",children:(0,s.jsx)("a",{href:"https://forms.gle/W3U4PZpJGFHWtHyA9",target:"_blank",children:"Get Free Trial"})})})]})}),(0,s.jsxs)(k.Z,{className:"mt-10 mb-5 mx-auto",layout:"vertical",onFinish:e=>{console.log("in handle submit. accessToken:",S,"token:",C,"formValues:",e),S&&C&&(e.user_email=N,T&&n&&(0,p.m_)(S,n,T,e.password).then(e=>{var t;let n="/ui/";n+="?userID="+((null===(t=e.data)||void 0===t?void 0:t.user_id)||e.user_id),document.cookie="token="+C,console.log("redirecting to:",n),window.location.href=n}))},children:[(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(k.Z.Item,{label:"Email Address",name:"user_email",children:(0,s.jsx)(m.Z,{type:"email",disabled:!0,value:N,defaultValue:N,className:"max-w-md"})}),(0,s.jsx)(k.Z.Item,{label:"Password",name:"password",rules:[{required:!0,message:"password required to sign up"}],help:"Create a password for your account",children:(0,s.jsx)(m.Z,{placeholder:"",type:"password",className:"max-w-md"})})]}),(0,s.jsx)("div",{className:"mt-10",children:(0,s.jsx)(j.ZP,{htmlType:"submit",children:"Sign Up"})})]})]})})}},3914:function(e,t,n){"use strict";function s(){let e=window.location.hostname,t=["Lax","Strict","None"];["/","/ui"].forEach(n=>{document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,";"),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,";"),t.forEach(t=>{let s="None"===t?" Secure;":"";document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; SameSite=").concat(t,";").concat(s),document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=".concat(n,"; domain=").concat(e,"; SameSite=").concat(t,";").concat(s)})}),console.log("After clearing cookies:",document.cookie)}function o(e){let t=document.cookie.split("; ").find(t=>t.startsWith(e+"="));return t?t.split("=")[1]:null}n.d(t,{b:function(){return s},e:function(){return o}})}},function(e){e.O(0,[665,42,899,250,971,117,744],function(){return e(e.s=32922)}),_N_E=e.O()}]);

ui/litellm-dashboard/out/_next/static/chunks/main-app-475d6efe4080647d.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{10264:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(10264)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{20169:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(20169)}),_N_E=e.O()}]);

ui/litellm-dashboard/out/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-75a5453f51d60261.js"/><script src="/ui/_next/static/chunks/fd9d1056-524b80e1a6b8bb06.js" async=""></script><script src="/ui/_next/static/chunks/117-883150efc583d711.js" async=""></script><script src="/ui/_next/static/chunks/main-app-4f7318ae681a6d94.js" async=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-75a5453f51d60261.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/a34f9d1faa5f3315-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"style\"]\n3:HL[\"/ui/_next/static/css/1f6915676624c422.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"4:I[12846,[],\"\"]\n6:I[19107,[],\"ClientPageRoot\"]\n7:I[38411,[\"665\",\"static/chunks/3014691f-0b72c78cfebbd712.js\",\"990\",\"static/chunks/13b76428-ebdf3012af0e4489.js\",\"42\",\"static/chunks/42-1cbed529ecb084e0.js\",\"261\",\"static/chunks/261-57d48f76eec1e568.js\",\"899\",\"static/chunks/899-9af4feaf6f21839c.js\",\"274\",\"static/chunks/274-bddaf0cf6c91e72f.js\",\"250\",\"static/chunks/250-dfc03a6fb4f0d254.js\",\"699\",\"static/chunks/699-87224ecba28f1f48.js\",\"931\",\"static/chunks/app/page-0f46d4a8b9bdf1c0.js\"],\"default\",1]\n8:I[4707,[],\"\"]\n9:I[36423,[],\"\"]\nb:I[61060,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L4\",null,{\"buildId\":\"Yb50LG5p7c9QpG54GIoFV\",\"assetPrefix\":\"/ui\",\"urlParts\":[\"\",\"\"],\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[[\"$L5\",[\"$\",\"$L6\",null,{\"props\":{\"params\":{},\"searchParams\":{}},\"Component\":\"$7\"}],null],null],null]},[[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/1f6915676624c422.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_cf7686\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[null,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script></body></html>
+<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-75a5453f51d60261.js"/><script src="/ui/_next/static/chunks/fd9d1056-524b80e1a6b8bb06.js" async=""></script><script src="/ui/_next/static/chunks/117-883150efc583d711.js" async=""></script><script src="/ui/_next/static/chunks/main-app-475d6efe4080647d.js" async=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-75a5453f51d60261.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/a34f9d1faa5f3315-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"style\"]\n3:HL[\"/ui/_next/static/css/6e6c0523f29030fd.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"4:I[12846,[],\"\"]\n6:I[19107,[],\"ClientPageRoot\"]\n7:I[37140,[\"665\",\"static/chunks/3014691f-0b72c78cfebbd712.js\",\"990\",\"static/chunks/13b76428-ebdf3012af0e4489.js\",\"42\",\"static/chunks/42-1cbed529ecb084e0.js\",\"261\",\"static/chunks/261-57d48f76eec1e568.js\",\"899\",\"static/chunks/899-9af4feaf6f21839c.js\",\"810\",\"static/chunks/810-493ce8d3227b491d.js\",\"250\",\"static/chunks/250-282480f9afa56ac6.js\",\"699\",\"static/chunks/699-87224ecba28f1f48.js\",\"931\",\"static/chunks/app/page-24bd7b05ba767df8.js\"],\"default\",1]\n8:I[4707,[],\"\"]\n9:I[36423,[],\"\"]\nb:I[61060,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L4\",null,{\"buildId\":\"zniqNKJW4P7vXGttXOEEQ\",\"assetPrefix\":\"/ui\",\"urlParts\":[\"\",\"\"],\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[[\"$L5\",[\"$\",\"$L6\",null,{\"props\":{\"params\":{},\"searchParams\":{}},\"Component\":\"$7\"}],null],null],null]},[[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/86f6cc749f6b8493.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/6e6c0523f29030fd.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_cf7686\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[null,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script></body></html>

ui/litellm-dashboard/out/index.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[38411,["665","static/chunks/3014691f-0b72c78cfebbd712.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","899","static/chunks/899-9af4feaf6f21839c.js","274","static/chunks/274-bddaf0cf6c91e72f.js","250","static/chunks/250-dfc03a6fb4f0d254.js","699","static/chunks/699-87224ecba28f1f48.js","931","static/chunks/app/page-0f46d4a8b9bdf1c0.js"],"default",1]
+3:I[37140,["665","static/chunks/3014691f-0b72c78cfebbd712.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","899","static/chunks/899-9af4feaf6f21839c.js","810","static/chunks/810-493ce8d3227b491d.js","250","static/chunks/250-282480f9afa56ac6.js","699","static/chunks/699-87224ecba28f1f48.js","931","static/chunks/app/page-24bd7b05ba767df8.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["Yb50LG5p7c9QpG54GIoFV",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/1f6915676624c422.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zniqNKJW4P7vXGttXOEEQ",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/6e6c0523f29030fd.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

ui/litellm-dashboard/out/model_hub.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[52829,["42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","250","static/chunks/250-dfc03a6fb4f0d254.js","699","static/chunks/699-87224ecba28f1f48.js","418","static/chunks/app/model_hub/page-cde2fb783e81a6c1.js"],"default",1]
+3:I[52829,["42","static/chunks/42-1cbed529ecb084e0.js","261","static/chunks/261-57d48f76eec1e568.js","250","static/chunks/250-282480f9afa56ac6.js","699","static/chunks/699-87224ecba28f1f48.js","418","static/chunks/app/model_hub/page-068a441595bd0fc3.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["Yb50LG5p7c9QpG54GIoFV",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/1f6915676624c422.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zniqNKJW4P7vXGttXOEEQ",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/6e6c0523f29030fd.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

ui/litellm-dashboard/out/onboarding.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[12011,["665","static/chunks/3014691f-0b72c78cfebbd712.js","42","static/chunks/42-1cbed529ecb084e0.js","899","static/chunks/899-9af4feaf6f21839c.js","250","static/chunks/250-dfc03a6fb4f0d254.js","461","static/chunks/app/onboarding/page-2bf7a26db5342dbf.js"],"default",1]
+3:I[12011,["665","static/chunks/3014691f-0b72c78cfebbd712.js","42","static/chunks/42-1cbed529ecb084e0.js","899","static/chunks/899-9af4feaf6f21839c.js","250","static/chunks/250-282480f9afa56ac6.js","461","static/chunks/app/onboarding/page-466610167078a21c.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["Yb50LG5p7c9QpG54GIoFV",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/1f6915676624c422.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zniqNKJW4P7vXGttXOEEQ",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/86f6cc749f6b8493.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/ui/_next/static/css/6e6c0523f29030fd.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_cf7686","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

ui/litellm-dashboard/src/app/onboarding/page.tsx

@@ -110,7 +110,7 @@ export default function Onboarding() {
           color="sky"
         >
           <Grid numItems={2} className="flex justify-between items-center">
-            <Col>SSO is under the Enterprise Tirer.</Col>
+            <Col>SSO is under the Enterprise Tier.</Col>
 
             <Col>
               <Button variant="primary" className="mb-2">

ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx

@@ -34,6 +34,7 @@ export const prepareModelAddRequest = async (
       }
 
       // Create a deployment for each mapping
+      const deployments = [];
       for (const mapping of modelMappings) {
         const litellmParamsObj: Record<string, any> = {};
         const modelInfoObj: Record<string, any> = {};
@@ -142,8 +143,10 @@ export const prepareModelAddRequest = async (
           }
         }
 
-        return { litellmParamsObj, modelInfoObj, modelName };
+        deployments.push({ litellmParamsObj, modelInfoObj, modelName });
       }
+
+      return deployments;
     } catch (error) {
       message.error("Failed to create model: " + error, 10);
     }
@@ -156,22 +159,25 @@ export const handleAddModelSubmit = async (
     callback?: () => void,
   ) => {
     try {
-      const result = await prepareModelAddRequest(values, accessToken, form);
+      const deployments = await prepareModelAddRequest(values, accessToken, form);
       
-      if (!result) {
-        return; // Exit if preparation failed
+      if (!deployments || deployments.length === 0) {
+        return; // Exit if preparation failed or no deployments
       }
       
-      const { litellmParamsObj, modelInfoObj, modelName } = result;
-      
-      const new_model: Model = {
-        model_name: modelName,
-        litellm_params: litellmParamsObj,
-        model_info: modelInfoObj,
-      };
-      
-      const response: any = await modelCreateCall(accessToken, new_model);
-      console.log(`response for model create call: ${response["data"]}`);
+      // Create each deployment
+      for (const deployment of deployments) {
+        const { litellmParamsObj, modelInfoObj, modelName } = deployment;
+        
+        const new_model: Model = {
+          model_name: modelName,
+          litellm_params: litellmParamsObj,
+          model_info: modelInfoObj,
+        };
+        
+        const response: any = await modelCreateCall(accessToken, new_model);
+        console.log(`response for model create call: ${response["data"]}`);
+      }
       
       callback && callback();
       form.resetFields();

ui/litellm-dashboard/src/components/add_model/model_connection_test.tsx

@@ -55,7 +55,7 @@ const ModelConnectionTest: React.FC<ModelConnectionTestProps> = ({
 
       console.log("Result from prepareModelAddRequest:", result);
 
-      const { litellmParamsObj, modelInfoObj, modelName: returnedModelName } = result;
+      const { litellmParamsObj, modelInfoObj, modelName: returnedModelName } = result[0];
 
       const response = await testConnectionRequest(accessToken, litellmParamsObj, modelInfoObj?.mode);
       if (response.status === "success") {

ui/litellm-dashboard/src/components/chat_ui.tsx

@@ -20,15 +20,28 @@ import {
   SelectItem,
   TextInput,
   Button,
+  Divider,
 } from "@tremor/react";
 
-import { message, Select } from "antd";
-import { modelAvailableCall } from "./networking";
-import openai from "openai";
-import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
+import { message, Select, Spin, Typography, Tooltip } from "antd";
+import { makeOpenAIChatCompletionRequest } from "./chat_ui/llm_calls/chat_completion";
+import { makeOpenAIImageGenerationRequest } from "./chat_ui/llm_calls/image_generation";
+import { fetchAvailableModels, ModelGroup  } from "./chat_ui/llm_calls/fetch_models";
+import { litellmModeMapping, ModelMode, EndpointType, getEndpointType } from "./chat_ui/mode_endpoint_mapping";
 import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
-import { Typography } from "antd";
 import { coy } from 'react-syntax-highlighter/dist/esm/styles/prism';
+import EndpointSelector from "./chat_ui/EndpointSelector";
+import { determineEndpointType } from "./chat_ui/EndpointUtils";
+import { 
+  SendOutlined, 
+  ApiOutlined, 
+  KeyOutlined, 
+  ClearOutlined, 
+  RobotOutlined, 
+  UserOutlined,
+  DeleteOutlined,
+  LoadingOutlined
+} from "@ant-design/icons";
 
 interface ChatUIProps {
   accessToken: string | null;
@@ -38,45 +51,6 @@ interface ChatUIProps {
   disabledPersonalKeyCreation: boolean;
 }
 
-async function generateModelResponse(
-  chatHistory: { role: string; content: string }[],
-  updateUI: (chunk: string, model: string) => void,
-  selectedModel: string,
-  accessToken: string
-) {
-  // base url should be the current base_url
-  const isLocal = process.env.NODE_ENV === "development";
-  if (isLocal !== true) {
-    console.log = function () {};
-  }
-  console.log("isLocal:", isLocal);
-  const proxyBaseUrl = isLocal
-    ? "http://localhost:4000"
-    : window.location.origin;
-  const client = new openai.OpenAI({
-    apiKey: accessToken, // Replace with your OpenAI API key
-    baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL
-    dangerouslyAllowBrowser: true, // using a temporary litellm proxy key
-  });
-
-  try {
-    const response = await client.chat.completions.create({
-      model: selectedModel,
-      stream: true,
-      messages: chatHistory as ChatCompletionMessageParam[],
-    });
-
-    for await (const chunk of response) {
-      console.log(chunk);
-      if (chunk.choices[0].delta.content) {
-        updateUI(chunk.choices[0].delta.content, chunk.model);
-      }
-    }
-  } catch (error) {
-    message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20);
-  }
-}
-
 const ChatUI: React.FC<ChatUIProps> = ({
   accessToken,
   token,
@@ -89,63 +63,55 @@ const ChatUI: React.FC<ChatUIProps> = ({
   );
   const [apiKey, setApiKey] = useState("");
   const [inputMessage, setInputMessage] = useState("");
-  const [chatHistory, setChatHistory] = useState<{ role: string; content: string; model?: string }[]>([]);
+  const [chatHistory, setChatHistory] = useState<{ role: string; content: string; model?: string; isImage?: boolean }[]>([]);
   const [selectedModel, setSelectedModel] = useState<string | undefined>(
     undefined
   );
   const [showCustomModelInput, setShowCustomModelInput] = useState<boolean>(false);
-  const [modelInfo, setModelInfo] = useState<any[]>([]);
+  const [modelInfo, setModelInfo] = useState<ModelGroup[]>([]);
   const customModelTimeout = useRef<NodeJS.Timeout | null>(null);
+  const [endpointType, setEndpointType] = useState<string>(EndpointType.CHAT);
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+  const abortControllerRef = useRef<AbortController | null>(null);
 
   const chatEndRef = useRef<HTMLDivElement>(null);
 
   useEffect(() => {
-    let useApiKey = apiKeySource === 'session' ? accessToken : apiKey;
-    console.log("useApiKey:", useApiKey);
-    if (!useApiKey || !token || !userRole || !userID) {
-      console.log("useApiKey or token or userRole or userID is missing = ", useApiKey, token, userRole, userID);
+    let userApiKey = apiKeySource === 'session' ? accessToken : apiKey;
+    if (!userApiKey || !token || !userRole || !userID) {
+      console.log("userApiKey or token or userRole or userID is missing = ", userApiKey, token, userRole, userID);
       return;
     }
 
-    
-
     // Fetch model info and set the default selected model
-    const fetchModelInfo = async () => {
+    const loadModels = async () => {
       try {
-        const fetchedAvailableModels = await modelAvailableCall(
-          useApiKey ?? '', // Use empty string if useApiKey is null,
-          userID,
-          userRole
+        if (!userApiKey) {
+          console.log("userApiKey is missing");
+          return;
+        }
+        const uniqueModels = await fetchAvailableModels(
+          userApiKey,
         );
   
-        console.log("model_info:", fetchedAvailableModels);
+        console.log("Fetched models:", uniqueModels);
   
-        if (fetchedAvailableModels?.data.length > 0) {
-          // Create a Map to store unique models using the model ID as key
-          const uniqueModelsMap = new Map();
-          
-          fetchedAvailableModels["data"].forEach((item: { id: string }) => {
-            uniqueModelsMap.set(item.id, {
-              value: item.id,
-              label: item.id
-            });
-          });
-
-          // Convert Map values back to array
-          const uniqueModels = Array.from(uniqueModelsMap.values());
-
-          // Sort models alphabetically
-          uniqueModels.sort((a, b) => a.label.localeCompare(b.label));
-
+        if (uniqueModels.length > 0) {
           setModelInfo(uniqueModels);
-          setSelectedModel(uniqueModels[0].value);
+          setSelectedModel(uniqueModels[0].model_group);
+          
+          // Auto-set endpoint based on the first model's mode
+          if (uniqueModels[0].mode) {
+            const initialEndpointType = determineEndpointType(uniqueModels[0].model_group, uniqueModels);
+            setEndpointType(initialEndpointType);
+          }
         }
       } catch (error) {
         console.error("Error fetching model info:", error);
       }
     };
   
-    fetchModelInfo();
+    loadModels();
   }, [accessToken, userID, userRole, apiKeySource, apiKey]);
   
 
@@ -162,11 +128,11 @@ const ChatUI: React.FC<ChatUIProps> = ({
     }
   }, [chatHistory]);
 
-  const updateUI = (role: string, chunk: string, model?: string) => {
+  const updateTextUI = (role: string, chunk: string, model?: string) => {
     setChatHistory((prevHistory) => {
       const lastMessage = prevHistory[prevHistory.length - 1];
 
-      if (lastMessage && lastMessage.role === role) {
+      if (lastMessage && lastMessage.role === role && !lastMessage.isImage) {
         return [
           ...prevHistory.slice(0, prevHistory.length - 1),
           { role, content: lastMessage.content + chunk, model },
@@ -177,12 +143,28 @@ const ChatUI: React.FC<ChatUIProps> = ({
     });
   };
 
+  const updateImageUI = (imageUrl: string, model: string) => {
+    setChatHistory((prevHistory) => [
+      ...prevHistory,
+      { role: "assistant", content: imageUrl, model, isImage: true }
+    ]);
+  };
+
   const handleKeyDown = (event: React.KeyboardEvent<HTMLInputElement>) => {
     if (event.key === 'Enter') {
       handleSendMessage();
     }
   };
 
+  const handleCancelRequest = () => {
+    if (abortControllerRef.current) {
+      abortControllerRef.current.abort();
+      abortControllerRef.current = null;
+      setIsLoading(false);
+      message.info("Request cancelled");
+    }
+  };
+
   const handleSendMessage = async () => {
     if (inputMessage.trim() === "") return;
 
@@ -197,27 +179,52 @@ const ChatUI: React.FC<ChatUIProps> = ({
       return;
     }
 
+    // Create new abort controller for this request
+    abortControllerRef.current = new AbortController();
+    const signal = abortControllerRef.current.signal;
+
     // Create message object without model field for API call
     const newUserMessage = { role: "user", content: inputMessage };
     
-    // Create chat history for API call - strip out model field
-    const apiChatHistory = [...chatHistory.map(({ role, content }) => ({ role, content })), newUserMessage];
-    
-    // Update UI with full message object (including model field for display)
+    // Update UI with full message object
     setChatHistory([...chatHistory, newUserMessage]);
+    setIsLoading(true);
 
     try {
       if (selectedModel) {
-        await generateModelResponse(
-          apiChatHistory,
-          (chunk, model) => updateUI("assistant", chunk, model),
-          selectedModel,
-          effectiveApiKey
-        );
+        // Use EndpointType enum for comparison
+        if (endpointType === EndpointType.CHAT) {
+          // Create chat history for API call - strip out model field and isImage field
+          const apiChatHistory = [...chatHistory.filter(msg => !msg.isImage).map(({ role, content }) => ({ role, content })), newUserMessage];
+          
+          await makeOpenAIChatCompletionRequest(
+            apiChatHistory,
+            (chunk, model) => updateTextUI("assistant", chunk, model),
+            selectedModel,
+            effectiveApiKey,
+            signal
+          );
+        } else if (endpointType === EndpointType.IMAGE) {
+          // For image generation
+          await makeOpenAIImageGenerationRequest(
+            inputMessage,
+            (imageUrl, model) => updateImageUI(imageUrl, model),
+            selectedModel,
+            effectiveApiKey,
+            signal
+          );
+        }
       }
     } catch (error) {
-      console.error("Error fetching model response", error);
-      updateUI("assistant", "Error fetching model response");
+      if (signal.aborted) {
+        console.log("Request was cancelled");
+      } else {
+        console.error("Error fetching response", error);
+        updateTextUI("assistant", "Error fetching response");
+      }
+    } finally {
+      setIsLoading(false);
+      abortControllerRef.current = null;
     }
 
     setInputMessage("");
@@ -238,193 +245,240 @@ const ChatUI: React.FC<ChatUIProps> = ({
     );
   }
 
-  const onChange = (value: string) => {
+  const onModelChange = (value: string) => {
     console.log(`selected ${value}`);
     setSelectedModel(value);
+    
+    // Use the utility function to determine the endpoint type
+    if (value !== 'custom') {
+      const newEndpointType = determineEndpointType(value, modelInfo);
+      setEndpointType(newEndpointType);
+    }
+    
     setShowCustomModelInput(value === 'custom');
   };
 
-  return (
-    <div style={{ width: "100%", position: "relative" }}>
-      <Grid className="gap-2 p-8 h-[80vh] w-full mt-2">
-        <Card>
-          
-          <TabGroup>
-            <TabList>
-              <Tab>Chat</Tab>
-            </TabList>
-            <TabPanels>
-              <TabPanel>
-                <div className="sm:max-w-2xl">
-                  <Grid numItems={2}>
-                    <Col>
-                      <Text>API Key Source</Text>
-                      <Select
-                        disabled={disabledPersonalKeyCreation}
-                        defaultValue="session"
-                        style={{ width: "100%" }}
-                        onChange={(value) => setApiKeySource(value as "session" | "custom")}
-                        options={[
-                          { value: 'session', label: 'Current UI Session' },
-                          { value: 'custom', label: 'Virtual Key' },
-                        ]}
-                      />
-                      {apiKeySource === 'custom' && (
-                        <TextInput
-                          className="mt-2"
-                          placeholder="Enter custom API key"
-                          type="password"
-                          onValueChange={setApiKey}
-                          value={apiKey}
-                        />
-                      )}
-                    </Col>
-                    <Col className="mx-2">
-                      <Text>Select Model:</Text>
-                      <Select
-                        placeholder="Select a Model"
-                        onChange={onChange}
-                        options={[
-                          ...modelInfo,
-                          { value: 'custom', label: 'Enter custom model' }
-                        ]}
-                        style={{ width: "350px" }}
-                        showSearch={true}
-                      />
-                      {showCustomModelInput && (
-                        <TextInput
-                          className="mt-2"
-                          placeholder="Enter custom model name"
-                          onValueChange={(value) => {
-                            // Using setTimeout to create a simple debounce effect
-                            if (customModelTimeout.current) {
-                              clearTimeout(customModelTimeout.current);
-                            }
-                            
-                            customModelTimeout.current = setTimeout(() => {
-                              setSelectedModel(value);
-                            }, 500); // 500ms delay after typing stops
-                          }}
-                        />
-                      )}
-                    </Col>
-                  </Grid>
+  const handleEndpointChange = (value: string) => {
+    setEndpointType(value);
+  };
 
-                  {/* Clear Chat Button */}
-                  <Button
-                    onClick={clearChatHistory}
-                    className="mt-4"
-                  >
-                    Clear Chat
-                  </Button>
-                </div>
-                <Table
-                  className="mt-5"
-                  style={{
-                    display: "block",
-                    maxHeight: "60vh",
-                    overflowY: "auto",
-                  }}
-                >
-                  <TableHead>
-                    <TableRow>
-                      <TableCell>
-                        {/* <Title>Chat</Title> */}
-                      </TableCell>
-                    </TableRow>
-                  </TableHead>
-                  <TableBody>
-                    {chatHistory.map((message, index) => (
-                      <TableRow key={index}>
-                        <TableCell>
-                          <div style={{ 
-                            display: 'flex',
-                            alignItems: 'center',
-                            gap: '8px',
-                            marginBottom: '4px'
-                          }}>
-                            <strong>{message.role}</strong>
-                            {message.role === "assistant" && message.model && (
-                              <span style={{
-                                fontSize: '12px',
-                                color: '#666',
-                                backgroundColor: '#f5f5f5',
-                                padding: '2px 6px',
-                                borderRadius: '4px',
-                                fontWeight: 'normal'
-                              }}>
-                                {message.model}
-                              </span>
-                            )}
-                          </div>
-                          <div style={{ 
-                            whiteSpace: "pre-wrap", 
-                            wordBreak: "break-word",
-                            maxWidth: "100%"
-                          }}>
-                            <ReactMarkdown
-                              components={{
-                                code({node, inline, className, children, ...props}: React.ComponentPropsWithoutRef<'code'> & {
-                                  inline?: boolean;
-                                  node?: any;
-                                }) {
-                                  const match = /language-(\w+)/.exec(className || '');
-                                  return !inline && match ? (
-                                    <SyntaxHighlighter
-                                      style={coy as any}
-                                      language={match[1]}
-                                      PreTag="div"
-                                      {...props}
-                                    >
-                                      {String(children).replace(/\n$/, '')}
-                                    </SyntaxHighlighter>
-                                  ) : (
-                                    <code className={className} {...props}>
-                                      {children}
-                                    </code>
-                                  );
-                                }
-                              }}
-                            >
-                              {message.content}
-                            </ReactMarkdown>
-                          </div>
-                        </TableCell>
-                      </TableRow>
-                    ))}
-                    <TableRow>
-                      <TableCell>
-                        <div ref={chatEndRef} style={{ height: "1px" }} />
-                      </TableCell>
-                    </TableRow>
-                  </TableBody>
-                </Table>
-                <div
-                  className="mt-3"
-                  style={{ position: "absolute", bottom: 5, width: "95%" }}
-                >
-                  <div className="flex" style={{ marginTop: "16px" }}>
-                    <TextInput
-                      type="text"
-                      value={inputMessage}
-                      onChange={(e) => setInputMessage(e.target.value)}
-                      onKeyDown={handleKeyDown}
-                      placeholder="Type your message..."
-                    />
-                    <Button
-                      onClick={handleSendMessage}
-                      className="ml-2"
-                    >
-                      Send
-                    </Button>
+  const antIcon = <LoadingOutlined style={{ fontSize: 24 }} spin />;
+
+  return (
+    <div className="w-full h-screen p-4 bg-white">
+    <Card className="w-full rounded-xl shadow-md overflow-hidden">
+      <div className="flex h-[80vh] w-full">
+        {/* Left Sidebar with Controls */}
+        <div className="w-1/4 p-4 border-r border-gray-200 bg-gray-50">
+          <div className="mb-6">
+            <div className="space-y-6">
+              <div>
+                <Text className="font-medium block mb-2 text-gray-700 flex items-center">
+                  <KeyOutlined className="mr-2" /> API Key Source
+                </Text>
+                <Select
+                  disabled={disabledPersonalKeyCreation}
+                  defaultValue="session"
+                  style={{ width: "100%" }}
+                  onChange={(value) => setApiKeySource(value as "session" | "custom")}
+                  options={[
+                    { value: 'session', label: 'Current UI Session' },
+                    { value: 'custom', label: 'Virtual Key' },
+                  ]}
+                  className="rounded-md"
+                />
+                {apiKeySource === 'custom' && (
+                  <TextInput
+                    className="mt-2"
+                    placeholder="Enter custom API key"
+                    type="password"
+                    onValueChange={setApiKey}
+                    value={apiKey}
+                    icon={KeyOutlined}
+                  />
+                )}
+              </div>
+              
+              <div>
+                <Text className="font-medium block mb-2 text-gray-700 flex items-center">
+                  <RobotOutlined className="mr-2" /> Select Model
+                </Text>
+                <Select
+                  placeholder="Select a Model"
+                  onChange={onModelChange}
+                  options={[
+                    ...modelInfo.map((option) => ({
+                      value: option.model_group,
+                      label: option.model_group
+                    })),
+                    { value: 'custom', label: 'Enter custom model' }
+                  ]}
+                  style={{ width: "100%" }}
+                  showSearch={true}
+                  className="rounded-md"
+                />
+                {showCustomModelInput && (
+                  <TextInput
+                    className="mt-2"
+                    placeholder="Enter custom model name"
+                    onValueChange={(value) => {
+                      // Using setTimeout to create a simple debounce effect
+                      if (customModelTimeout.current) {
+                        clearTimeout(customModelTimeout.current);
+                      }
+                      
+                      customModelTimeout.current = setTimeout(() => {
+                        setSelectedModel(value);
+                      }, 500); // 500ms delay after typing stops
+                    }}
+                  />
+                )}
+              </div>
+              
+              <div>
+                <Text className="font-medium block mb-2 text-gray-700 flex items-center">
+                  <ApiOutlined className="mr-2" /> Endpoint Type
+                </Text>
+                <EndpointSelector 
+                  endpointType={endpointType}
+                  onEndpointChange={handleEndpointChange}
+                  className="mb-4"
+                />
+              </div>
+              
+              <Button
+                onClick={clearChatHistory}
+                className="w-full bg-gray-100 hover:bg-gray-200 text-gray-700 border-gray-300 mt-4"
+                icon={ClearOutlined}
+              >
+                Clear Chat
+              </Button>
+            </div>
+          </div>
+        </div>
+        
+        {/* Main Chat Area */}
+        <div className="w-3/4 flex flex-col bg-white">
+          <div className="flex-1 overflow-auto p-4 pb-0">
+            {chatHistory.length === 0 && (
+              <div className="h-full flex flex-col items-center justify-center text-gray-400">
+                <RobotOutlined style={{ fontSize: '48px', marginBottom: '16px' }} />
+                <Text>Start a conversation or generate an image</Text>
+              </div>
+            )}
+            
+            {chatHistory.map((message, index) => (
+              <div 
+                key={index} 
+                className={`mb-4 ${message.role === "user" ? "text-right" : "text-left"}`}
+              >
+                <div className="inline-block max-w-[80%] rounded-lg shadow-sm p-3.5 px-4" style={{
+                  backgroundColor: message.role === "user" ? '#f0f8ff' : '#ffffff',
+                  border: message.role === "user" ? '1px solid #e6f0fa' : '1px solid #f0f0f0',
+                  textAlign: 'left'
+                }}>
+                  <div className="flex items-center gap-2 mb-1.5">
+                    <div className="flex items-center justify-center w-6 h-6 rounded-full mr-1" style={{
+                      backgroundColor: message.role === "user" ? '#e6f0fa' : '#f5f5f5',
+                    }}>
+                      {message.role === "user" ? 
+                        <UserOutlined style={{ fontSize: '12px', color: '#2563eb' }} /> : 
+                        <RobotOutlined style={{ fontSize: '12px', color: '#4b5563' }} />
+                      }
+                    </div>
+                    <strong className="text-sm capitalize">{message.role}</strong>
+                    {message.role === "assistant" && message.model && (
+                      <span className="text-xs px-2 py-0.5 rounded bg-gray-100 text-gray-600 font-normal">
+                        {message.model}
+                      </span>
+                    )}
+                  </div>
+                  <div className="whitespace-pre-wrap break-words max-w-full message-content">
+                    {message.isImage ? (
+                      <img 
+                        src={message.content} 
+                        alt="Generated image" 
+                        className="max-w-full rounded-md border border-gray-200 shadow-sm" 
+                        style={{ maxHeight: '500px' }} 
+                      />
+                    ) : (
+                      <ReactMarkdown
+                        components={{
+                          code({node, inline, className, children, ...props}: React.ComponentPropsWithoutRef<'code'> & {
+                            inline?: boolean;
+                            node?: any;
+                          }) {
+                            const match = /language-(\w+)/.exec(className || '');
+                            return !inline && match ? (
+                              <SyntaxHighlighter
+                                style={coy as any}
+                                language={match[1]}
+                                PreTag="div"
+                                className="rounded-md my-2"
+                                {...props}
+                              >
+                                {String(children).replace(/\n$/, '')}
+                              </SyntaxHighlighter>
+                            ) : (
+                              <code className={`${className} px-1.5 py-0.5 rounded bg-gray-100 text-sm font-mono`} {...props}>
+                                {children}
+                              </code>
+                            );
+                          }
+                        }}
+                      >
+                        {message.content}
+                      </ReactMarkdown>
+                    )}
                   </div>
                 </div>
-              </TabPanel>
-              
-            </TabPanels>
-          </TabGroup>
-        </Card>
-      </Grid>
+              </div>
+            ))}
+            {isLoading && (
+              <div className="flex justify-center items-center my-4">
+                <Spin indicator={antIcon} />
+              </div>
+            )}
+            <div ref={chatEndRef} style={{ height: "1px" }} />
+          </div>
+          
+          <div className="p-4 border-t border-gray-200 bg-white">
+            <div className="flex items-center">
+              <TextInput
+                type="text"
+                value={inputMessage}
+                onChange={(e) => setInputMessage(e.target.value)}
+                onKeyDown={handleKeyDown}
+                placeholder={
+                  endpointType === EndpointType.CHAT 
+                    ? "Type your message..." 
+                    : "Describe the image you want to generate..."
+                }
+                disabled={isLoading}
+                className="flex-1"
+              />
+              {isLoading ? (
+                <Button
+                  onClick={handleCancelRequest}
+                  className="ml-2 bg-red-50 hover:bg-red-100 text-red-600 border-red-200"
+                  icon={DeleteOutlined}
+                >
+                  Cancel
+                </Button>
+              ) : (
+                <Button
+                  onClick={handleSendMessage}
+                  className="ml-2 text-white"
+                  icon={endpointType === EndpointType.CHAT ? SendOutlined : RobotOutlined}
+                >
+                  {endpointType === EndpointType.CHAT ? "Send" : "Generate"}
+                </Button>
+              )}
+            </div>
+          </div>
+        </div>
+      </div>
+    </Card>
     </div>
   );
 };

ui/litellm-dashboard/src/components/chat_ui/EndpointSelector.tsx

@@ -0,0 +1,40 @@
+import React from "react";
+import { Select } from "antd";
+import { Text } from "@tremor/react";
+import { EndpointType } from "./mode_endpoint_mapping";
+
+interface EndpointSelectorProps {
+  endpointType: string; // Accept string to avoid type conflicts
+  onEndpointChange: (value: string) => void;
+  className?: string;
+}
+
+/**
+ * A reusable component for selecting API endpoints
+ */
+const EndpointSelector: React.FC<EndpointSelectorProps> = ({
+  endpointType,
+  onEndpointChange,
+  className,
+}) => {
+  // Map endpoint types to their display labels
+  const endpointOptions = [
+    { value: EndpointType.CHAT, label: '/chat/completions' },
+    { value: EndpointType.IMAGE, label: '/images/generations' }
+  ];
+
+  return (
+    <div className={className}>
+      <Text>Endpoint Type:</Text>
+      <Select
+        value={endpointType}
+        style={{ width: "100%" }}
+        onChange={onEndpointChange}
+        options={endpointOptions}
+        className="rounded-md"
+      />
+    </div>
+  );
+};
+
+export default EndpointSelector; 

ui/litellm-dashboard/src/components/chat_ui/EndpointUtils.tsx

@@ -0,0 +1,27 @@
+import { ModelGroup } from "./llm_calls/fetch_models";
+import { ModelMode, EndpointType, getEndpointType } from "./mode_endpoint_mapping";
+
+/**
+ * Determines the appropriate endpoint type based on the selected model
+ * 
+ * @param selectedModel - The model identifier string
+ * @param modelInfo - Array of model information
+ * @returns The appropriate endpoint type
+ */
+export const determineEndpointType = (
+  selectedModel: string, 
+  modelInfo: ModelGroup[]
+): EndpointType => {
+  // Find the model information for the selected model
+  const selectedModelInfo = modelInfo.find(
+    (option) => option.model_group === selectedModel
+  );
+  
+  // If model info is found and it has a mode, determine the endpoint type
+  if (selectedModelInfo?.mode) {
+    return getEndpointType(selectedModelInfo.mode);
+  }
+  
+  // Default to chat endpoint if no match is found
+  return EndpointType.CHAT;
+}; 

ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx

@@ -0,0 +1,49 @@
+import openai from "openai";
+import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
+import { message } from "antd";
+
+export async function makeOpenAIChatCompletionRequest(
+    chatHistory: { role: string; content: string }[],
+    updateUI: (chunk: string, model: string) => void,
+    selectedModel: string,
+    accessToken: string,
+    signal?: AbortSignal
+  ) {
+    // base url should be the current base_url
+    const isLocal = process.env.NODE_ENV === "development";
+    if (isLocal !== true) {
+      console.log = function () {};
+    }
+    console.log("isLocal:", isLocal);
+    const proxyBaseUrl = isLocal
+      ? "http://localhost:4000"
+      : window.location.origin;
+    const client = new openai.OpenAI({
+      apiKey: accessToken, // Replace with your OpenAI API key
+      baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL
+      dangerouslyAllowBrowser: true, // using a temporary litellm proxy key
+    });
+  
+    try {
+      const response = await client.chat.completions.create({
+        model: selectedModel,
+        stream: true,
+        messages: chatHistory as ChatCompletionMessageParam[],
+      }, { signal });
+  
+      for await (const chunk of response) {
+        console.log(chunk);
+        if (chunk.choices[0].delta.content) {
+          updateUI(chunk.choices[0].delta.content, chunk.model);
+        }
+      }
+    } catch (error) {
+      if (signal?.aborted) {
+        console.log("Chat completion request was cancelled");
+      } else {
+        message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20);
+      }
+      throw error; // Re-throw to allow the caller to handle the error
+    }
+  }
+  

ui/litellm-dashboard/src/components/chat_ui/llm_calls/fetch_models.tsx

@@ -0,0 +1,35 @@
+// fetch_models.ts
+
+import { modelHubCall } from "../../networking";
+
+export interface ModelGroup {
+  model_group: string;
+  mode?: string;
+}
+
+/**
+ * Fetches available models using modelHubCall and formats them for the selection dropdown.
+ */
+export const fetchAvailableModels = async (
+  accessToken: string
+): Promise<ModelGroup[]> => {
+  try {
+    const fetchedModels = await modelHubCall(accessToken);
+    console.log("model_info:", fetchedModels);
+
+    if (fetchedModels?.data.length > 0) {
+      const models: ModelGroup[] = fetchedModels.data.map((item: any) => ({
+        model_group: item.model_group, // Display the model_group to the user
+        mode: item?.mode, // Save the mode for auto-selection of endpoint type
+      }));
+
+      // Sort models alphabetically by label
+      models.sort((a, b) => a.model_group.localeCompare(b.model_group));
+      return models;
+    }
+    return [];
+  } catch (error) {
+    console.error("Error fetching model info:", error);
+    throw error;
+  }
+};

ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx

@@ -0,0 +1,57 @@
+import openai from "openai";
+import { message } from "antd";
+
+export async function makeOpenAIImageGenerationRequest(
+  prompt: string,
+  updateUI: (imageUrl: string, model: string) => void,
+  selectedModel: string,
+  accessToken: string,
+  signal?: AbortSignal
+) {
+  // base url should be the current base_url
+  const isLocal = process.env.NODE_ENV === "development";
+  if (isLocal !== true) {
+    console.log = function () {};
+  }
+  console.log("isLocal:", isLocal);
+  const proxyBaseUrl = isLocal
+    ? "http://localhost:4000"
+    : window.location.origin;
+  const client = new openai.OpenAI({
+    apiKey: accessToken,
+    baseURL: proxyBaseUrl,
+    dangerouslyAllowBrowser: true,
+  });
+
+  try {
+    const response = await client.images.generate({
+      model: selectedModel,
+      prompt: prompt,
+    }, { signal });
+
+    console.log(response.data);
+    
+    if (response.data && response.data[0]) {
+      // Handle either URL or base64 data from response
+      if (response.data[0].url) {
+        // Use the URL directly
+        updateUI(response.data[0].url, selectedModel);
+      } else if (response.data[0].b64_json) {
+        // Convert base64 to data URL format
+        const base64Data = response.data[0].b64_json;
+        updateUI(`data:image/png;base64,${base64Data}`, selectedModel);
+      } else {
+        throw new Error("No image data found in response");
+      }
+    } else {
+      throw new Error("Invalid response format");
+    }
+  } catch (error) {
+    if (signal?.aborted) {
+      console.log("Image generation request was cancelled");
+    } else {
+      message.error(`Error occurred while generating image. Please try again. Error: ${error}`, 20);
+    }
+    throw error; // Re-throw to allow the caller to handle the error
+  }
+}

ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx

@@ -0,0 +1,34 @@
+// litellmMapping.ts
+
+// Define an enum for the modes as returned in model_info
+export enum ModelMode {
+    IMAGE_GENERATION = "image_generation",
+    CHAT = "chat",
+    // add additional modes as needed
+  }
+  
+  // Define an enum for the endpoint types your UI calls
+  export enum EndpointType {
+    IMAGE = "image",
+    CHAT = "chat",
+    // add additional endpoint types if required
+  }
+  
+  // Create a mapping between the model mode and the corresponding endpoint type
+  export const litellmModeMapping: Record<ModelMode, EndpointType> = {
+    [ModelMode.IMAGE_GENERATION]: EndpointType.IMAGE,
+    [ModelMode.CHAT]: EndpointType.CHAT,
+  };
+
+  export const getEndpointType = (mode: string): EndpointType => {
+    // Check if the string mode exists as a key in ModelMode enum
+    console.log("getEndpointType:", mode);
+    if (Object.values(ModelMode).includes(mode as ModelMode)) {
+      const endpointType = litellmModeMapping[mode as ModelMode];
+      console.log("endpointType:", endpointType);
+      return endpointType;
+    }
+
+    // else default to chat
+    return EndpointType.CHAT;
+  };

ui/litellm-dashboard/src/components/create_key_button.tsx

@@ -92,7 +92,7 @@ const getPredefinedTags = (data: any[] | null) => {
   return uniqueTags;
 }
 
-export const fetchTeamModels = async (userID: string, userRole: string, accessToken: string, teamID: string): Promise<string[]> => {
+export const fetchTeamModels = async (userID: string, userRole: string, accessToken: string, teamID: string | null): Promise<string[]> => {
   try {
     if (userID === null || userRole === null) {
       return [];
@@ -177,6 +177,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({
   const handleCancel = () => {
     setIsModalVisible(false);
     setApiKey(null);
+    setSelectedCreateKeyTeam(null);
     form.resetFields();
   };
 
@@ -291,14 +292,14 @@ const CreateKey: React.FC<CreateKeyProps> = ({
   };
 
   useEffect(() => {
-    if (userID && userRole && accessToken && selectedCreateKeyTeam) {
-      fetchTeamModels(userID, userRole, accessToken, selectedCreateKeyTeam.team_id).then((models) => {
-        let allModels = Array.from(new Set([...selectedCreateKeyTeam.models, ...models]));
+    if (userID && userRole && accessToken) {
+      fetchTeamModels(userID, userRole, accessToken, selectedCreateKeyTeam?.team_id ?? null).then((models) => {
+        let allModels = Array.from(new Set([...(selectedCreateKeyTeam?.models ?? []), ...models]));
         setModelsToPick(allModels);
       });
     }
     form.setFieldValue('models', []);
-  }, [selectedCreateKeyTeam]);
+  }, [selectedCreateKeyTeam, accessToken, userID, userRole]);
 
   // Add a callback function to handle user creation
   const handleUserCreated = (userId: string) => {

ui/litellm-dashboard/src/components/create_user_button.tsx

@@ -53,7 +53,7 @@ const Createuser: React.FC<CreateuserProps> = ({
   const [uiSettings, setUISettings] = useState<UISettings | null>(null);
   const [form] = Form.useForm();
   const [isModalVisible, setIsModalVisible] = useState(false);
-  const [apiuser, setApiuser] = useState<string | null>(null);
+  const [apiuser, setApiuser] = useState<boolean>(false);
   const [userModels, setUserModels] = useState<string[]>([]);
   const [isInvitationLinkModalVisible, setIsInvitationLinkModalVisible] =
     useState(false);
@@ -113,7 +113,7 @@ const Createuser: React.FC<CreateuserProps> = ({
 
   const handleCancel = () => {
     setIsModalVisible(false);
-    setApiuser(null);
+    setApiuser(false);
     form.resetFields();
   };
 
@@ -130,7 +130,7 @@ const Createuser: React.FC<CreateuserProps> = ({
       console.log("formValues in create user:", formValues);
       const response = await userCreateCall(accessToken, null, formValues);
       console.log("user create Response:", response);
-      setApiuser(response["key"]);
+      setApiuser(true);
       const user_id = response.data?.user_id || response.user_id;
 
       // Call the callback if provided (for embedded mode)