From e44318c605b8a78f113922b7c0ac7973db54ea89 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 3 Apr 2025 14:32:20 -0700 Subject: [PATCH 01/10] refactor to have 1 folder for llm api calls --- .../src/components/chat_ui.tsx | 44 +------------------ .../chat_ui/llm_calls/chat_completion.tsx | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 42 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index c505a954b8..1491a57b49 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -24,8 +24,7 @@ import { import { message, Select } from "antd"; import { modelAvailableCall } from "./networking"; -import openai from "openai"; -import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; +import { makeOpenAIChatCompletionRequest } from "./chat_ui/llm_calls/chat_completion"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { Typography } from "antd"; import { coy } from 'react-syntax-highlighter/dist/esm/styles/prism'; @@ -38,45 +37,6 @@ interface ChatUIProps { disabledPersonalKeyCreation: boolean; } -async function generateModelResponse( - chatHistory: { role: string; content: string }[], - updateUI: (chunk: string, model: string) => void, - selectedModel: string, - accessToken: string -) { - // base url should be the current base_url - const isLocal = process.env.NODE_ENV === "development"; - if (isLocal !== true) { - console.log = function () {}; - } - console.log("isLocal:", isLocal); - const proxyBaseUrl = isLocal - ? "http://localhost:4000" - : window.location.origin; - const client = new openai.OpenAI({ - apiKey: accessToken, // Replace with your OpenAI API key - baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL - dangerouslyAllowBrowser: true, // using a temporary litellm proxy key - }); - - try { - const response = await client.chat.completions.create({ - model: selectedModel, - stream: true, - messages: chatHistory as ChatCompletionMessageParam[], - }); - - for await (const chunk of response) { - console.log(chunk); - if (chunk.choices[0].delta.content) { - updateUI(chunk.choices[0].delta.content, chunk.model); - } - } - } catch (error) { - message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20); - } -} - const ChatUI: React.FC = ({ accessToken, token, @@ -208,7 +168,7 @@ const ChatUI: React.FC = ({ try { if (selectedModel) { - await generateModelResponse( + await makeOpenAIChatCompletionRequest( apiChatHistory, (chunk, model) => updateUI("assistant", chunk, model), selectedModel, diff --git a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx new file mode 100644 index 0000000000..25cc641243 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx @@ -0,0 +1,44 @@ + +import openai from "openai"; +import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; +import { message } from "antd"; + +export async function makeOpenAIChatCompletionRequest( + chatHistory: { role: string; content: string }[], + updateUI: (chunk: string, model: string) => void, + selectedModel: string, + accessToken: string + ) { + // base url should be the current base_url + const isLocal = process.env.NODE_ENV === "development"; + if (isLocal !== true) { + console.log = function () {}; + } + console.log("isLocal:", isLocal); + const proxyBaseUrl = isLocal + ? "http://localhost:4000" + : window.location.origin; + const client = new openai.OpenAI({ + apiKey: accessToken, // Replace with your OpenAI API key + baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL + dangerouslyAllowBrowser: true, // using a temporary litellm proxy key + }); + + try { + const response = await client.chat.completions.create({ + model: selectedModel, + stream: true, + messages: chatHistory as ChatCompletionMessageParam[], + }); + + for await (const chunk of response) { + console.log(chunk); + if (chunk.choices[0].delta.content) { + updateUI(chunk.choices[0].delta.content, chunk.model); + } + } + } catch (error) { + message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20); + } + } + \ No newline at end of file From 6ffe3f1e46a78838f04bb8e6a105c4e92c88365b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 3 Apr 2025 14:43:56 -0700 Subject: [PATCH 02/10] working image generation on chat ui --- .../src/components/chat_ui.tsx | 134 ++++++++++++------ .../chat_ui/llm_calls/image_generation.tsx | 51 +++++++ 2 files changed, 138 insertions(+), 47 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index 1491a57b49..f9c731461c 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -25,6 +25,7 @@ import { import { message, Select } from "antd"; import { modelAvailableCall } from "./networking"; import { makeOpenAIChatCompletionRequest } from "./chat_ui/llm_calls/chat_completion"; +import { makeOpenAIImageGenerationRequest } from "./chat_ui/llm_calls/image_generation"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { Typography } from "antd"; import { coy } from 'react-syntax-highlighter/dist/esm/styles/prism'; @@ -49,13 +50,14 @@ const ChatUI: React.FC = ({ ); const [apiKey, setApiKey] = useState(""); const [inputMessage, setInputMessage] = useState(""); - const [chatHistory, setChatHistory] = useState<{ role: string; content: string; model?: string }[]>([]); + const [chatHistory, setChatHistory] = useState<{ role: string; content: string; model?: string; isImage?: boolean }[]>([]); const [selectedModel, setSelectedModel] = useState( undefined ); const [showCustomModelInput, setShowCustomModelInput] = useState(false); const [modelInfo, setModelInfo] = useState([]); const customModelTimeout = useRef(null); + const [endpointType, setEndpointType] = useState<'chat' | 'image'>('chat'); const chatEndRef = useRef(null); @@ -67,8 +69,6 @@ const ChatUI: React.FC = ({ return; } - - // Fetch model info and set the default selected model const fetchModelInfo = async () => { try { @@ -122,11 +122,11 @@ const ChatUI: React.FC = ({ } }, [chatHistory]); - const updateUI = (role: string, chunk: string, model?: string) => { + const updateTextUI = (role: string, chunk: string, model?: string) => { setChatHistory((prevHistory) => { const lastMessage = prevHistory[prevHistory.length - 1]; - if (lastMessage && lastMessage.role === role) { + if (lastMessage && lastMessage.role === role && !lastMessage.isImage) { return [ ...prevHistory.slice(0, prevHistory.length - 1), { role, content: lastMessage.content + chunk, model }, @@ -137,6 +137,13 @@ const ChatUI: React.FC = ({ }); }; + const updateImageUI = (imageUrl: string, model: string) => { + setChatHistory((prevHistory) => [ + ...prevHistory, + { role: "assistant", content: imageUrl, model, isImage: true } + ]); + }; + const handleKeyDown = (event: React.KeyboardEvent) => { if (event.key === 'Enter') { handleSendMessage(); @@ -160,24 +167,34 @@ const ChatUI: React.FC = ({ // Create message object without model field for API call const newUserMessage = { role: "user", content: inputMessage }; - // Create chat history for API call - strip out model field - const apiChatHistory = [...chatHistory.map(({ role, content }) => ({ role, content })), newUserMessage]; - - // Update UI with full message object (including model field for display) + // Update UI with full message object setChatHistory([...chatHistory, newUserMessage]); try { if (selectedModel) { - await makeOpenAIChatCompletionRequest( - apiChatHistory, - (chunk, model) => updateUI("assistant", chunk, model), - selectedModel, - effectiveApiKey - ); + if (endpointType === 'chat') { + // Create chat history for API call - strip out model field and isImage field + const apiChatHistory = [...chatHistory.filter(msg => !msg.isImage).map(({ role, content }) => ({ role, content })), newUserMessage]; + + await makeOpenAIChatCompletionRequest( + apiChatHistory, + (chunk, model) => updateTextUI("assistant", chunk, model), + selectedModel, + effectiveApiKey + ); + } else { + // For image generation + await makeOpenAIImageGenerationRequest( + inputMessage, + (imageUrl, model) => updateImageUI(imageUrl, model), + selectedModel, + effectiveApiKey + ); + } } } catch (error) { - console.error("Error fetching model response", error); - updateUI("assistant", "Error fetching model response"); + console.error("Error fetching response", error); + updateTextUI("assistant", "Error fetching response"); } setInputMessage(""); @@ -198,12 +215,16 @@ const ChatUI: React.FC = ({ ); } - const onChange = (value: string) => { + const onModelChange = (value: string) => { console.log(`selected ${value}`); setSelectedModel(value); setShowCustomModelInput(value === 'custom'); }; + const handleEndpointChange = (value: string) => { + setEndpointType(value as 'chat' | 'image'); + }; + return (
@@ -240,10 +261,21 @@ const ChatUI: React.FC = ({ )} + Endpoint Type: + = ({ wordBreak: "break-word", maxWidth: "100%" }}> - & { - inline?: boolean; - node?: any; - }) { - const match = /language-(\w+)/.exec(className || ''); - return !inline && match ? ( - - {String(children).replace(/\n$/, '')} - - ) : ( - - {children} - - ); - } - }} - > - {message.content} - + {message.isImage ? ( + Generated image + ) : ( + & { + inline?: boolean; + node?: any; + }) { + const match = /language-(\w+)/.exec(className || ''); + return !inline && match ? ( + + {String(children).replace(/\n$/, '')} + + ) : ( + + {children} + + ); + } + }} + > + {message.content} + + )}
@@ -369,13 +409,13 @@ const ChatUI: React.FC = ({ value={inputMessage} onChange={(e) => setInputMessage(e.target.value)} onKeyDown={handleKeyDown} - placeholder="Type your message..." + placeholder={endpointType === 'chat' ? "Type your message..." : "Describe the image you want to generate..."} /> diff --git a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx new file mode 100644 index 0000000000..1824b83d0b --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx @@ -0,0 +1,51 @@ +import openai from "openai"; +import { message } from "antd"; + +export async function makeOpenAIImageGenerationRequest( + prompt: string, + updateUI: (imageUrl: string, model: string) => void, + selectedModel: string, + accessToken: string +) { + // base url should be the current base_url + const isLocal = process.env.NODE_ENV === "development"; + if (isLocal !== true) { + console.log = function () {}; + } + console.log("isLocal:", isLocal); + const proxyBaseUrl = isLocal + ? "http://localhost:4000" + : window.location.origin; + const client = new openai.OpenAI({ + apiKey: accessToken, + baseURL: proxyBaseUrl, + dangerouslyAllowBrowser: true, + }); + + try { + const response = await client.images.generate({ + model: selectedModel, + prompt: prompt, + }); + + console.log(response.data); + + if (response.data && response.data[0]) { + // Handle either URL or base64 data from response + if (response.data[0].url) { + // Use the URL directly + updateUI(response.data[0].url, selectedModel); + } else if (response.data[0].b64_json) { + // Convert base64 to data URL format + const base64Data = response.data[0].b64_json; + updateUI(`data:image/png;base64,${base64Data}`, selectedModel); + } else { + throw new Error("No image data found in response"); + } + } else { + throw new Error("Invalid response format"); + } + } catch (error) { + message.error(`Error occurred while generating image. Please try again. Error: ${error}`, 20); + } +} From b361329e07e076c5cd28203540a5ff9ed42038ee Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 3 Apr 2025 19:27:44 -0700 Subject: [PATCH 03/10] use 1 file for fetch model options --- .../src/components/chat_ui.tsx | 30 +++-------- .../chat_ui/llm_calls/fetch_models.tsx | 54 +++++++++++++++++++ 2 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/chat_ui/llm_calls/fetch_models.tsx diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index f9c731461c..547e926559 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -23,9 +23,9 @@ import { } from "@tremor/react"; import { message, Select } from "antd"; -import { modelAvailableCall } from "./networking"; import { makeOpenAIChatCompletionRequest } from "./chat_ui/llm_calls/chat_completion"; import { makeOpenAIImageGenerationRequest } from "./chat_ui/llm_calls/image_generation"; +import { fetchAvailableModels } from "./chat_ui/llm_calls/fetch_models"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { Typography } from "antd"; import { coy } from 'react-syntax-highlighter/dist/esm/styles/prism'; @@ -70,33 +70,17 @@ const ChatUI: React.FC = ({ } // Fetch model info and set the default selected model - const fetchModelInfo = async () => { + const loadModels = async () => { try { - const fetchedAvailableModels = await modelAvailableCall( - useApiKey ?? '', // Use empty string if useApiKey is null, + const uniqueModels = await fetchAvailableModels( + useApiKey, userID, userRole ); - console.log("model_info:", fetchedAvailableModels); + console.log("Fetched models:", uniqueModels); - if (fetchedAvailableModels?.data.length > 0) { - // Create a Map to store unique models using the model ID as key - const uniqueModelsMap = new Map(); - - fetchedAvailableModels["data"].forEach((item: { id: string }) => { - uniqueModelsMap.set(item.id, { - value: item.id, - label: item.id - }); - }); - - // Convert Map values back to array - const uniqueModels = Array.from(uniqueModelsMap.values()); - - // Sort models alphabetically - uniqueModels.sort((a, b) => a.label.localeCompare(b.label)); - + if (uniqueModels.length > 0) { setModelInfo(uniqueModels); setSelectedModel(uniqueModels[0].value); } @@ -105,7 +89,7 @@ const ChatUI: React.FC = ({ } }; - fetchModelInfo(); + loadModels(); }, [accessToken, userID, userRole, apiKeySource, apiKey]); diff --git a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/fetch_models.tsx new file mode 100644 index 0000000000..579af1add2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/fetch_models.tsx @@ -0,0 +1,54 @@ +import { modelAvailableCall } from "../../networking"; + +interface ModelOption { + value: string; + label: string; +} + +/** + * Fetches available models for the user and formats them as options + * for selection dropdowns + */ +export const fetchAvailableModels = async ( + apiKey: string | null, + userID: string, + userRole: string, + teamID: string | null = null +): Promise => { + try { + const fetchedAvailableModels = await modelAvailableCall( + apiKey ?? '', // Use empty string if apiKey is null + userID, + userRole, + false, + teamID + ); + + console.log("model_info:", fetchedAvailableModels); + + if (fetchedAvailableModels?.data.length > 0) { + // Create a Map to store unique models using the model ID as key + const uniqueModelsMap = new Map(); + + fetchedAvailableModels["data"].forEach((item: { id: string }) => { + uniqueModelsMap.set(item.id, { + value: item.id, + label: item.id + }); + }); + + // Convert Map values back to array + const uniqueModels = Array.from(uniqueModelsMap.values()); + + // Sort models alphabetically + uniqueModels.sort((a, b) => a.label.localeCompare(b.label)); + + return uniqueModels; + } + + return []; + } catch (error) { + console.error("Error fetching model info:", error); + throw error; + } +}; \ No newline at end of file From 747894864c78cc598c9a1ff7b7cd03f09c7b2414 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 3 Apr 2025 20:05:11 -0700 Subject: [PATCH 04/10] use litellm mapping --- .../chat_ui/mode_endpoint_mapping.tsx | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx diff --git a/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx b/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx new file mode 100644 index 0000000000..a86fb4119b --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx @@ -0,0 +1,22 @@ +// litellmMapping.ts + +// Define an enum for the modes as returned in model_info +export enum ModelMode { + IMAGE_GENERATION = "image_generation", + CHAT = "chat", + // add additional modes as needed + } + + // Define an enum for the endpoint types your UI calls + export enum EndpointType { + IMAGE = "image", + CHAT = "chat", + // add additional endpoint types if required + } + + // Create a mapping between the model mode and the corresponding endpoint type + export const litellmModeMapping: Record = { + [ModelMode.IMAGE_GENERATION]: EndpointType.IMAGE, + [ModelMode.CHAT]: EndpointType.CHAT, + }; + \ No newline at end of file From 72d7b268116a99eaf2028a501f2706a84b0d644e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 3 Apr 2025 21:00:42 -0700 Subject: [PATCH 05/10] fix allow selecting endpoint on test key page --- .../src/components/chat_ui.tsx | 50 ++++++++++------ .../chat_ui/llm_calls/fetch_models.tsx | 59 +++++++------------ 2 files changed, 53 insertions(+), 56 deletions(-) diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index 547e926559..bf483a790e 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -25,7 +25,8 @@ import { import { message, Select } from "antd"; import { makeOpenAIChatCompletionRequest } from "./chat_ui/llm_calls/chat_completion"; import { makeOpenAIImageGenerationRequest } from "./chat_ui/llm_calls/image_generation"; -import { fetchAvailableModels } from "./chat_ui/llm_calls/fetch_models"; +import { fetchAvailableModels, ModelGroup } from "./chat_ui/llm_calls/fetch_models"; +import { litellmModeMapping, ModelMode, EndpointType } from "./chat_ui/mode_endpoint_mapping"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { Typography } from "antd"; import { coy } from 'react-syntax-highlighter/dist/esm/styles/prism'; @@ -55,7 +56,7 @@ const ChatUI: React.FC = ({ undefined ); const [showCustomModelInput, setShowCustomModelInput] = useState(false); - const [modelInfo, setModelInfo] = useState([]); + const [modelInfo, setModelInfo] = useState([]); const customModelTimeout = useRef(null); const [endpointType, setEndpointType] = useState<'chat' | 'image'>('chat'); @@ -74,15 +75,18 @@ const ChatUI: React.FC = ({ try { const uniqueModels = await fetchAvailableModels( useApiKey, - userID, - userRole ); console.log("Fetched models:", uniqueModels); if (uniqueModels.length > 0) { setModelInfo(uniqueModels); - setSelectedModel(uniqueModels[0].value); + setSelectedModel(uniqueModels[0].model_group); + // Auto-set endpoint based on the first model's mode if available + const firstMode = uniqueModels[0].mode as ModelMode; + if (firstMode && litellmModeMapping[firstMode]) { + setEndpointType(litellmModeMapping[firstMode] as EndpointType); + } } } catch (error) { console.error("Error fetching model info:", error); @@ -202,6 +206,14 @@ const ChatUI: React.FC = ({ const onModelChange = (value: string) => { console.log(`selected ${value}`); setSelectedModel(value); + // Look up the selected model to auto-select the endpoint type + const selectedOption = modelInfo.find((option) => option.model_group === value); + if (selectedOption && selectedOption.mode) { + const mode = selectedOption.mode as ModelMode; + if (litellmModeMapping[mode]) { + setEndpointType(litellmModeMapping[mode] as EndpointType); + } + } setShowCustomModelInput(value === 'custom'); }; @@ -245,23 +257,15 @@ const ChatUI: React.FC = ({ )} - Endpoint Type: - ({ + value: option.model_group, + label: option.model_group + })), { value: 'custom', label: 'Enter custom model' } ]} style={{ width: "350px" }} @@ -283,7 +287,19 @@ const ChatUI: React.FC = ({ }} /> )} + Endpoint Type: + - @@ -409,13 +406,17 @@ const ChatUI: React.FC = ({ value={inputMessage} onChange={(e) => setInputMessage(e.target.value)} onKeyDown={handleKeyDown} - placeholder={endpointType === 'chat' ? "Type your message..." : "Describe the image you want to generate..."} + placeholder={ + endpointType === EndpointType.CHAT + ? "Type your message..." + : "Describe the image you want to generate..." + } /> diff --git a/ui/litellm-dashboard/src/components/chat_ui/EndpointSelector.tsx b/ui/litellm-dashboard/src/components/chat_ui/EndpointSelector.tsx new file mode 100644 index 0000000000..a0f23da98d --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/EndpointSelector.tsx @@ -0,0 +1,39 @@ +import React from "react"; +import { Select } from "antd"; +import { Text } from "@tremor/react"; +import { EndpointType } from "./mode_endpoint_mapping"; + +interface EndpointSelectorProps { + endpointType: string; // Accept string to avoid type conflicts + onEndpointChange: (value: string) => void; + className?: string; +} + +/** + * A reusable component for selecting API endpoints + */ +const EndpointSelector: React.FC = ({ + endpointType, + onEndpointChange, + className, +}) => { + // Map endpoint types to their display labels + const endpointOptions = [ + { value: EndpointType.CHAT, label: '/chat/completions' }, + { value: EndpointType.IMAGE, label: '/images/generations' } + ]; + + return ( +
+ Endpoint Type: + setApiKeySource(value as "session" | "custom")} + options={[ + { value: 'session', label: 'Current UI Session' }, + { value: 'custom', label: 'Virtual Key' }, + ]} + /> + {apiKeySource === 'custom' && ( + + )} +
- - - Chat - - - -
- - - API Key Source - ({ + value: option.model_group, + label: option.model_group + })), + { value: 'custom', label: 'Enter custom model' } + ]} + style={{ width: "100%" }} + showSearch={true} + /> + {showCustomModelInput && ( + { + // Using setTimeout to create a simple debounce effect + if (customModelTimeout.current) { + clearTimeout(customModelTimeout.current); + } + + customModelTimeout.current = setTimeout(() => { + setSelectedModel(value); + }, 500); // 500ms delay after typing stops + }} + /> + )} +
+ +
+ +
+ + + + + {/* Main Chat Area */} +
+
+ {chatHistory.map((message, index) => ( +
+
+
+ {message.role} + {message.role === "assistant" && message.model && ( + + {message.model} + + )} +
+
+ {message.isImage ? ( + Generated image - {apiKeySource === 'custom' && ( - - )} - - - Select Model: - setApiKeySource(value as "session" | "custom")} - options={[ - { value: 'session', label: 'Current UI Session' }, - { value: 'custom', label: 'Virtual Key' }, - ]} - /> - {apiKeySource === 'custom' && ( - - )} +
+
+ + API Key Source + + ({ + value: option.model_group, + label: option.model_group + })), + { value: 'custom', label: 'Enter custom model' } + ]} + style={{ width: "100%" }} + showSearch={true} + className="rounded-md" + /> + {showCustomModelInput && ( + { + // Using setTimeout to create a simple debounce effect + if (customModelTimeout.current) { + clearTimeout(customModelTimeout.current); + } + + customModelTimeout.current = setTimeout(() => { + setSelectedModel(value); + }, 500); // 500ms delay after typing stops + }} + /> + )} +
+ +
+ + Endpoint Type + + +
+ + +
- -
- Select Model: -
); diff --git a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx index 25cc641243..e40eb3e696 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx @@ -1,4 +1,3 @@ - import openai from "openai"; import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; import { message } from "antd"; @@ -7,7 +6,8 @@ export async function makeOpenAIChatCompletionRequest( chatHistory: { role: string; content: string }[], updateUI: (chunk: string, model: string) => void, selectedModel: string, - accessToken: string + accessToken: string, + signal?: AbortSignal ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -29,7 +29,7 @@ export async function makeOpenAIChatCompletionRequest( model: selectedModel, stream: true, messages: chatHistory as ChatCompletionMessageParam[], - }); + }, { signal }); for await (const chunk of response) { console.log(chunk); @@ -38,7 +38,12 @@ export async function makeOpenAIChatCompletionRequest( } } } catch (error) { - message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20); + if (signal?.aborted) { + console.log("Chat completion request was cancelled"); + } else { + message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20); + } + throw error; // Re-throw to allow the caller to handle the error } } \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx index 1824b83d0b..d972870a42 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/image_generation.tsx @@ -5,7 +5,8 @@ export async function makeOpenAIImageGenerationRequest( prompt: string, updateUI: (imageUrl: string, model: string) => void, selectedModel: string, - accessToken: string + accessToken: string, + signal?: AbortSignal ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -26,7 +27,7 @@ export async function makeOpenAIImageGenerationRequest( const response = await client.images.generate({ model: selectedModel, prompt: prompt, - }); + }, { signal }); console.log(response.data); @@ -46,6 +47,11 @@ export async function makeOpenAIImageGenerationRequest( throw new Error("Invalid response format"); } } catch (error) { - message.error(`Error occurred while generating image. Please try again. Error: ${error}`, 20); + if (signal?.aborted) { + console.log("Image generation request was cancelled"); + } else { + message.error(`Error occurred while generating image. Please try again. Error: ${error}`, 20); + } + throw error; // Re-throw to allow the caller to handle the error } } From f6c2b86903fd54036d296e4be1a29adf8559474b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 3 Apr 2025 22:21:11 -0700 Subject: [PATCH 09/10] fix typo --- ui/litellm-dashboard/src/components/chat_ui.tsx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index 8a48608bde..5b23ec11e3 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -77,10 +77,9 @@ const ChatUI: React.FC = ({ const chatEndRef = useRef(null); useEffect(() => { - let useApiKey = apiKeySource === 'session' ? accessToken : apiKey; - console.log("useApiKey:", useApiKey); - if (!useApiKey || !token || !userRole || !userID) { - console.log("useApiKey or token or userRole or userID is missing = ", useApiKey, token, userRole, userID); + let userApiKey = apiKeySource === 'session' ? accessToken : apiKey; + if (!userApiKey || !token || !userRole || !userID) { + console.log("userApiKey or token or userRole or userID is missing = ", userApiKey, token, userRole, userID); return; } @@ -88,7 +87,7 @@ const ChatUI: React.FC = ({ const loadModels = async () => { try { const uniqueModels = await fetchAvailableModels( - useApiKey, + userApiKey, ); console.log("Fetched models:", uniqueModels); From c8468b71c854f814afc555d1591898d6de98fac8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 3 Apr 2025 22:32:56 -0700 Subject: [PATCH 10/10] fix linting ui --- ui/litellm-dashboard/src/components/chat_ui.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index 5b23ec11e3..7f2b0e1a26 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -86,6 +86,10 @@ const ChatUI: React.FC = ({ // Fetch model info and set the default selected model const loadModels = async () => { try { + if (!userApiKey) { + console.log("userApiKey is missing"); + return; + } const uniqueModels = await fetchAvailableModels( userApiKey, );