From f48f4a767c6d88a3458bf1397a630e844f288d58 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 17:16:33 -0700 Subject: [PATCH 1/5] feat - return slow responses on admin UI --- litellm/proxy/proxy_server.py | 59 +++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 36548dbeda..72dd9e6135 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7607,6 +7607,65 @@ async def model_metrics( } +@router.get( + "/model/metrics/slow_responses", + description="View number of hanging requests per model_group", + tags=["model management"], + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], +) +async def model_metrics_slow_responses( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + _selected_model_group: Optional[str] = "gpt-4-32k", + startTime: Optional[datetime] = None, + endTime: Optional[datetime] = None, +): + global prisma_client, llm_router, proxy_logging_obj + if prisma_client is None: + raise ProxyException( + message="Prisma Client is not initialized", + type="internal_error", + param="None", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + startTime = startTime or datetime.now() - timedelta(days=30) + endTime = endTime or datetime.now() + + alerting_threshold = ( + proxy_logging_obj.slack_alerting_instance.alerting_threshold or 300 + ) + alerting_threshold = int(alerting_threshold) + + sql_query = """ +SELECT + api_base, + COUNT(*) AS count +FROM + "LiteLLM_SpendLogs" +WHERE + ("endTime" - "startTime") >= (INTERVAL '1 SECOND' * CAST($1 AS INTEGER)) + AND "model" = $2 + AND "cache_hit" != 'True' +GROUP BY + api_base +ORDER BY + count DESC; + + """ + + db_response = await prisma_client.db.query_raw( + sql_query, alerting_threshold, _selected_model_group + ) + + if db_response is not None: + for row in db_response: + _api_base = row.get("api_base") or "" + if "/openai/" in _api_base: + _api_base = _api_base.split("/openai/")[0] + row["api_base"] = _api_base + return db_response + + @router.get( "/model/metrics/exceptions", description="View number of failed requests per model on config.yaml", From 26eda88b26a9bde157c1c83c6c504a50a9be4266 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 17:18:14 -0700 Subject: [PATCH 2/5] feat - show slow count and total count --- litellm/proxy/proxy_server.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 72dd9e6135..06651e7b63 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7639,18 +7639,20 @@ async def model_metrics_slow_responses( sql_query = """ SELECT api_base, - COUNT(*) AS count + COUNT(*) AS total_count, + SUM(CASE + WHEN ("endTime" - "startTime") >= (INTERVAL '1 SECOND' * CAST($1 AS INTEGER)) THEN 1 + ELSE 0 + END) AS slow_count FROM "LiteLLM_SpendLogs" WHERE - ("endTime" - "startTime") >= (INTERVAL '1 SECOND' * CAST($1 AS INTEGER)) - AND "model" = $2 + "model" = $2 AND "cache_hit" != 'True' GROUP BY api_base ORDER BY - count DESC; - + slow_count DESC; """ db_response = await prisma_client.db.query_raw( From 5041b924acee874d7b2ca785c57d2fbb999e4750 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 17:23:37 -0700 Subject: [PATCH 3/5] ui - show slow responses on admin UI --- .../src/components/model_dashboard.tsx | 96 ++++++++++--------- .../src/components/networking.tsx | 40 ++++++++ 2 files changed, 93 insertions(+), 43 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index c9fcdf228b..f8af6f5bf9 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -18,7 +18,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking"; +import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall } from "./networking"; import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, @@ -205,6 +205,7 @@ const ModelDashboard: React.FC = ({ const [modelExceptions, setModelExceptions] = useState([]); const [allExceptions, setAllExceptions] = useState([]); const [failureTableData, setFailureTableData] = useState([]); + const [slowResponsesData, setSlowResponsesData] = useState([]); const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { const [form] = Form.useForm(); @@ -479,39 +480,51 @@ const handleEditSubmit = async (formValues: Record) => { setAllExceptions(modelExceptionsResponse.exception_types); - let modelMetricsData = modelMetricsResponse.data; - let successdeploymentToSuccess: Record = {}; - for (let i = 0; i < modelMetricsData.length; i++) { - let element = modelMetricsData[i]; - let _model_name = element.model; - let _num_requests = element.num_requests; - successdeploymentToSuccess[_model_name] = _num_requests - } - console.log("successdeploymentToSuccess:", successdeploymentToSuccess) + const slowResponses = await modelMetricsSlowResponsesCall( + accessToken, + userID, + userRole, + null + ) + + console.log("slowResponses:", slowResponses) + + setSlowResponsesData(slowResponses); + + + // let modelMetricsData = modelMetricsResponse.data; + // let successdeploymentToSuccess: Record = {}; + // for (let i = 0; i < modelMetricsData.length; i++) { + // let element = modelMetricsData[i]; + // let _model_name = element.model; + // let _num_requests = element.num_requests; + // successdeploymentToSuccess[_model_name] = _num_requests + // } + // console.log("successdeploymentToSuccess:", successdeploymentToSuccess) - let failureTableData = []; - let _failureData = modelExceptionsResponse.data; - for (let i = 0; i < _failureData.length; i++) { - const model = _failureData[i]; - let _model_name = model.model; - let total_exceptions = model.total_exceptions; - let total_Requests = successdeploymentToSuccess[_model_name]; - if (total_Requests == null) { - total_Requests = 0 - } - let _data = { - model: _model_name, - total_exceptions: total_exceptions, - total_Requests: total_Requests, - failure_rate: total_Requests / total_exceptions - } - failureTableData.push(_data); - // sort failureTableData by failure_rate - failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); + // let failureTableData = []; + // let _failureData = modelExceptionsResponse.data; + // for (let i = 0; i < _failureData.length; i++) { + // const model = _failureData[i]; + // let _model_name = model.model; + // let total_exceptions = model.total_exceptions; + // let total_Requests = successdeploymentToSuccess[_model_name]; + // if (total_Requests == null) { + // total_Requests = 0 + // } + // let _data = { + // model: _model_name, + // total_exceptions: total_exceptions, + // total_Requests: total_Requests, + // failure_rate: total_Requests / total_exceptions + // } + // failureTableData.push(_data); + // // sort failureTableData by failure_rate + // failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); - setFailureTableData(failureTableData); - console.log("failureTableData:", failureTableData); - } + // setFailureTableData(failureTableData); + // console.log("failureTableData:", failureTableData); + // } } catch (error) { console.error("There was an error fetching the model data", error); @@ -1126,8 +1139,6 @@ const handleEditSubmit = async (formValues: Record) => { /> )} - - @@ -1135,20 +1146,19 @@ const handleEditSubmit = async (formValues: Record) => { - Model - Success Requests - Error Requests - Failure % + Deployment + Success Responses + Slow Responses - {failureTableData.map((metric, idx) => ( + {slowResponsesData.map((metric, idx) => ( - {metric.model} - {metric.total_Requests} - {metric.total_exceptions} - {metric.failure_rate}% + {metric.api_base} + {metric.total_count} + {metric.slow_count} + ))} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index ed17b7daa6..35880949b2 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -475,6 +475,46 @@ export const modelMetricsCall = async ( }; + +export const modelMetricsSlowResponsesCall = async ( + accessToken: String, + userID: String, + userRole: String, + modelGroup: String | null, +) => { + /** + * Get all models on proxy + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`; + if (modelGroup) { + url = `${url}?_selected_model_group=${modelGroup}` + } + // message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 20); + throw new Error("Network response was not ok"); + } + const data = await response.json(); + // message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + + export const modelExceptionsCall = async ( accessToken: String, userID: String, From 44f74c65e333579400834dbff6efa0855309a803 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 17:25:42 -0700 Subject: [PATCH 4/5] fix update slow responses based on selected model group --- .../src/components/model_dashboard.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index f8af6f5bf9..aca424ab27 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -704,6 +704,18 @@ const handleEditSubmit = async (formValues: Record) => { setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); + + const slowResponses = await modelMetricsSlowResponsesCall( + accessToken, + userID, + userRole, + modelGroup + ) + + console.log("slowResponses:", slowResponses) + + setSlowResponsesData(slowResponses); + } catch (error) { console.error("Failed to fetch model metrics", error); } From 789ef4566c1792596bae14ccdcfa2c51218ad2d2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 17:29:52 -0700 Subject: [PATCH 5/5] ui - show slow responses --- ui/litellm-dashboard/src/components/model_dashboard.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index aca424ab27..f50447381d 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1135,7 +1135,7 @@ const handleEditSubmit = async (formValues: Record) => { - + Avg Latency per Token

(seconds/token)

average Latency for successfull requests divided by the total tokens { modelMetrics && modelMetricsCategories && ( @@ -1154,13 +1154,13 @@ const handleEditSubmit = async (formValues: Record) => { - +
Deployment Success Responses - Slow Responses + Slow Responses

Success Responses taking 600+s