From 3b823c75874b75bc54e0cd060899046c515f886f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 16:30:25 -0700 Subject: [PATCH 01/10] fix - by default log raw curl command on langfuse --- litellm/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 4dcee6be45..4ad6ff9899 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1270,7 +1270,11 @@ class Logging: api_base ) # used for alerting masked_headers = { - k: (v[:-20] + "*" * 20) if (isinstance(v, str) and len(v) > 20) else v + k: ( + (v[:-44] + "*" * 44) + if (isinstance(v, str) and len(v) > 44) + else "*****" + ) for k, v in headers.items() } formatted_headers = " ".join( @@ -1309,8 +1313,7 @@ class Logging: # check if user wants the raw request logged to their logging provider (like LangFuse) _litellm_params = self.model_call_details.get("litellm_params", {}) _metadata = _litellm_params.get("metadata", {}) or {} - if _metadata.get("log_raw_request", False) is True: - _metadata["raw_request"] = curl_command + _metadata["raw_request"] = curl_command if self.logger_fn and callable(self.logger_fn): try: From b8c9326df4411501398564ac9d824561368dd8c7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 16:33:42 -0700 Subject: [PATCH 02/10] add type for AllCallbacks --- litellm/proxy/_types.py | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 20ff285a73..a3c659f8bf 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1346,3 +1346,51 @@ class InvitationModel(LiteLLMBase): class ConfigFieldInfo(LiteLLMBase): field_name: str field_value: Any + + +class CallbackOnUI(LiteLLMBase): + litellm_callback_name: str + litellm_callback_params: Optional[list] + ui_callback_name: str + + +class AllCallbacks(LiteLLMBase): + langfuse: CallbackOnUI = CallbackOnUI( + litellm_callback_name="langfuse", + ui_callback_name="Langfuse", + litellm_callback_params=[ + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + ], + ) + + opentelemetry: CallbackOnUI = CallbackOnUI( + litellm_callback_name="otel", + ui_callback_name="OpenTelemetry", + litellm_callback_params=[ + "OTEL_EXPORTER", + "OTEL_ENDPOINT", + "OTEL_HEADERS", + ], + ) + + openmeter: CallbackOnUI = CallbackOnUI( + litellm_callback_name="openmeter", + ui_callback_name="openmeter", + litellm_callback_params=[ + "OPENMETER_API_ENDPOINT", + "OPENMETER_API_KEY", + ], + ) + + custom_callback_api: CallbackOnUI = CallbackOnUI( + litellm_callback_name="custom_callback_api", + litellm_callback_params=["GENERIC_LOGGER_ENDPOINT"], + ui_callback_name="Custom Callback API", + ) + + datadog: CallbackOnUI = CallbackOnUI( + litellm_callback_name="datadog", + litellm_callback_params=["DD_API_KEY", "DD_SITE"], + ui_callback_name="datadog", + ) From 82a3b3e615e0523e29296b3917e7eaf165e8f46d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 17:18:40 -0700 Subject: [PATCH 03/10] ui - add callbacks through admin ui --- .../src/components/settings.tsx | 167 ++++++++++++++++-- 1 file changed, 157 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 91f35c2496..42c969359c 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect } from "react"; import { Card, - Title, + Subtitle, Table, TableHead, @@ -23,15 +23,20 @@ import { TabList, Tab, Callout, + SelectItem, } from "@tremor/react"; + +import { Modal, Typography, Form, Input, Select, Button as Button2, message } from "antd"; + +const { Title, Paragraph } = Typography; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck, } from "./networking"; -import { Modal, Form, Input, Select, Button as Button2, message } from "antd"; import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; import AlertingSettings from "./alerting/alerting_settings"; +import FormItem from "antd/es/form/FormItem"; interface SettingsPageProps { accessToken: string | null; userRole: string | null; @@ -39,6 +44,15 @@ interface SettingsPageProps { premiumUser: boolean; } + +interface genericCallbackParams { + + litellm_callback_name: string // what to send in request + ui_callback_name: string // what to show on UI + litellm_callback_params: string[] | null // known required params for this callback +} + + interface AlertingVariables { SLACK_WEBHOOK_URL: string | null; LANGFUSE_PUBLIC_KEY: string | null; @@ -92,7 +106,7 @@ const Settings: React.FC = ({ premiumUser, }) => { const [callbacks, setCallbacks] = - useState(defaultLoggingObject); + useState([]); const [alerts, setAlerts] = useState([]); const [isModalVisible, setIsModalVisible] = useState(false); const [form] = Form.useForm(); @@ -104,6 +118,12 @@ const Settings: React.FC = ({ >({}); const [activeAlerts, setActiveAlerts] = useState([]); + const [showAddCallbacksModal, setShowAddCallbacksModal] = useState(false); + const [allCallbacks, setAllCallbacks] = useState([]); + + const [selectedCallbacktoAdd, setSelectedCallbacktoAdd] = useState(null); + const [selectedCallbackParams, setSelectedCallbackParams] = useState([]); + const handleSwitchChange = (alertName: string) => { if (activeAlerts.includes(alertName)) { setActiveAlerts(activeAlerts.filter((alert) => alert !== alertName)); @@ -128,8 +148,7 @@ const Settings: React.FC = ({ } getCallbacksCall(accessToken, userID, userRole).then((data) => { console.log("callbacks", data); - let updatedCallbacks: any[] = defaultLoggingObject; - + let updatedCallbacks: any[] = []; updatedCallbacks = updatedCallbacks.map((item: any) => { const callback = data.callbacks.find( (cb: any) => cb.name === item.name @@ -145,6 +164,8 @@ const Settings: React.FC = ({ }); setCallbacks(updatedCallbacks); + + setAllCallbacks(data.available_callbacks); // setCallbacks(callbacks_data); let alerts_data = data.alerts; @@ -225,6 +246,56 @@ const Settings: React.FC = ({ message.success("Email settings updated successfully"); } + + const addNewCallbackCall = async (formValues: Record) => { + if (!accessToken) { + return; + } + let new_callback = formValues?.callback + + let env_vars: Record = {}; + // add all other variables + Object.entries(formValues).forEach(([key, value]) => { + if (key !== "callback") { + env_vars[key] = value + } + }); + + let payload = { + environment_variables: env_vars, + litellm_settings: { + success_callback: [new_callback], + }, + } + + + try { + let newCallback = await setCallbacksCall(accessToken, payload); + message.success(`Callback ${new_callback} added successfully`); + setIsModalVisible(false); + form.resetFields(); + setSelectedCallback(null); + } catch (error) { + message.error("Failed to add callback: " + error, 20); + } + } + + + + const handleSelectedCallbackChange = (callbackObject: genericCallbackParams) => { + + console.log("inside handleSelectedCallbackChange", callbackObject); + setSelectedCallback(callbackObject.litellm_callback_name); + + console.log("all callbacks", allCallbacks); + if (selectedCallback) { + setSelectedCallbackParams(callbackObject.litellm_callback_params); + console.log("selectedCallbackParams", selectedCallbackParams); + } else { + setSelectedCallbackParams([]); + } + }; + const handleSaveAlerts = () => { if (!accessToken) { return; @@ -398,10 +469,7 @@ const Settings: React.FC = ({ return (
- + Logging Callbacks @@ -411,7 +479,8 @@ const Settings: React.FC = ({ - + Active Logging Callbacks + @@ -471,6 +540,10 @@ const Settings: React.FC = ({
+ @@ -770,7 +843,81 @@ const Settings: React.FC = ({
+ setShowAddCallbacksModal(false)} + footer={null} + > + + +
+ + <> + + + + + + { + selectedCallbackParams && selectedCallbackParams.map((param) => ( + + + + )) + } + +
+ Save +
+ + +
+ + + + + + + + + + + ); }; From 5fd38faa64f785f726ae171b5f205f3c27eb05b3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 18:06:26 -0700 Subject: [PATCH 04/10] ui - edit callbacks and test on ui --- .../src/components/settings.tsx | 235 ++++++++---------- 1 file changed, 105 insertions(+), 130 deletions(-) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 42c969359c..900349555c 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -24,8 +24,13 @@ import { Tab, Callout, SelectItem, + Icon, } from "@tremor/react"; +import { + PencilAltIcon +} from "@heroicons/react/outline"; + import { Modal, Typography, Form, Input, Select, Button as Button2, message } from "antd"; const { Title, Paragraph } = Typography; @@ -124,6 +129,9 @@ const Settings: React.FC = ({ const [selectedCallbacktoAdd, setSelectedCallbacktoAdd] = useState(null); const [selectedCallbackParams, setSelectedCallbackParams] = useState([]); + const [showEditCallback, setShowEditCallback] = useState(false); + const [selectedEditCallback, setSelectedEditCallback] = useState(null); + const handleSwitchChange = (alertName: string) => { if (activeAlerts.includes(alertName)) { setActiveAlerts(activeAlerts.filter((alert) => alert !== alertName)); @@ -148,23 +156,8 @@ const Settings: React.FC = ({ } getCallbacksCall(accessToken, userID, userRole).then((data) => { console.log("callbacks", data); - let updatedCallbacks: any[] = []; - updatedCallbacks = updatedCallbacks.map((item: any) => { - const callback = data.callbacks.find( - (cb: any) => cb.name === item.name - ); - if (callback) { - return { - ...item, - variables: { ...item.variables, ...callback.variables }, - }; - } else { - return item; - } - }); - - setCallbacks(updatedCallbacks); + setCallbacks(data.callbacks); setAllCallbacks(data.available_callbacks); // setCallbacks(callbacks_data); @@ -246,6 +239,36 @@ const Settings: React.FC = ({ message.success("Email settings updated successfully"); } + const updateCallbackCall = async (formValues: Record) => { + if (!accessToken) { + return; + } + + let env_vars: Record = {}; + // add all other variables + Object.entries(formValues).forEach(([key, value]) => { + if (key !== "callback") { + env_vars[key] = value + } + }); + let payload = { + environment_variables: env_vars, + } + + + try { + let newCallback = await setCallbacksCall(accessToken, payload); + message.success(`Callback added successfully`); + setIsModalVisible(false); + form.resetFields(); + setSelectedCallback(null); + } catch (error) { + message.error("Failed to add callback: " + error, 20); + } + } + + + const addNewCallbackCall = async (formValues: Record) => { if (!accessToken) { @@ -288,7 +311,7 @@ const Settings: React.FC = ({ setSelectedCallback(callbackObject.litellm_callback_name); console.log("all callbacks", allCallbacks); - if (selectedCallback) { + if (callbackObject && callbackObject.litellm_callback_params) { setSelectedCallbackParams(callbackObject.litellm_callback_params); console.log("selectedCallbackParams", selectedCallbackParams); } else { @@ -480,72 +503,60 @@ const Settings: React.FC = ({ Active Logging Callbacks - + + + + + - Callback - Callback Env Vars + Callback Name + {/* Callback Env Vars */} + {callbacks - .filter((callback) => callback.name !== "slack") .map((callback, index) => ( - + + - {callback.name} + {callback.name} -
    - {Object.entries(callback.variables ?? {}) - .filter(([key, value]) => - key.toLowerCase().includes(callback.name) - ) - .map(([key, value]) => ( -
  • - {key} - {key === "LANGFUSE_HOST" ? ( -

    - default value=https://cloud.langfuse.com -

    - ) : ( -
    - )} - -
    • - ))} -
- - + + { + setSelectedEditCallback(callback); + setShowEditCallback(true); + }} + /> + + ))}
+ - @@ -777,72 +788,7 @@ const Settings: React.FC = ({ - -
- - - - - {selectedCallback === "langfuse" && ( - <> - - - - - - - - - )} - - {selectedCallback == "openmeter" && ( - <> - - - - - )} - -
- Save -
-
- + = ({ + + setShowEditCallback(false)} + footer={null} + > +
+ <> + { + selectedEditCallback && selectedEditCallback.variables && Object.entries(selectedEditCallback.variables).map(([param, value]) => ( + + + + )) + } + +
+ Save +
- - - +
+ From 07735962350959db703ed03ccd585d3cddb44c91 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 18:08:27 -0700 Subject: [PATCH 05/10] fix - improve logic for all_available_callbacks --- litellm/proxy/proxy_server.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6d6a1e36bd..eda8af66c9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3129,9 +3129,9 @@ class ProxyConfig: if "alerting" in _general_settings: if ( general_settings is not None - and general_settings["alerting"] is not None + and general_settings.get("alerting", None) is not None and isinstance(general_settings["alerting"], list) - and _general_settings["alerting"] is not None + and _general_settings.get("alerting", None) is not None and isinstance(_general_settings["alerting"], list) ): for alert in _general_settings["alerting"]: @@ -13435,6 +13435,14 @@ async def update_config(config_info: ConfigYAML): if prisma_client is None: raise Exception("No DB Connected") + if store_model_in_db is not True: + raise HTTPException( + status_code=500, + detail={ + "error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature." + }, + ) + updated_settings = config_info.json(exclude_none=True) updated_settings = prisma_client.jsonify_object(updated_settings) for k, v in updated_settings.items(): @@ -13858,6 +13866,8 @@ async def get_config(): try: import base64 + all_available_callbacks = AllCallbacks() + config_data = await proxy_config.get_config() _litellm_settings = config_data.get("litellm_settings", {}) _general_settings = config_data.get("general_settings", {}) @@ -13992,11 +14002,13 @@ async def get_config(): _router_settings = {} else: _router_settings = llm_router.get_settings() + return { "status": "success", "callbacks": _data_to_return, "alerts": alerting_data, "router_settings": _router_settings, + "available_callbacks": all_available_callbacks, } except Exception as e: traceback.print_exc() From da6702de7f58664636964b8ab4b3356eae10c450 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 18:09:12 -0700 Subject: [PATCH 06/10] fix types on callback names and vars required --- litellm/proxy/_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a3c659f8bf..62e0d1c0de 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1376,7 +1376,7 @@ class AllCallbacks(LiteLLMBase): openmeter: CallbackOnUI = CallbackOnUI( litellm_callback_name="openmeter", - ui_callback_name="openmeter", + ui_callback_name="OpenMeter", litellm_callback_params=[ "OPENMETER_API_ENDPOINT", "OPENMETER_API_KEY", @@ -1392,5 +1392,5 @@ class AllCallbacks(LiteLLMBase): datadog: CallbackOnUI = CallbackOnUI( litellm_callback_name="datadog", litellm_callback_params=["DD_API_KEY", "DD_SITE"], - ui_callback_name="datadog", + ui_callback_name="Datadog", ) From 03b2757eb4322a4ae85385d5c2d7ed210a6c489f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 18:10:57 -0700 Subject: [PATCH 07/10] fix ui new Title component --- ui/litellm-dashboard/src/components/settings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 900349555c..03a1a95aa6 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -650,7 +650,7 @@ const Settings: React.FC = ({ - Email Settings + Email Settings LiteLLM Docs: email alerts
From 8614be7313949e23236f0b82eba016c2d9ef5fd5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 18:22:43 -0700 Subject: [PATCH 08/10] ui -fix types for allowed logging callbacks --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 62e0d1c0de..9b370d763a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1364,7 +1364,7 @@ class AllCallbacks(LiteLLMBase): ], ) - opentelemetry: CallbackOnUI = CallbackOnUI( + otel: CallbackOnUI = CallbackOnUI( litellm_callback_name="otel", ui_callback_name="OpenTelemetry", litellm_callback_params=[ From dcc3477cd56a0ca3986883bf264879154a8c2737 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 4 Jun 2024 18:25:00 -0700 Subject: [PATCH 09/10] ui - link to litellm for logging providers --- ui/litellm-dashboard/src/components/settings.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 03a1a95aa6..0ca13e6ebd 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -796,6 +796,8 @@ const Settings: React.FC = ({ onCancel= {() => setShowAddCallbacksModal(false)} footer={null} > + + LiteLLM Docs: Logging
Date: Tue, 4 Jun 2024 18:28:42 -0700 Subject: [PATCH 10/10] feat - log to s3 buckets through litellm Admin UI --- litellm/proxy/_types.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9b370d763a..e5ea35a398 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1374,6 +1374,16 @@ class AllCallbacks(LiteLLMBase): ], ) + s3: CallbackOnUI = CallbackOnUI( + litellm_callback_name="s3", + ui_callback_name="s3 Bucket (AWS)", + litellm_callback_params=[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_REGION_NAME", + ], + ) + openmeter: CallbackOnUI = CallbackOnUI( litellm_callback_name="openmeter", ui_callback_name="OpenMeter",