diff --git a/docs/my-website/docs/observability/openmeter.md b/docs/my-website/docs/observability/openmeter.md new file mode 100644 index 0000000000..64d9c39d21 --- /dev/null +++ b/docs/my-website/docs/observability/openmeter.md @@ -0,0 +1,97 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# OpenMeter - Usage-Based Billing + +[OpenMeter](https://openmeter.io/) is an Open Source Usage-Based Billing solution for AI/Cloud applications. It integrates with Stripe for easy billing. + + + +:::info +We want to learn how we can make the callbacks better! Meet the LiteLLM [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or +join our [discord](https://discord.gg/wuPM9dRgDw) +::: + + +## Quick Start +Use just 2 lines of code, to instantly log your responses **across all providers** with OpenMeter + +Get your OpenMeter API Key from https://openmeter.cloud/meters + +```python +litellm.success_callback = ["openmeter"] # logs cost + usage of successful calls to openmeter +``` + + + + + +```python +# pip install langfuse +import litellm +import os + +# from https://openmeter.cloud +os.environ["OPENMETER_API_ENDPOINT"] = "" +os.environ["OPENMETER_API_KEY"] = "" + +# LLM API Keys +os.environ['OPENAI_API_KEY']="" + +# set langfuse as a callback, litellm will send the data to langfuse +litellm.success_callback = ["openmeter"] + +# openai call +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Hi šŸ‘‹ - i'm openai"} + ] +) +``` + + + + +1. Add to Config.yaml +```yaml +model_list: +- litellm_params: + api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint + +litellm_settings: + success_callback: ["openmeter"] # šŸ‘ˆ KEY CHANGE +``` + +2. Start Proxy + +``` +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "fake-openai-endpoint", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + } +' +``` + + + + + + \ No newline at end of file diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 60a5d060a5..1c3b4f81c2 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -10,6 +10,7 @@ Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTeleme - [Async Custom Callbacks](#custom-callback-class-async) - [Async Custom Callback APIs](#custom-callback-apis-async) - [Logging to Langfuse](#logging-proxy-inputoutput---langfuse) +- [Logging to OpenMeter](#logging-proxy-inputoutput---langfuse) - [Logging to s3 Buckets](#logging-proxy-inputoutput---s3-buckets) - [Logging to DataDog](#logging-proxy-inputoutput---datadog) - [Logging to DynamoDB](#logging-proxy-inputoutput---dynamodb) @@ -591,6 +592,59 @@ litellm_settings: +## Logging Proxy Cost + Usage - OpenMeter + +Bill customers according to their LLM API usage with [OpenMeter](../observability/openmeter.md) + +**Required Env Variables** + +```bash +# from https://openmeter.cloud +export OPENMETER_API_ENDPOINT="" # defaults to https://openmeter.cloud +export OPENMETER_API_KEY="" +``` + +### Quick Start + +1. Add to Config.yaml +```yaml +model_list: +- litellm_params: + api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint + +litellm_settings: + success_callback: ["openmeter"] # šŸ‘ˆ KEY CHANGE +``` + +2. Start Proxy + +``` +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "fake-openai-endpoint", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + } +' +``` + + + + ## Logging Proxy Input/Output - DataDog We will use the `--config` to set `litellm.success_callback = ["datadog"]` this will log all successfull LLM calls to DataDog diff --git a/docs/my-website/img/openmeter.png b/docs/my-website/img/openmeter.png new file mode 100644 index 0000000000..29fa965573 Binary files /dev/null and b/docs/my-website/img/openmeter.png differ diff --git a/docs/my-website/img/openmeter_img_2.png b/docs/my-website/img/openmeter_img_2.png new file mode 100644 index 0000000000..e96edc2e49 Binary files /dev/null and b/docs/my-website/img/openmeter_img_2.png differ diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index e4f4e806e1..ef2ddb57e5 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -170,6 +170,7 @@ const sidebars = { "observability/custom_callback", "observability/langfuse_integration", "observability/sentry", + "observability/openmeter", "observability/promptlayer_integration", "observability/wandb_integration", "observability/langsmith_integration", @@ -177,7 +178,6 @@ const sidebars = { "observability/traceloop_integration", "observability/athina_integration", "observability/lunary_integration", - "observability/athina_integration", "observability/helicone_integration", "observability/supabase_integration", `observability/telemetry`, diff --git a/litellm/__init__.py b/litellm/__init__.py index a3d61bce16..5cc4d2316e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -22,6 +22,7 @@ success_callback: List[Union[str, Callable]] = [] failure_callback: List[Union[str, Callable]] = [] service_callback: List[Union[str, Callable]] = [] callbacks: List[Callable] = [] +_custom_logger_compatible_callbacks: list = ["openmeter"] _langfuse_default_tags: Optional[ List[ Literal[ diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index e8bc72c2d7..304edbbc93 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -118,7 +118,6 @@ class LangFuseLogger: pass # end of processing langfuse ######################## - print_verbose(f"response obj type: {type(response_obj)}") if ( level == "ERROR" and status_message is not None @@ -151,7 +150,6 @@ class LangFuseLogger: trace_id = None generation_id = None if self._is_langfuse_v2(): - print_verbose("INSIDE V2 LANGFUSE") trace_id, generation_id = self._log_langfuse_v2( user_id, metadata, @@ -378,9 +376,6 @@ class LangFuseLogger: "headers": clean_headers, } - print_verbose(f"trace_params: {trace_params}") - - print_verbose(f"trace_params: {trace_params}") trace = self.Langfuse.trace(**trace_params) generation_id = None @@ -430,11 +425,8 @@ class LangFuseLogger: "completion_start_time", None ) - print_verbose(f"generation_params: {generation_params}") - generation_client = trace.generation(**generation_params) - - print_verbose(f"LANGFUSE TRACE ID - {generation_client.trace_id}") + return generation_client.trace_id, generation_id except Exception as e: verbose_logger.debug(f"Langfuse Layer Error - {traceback.format_exc()}") diff --git a/litellm/integrations/openmeter.py b/litellm/integrations/openmeter.py new file mode 100644 index 0000000000..2ed551c8d8 --- /dev/null +++ b/litellm/integrations/openmeter.py @@ -0,0 +1,123 @@ +# What is this? +## On Success events log cost to OpenMeter - https://github.com/BerriAI/litellm/issues/1268 + +import dotenv, os, json +import requests +import litellm + +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +import uuid + + +def get_utc_datetime(): + import datetime as dt + from datetime import datetime + + if hasattr(dt, "UTC"): + return datetime.now(dt.UTC) # type: ignore + else: + return datetime.utcnow() # type: ignore + + +class OpenMeterLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.validate_environment() + self.async_http_handler = AsyncHTTPHandler() + self.sync_http_handler = HTTPHandler() + + def validate_environment(self): + """ + Expects + OPENMETER_API_ENDPOINT, + OPENMETER_API_KEY, + + in the environment + """ + missing_keys = [] + if litellm.get_secret("OPENMETER_API_KEY", None) is None: + missing_keys.append("OPENMETER_API_KEY") + + if len(missing_keys) > 0: + raise Exception("Missing keys={} in environment.".format(missing_keys)) + + def _common_logic(self, kwargs: dict, response_obj): + call_id = response_obj.get("id", kwargs.get("litellm_call_id")) + dt = get_utc_datetime().isoformat() + cost = kwargs.get("response_cost", None) + model = kwargs.get("model") + usage = {} + if ( + isinstance(response_obj, litellm.ModelResponse) + or isinstance(response_obj, litellm.EmbeddingResponse) + ) and hasattr(response_obj, "usage"): + usage = { + "prompt_tokens": response_obj["usage"].get("prompt_tokens", 0), + "completion_tokens": response_obj["usage"].get("completion_tokens", 0), + "total_tokens": response_obj["usage"].get("total_tokens"), + } + + return { + "specversion": "1.0", + "type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"), + "id": call_id, + "time": dt, + "subject": kwargs.get("user", ""), # end-user passed in via 'user' param + "source": "litellm-proxy", + "data": {"model": model, "cost": cost, **usage}, + } + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + _url = litellm.get_secret( + "OPENMETER_API_ENDPOINT", default_value="https://openmeter.cloud" + ) + if _url.endswith("/"): + _url += "api/v1/events" + else: + _url += "/api/v1/events" + + api_key = litellm.get_secret("OPENMETER_API_KEY") + + _data = self._common_logic(kwargs=kwargs, response_obj=response_obj) + self.sync_http_handler.post( + url=_url, + data=_data, + headers={ + "Content-Type": "application/cloudevents+json", + "Authorization": "Bearer {}".format(api_key), + }, + ) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + _url = litellm.get_secret( + "OPENMETER_API_ENDPOINT", default_value="https://openmeter.cloud" + ) + if _url.endswith("/"): + _url += "api/v1/events" + else: + _url += "/api/v1/events" + + api_key = litellm.get_secret("OPENMETER_API_KEY") + + _data = self._common_logic(kwargs=kwargs, response_obj=response_obj) + _headers = { + "Content-Type": "application/cloudevents+json", + "Authorization": "Bearer {}".format(api_key), + } + + try: + response = await self.async_http_handler.post( + url=_url, + data=json.dumps(_data), + headers=_headers, + ) + + response.raise_for_status() + except Exception as e: + print(f"\nAn Exception Occurred - {str(e)}") + if hasattr(response, "text"): + print(f"\nError Message: {response.text}") + raise e diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml index 9db128d0e2..9f2f6ec174 100644 --- a/litellm/proxy/_super_secret_config.yaml +++ b/litellm/proxy/_super_secret_config.yaml @@ -1,19 +1,15 @@ model_list: - litellm_params: - api_base: http://0.0.0.0:8080 + api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ api_key: my-fake-key model: openai/my-fake-model - rpm: 100 - model_name: fake-openai-endpoint -- litellm_params: - api_base: http://0.0.0.0:8081 - api_key: my-fake-key - model: openai/my-fake-model-2 - rpm: 100 model_name: fake-openai-endpoint router_settings: num_retries: 0 enable_pre_call_checks: true redis_host: os.environ/REDIS_HOST redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT \ No newline at end of file + redis_port: os.environ/REDIS_PORT + +litellm_settings: + success_callback: ["openmeter"] \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06651e7b63..9cc8719665 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2693,9 +2693,10 @@ class ProxyConfig: environment_variables = config_data.get("environment_variables", {}) for k, v in environment_variables.items(): try: - decoded_b64 = base64.b64decode(v) - value = decrypt_value(value=decoded_b64, master_key=master_key) # type: ignore - os.environ[k] = value + if v is not None: + decoded_b64 = base64.b64decode(v) + value = decrypt_value(value=decoded_b64, master_key=master_key) # type: ignore + os.environ[k] = value except Exception as e: verbose_proxy_logger.error( "Error setting env variable: %s - %s", k, str(e) @@ -8705,6 +8706,13 @@ async def update_config(config_info: ConfigYAML): _existing_settings = config["general_settings"] for k, v in updated_general_settings.items(): # overwrite existing settings with updated values + if k == "alert_to_webhook_url": + # check if slack is already enabled. if not, enable it + if "slack" not in _existing_settings: + if "alerting" not in _existing_settings: + _existing_settings["alerting"] = ["slack"] + elif isinstance(_existing_settings["alerting"], list): + _existing_settings["alerting"].append("slack") _existing_settings[k] = v config["general_settings"] = _existing_settings @@ -8819,7 +8827,25 @@ async def get_config(): """ for _callback in _success_callbacks: - if _callback == "langfuse": + if _callback == "openmeter": + env_vars = [ + "OPENMETER_API_KEY", + ] + env_vars_dict = {} + for _var in env_vars: + env_variable = environment_variables.get(_var, None) + if env_variable is None: + env_vars_dict[_var] = None + else: + # decode + decrypt the value + decoded_b64 = base64.b64decode(env_variable) + _decrypted_value = decrypt_value( + value=decoded_b64, master_key=master_key + ) + env_vars_dict[_var] = _decrypted_value + + _data_to_return.append({"name": _callback, "variables": env_vars_dict}) + elif _callback == "langfuse": _langfuse_vars = [ "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", @@ -8959,9 +8985,9 @@ async def test_endpoint(request: Request): ) async def health_services_endpoint( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - service: Literal["slack_budget_alerts", "langfuse", "slack"] = fastapi.Query( - description="Specify the service being hit." - ), + service: Literal[ + "slack_budget_alerts", "langfuse", "slack", "openmeter" + ] = fastapi.Query(description="Specify the service being hit."), ): """ Hidden endpoint. @@ -8975,7 +9001,7 @@ async def health_services_endpoint( raise HTTPException( status_code=400, detail={"error": "Service must be specified."} ) - if service not in ["slack_budget_alerts", "langfuse", "slack"]: + if service not in ["slack_budget_alerts", "langfuse", "slack", "openmeter"]: raise HTTPException( status_code=400, detail={ @@ -8983,6 +9009,18 @@ async def health_services_endpoint( }, ) + if service == "openmeter": + _ = await litellm.acompletion( + model="openai/litellm-mock-response-model", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + user="litellm:/health/services", + mock_response="This is a mock response", + ) + return { + "status": "success", + "message": "Mock LLM request made - check openmeter.", + } + if service == "langfuse": from litellm.integrations.langfuse import LangFuseLogger @@ -8999,63 +9037,66 @@ async def health_services_endpoint( "message": "Mock LLM request made - check langfuse.", } - if "slack" in general_settings.get("alerting", []): - # test_message = f"""\nšŸšØ `ProjectedLimitExceededError` šŸ’ø\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700""" - # check if user has opted into unique_alert_webhooks - if ( - proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url - is not None - ): - for ( - alert_type - ) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url: - """ - "llm_exceptions", - "llm_too_slow", - "llm_requests_hanging", - "budget_alerts", - "db_exceptions", - """ - # only test alert if it's in active alert types - if ( - proxy_logging_obj.slack_alerting_instance.alert_types - is not None - and alert_type - not in proxy_logging_obj.slack_alerting_instance.alert_types - ): - continue - test_message = "default test message" - if alert_type == "llm_exceptions": - test_message = f"LLM Exception test alert" - elif alert_type == "llm_too_slow": - test_message = f"LLM Too Slow test alert" - elif alert_type == "llm_requests_hanging": - test_message = f"LLM Requests Hanging test alert" - elif alert_type == "budget_alerts": - test_message = f"Budget Alert test alert" - elif alert_type == "db_exceptions": - test_message = f"DB Exception test alert" + if service == "slack" or service == "slack_budget_alerts": + if "slack" in general_settings.get("alerting", []): + # test_message = f"""\nšŸšØ `ProjectedLimitExceededError` šŸ’ø\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700""" + # check if user has opted into unique_alert_webhooks + if ( + proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url + is not None + ): + for ( + alert_type + ) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url: + """ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + """ + # only test alert if it's in active alert types + if ( + proxy_logging_obj.slack_alerting_instance.alert_types + is not None + and alert_type + not in proxy_logging_obj.slack_alerting_instance.alert_types + ): + continue + test_message = "default test message" + if alert_type == "llm_exceptions": + test_message = f"LLM Exception test alert" + elif alert_type == "llm_too_slow": + test_message = f"LLM Too Slow test alert" + elif alert_type == "llm_requests_hanging": + test_message = f"LLM Requests Hanging test alert" + elif alert_type == "budget_alerts": + test_message = f"Budget Alert test alert" + elif alert_type == "db_exceptions": + test_message = f"DB Exception test alert" + await proxy_logging_obj.alerting_handler( + message=test_message, level="Low", alert_type=alert_type + ) + else: await proxy_logging_obj.alerting_handler( - message=test_message, level="Low", alert_type=alert_type + message="This is a test slack alert message", + level="Low", + alert_type="budget_alerts", ) + return { + "status": "success", + "message": "Mock Slack Alert sent, verify Slack Alert Received on your channel", + } else: - await proxy_logging_obj.alerting_handler( - message="This is a test slack alert message", - level="Low", - alert_type="budget_alerts", + raise HTTPException( + status_code=422, + detail={ + "error": '"{}" not in proxy config: general_settings. Unable to test this.'.format( + service + ) + }, ) - return { - "status": "success", - "message": "Mock Slack Alert sent, verify Slack Alert Received on your channel", - } - else: - raise HTTPException( - status_code=422, - detail={ - "error": '"slack" not in proxy config: general_settings. Unable to test this.' - }, - ) except Exception as e: if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 97f679fd7f..b5db81b319 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1777,7 +1777,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time): usage = response_obj["usage"] if type(usage) == litellm.Usage: usage = dict(usage) - id = response_obj.get("id", str(uuid.uuid4())) + id = response_obj.get("id", kwargs.get("litellm_call_id")) api_key = metadata.get("user_api_key", "") if api_key is not None and isinstance(api_key, str) and api_key.startswith("sk-"): # hash the api_key diff --git a/litellm/utils.py b/litellm/utils.py index 6243195ef8..96bccb5fe4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -70,6 +70,7 @@ from .integrations.langsmith import LangsmithLogger from .integrations.weights_biases import WeightsBiasesLogger from .integrations.custom_logger import CustomLogger from .integrations.langfuse import LangFuseLogger +from .integrations.openmeter import OpenMeterLogger from .integrations.datadog import DataDogLogger from .integrations.prometheus import PrometheusLogger from .integrations.prometheus_services import PrometheusServicesLogger @@ -130,6 +131,7 @@ langsmithLogger = None weightsBiasesLogger = None customLogger = None langFuseLogger = None +openMeterLogger = None dataDogLogger = None prometheusLogger = None dynamoLogger = None @@ -1922,6 +1924,51 @@ class Logging: end_time=end_time, print_verbose=print_verbose, ) + if ( + callback == "openmeter" + and self.model_call_details.get("litellm_params", {}).get( + "acompletion", False + ) + == False + and self.model_call_details.get("litellm_params", {}).get( + "aembedding", False + ) + == False + and self.model_call_details.get("litellm_params", {}).get( + "aimage_generation", False + ) + == False + and self.model_call_details.get("litellm_params", {}).get( + "atranscription", False + ) + == False + ): + global openMeterLogger + if openMeterLogger is None: + print_verbose("Instantiates openmeter client") + openMeterLogger = OpenMeterLogger() + if self.stream and complete_streaming_response is None: + openMeterLogger.log_stream_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + else: + if self.stream and complete_streaming_response: + self.model_call_details["complete_response"] = ( + self.model_call_details.get( + "complete_streaming_response", {} + ) + ) + result = self.model_call_details["complete_response"] + openMeterLogger.log_success_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + if ( isinstance(callback, CustomLogger) and self.model_call_details.get("litellm_params", {}).get( @@ -2083,7 +2130,6 @@ class Logging: self.redact_message_input_output_from_logging(result=result) - print_verbose(f"Async success callbacks: {callbacks}") for callback in callbacks: # check if callback can run for this request litellm_params = self.model_call_details.get("litellm_params", {}) @@ -2121,6 +2167,35 @@ class Logging: await litellm.cache.async_add_cache(result, **kwargs) else: litellm.cache.add_cache(result, **kwargs) + if callback == "openmeter": + global openMeterLogger + if self.stream == True: + if ( + "async_complete_streaming_response" + in self.model_call_details + ): + await openMeterLogger.async_log_success_event( + kwargs=self.model_call_details, + response_obj=self.model_call_details[ + "async_complete_streaming_response" + ], + start_time=start_time, + end_time=end_time, + ) + else: + await openMeterLogger.async_log_stream_event( # [TODO]: move this to being an async log stream event function + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + else: + await openMeterLogger.async_log_success_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) if isinstance(callback, CustomLogger): # custom logger class if self.stream == True: if ( @@ -2594,7 +2669,7 @@ def function_setup( if inspect.iscoroutinefunction(callback): litellm._async_success_callback.append(callback) removed_async_items.append(index) - elif callback == "dynamodb": + elif callback == "dynamodb" or callback == "openmeter": # dynamo is an async callback, it's used for the proxy and needs to be async # we only support async dynamo db logging for acompletion/aembedding since that's used on proxy litellm._async_success_callback.append(callback) @@ -6777,11 +6852,11 @@ def validate_environment(model: Optional[str] = None) -> dict: def set_callbacks(callback_list, function_id=None): - global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, aispendLogger, berrispendLogger, supabaseClient, liteDebuggerClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, langsmithLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger + global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, aispendLogger, berrispendLogger, supabaseClient, liteDebuggerClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, langsmithLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger try: for callback in callback_list: - print_verbose(f"callback: {callback}") + print_verbose(f"init callback list: {callback}") if callback == "sentry": try: import sentry_sdk @@ -6844,6 +6919,8 @@ def set_callbacks(callback_list, function_id=None): promptLayerLogger = PromptLayerLogger() elif callback == "langfuse": langFuseLogger = LangFuseLogger() + elif callback == "openmeter": + openMeterLogger = OpenMeterLogger() elif callback == "datadog": dataDogLogger = DataDogLogger() elif callback == "prometheus": diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 3902b8c101..53de362863 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -38,6 +38,7 @@ interface AlertingVariables { LANGFUSE_PUBLIC_KEY: string | null, LANGFUSE_SECRET_KEY: string | null, LANGFUSE_HOST: string | null + OPENMETER_API_KEY: string | null } interface AlertingObject { @@ -45,12 +46,45 @@ interface AlertingObject { variables: AlertingVariables } +const defaultLoggingObject: AlertingObject[] = [ + { + "name": "slack", + "variables": { + "LANGFUSE_HOST": null, + "LANGFUSE_PUBLIC_KEY": null, + "LANGFUSE_SECRET_KEY": null, + "OPENMETER_API_KEY": null, + "SLACK_WEBHOOK_URL": null + } + }, + { + "name": "langfuse", + "variables": { + "LANGFUSE_HOST": null, + "LANGFUSE_PUBLIC_KEY": null, + "LANGFUSE_SECRET_KEY": null, + "OPENMETER_API_KEY": null, + "SLACK_WEBHOOK_URL": null + } + }, + { + "name": "openmeter", + "variables": { + "LANGFUSE_HOST": null, + "LANGFUSE_PUBLIC_KEY": null, + "LANGFUSE_SECRET_KEY": null, + "OPENMETER_API_KEY": null, + "SLACK_WEBHOOK_URL": null + } + } +] + const Settings: React.FC = ({ accessToken, userRole, userID, }) => { - const [callbacks, setCallbacks] = useState([]); + const [callbacks, setCallbacks] = useState(defaultLoggingObject); const [alerts, setAlerts] = useState([]); const [isModalVisible, setIsModalVisible] = useState(false); const [form] = Form.useForm(); @@ -81,8 +115,19 @@ const Settings: React.FC = ({ } getCallbacksCall(accessToken, userID, userRole).then((data) => { console.log("callbacks", data); - let callbacks_data = data.callbacks; - setCallbacks(callbacks_data); + let updatedCallbacks: any[] = defaultLoggingObject; + + updatedCallbacks = updatedCallbacks.map((item: any) => { + const callback = data.callbacks.find((cb: any) => cb.name === item.name); + if (callback) { + return { ...item, variables: { ...item.variables, ...callback.variables } }; + } else { + return item; + } + }); + + setCallbacks(updatedCallbacks) + // setCallbacks(callbacks_data); let alerts_data = data.alerts; console.log("alerts_data", alerts_data); @@ -175,6 +220,9 @@ const Settings: React.FC = ({ const payload = { environment_variables: updatedVariables, + litellm_settings: { + "success_callback": [callback.name] + } }; try { @@ -212,7 +260,8 @@ const Settings: React.FC = ({ "SLACK_WEBHOOK_URL": null, "LANGFUSE_HOST": null, "LANGFUSE_PUBLIC_KEY": values.langfusePublicKey, - "LANGFUSE_SECRET_KEY": values.langfusePrivateKey + "LANGFUSE_SECRET_KEY": values.langfusePrivateKey, + OPENMETER_API_KEY: null } } // add langfuse to callbacks @@ -239,10 +288,34 @@ const Settings: React.FC = ({ "SLACK_WEBHOOK_URL": values.slackWebhookUrl, "LANGFUSE_HOST": null, "LANGFUSE_PUBLIC_KEY": null, - "LANGFUSE_SECRET_KEY": null + "LANGFUSE_SECRET_KEY": null, + "OPENMETER_API_KEY": null } } setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]); + } else if (values.callback == "openmeter") { + console.log(`values.openMeterApiKey: ${values.openMeterApiKey}`) + payload = { + environment_variables: { + OPENMETER_API_KEY: values.openMeterApiKey, + }, + litellm_settings: { + success_callback: [values.callback] + } + }; + setCallbacksCall(accessToken, payload); + let newCallback: AlertingObject = { + "name": values.callback, + "variables": { + "SLACK_WEBHOOK_URL": null, + "LANGFUSE_HOST": null, + "LANGFUSE_PUBLIC_KEY": null, + "LANGFUSE_SECRET_KEY": null, + OPENMETER_API_KEY: values.openMeterAPIKey + } + } + // add langfuse to callbacks + setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]); } else { payload = { error: 'Invalid callback value' @@ -282,24 +355,24 @@ const Settings: React.FC = ({ - {callbacks.map((callback, index) => ( + {callbacks.filter((callback) => callback.name !== "slack").map((callback, index) => ( {callback.name}
    - {Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => ( -
  • - {key} - {key === "LANGFUSE_HOST" ? ( -

    default value=https://cloud.langfuse.com

    - ) : ( -
    - )} - -
    • - ))} + {Object.entries(callback.variables ?? {}).filter(([key, value]) => key.toLowerCase().includes(callback.name)).map(([key, value]) => ( +
  • + {key} + {key === "LANGFUSE_HOST" ? ( +

    default value=https://cloud.langfuse.com

    + ) : ( +
    + )} + +
    • + ))}
@@ -392,9 +462,10 @@ const Settings: React.FC = ({ > - + {selectedCallback === 'langfuse' && ( <> = ({ )} + { + selectedCallback == "openmeter" && <> + + + + + } +
Save