From 715387c3c0f7dcd2181b74a018cc4ab697aa4a50 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 15:59:42 -0700 Subject: [PATCH 01/11] add message_logging on Custom Logger --- litellm/integrations/custom_logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index ce0caf32bc..e5d3dfd56c 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -15,7 +15,8 @@ from litellm.types.utils import AdapterCompletionStreamWrapper, ModelResponse class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class # Class variables or attributes - def __init__(self) -> None: + def __init__(self, message_logging: bool = True) -> None: + self.message_logging = message_logging pass def log_pre_api_call(self, model, messages, kwargs): From b86075ef9a95bbd0cec2208dc4a0c1a19f53c301 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:00:47 -0700 Subject: [PATCH 02/11] refactor redact_message_input_output_from_custom_logger --- litellm/litellm_core_utils/redact_messages.py | 89 +++++++++++-------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 7f342e2711..631810aaab 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -11,6 +11,7 @@ import copy from typing import TYPE_CHECKING, Any import litellm +from litellm.integrations.custom_logger import CustomLogger if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -22,6 +23,56 @@ else: LiteLLMLoggingObject = Any +def redact_message_input_output_from_custom_logger( + litellm_logging_obj: LiteLLMLoggingObject, result, custom_logger: CustomLogger +): + if ( + hasattr(custom_logger, "message_logging") + and custom_logger.message_logging is not True + ): + return perform_redaction(litellm_logging_obj, result) + return result + + +def perform_redaction(litellm_logging_obj: LiteLLMLoggingObject, result): + """ + Performs the actual redaction on the logging object and result. + """ + # Redact model_call_details + litellm_logging_obj.model_call_details["messages"] = [ + {"role": "user", "content": "redacted-by-litellm"} + ] + litellm_logging_obj.model_call_details["prompt"] = "" + litellm_logging_obj.model_call_details["input"] = "" + + # Redact streaming response + if ( + litellm_logging_obj.stream is True + and "complete_streaming_response" in litellm_logging_obj.model_call_details + ): + _streaming_response = litellm_logging_obj.model_call_details[ + "complete_streaming_response" + ] + for choice in _streaming_response.choices: + if isinstance(choice, litellm.Choices): + choice.message.content = "redacted-by-litellm" + elif isinstance(choice, litellm.utils.StreamingChoices): + choice.delta.content = "redacted-by-litellm" + + # Redact result + if result is not None and isinstance(result, litellm.ModelResponse): + _result = copy.deepcopy(result) + if hasattr(_result, "choices") and _result.choices is not None: + for choice in _result.choices: + if isinstance(choice, litellm.Choices): + choice.message.content = "redacted-by-litellm" + elif isinstance(choice, litellm.utils.StreamingChoices): + choice.delta.content = "redacted-by-litellm" + return _result + + return result + + def redact_message_input_output_from_logging( litellm_logging_obj: LiteLLMLoggingObject, result ): @@ -50,43 +101,7 @@ def redact_message_input_output_from_logging( ): return result - # remove messages, prompts, input, response from logging - litellm_logging_obj.model_call_details["messages"] = [ - {"role": "user", "content": "redacted-by-litellm"} - ] - litellm_logging_obj.model_call_details["prompt"] = "" - litellm_logging_obj.model_call_details["input"] = "" - - # response cleaning - # ChatCompletion Responses - if ( - litellm_logging_obj.stream is True - and "complete_streaming_response" in litellm_logging_obj.model_call_details - ): - _streaming_response = litellm_logging_obj.model_call_details[ - "complete_streaming_response" - ] - for choice in _streaming_response.choices: - if isinstance(choice, litellm.Choices): - choice.message.content = "redacted-by-litellm" - elif isinstance(choice, litellm.utils.StreamingChoices): - choice.delta.content = "redacted-by-litellm" - else: - if result is not None: - if isinstance(result, litellm.ModelResponse): - # only deep copy litellm.ModelResponse - _result = copy.deepcopy(result) - if hasattr(_result, "choices") and _result.choices is not None: - for choice in _result.choices: - if isinstance(choice, litellm.Choices): - choice.message.content = "redacted-by-litellm" - elif isinstance(choice, litellm.utils.StreamingChoices): - choice.delta.content = "redacted-by-litellm" - - return _result - - # by default return result - return result + return perform_redaction(litellm_logging_obj, result) def redact_user_api_key_info(metadata: dict) -> dict: From 12d8c0d0a408c0f6db9f705dc174cd8e7e1138d6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:02:24 -0700 Subject: [PATCH 03/11] use redact_message_input_output_from_custom_logger --- litellm/litellm_core_utils/litellm_logging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f57dd3b812..b1db82a775 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -28,6 +28,7 @@ from litellm.cost_calculator import _select_model_name_for_cost_calc from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( + redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, ) from litellm.rerank_api.types import RerankResponse @@ -1395,6 +1396,9 @@ class Logging: call_type=self.call_type, ) elif isinstance(callback, CustomLogger): + result = redact_message_input_output_from_custom_logger( + result=result, litellm_logging_obj=self, custom_logger=callback + ) self.model_call_details, result = await callback.async_logging_hook( kwargs=self.model_call_details, result=result, From b36f9642179a6daac5398d397c54c9259355f1e1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:03:39 -0700 Subject: [PATCH 04/11] fix init custom logger when init OTEL runs --- litellm/integrations/opentelemetry.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index d35c7f304a..e2c3d6f3b8 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -71,7 +71,10 @@ class OpenTelemetryConfig: class OpenTelemetry(CustomLogger): def __init__( - self, config=OpenTelemetryConfig.from_env(), callback_name: Optional[str] = None + self, + config=OpenTelemetryConfig.from_env(), + callback_name: Optional[str] = None, + **kwargs, ): from opentelemetry import trace from opentelemetry.sdk.resources import Resource @@ -101,6 +104,9 @@ class OpenTelemetry(CustomLogger): otel_exporter_logger = logging.getLogger("opentelemetry.sdk.trace.export") otel_exporter_logger.setLevel(logging.DEBUG) + # init CustomLogger params + super().__init__(**kwargs) + def log_success_event(self, kwargs, response_obj, start_time, end_time): self._handle_sucess(kwargs, response_obj, start_time, end_time) @@ -261,6 +267,8 @@ class OpenTelemetry(CustomLogger): if litellm.turn_off_message_logging is True: pass + elif self.message_logging is not True: + pass else: # Span 2: Raw Request / Response to LLM raw_request_span = self.tracer.start_span( From 7c9591881c9981c53a53aa7afeba86a2d18c70ed Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:05:48 -0700 Subject: [PATCH 05/11] use callback_settings when intializing otel --- litellm/proxy/common_utils/callback_utils.py | 8 +++-- litellm/proxy/proxy_config.yaml | 6 +++- litellm/proxy/proxy_server.py | 6 +++- litellm/tests/test_async_opentelemetry.py | 35 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 4ccf61e234..4d0fd23030 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -17,7 +17,7 @@ def initialize_callbacks_on_proxy( litellm_settings: dict, callback_specific_params: dict = {}, ): - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import callback_settings, prisma_client verbose_proxy_logger.debug( f"{blue_color_code}initializing callbacks={value} on proxy{reset_color_code}" @@ -34,7 +34,11 @@ def initialize_callbacks_on_proxy( from litellm.integrations.opentelemetry import OpenTelemetry from litellm.proxy import proxy_server - open_telemetry_logger = OpenTelemetry() + _otel_settings = {} + if "otel" in callback_settings: + _otel_settings = callback_settings["otel"] + + open_telemetry_logger = OpenTelemetry(**_otel_settings) # Add Otel as a service callback if "otel" not in litellm.service_callback: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index b407b0d7ad..e385a23d7e 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -16,7 +16,11 @@ guardrails: output_parse_pii: True litellm_settings: - callbacks: ["prometheus"] + callbacks: ["otel"] + +callback_settings: + otel: + message_logging: False general_settings: master_key: sk-1234 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b6ebbe1df8..341f2b5d62 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -478,6 +478,7 @@ experimental = False llm_router: Optional[litellm.Router] = None llm_model_list: Optional[list] = None general_settings: dict = {} +callback_settings: dict = {} log_file = "api_log.json" worker_config = None master_key = None @@ -1491,7 +1492,7 @@ class ProxyConfig: """ Load config values into proxy global state """ - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, use_background_health_checks, health_check_interval, use_queue, custom_db_client, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, use_background_health_checks, health_check_interval, use_queue, custom_db_client, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings # Load existing config if os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is not None: @@ -1533,6 +1534,9 @@ class ProxyConfig: _license_check.license_str = os.getenv("LITELLM_LICENSE", None) premium_user = _license_check.is_premium() + ## Callback settings + callback_settings = config.get("callback_settings", None) + ## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..) litellm_settings = config.get("litellm_settings", None) if litellm_settings is None: diff --git a/litellm/tests/test_async_opentelemetry.py b/litellm/tests/test_async_opentelemetry.py index aee434f2a4..1fac0bb67e 100644 --- a/litellm/tests/test_async_opentelemetry.py +++ b/litellm/tests/test_async_opentelemetry.py @@ -12,6 +12,41 @@ from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfi verbose_logger.setLevel(logging.DEBUG) +class TestOpenTelemetry(OpenTelemetry): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.kwargs = None + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + print("in async_log_success_event for TestOpenTelemetry kwargs=", self.kwargs) + self.kwargs = kwargs + await super().async_log_success_event( + kwargs, response_obj, start_time, end_time + ) + + +@pytest.mark.asyncio +async def test_otel_with_message_logging_off(): + from litellm.integrations.opentelemetry import OpenTelemetry + + otel_logger = TestOpenTelemetry( + message_logging=False, config=OpenTelemetryConfig(exporter="console") + ) + + litellm.callbacks = [otel_logger] + + response = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + ) + print("response", response) + + assert otel_logger.kwargs["messages"] == [ + {"role": "user", "content": "redacted-by-litellm"} + ] + + @pytest.mark.asyncio @pytest.mark.skip(reason="Local only test. WIP.") async def test_async_otel_callback(): From 4592d80f43cc9f4c1fb85163a74c1b501cecfdae Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:10:13 -0700 Subject: [PATCH 06/11] add doc on redacting otel message / response --- docs/my-website/docs/proxy/logging.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 9492920d0d..f7b650f7a5 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -744,6 +744,20 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ** 🎉 Expect to see this trace logged in your OTEL collector** +### Redacting Messages, Response Content from OTEL Logging + +Set `message_logging=False` for `otel`, no messages / response will be logged + +```yaml +litellm_settings: + callbacks: ["otel"] + +## 👇 Key Change +callback_settings: + otel: + message_logging: False +``` + ### Context propagation across Services `Traceparent HTTP Header` ❓ Use this when you want to **pass information about the incoming request in a distributed tracing system** From b60361fca1e5a445fbb74bf0d88688141c43384e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:20:47 -0700 Subject: [PATCH 07/11] fix otel test --- litellm/proxy/common_utils/callback_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 4d0fd23030..2dd28c1f5f 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -35,7 +35,7 @@ def initialize_callbacks_on_proxy( from litellm.proxy import proxy_server _otel_settings = {} - if "otel" in callback_settings: + if isinstance(callback_settings, dict) and "otel" in callback_settings: _otel_settings = callback_settings["otel"] open_telemetry_logger = OpenTelemetry(**_otel_settings) From e25786ed8e4f54cde1fa3f3708d5db9b17d9cd00 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 17:01:20 -0700 Subject: [PATCH 08/11] fix test otel message logging off --- litellm/tests/test_async_opentelemetry.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/tests/test_async_opentelemetry.py b/litellm/tests/test_async_opentelemetry.py index 1fac0bb67e..55b60b302d 100644 --- a/litellm/tests/test_async_opentelemetry.py +++ b/litellm/tests/test_async_opentelemetry.py @@ -42,6 +42,8 @@ async def test_otel_with_message_logging_off(): ) print("response", response) + await asyncio.sleep(4) + assert otel_logger.kwargs["messages"] == [ {"role": "user", "content": "redacted-by-litellm"} ] From 407bdf10cece582084dc8e61e2834de9a0b953c5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 17:43:11 -0700 Subject: [PATCH 09/11] run test in verbose mode --- litellm/tests/test_async_opentelemetry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/tests/test_async_opentelemetry.py b/litellm/tests/test_async_opentelemetry.py index 55b60b302d..ed77424b44 100644 --- a/litellm/tests/test_async_opentelemetry.py +++ b/litellm/tests/test_async_opentelemetry.py @@ -27,6 +27,7 @@ class TestOpenTelemetry(OpenTelemetry): @pytest.mark.asyncio async def test_otel_with_message_logging_off(): + litellm.set_verbose = True from litellm.integrations.opentelemetry import OpenTelemetry otel_logger = TestOpenTelemetry( From 16b6b56c8b6b3663683d9db6f414ef5660cbe6b9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 17:51:35 -0700 Subject: [PATCH 10/11] fix otel logging test --- litellm/tests/test_async_opentelemetry.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/tests/test_async_opentelemetry.py b/litellm/tests/test_async_opentelemetry.py index ed77424b44..9d180306fd 100644 --- a/litellm/tests/test_async_opentelemetry.py +++ b/litellm/tests/test_async_opentelemetry.py @@ -18,7 +18,7 @@ class TestOpenTelemetry(OpenTelemetry): self.kwargs = None async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - print("in async_log_success_event for TestOpenTelemetry kwargs=", self.kwargs) + print("in async_log_success_event for TestOpenTelemetry kwargs=", kwargs) self.kwargs = kwargs await super().async_log_success_event( kwargs, response_obj, start_time, end_time @@ -26,9 +26,8 @@ class TestOpenTelemetry(OpenTelemetry): @pytest.mark.asyncio -async def test_otel_with_message_logging_off(): +async def test_awesome_otel_with_message_logging_off(): litellm.set_verbose = True - from litellm.integrations.opentelemetry import OpenTelemetry otel_logger = TestOpenTelemetry( message_logging=False, config=OpenTelemetryConfig(exporter="console") From 569f3ddda9f562dae4564f57f58e7a13416c4d8d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 17:59:07 -0700 Subject: [PATCH 11/11] fix test_awesome_otel_with_message_logging_off --- litellm/tests/test_async_opentelemetry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_async_opentelemetry.py b/litellm/tests/test_async_opentelemetry.py index 9d180306fd..e94adb8335 100644 --- a/litellm/tests/test_async_opentelemetry.py +++ b/litellm/tests/test_async_opentelemetry.py @@ -34,6 +34,8 @@ async def test_awesome_otel_with_message_logging_off(): ) litellm.callbacks = [otel_logger] + litellm.success_callback = [] + litellm.failure_callback = [] response = await litellm.acompletion( model="gpt-3.5-turbo", @@ -42,7 +44,7 @@ async def test_awesome_otel_with_message_logging_off(): ) print("response", response) - await asyncio.sleep(4) + await asyncio.sleep(5) assert otel_logger.kwargs["messages"] == [ {"role": "user", "content": "redacted-by-litellm"}