Merge pull request #14179 from zhirafovod/otel_logs_metrics

docs/my-website/docs/proxy/config_settings.md

@@ -573,6 +573,10 @@ router_settings:
 | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
 | LITELLM_LOG | Enable detailed logging for LiteLLM
 | LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
+| LITELLM_LOGGER_NAME | Name for OTEL logger 
+| LITELLM_METER_NAME | Name for OTEL Meter 
+| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
+| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
 | LITELLM_MASTER_KEY | Master key for proxy authentication
 | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
 | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60

litellm/integrations/opentelemetry.py

@@ -15,6 +15,8 @@ from litellm.types.utils import (
     StandardLoggingPayload,
 )
 
+# OpenTelemetry imports moved to individual functions to avoid import errors when not installed
+
 if TYPE_CHECKING:
     from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter
     from opentelemetry.trace import Context as _Context
@@ -41,6 +43,8 @@ else:
     Context = Any
 
 LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
+LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
+LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
 # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
 RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
 LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@@ -83,6 +87,8 @@ class OpenTelemetryConfig:
     exporter: Union[str, SpanExporter] = "console"
     endpoint: Optional[str] = None
     headers: Optional[str] = None
+    enable_metrics: bool = False
+    enable_events: bool = False
 
     @classmethod
     def from_env(cls):
@@ -104,6 +110,14 @@ class OpenTelemetryConfig:
         headers = os.getenv(
             "OTEL_EXPORTER_OTLP_HEADERS", os.getenv("OTEL_HEADERS")
         )  # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***"
+        enable_metrics: bool = (
+            os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false").lower()
+            == "true"
+        )
+        enable_events: bool = (
+            os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower()
+            == "true"
+        )
 
         if exporter == "in_memory":
             return cls(exporter=InMemorySpanExporter())
@@ -111,6 +125,8 @@ class OpenTelemetryConfig:
             exporter=exporter,
             endpoint=endpoint,
             headers=headers,  # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***"
+            enable_metrics=enable_metrics,
+            enable_events=enable_events,
         )
 
 
@@ -119,27 +135,22 @@ class OpenTelemetry(CustomLogger):
         self,
         config: Optional[OpenTelemetryConfig] = None,
         callback_name: Optional[str] = None,
+        # injection points for testing
+        tracer_provider: Optional[Any] = None,
+        logger_provider: Optional[Any] = None,
+        meter_provider: Optional[Any] = None,
         **kwargs,
     ):
-        from opentelemetry import trace
-        from opentelemetry.sdk.trace import TracerProvider
-        from opentelemetry.trace import SpanKind
 
         if config is None:
             config = OpenTelemetryConfig.from_env()
 
         self.config = config
+        self.callback_name = callback_name
         self.OTEL_EXPORTER = self.config.exporter
         self.OTEL_ENDPOINT = self.config.endpoint
         self.OTEL_HEADERS = self.config.headers
-        provider = TracerProvider(resource=_get_litellm_resource())
-        provider.add_span_processor(self._get_span_processor())
-        self.callback_name = callback_name
-
-        trace.set_tracer_provider(provider)
-        self.tracer = trace.get_tracer(LITELLM_TRACER_NAME)
-
-        self.span_kind = SpanKind
+        self._init_tracing(tracer_provider)
 
         _debug_otel = str(os.getenv("DEBUG_OTEL", "False")).lower()
 
@@ -156,6 +167,8 @@ class OpenTelemetry(CustomLogger):
 
         # init CustomLogger params
         super().__init__(**kwargs)
+        self._init_metrics(meter_provider)
+        self._init_logs(logger_provider)
         self._init_otel_logger_on_litellm_proxy()
 
     def _init_otel_logger_on_litellm_proxy(self):
@@ -178,14 +191,109 @@ class OpenTelemetry(CustomLogger):
             litellm.service_callback.append("otel")
         setattr(proxy_server, "open_telemetry_logger", self)
 
+    def _init_tracing(self, tracer_provider):
+        from opentelemetry import trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.trace import SpanKind
+
+        # use provided tracer or create a new one
+        if tracer_provider is None:
+            tracer_provider = TracerProvider(resource=_get_litellm_resource())
+            # Only add OTLP span processor if we created the tracer provider ourselves
+            tracer_provider.add_span_processor(self._get_span_processor())
+
+        # register global provider and grab our tracer
+        trace.set_tracer_provider(tracer_provider)
+        self.tracer = trace.get_tracer(LITELLM_TRACER_NAME)
+        self.span_kind = SpanKind
+
+    def _init_metrics(self, meter_provider):
+        if not self.config.enable_metrics:
+            self._operation_duration_histogram = None
+            self._token_usage_histogram = None
+            self._cost_histogram = None
+            return
+
+        from opentelemetry import metrics
+        from opentelemetry.sdk.metrics import Histogram, MeterProvider
+
+        # Only create OTLP infrastructure if no custom meter provider is provided
+        if meter_provider is None:
+            from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+                OTLPMetricExporter,
+            )
+            from opentelemetry.sdk.metrics.export import (
+                AggregationTemporality,
+                PeriodicExportingMetricReader,
+            )
+
+            _metric_exporter = OTLPMetricExporter(
+                endpoint=self.config.endpoint,
+                headers=OpenTelemetry._get_headers_dictionary(self.config.headers),
+                preferred_temporality={Histogram: AggregationTemporality.DELTA},
+            )
+            _metric_reader = PeriodicExportingMetricReader(
+                _metric_exporter, export_interval_millis=10000
+            )
+
+            meter_provider = MeterProvider(
+                metric_readers=[_metric_reader], resource=_get_litellm_resource()
+            )
+            meter = meter_provider.get_meter(__name__)
+        else:
+            # Use the provided meter provider as-is, without creating additional OTLP infrastructure
+            meter = meter_provider.get_meter(__name__)
+
+        metrics.set_meter_provider(meter_provider)
+
+        self._operation_duration_histogram = meter.create_histogram(
+            name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38
+            description="GenAI operation duration",
+            unit="s",
+        )
+        self._token_usage_histogram = meter.create_histogram(
+            name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38
+            description="GenAI token usage",
+            unit="{token}",
+        )
+        self._cost_histogram = meter.create_histogram(
+            name="gen_ai.client.token.cost",
+            description="GenAI request cost",
+            unit="USD",
+        )
+
+    def _init_logs(self, logger_provider):
+        # nothing to do if events disabled
+        if not self.config.enable_events:
+            return
+
+        from opentelemetry._logs import set_logger_provider
+        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+        from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
+        from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+
+        # set up log pipeline
+        if logger_provider is None:
+            logger_provider = OTLoggerProvider()
+            # Only add OTLP exporter if we created the logger provider ourselves
+            logger_provider.add_log_record_processor(
+                BatchLogRecordProcessor(
+                    OTLPLogExporter(
+                        endpoint=self.config.endpoint,
+                        headers=self._get_headers_dictionary(self.config.headers),
+                    )
+                )
+            )
+        set_logger_provider(logger_provider)
+
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
-        self._handle_sucess(kwargs, response_obj, start_time, end_time)
+        self._handle_success(kwargs, response_obj, start_time, end_time)
 
     def log_failure_event(self, kwargs, response_obj, start_time, end_time):
         self._handle_failure(kwargs, response_obj, start_time, end_time)
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
-        self._handle_sucess(kwargs, response_obj, start_time, end_time)
+        self._handle_success(kwargs, response_obj, start_time, end_time)
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         self._handle_failure(kwargs, response_obj, start_time, end_time)
@@ -372,9 +480,9 @@ class OpenTelemetry(CustomLogger):
 
     def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
         """Extract dynamic headers from kwargs if available."""
-        standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
-            kwargs.get("standard_callback_dynamic_params")
-        )
+        standard_callback_dynamic_params: Optional[
+            StandardCallbackDynamicParams
+        ] = kwargs.get("standard_callback_dynamic_params")
 
         if not standard_callback_dynamic_params:
             return None
@@ -414,50 +522,185 @@ class OpenTelemetry(CustomLogger):
     # End of Team/Key Based Logging Control Flow
     #########################################################
 
-    def _handle_sucess(self, kwargs, response_obj, start_time, end_time):
-        from opentelemetry import trace
-        from opentelemetry.trace import Status, StatusCode
+    def _handle_success(self, kwargs, response_obj, start_time, end_time):
 
         verbose_logger.debug(
             "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s",
             kwargs,
             self.config,
         )
+        ctx, parent_span = self._get_span_context(kwargs)
+
+        # 1. Primary span
+        span = self._start_primary_span(kwargs, response_obj, start_time, end_time, ctx)
+
+        # 2. Raw‐request sub-span (if enabled)
+        self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span)
+
+        # 3. Guardrail span
+        self._create_guardrail_span(kwargs=kwargs, context=ctx)
+
+        # 4. Metrics & cost recording
+        self._record_metrics(kwargs, response_obj, start_time, end_time)
+
+        # 5. Semantic logs. 
+        if self.config.enable_events:
+            self._emit_semantic_logs(kwargs, response_obj, span)
+
+        # 6. End parent span
+        if parent_span is not None:
+            parent_span.end(end_time=self._to_ns(datetime.now()))
+
+    def _start_primary_span(self, kwargs, response_obj, start_time, end_time, context):
+        from opentelemetry.trace import Status, StatusCode
 
-        _parent_context, parent_otel_span = self._get_span_context(kwargs)
-        # Span 1: Request sent to litellm SDK
         otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
         span = otel_tracer.start_span(
             name=self._get_span_name(kwargs),
             start_time=self._to_ns(start_time),
-            context=_parent_context,
+            context=context,
         )
         span.set_status(Status(StatusCode.OK))
         self.set_attributes(span, kwargs, response_obj)
+        span.end(end_time=self._to_ns(end_time))
+        return span
 
-        if litellm.turn_off_message_logging is True:
-            pass
-        elif self.message_logging is not True:
-            pass
-        else:
-            # Span 2: Raw Request / Response to LLM
-            raw_request_span = otel_tracer.start_span(
-                name=RAW_REQUEST_SPAN_NAME,
-                start_time=self._to_ns(start_time),
-                context=trace.set_span_in_context(span),
+    def _maybe_log_raw_request(
+        self, kwargs, response_obj, start_time, end_time, parent_span
+    ):
+        from opentelemetry import trace
+        from opentelemetry.trace import Status, StatusCode
+
+        # only log raw LLM request/response if message_logging is on and not globally turned off
+        if litellm.turn_off_message_logging or not self.message_logging:
+            return
+
+        otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
+        raw_span = otel_tracer.start_span(
+            name=RAW_REQUEST_SPAN_NAME,
+            start_time=self._to_ns(start_time),
+            context=trace.set_span_in_context(parent_span),
+        )
+        raw_span.set_status(Status(StatusCode.OK))
+        self.set_raw_request_attributes(raw_span, kwargs, response_obj)
+        raw_span.end(end_time=self._to_ns(end_time))
+
+    def _record_metrics(self, kwargs, response_obj, start_time, end_time):
+        duration_s = (end_time - start_time).total_seconds()
+        params = kwargs.get("litellm_params") or {}
+        provider = params.get("custom_llm_provider", "Unknown")
+
+        common_attrs = {
+            "gen_ai.operation.name": "chat",
+            "gen_ai.system": provider,
+            "gen_ai.request.model": kwargs.get("model"),
+            "gen_ai.framework": "litellm",
+        }
+
+        std_log = kwargs.get("standard_logging_object")
+        md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {})
+        for key in [
+            "user_api_key_hash",
+            "user_api_key_alias",
+            "user_api_key_team_id",
+            "user_api_key_org_id",
+            "user_api_key_user_id",
+            "user_api_key_team_alias",
+            "user_api_key_user_email",
+            "spend_logs_metadata",
+            "requester_ip_address",
+            "requester_metadata",
+            "user_api_key_end_user_id",
+            "prompt_management_metadata",
+            "applied_guardrails",
+            "mcp_tool_call_metadata",
+            "vector_store_request_metadata",
+        ]:
+            if md.get(key) is not None:
+                common_attrs[f"metadata.{key}"] = str(md[key])
+
+        if self._operation_duration_histogram:
+            self._operation_duration_histogram.record(
+                duration_s, attributes=common_attrs
+            )
+            if (
+                response_obj
+                and (usage := response_obj.get("usage"))
+                and self._token_usage_histogram
+            ):
+                in_attrs = {**common_attrs, "gen_ai.token.type": "input"}
+                out_attrs = {**common_attrs, "gen_ai.token.type": "completion"}
+                self._token_usage_histogram.record(
+                    usage.get("prompt_tokens", 0), attributes=in_attrs
+                )
+                self._token_usage_histogram.record(
+                    usage.get("completion_tokens", 0), attributes=out_attrs
+                )
+
+        cost = kwargs.get("response_cost")
+        if self._cost_histogram and cost:
+            self._cost_histogram.record(cost, attributes=common_attrs)
+
+    def _emit_semantic_logs(self, kwargs, response_obj, span: Span):
+        if not self.config.enable_events:
+            return
+
+        from opentelemetry._logs import get_logger, LogRecord
+        otel_logger = get_logger(LITELLM_LOGGER_NAME)
+
+        parent_ctx = span.get_span_context()
+        provider = (kwargs.get("litellm_params") or {}).get(
+            "custom_llm_provider", "Unknown"
+        )
+
+        # per-message events
+        for msg in kwargs.get("messages", []):
+            role = msg.get("role", "user")
+            attrs = {"event_name": "gen_ai.content.prompt", "gen_ai.system": provider}
+            if role == "tool" and msg.get("id"):
+                attrs["id"] = msg["id"]
+            if self.message_logging and msg.get("content"):
+                attrs["gen_ai.prompt"] = msg["content"]
+
+            otel_logger.emit(
+                LogRecord(
+                    attributes=attrs,
+                    body=msg.copy(),
+                    trace_id=parent_ctx.trace_id,
+                    span_id=parent_ctx.span_id,
+                    trace_flags=parent_ctx.trace_flags,
+                )
             )
 
-            raw_request_span.set_status(Status(StatusCode.OK))
-            self.set_raw_request_attributes(raw_request_span, kwargs, response_obj)
-            raw_request_span.end(end_time=self._to_ns(end_time))
+        # per-choice events
+        for idx, choice in enumerate(response_obj.get("choices", [])):
+            attrs = {
+                "event_name": "gen_ai.content.completion",
+                "gen_ai.system": provider,
+                "index": idx,
+                "finish_reason": choice.get("finish_reason"),
+            }
+            body_msg = choice.get("message", {})
+            if self.message_logging and body_msg.get("content"):
+                attrs["message.content"] = body_msg["content"]
+            body = {
+                "index": idx,
+                "finish_reason": choice.get("finish_reason"),
+                "message": {"role": body_msg.get("role", "assistant")},
+            }
+            if self.message_logging and body_msg.get("content"):
+                body["message"]["content"] = body_msg["content"]
 
-        span.end(end_time=self._to_ns(end_time))
+            otel_logger.emit(
+                LogRecord(
+                    attributes=attrs,
+                    body=body,
+                    trace_id=parent_ctx.trace_id,
+                    span_id=parent_ctx.span_id,
+                    trace_flags=parent_ctx.trace_flags,
+                )
+            )
 
-        # Create span for guardrail information
-        self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
-
-        if parent_otel_span is not None:
-            parent_otel_span.end(end_time=self._to_ns(datetime.now()))
 
     def _create_guardrail_span(
         self, kwargs: Optional[dict], context: Optional[Context]

tests/test_litellm/integrations/open_telemetry/data/captured_response.json

@@ -0,0 +1 @@
+{"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}

tests/test_litellm/integrations/test_opentelemetry.py

@@ -1,15 +1,142 @@
+import json
 import os
 import sys
 import unittest
 from unittest.mock import MagicMock, patch
+from datetime import datetime, timedelta
+import time
 
 # Adds the grandparent directory to sys.path to allow importing project modules
 sys.path.insert(0, os.path.abspath("../.."))
 from litellm.integrations.opentelemetry import OpenTelemetry
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
+from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor, InMemoryLogExporter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+
+class TestOpenTelemetryGuardrails(unittest.TestCase):
+    @patch("litellm.integrations.opentelemetry.datetime")
+    def test_create_guardrail_span_with_valid_info(self, mock_datetime):
+        # Setup
+        otel = OpenTelemetry()
+        otel.tracer = MagicMock()
+        mock_span = MagicMock()
+        otel.tracer.start_span.return_value = mock_span
+
+        # Create guardrail information
+        guardrail_info = {
+            "guardrail_name": "test_guardrail",
+            "guardrail_mode": "input",
+            "masked_entity_count": {"CREDIT_CARD": 2},
+            "guardrail_response": "filtered_content",
+            "start_time": 1609459200.0,
+            "end_time": 1609459201.0,
+        }
+
+        # Create a kwargs dict with standard_logging_object containing guardrail information
+        kwargs = {"standard_logging_object": {"guardrail_information": guardrail_info}}
+
+        # Call the method
+        otel._create_guardrail_span(kwargs=kwargs, context=None)
+
+        # Assertions
+        otel.tracer.start_span.assert_called_once()
+
+        # print all calls to mock_span.set_attribute
+        print("Calls to mock_span.set_attribute:")
+        for call in mock_span.set_attribute.call_args_list:
+            print(call)
+
+        # Check that the span has the correct attributes set
+        mock_span.set_attribute.assert_any_call("guardrail_name", "test_guardrail")
+        mock_span.set_attribute.assert_any_call("guardrail_mode", "input")
+        mock_span.set_attribute.assert_any_call(
+            "guardrail_response", "filtered_content"
+        )
+        mock_span.set_attribute.assert_any_call(
+            "masked_entity_count", safe_dumps({"CREDIT_CARD": 2})
+        )
+
+        # Verify that the span was ended
+        mock_span.end.assert_called_once()
+
+    def test_create_guardrail_span_with_no_info(self):
+        # Setup
+        otel = OpenTelemetry()
+        otel.tracer = MagicMock()
+
+        # Test with no guardrail information
+        kwargs = {"standard_logging_object": {}}
+        otel._create_guardrail_span(kwargs=kwargs, context=None)
+
+        # Verify that start_span was never called
+        otel.tracer.start_span.assert_not_called()
+
 
 class TestOpenTelemetry(unittest.TestCase):
+    POLL_INTERVAL = 0.05
+    POLL_TIMEOUT = 2.0
+    MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0"
+    HERE = os.path.dirname(__file__)
+
+    def wait_for_spans(self, exporter: InMemorySpanExporter, prefix: str):
+        """Poll until we see at least one span with an attribute key starting with `prefix`."""
+        deadline = time.time() + self.POLL_TIMEOUT
+        while time.time() < deadline:
+            spans = exporter.get_finished_spans()
+            matches = [
+                s
+                for s in spans
+                if s.attributes and any(str(k).startswith(prefix) for k in s.attributes)
+            ]
+            if matches:
+                return matches
+            time.sleep(self.POLL_INTERVAL)
+        return []
+
+    def wait_for_metric(self, reader: InMemoryMetricReader, name: str):
+        """Poll until we see a metric with the given name."""
+        deadline = time.time() + self.POLL_TIMEOUT
+        while time.time() < deadline:
+            data = reader.get_metrics_data()
+            # guard against None or missing attribute
+            if not data or not hasattr(data, "resource_metrics"):
+                time.sleep(self.POLL_INTERVAL)
+                continue
+
+            for rm in data.resource_metrics:
+                for sm in rm.scope_metrics:
+                    for m in sm.metrics:
+                        if m.name == name:
+                            return m
+
+            time.sleep(self.POLL_INTERVAL)
+        return None
+
+    def wait_for_log(self, reader: InMemoryLogExporter, name: str):
+        """Poll until we see a log with the given name."""
+        deadline = time.time() + self.POLL_TIMEOUT
+        while time.time() < deadline:
+            logs = reader.get_finished_logs()
+            if not logs:
+                time.sleep(self.POLL_INTERVAL)
+                continue
+            matches = [
+                log
+                for log in logs
+                # if log.attributes and any(str(k).startswith(prefix) for k in log.attributes)
+            ]
+            if matches:
+                return matches
+            time.sleep(self.POLL_INTERVAL)
+        return []
+
     @patch("litellm.integrations.opentelemetry.datetime")
     def test_create_guardrail_span_with_valid_info(self, mock_datetime):
         # Setup
@@ -79,7 +206,6 @@ class TestOpenTelemetry(unittest.TestCase):
         ) as mock_get_headers, patch.object(
             otel, "_get_tracer_with_dynamic_headers"
         ) as mock_get_tracer:
-
             # Test case 1: With dynamic headers
             mock_get_headers.return_value = {
                 "arize-space-id": "test-space",
@@ -399,3 +525,229 @@ class TestOpenTelemetry(unittest.TestCase):
         self.assertEqual(attributes.get("service.name"), "litellm-service")
         # But other attributes from OTEL_RESOURCE_ATTRIBUTES should still be present
         self.assertEqual(attributes.get("extra.attr"), "extra-value")
+
+    def test_handle_success_generates_spans_metrics_and_events(self):
+        # force both metrics & events on
+        os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"] = "true"
+        os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true"
+
+        # ─── build in‐memory OTEL providers/exporters ─────────────────────────────
+        span_exporter = InMemorySpanExporter()
+        tracer_provider = TracerProvider()
+        tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
+
+        log_exporter = InMemoryLogExporter()
+        logger_provider = OTLoggerProvider()
+        logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
+
+        metric_reader = InMemoryMetricReader()
+        meter_provider = MeterProvider(metric_readers=[metric_reader])
+
+        # ─── instantiate our OpenTelemetry logger with test providers ───────────
+        otel = OpenTelemetry(
+            tracer_provider=tracer_provider,
+            meter_provider=meter_provider,
+            logger_provider=logger_provider,
+        )
+
+        # OpenTelemetry attempts to set a global tracer provider, which can be set only once.
+        # so we hack here to set a local tracer deriver from the provider we created.
+        otel.tracer = tracer_provider.get_tracer(__name__)
+
+        # ─── minimal input / output for a chat call ──────────────────────────────
+        start = datetime.utcnow()
+        end = start + timedelta(seconds=1)
+
+        with open(
+            os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json")
+        ) as f:
+            kwargs = json.load(f)
+        with open(
+            os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json")
+        ) as f:
+            response_obj = json.load(f)
+
+        # ─── exercise the hook ───────────────────────────────────────────────────
+        otel._handle_success(kwargs, response_obj, start, end)
+
+        # ─── assert spans ────────────────────────────────────────────────────────
+        spans = self.wait_for_spans(span_exporter, "gen_ai.")
+        self.assertTrue(spans, "Expected at least one gen_ai span")
+
+        # verify our top‐level litellm_request span is present
+        names = [s.name for s in spans]
+        self.assertIn("litellm_request", names)
+
+        # ─── assert metrics ──────────────────────────────────────────────────────
+        duration_metric = self.wait_for_metric(
+            metric_reader, "gen_ai.client.operation.duration"
+        )
+        self.assertIsNotNone(duration_metric, "duration histogram was not recorded")
+
+        # check that our model attribute made it onto at least one data point
+        found_dp = False
+        if (
+            duration_metric
+            and hasattr(duration_metric, "data")
+            and hasattr(duration_metric.data, "data_points")
+        ):
+            found_dp = any(
+                dp.attributes.get("gen_ai.request.model") == self.MODEL
+                for dp in duration_metric.data.data_points
+            )
+        self.assertTrue(
+            found_dp, "expected gen_ai.request.model attribute on a data point"
+        )
+
+        # ─── assert logs ───────────────────────────────────────────────────────
+        logs = []
+        logs = self.wait_for_log(log_exporter, "gen_ai.")
+        self.assertTrue(logs, "Expected at least one gen_ai log")
+
+        user_logs = [log for log in logs if log.log_record.attributes.get("event_name") == "gen_ai.content.prompt"]
+        self.assertTrue(user_logs, "did not see a gen_ai.content.prompt log")
+        # check log bodies
+        user_prompt = user_logs[0].log_record.attributes.get("gen_ai.prompt")
+        self.assertEqual("What is the capital of France?", user_prompt, "did not see a prompt message")
+
+        choice_logs = [log for log in logs if log.log_record.attributes.get("event_name") == "gen_ai.content.completion"]
+        self.assertTrue(choice_logs, "did not see a gen_ai.content.completion event")
+
+        choice_response = choice_logs[0].log_record.body
+        self.assertIsNotNone(choice_response, "did not see a response message")
+        self.assertEqual("stop", choice_response.get("finish_reason"), "did not see expected finish reason")
+
+
+    def test_handle_success_spans_only(self):
+        # make sure neither events nor metrics is on
+        os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None)
+        os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", None)
+
+        # ─── build in‐memory OTEL providers/exporters ─────────────────────────────
+        span_exporter = InMemorySpanExporter()
+        tracer_provider = TracerProvider()
+        tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
+
+        # no logs / no metrics
+        log_exporter = InMemoryLogExporter()
+        logger_provider = OTLoggerProvider()
+        logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
+        metric_reader = InMemoryMetricReader()
+        meter_provider = MeterProvider(metric_readers=[metric_reader])
+
+        # ─── instantiate our OpenTelemetry logger with test providers ───────────
+        otel = OpenTelemetry(
+            tracer_provider=tracer_provider,
+            meter_provider=meter_provider,
+            logger_provider=logger_provider,  # pass even if events disabled (safe)
+        )
+        # bind our tracer to the test tracer provider (global registration is a no-op after the first time)
+        otel.tracer = tracer_provider.get_tracer(__name__)
+
+        # ─── minimal input / output for a chat call ──────────────────────────────
+        start = datetime.utcnow()
+        end = start + timedelta(seconds=1)
+        with open(
+            os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json")
+        ) as f:
+            kwargs = json.load(f)
+        with open(
+            os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json")
+        ) as f:
+            response_obj = json.load(f)
+
+        # ─── exercise the hook ───────────────────────────────────────────────────
+        otel._handle_success(kwargs, response_obj, start, end)
+
+        # ─── assert spans only ───────────────────────────────────────────────────
+        spans = span_exporter.get_finished_spans()
+        self.assertTrue(spans, "Expected at least one span")
+        # must have the top‐level litellm_request span
+        # self.assertIn(
+        #     LITELLM_REQUEST_SPAN_NAME,
+        #     [s.name for s in spans],
+        #     "litellm_request span missing",
+        # )
+        # model attribute should be on that span
+        found = any(
+            s.attributes
+            and s.attributes.get("gen_ai.request.model") == self.MODEL
+            for s in spans
+        )
+        self.assertTrue(found, "expected gen_ai.request.model on span attributes")
+
+        # no metrics recorded
+        self.assertIsNone(
+            self.wait_for_metric(metric_reader, "gen_ai.client.operation.duration"),
+            "Did not expect any metrics",
+        )
+        # no logs emitted
+        logs = log_exporter.get_finished_logs()
+        self.assertFalse(logs, "Did not expect any logs")
+
+    def test_handle_success_spans_and_metrics(self):
+        # only metrics on
+        os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None)
+        os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true"
+
+        # ─── build in‐memory OTEL providers/exporters ─────────────────────────────
+        span_exporter = InMemorySpanExporter()
+        tracer_provider = TracerProvider()
+        tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
+
+        log_exporter = InMemoryLogExporter()
+        logger_provider = OTLoggerProvider()
+        logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
+        metric_reader = InMemoryMetricReader()
+        meter_provider = MeterProvider(metric_readers=[metric_reader])
+
+        # ─── instantiate our OpenTelemetry logger with test providers ───────────
+        otel = OpenTelemetry(
+            tracer_provider=tracer_provider,
+            meter_provider=meter_provider,
+            logger_provider=logger_provider,  # needed if events were enabled
+        )
+        otel.tracer = tracer_provider.get_tracer(__name__)
+
+        # ─── minimal input / output for a chat call ──────────────────────────────
+        start = datetime.utcnow()
+        end = start + timedelta(seconds=1)
+        with open(
+            os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json")
+        ) as f:
+            kwargs = json.load(f)
+        with open(
+            os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json")
+        ) as f:
+            response_obj = json.load(f)
+
+        # ─── exercise the hook ───────────────────────────────────────────────────
+        otel._handle_success(kwargs, response_obj, start, end)
+
+        # ─── assert spans ────────────────────────────────────────────────────────
+        spans = span_exporter.get_finished_spans()
+        self.assertTrue(spans, "Expected at least one span")
+
+        # ─── assert metrics ──────────────────────────────────────────────────────
+        duration_metric = self.wait_for_metric(
+            metric_reader, "gen_ai.client.operation.duration"
+        )
+        self.assertIsNotNone(duration_metric, "duration histogram was not recorded")
+        # model attribute should be present on a data point
+        found_dp = False
+        if (
+            duration_metric
+            and hasattr(duration_metric, "data")
+            and hasattr(duration_metric.data, "data_points")
+        ):
+            found_dp = any(
+                dp.attributes.get("gen_ai.request.model") == self.MODEL
+                for dp in duration_metric.data.data_points
+            )
+        self.assertTrue(
+            found_dp, "expected gen_ai.request.model attribute on a data point"
+        )
+
+        # ─── no events when only metrics enabled ─────────────────────────────────
+        logs = log_exporter.get_finished_logs()
+        self.assertFalse(logs, "Did not expect any logs")