working streaming logging + cost tracking

litellm/litellm_core_utils/litellm_logging.py

@@ -1117,7 +1117,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
             ## BUILD COMPLETE STREAMED RESPONSE
             complete_streaming_response: Optional[
-                Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent]
+                Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]
             ] = None
             if "complete_streaming_response" in self.model_call_details:
                 return  # break out of this.
@@ -1639,7 +1639,7 @@ class Logging(LiteLLMLoggingBaseClass):
         if "async_complete_streaming_response" in self.model_call_details:
             return  # break out of this.
         complete_streaming_response: Optional[
-            Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent]
+            Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]
         ] = self._get_assembled_streaming_response(
             result=result,
             start_time=start_time,
@@ -2360,13 +2360,13 @@ class Logging(LiteLLMLoggingBaseClass):
         end_time: datetime.datetime,
         is_async: bool,
         streaming_chunks: List[Any],
-    ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent]]:
+    ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]]:
         if isinstance(result, ModelResponse):
             return result
         elif isinstance(result, TextCompletionResponse):
             return result
         elif isinstance(result, ResponseCompletedEvent):
-            return result
+            return result.response
         elif isinstance(result, ModelResponseStream):
             complete_streaming_response: Optional[
                 Union[ModelResponse, TextCompletionResponse]

tests/llm_responses_api_testing/test_openai_responses_api.py

@@ -9,6 +9,7 @@ import litellm
 from litellm.integrations.custom_logger import CustomLogger
 import json
 from litellm.types.utils import StandardLoggingPayload
+from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse
 
 
 @pytest.mark.asyncio
@@ -43,23 +44,66 @@ class TestCustomLogger(CustomLogger):
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         print("in async_log_success_event")
+        print("kwargs=", json.dumps(kwargs, indent=4, default=str))
         self.standard_logging_object = kwargs["standard_logging_object"]
         pass
 
 
+def validate_standard_logging_payload(
+    slp: StandardLoggingPayload, response: ResponsesAPIResponse, request_model: str
+):
+    """
+    Validate that a StandardLoggingPayload object matches the expected response
+
+    Args:
+        slp (StandardLoggingPayload): The standard logging payload object to validate
+        response (dict): The litellm response to compare against
+        request_model (str): The model name that was requested
+    """
+    # Validate payload exists
+    assert slp is not None, "Standard logging payload should not be None"
+
+    # Validate token counts
+    print("response=", json.dumps(response, indent=4, default=str))
+    assert (
+        slp["prompt_tokens"] == response["usage"]["input_tokens"]
+    ), "Prompt tokens mismatch"
+    assert (
+        slp["completion_tokens"] == response["usage"]["output_tokens"]
+    ), "Completion tokens mismatch"
+    assert (
+        slp["total_tokens"]
+        == response["usage"]["input_tokens"] + response["usage"]["output_tokens"]
+    ), "Total tokens mismatch"
+
+    # Validate spend and response metadata
+    assert slp["response_cost"] > 0, "Response cost should be greater than 0"
+    assert slp["id"] == response["id"], "Response ID mismatch"
+    assert slp["model"] == request_model, "Model name mismatch"
+
+    # Validate messages
+    assert slp["messages"] == [{"content": "hi", "role": "user"}], "Messages mismatch"
+
+    # Validate complete response structure
+    validate_responses_match(slp["response"], response)
+
+
 @pytest.mark.asyncio
 async def test_basic_openai_responses_api_streaming_with_logging():
     litellm._turn_on_debug()
     litellm.set_verbose = True
     test_custom_logger = TestCustomLogger()
     litellm.callbacks = [test_custom_logger]
+    request_model = "gpt-4o"
     response = await litellm.aresponses(
-        model="gpt-4o",
+        model=request_model,
         input="hi",
         stream=True,
     )
-
+    final_response: Optional[ResponseCompletedEvent] = None
     async for event in response:
+        if event.type == "response.completed":
+            final_response = event
         print("litellm response=", json.dumps(event, indent=4, default=str))
 
     print("sleeping for 2 seconds...")
@@ -69,6 +113,15 @@ async def test_basic_openai_responses_api_streaming_with_logging():
         json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str),
     )
 
+    assert final_response is not None
+    assert test_custom_logger.standard_logging_object is not None
+
+    validate_standard_logging_payload(
+        slp=test_custom_logger.standard_logging_object,
+        response=final_response.response,
+        request_model=request_model,
+    )
+
 
 def validate_responses_match(slp_response, litellm_response):
     """Validate that the standard logging payload OpenAI response matches the litellm response"""
@@ -129,37 +182,9 @@ async def test_basic_openai_responses_api_non_streaming_with_logging():
         json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str),
     )
 
+    assert response is not None
     assert test_custom_logger.standard_logging_object is not None
 
-    # validate token counts match OpenAI response
-    assert (
-        test_custom_logger.standard_logging_object["prompt_tokens"]
-        == response["usage"]["input_tokens"]
-    )
-    assert (
-        test_custom_logger.standard_logging_object["completion_tokens"]
-        == response["usage"]["output_tokens"]
-    )
-    assert (
-        test_custom_logger.standard_logging_object["total_tokens"]
-        == response["usage"]["input_tokens"] + response["usage"]["output_tokens"]
-    )
-
-    # validate spend > 0
-    assert test_custom_logger.standard_logging_object["response_cost"] > 0
-
-    # validate response id matches OpenAI
-    assert test_custom_logger.standard_logging_object["id"] == response["id"]
-
-    # validate model matches
-    assert test_custom_logger.standard_logging_object["model"] == request_model
-
-    # validate messages matches
-    assert test_custom_logger.standard_logging_object["messages"] == [
-        {"content": "hi", "role": "user"}
-    ]
-
-    # Add validation after existing assertions
-    validate_responses_match(
-        test_custom_logger.standard_logging_object["response"], response
+    validate_standard_logging_payload(
+        test_custom_logger.standard_logging_object, response, request_model
     )