diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9acd70db6f..18af639918 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1117,7 +1117,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent] + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse] ] = None if "complete_streaming_response" in self.model_call_details: return # break out of this. @@ -1639,7 +1639,7 @@ class Logging(LiteLLMLoggingBaseClass): if "async_complete_streaming_response" in self.model_call_details: return # break out of this. complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent] + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse] ] = self._get_assembled_streaming_response( result=result, start_time=start_time, @@ -2360,13 +2360,13 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: List[Any], - ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent]]: + ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]]: if isinstance(result, ModelResponse): return result elif isinstance(result, TextCompletionResponse): return result elif isinstance(result, ResponseCompletedEvent): - return result + return result.response elif isinstance(result, ModelResponseStream): complete_streaming_response: Optional[ Union[ModelResponse, TextCompletionResponse] diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index efa6dcf405..9745269bef 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -9,6 +9,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger import json from litellm.types.utils import StandardLoggingPayload +from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse @pytest.mark.asyncio @@ -43,23 +44,66 @@ class TestCustomLogger(CustomLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): print("in async_log_success_event") + print("kwargs=", json.dumps(kwargs, indent=4, default=str)) self.standard_logging_object = kwargs["standard_logging_object"] pass +def validate_standard_logging_payload( + slp: StandardLoggingPayload, response: ResponsesAPIResponse, request_model: str +): + """ + Validate that a StandardLoggingPayload object matches the expected response + + Args: + slp (StandardLoggingPayload): The standard logging payload object to validate + response (dict): The litellm response to compare against + request_model (str): The model name that was requested + """ + # Validate payload exists + assert slp is not None, "Standard logging payload should not be None" + + # Validate token counts + print("response=", json.dumps(response, indent=4, default=str)) + assert ( + slp["prompt_tokens"] == response["usage"]["input_tokens"] + ), "Prompt tokens mismatch" + assert ( + slp["completion_tokens"] == response["usage"]["output_tokens"] + ), "Completion tokens mismatch" + assert ( + slp["total_tokens"] + == response["usage"]["input_tokens"] + response["usage"]["output_tokens"] + ), "Total tokens mismatch" + + # Validate spend and response metadata + assert slp["response_cost"] > 0, "Response cost should be greater than 0" + assert slp["id"] == response["id"], "Response ID mismatch" + assert slp["model"] == request_model, "Model name mismatch" + + # Validate messages + assert slp["messages"] == [{"content": "hi", "role": "user"}], "Messages mismatch" + + # Validate complete response structure + validate_responses_match(slp["response"], response) + + @pytest.mark.asyncio async def test_basic_openai_responses_api_streaming_with_logging(): litellm._turn_on_debug() litellm.set_verbose = True test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] + request_model = "gpt-4o" response = await litellm.aresponses( - model="gpt-4o", + model=request_model, input="hi", stream=True, ) - + final_response: Optional[ResponseCompletedEvent] = None async for event in response: + if event.type == "response.completed": + final_response = event print("litellm response=", json.dumps(event, indent=4, default=str)) print("sleeping for 2 seconds...") @@ -69,6 +113,15 @@ async def test_basic_openai_responses_api_streaming_with_logging(): json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), ) + assert final_response is not None + assert test_custom_logger.standard_logging_object is not None + + validate_standard_logging_payload( + slp=test_custom_logger.standard_logging_object, + response=final_response.response, + request_model=request_model, + ) + def validate_responses_match(slp_response, litellm_response): """Validate that the standard logging payload OpenAI response matches the litellm response""" @@ -129,37 +182,9 @@ async def test_basic_openai_responses_api_non_streaming_with_logging(): json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), ) + assert response is not None assert test_custom_logger.standard_logging_object is not None - # validate token counts match OpenAI response - assert ( - test_custom_logger.standard_logging_object["prompt_tokens"] - == response["usage"]["input_tokens"] - ) - assert ( - test_custom_logger.standard_logging_object["completion_tokens"] - == response["usage"]["output_tokens"] - ) - assert ( - test_custom_logger.standard_logging_object["total_tokens"] - == response["usage"]["input_tokens"] + response["usage"]["output_tokens"] - ) - - # validate spend > 0 - assert test_custom_logger.standard_logging_object["response_cost"] > 0 - - # validate response id matches OpenAI - assert test_custom_logger.standard_logging_object["id"] == response["id"] - - # validate model matches - assert test_custom_logger.standard_logging_object["model"] == request_model - - # validate messages matches - assert test_custom_logger.standard_logging_object["messages"] == [ - {"content": "hi", "role": "user"} - ] - - # Add validation after existing assertions - validate_responses_match( - test_custom_logger.standard_logging_object["response"], response + validate_standard_logging_payload( + test_custom_logger.standard_logging_object, response, request_model )