mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-19 16:19:29 +00:00
fix: preserve usage/cached_tokens in Responses API streaming bridge (#22194)
The response.completed handler in the completion→responses streaming bridge was discarding the usage object, causing prompt_tokens_details (and cached_tokens) to always be None when streaming with models that use the Responses API (e.g. gpt-5.2-codex, gpt-5.3-codex). Extract usage from the response.completed event and translate it via the existing _transform_response_api_usage_to_chat_usage helper. Fixes #22192
This commit is contained in:
@@ -1088,6 +1088,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
||||
|
||||
finish_reason = "tool_calls" if has_function_calls else "stop"
|
||||
|
||||
usage = None
|
||||
if response_data.get("usage"):
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
response_data.get("usage")
|
||||
)
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
@@ -1095,7 +1101,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
||||
delta=Delta(content=""),
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
]
|
||||
],
|
||||
usage=usage
|
||||
)
|
||||
else:
|
||||
pass
|
||||
|
||||
+51
@@ -738,6 +738,57 @@ def test_response_completed_with_message_only_emits_stop_finish_reason():
|
||||
)
|
||||
|
||||
|
||||
|
||||
def test_response_completed_preserves_usage_with_cached_tokens():
|
||||
"""
|
||||
Test that response.completed correctly translates Responses API usage
|
||||
(input_tokens_details) to chat completion usage (prompt_tokens_details).
|
||||
|
||||
This is a regression test for an issue where streaming with models that
|
||||
use the Responses API bridge (e.g. gpt-5.2-codex) would drop
|
||||
prompt_tokens_details, causing cached_tokens to always be None.
|
||||
"""
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
|
||||
iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True)
|
||||
|
||||
chunk = {
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_789",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_abc",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Six"}],
|
||||
"status": "completed",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 1226,
|
||||
"output_tokens": 5,
|
||||
"total_tokens": 1231,
|
||||
"input_tokens_details": {"cached_tokens": 1024},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk)
|
||||
|
||||
assert result.usage is not None, "usage should be set on response.completed chunk"
|
||||
assert result.usage.prompt_tokens == 1226, "prompt_tokens should map from input_tokens"
|
||||
assert result.usage.completion_tokens == 5, "completion_tokens should map from output_tokens"
|
||||
assert result.usage.prompt_tokens_details is not None, "prompt_tokens_details should be set"
|
||||
assert result.usage.prompt_tokens_details.cached_tokens == 1024, (
|
||||
"cached_tokens should be preserved from input_tokens_details"
|
||||
)
|
||||
|
||||
|
||||
def test_function_call_done_emits_is_finished():
|
||||
"""
|
||||
Test that OUTPUT_ITEM_DONE for a function_call still emits is_finished=True.
|
||||
|
||||
Reference in New Issue
Block a user