From 4ecd8bf81b0a34992e52e58bece7a9bf8d8ae9e6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 21 Apr 2026 20:46:19 -0700 Subject: [PATCH] fix(chatgpt): preserve responses routing and recover empty output (#25403) (#26219) - preserve existing shared backend `mode` when router deployment registration reuses a provider/model key already in `litellm.model_cost` (prevents alias with `mode: chat` from downgrading shared `chatgpt/gpt-5.4` from `responses` to `chat` and triggering 403s on /v1/chat/completions) - teach the ChatGPT Responses parser to recover `response.output_item.done` entries when `response.completed.output` is empty - add defensive /responses -> /chat/completions bridge fallback that reconstructs output items from raw SSE when `raw_response.output` is empty - regression coverage for shared alias routing, empty completed.output parsing, and SSE bridge recovery Closes #25403 Co-authored-by: afoninsky Co-authored-by: Claude Opus 4.7 (1M context) --- .../transformation.py | 141 ++++++++++- .../llms/chatgpt/responses/transformation.py | 17 +- litellm/router.py | 12 + ...responses_transformation_transformation.py | 234 ++++++++++++++++++ .../test_chatgpt_responses_transformation.py | 83 +++++++ .../test_router_model_cost_isolation.py | 78 ++++++ 6 files changed, 560 insertions(+), 5 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index e3cbf422e5..8287d5b6ab 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -26,6 +26,7 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse from litellm._logging import verbose_logger +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, @@ -97,7 +98,7 @@ def _build_reasoning_item( def _reasoning_item_to_response_input( - r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]] + r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]], ) -> Dict[str, Any]: """Convert a stored ChatCompletionReasoningItem back to a Responses API input item.""" r_input: Dict[str, Any] = { @@ -601,6 +602,125 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return choices + @classmethod + def _recover_output_items_from_raw_sse( + cls, raw_sse: Optional[str] + ) -> List[Dict[str, Any]]: + if not raw_sse or not isinstance(raw_sse, str): + return [] + + recovered_output_items: Dict[int, Dict[str, Any]] = {} + recovered_text_only_items: Dict[int, Dict[str, Any]] = {} + + for chunk in raw_sse.splitlines(): + stripped_chunk = ( + CustomStreamWrapper._strip_sse_data_from_chunk(chunk.strip()) or "" + ).strip() + if ( + not stripped_chunk + or stripped_chunk == "[DONE]" + or stripped_chunk.startswith("event:") + ): + continue + + try: + parsed_chunk = json.loads(stripped_chunk) + except json.JSONDecodeError: + continue + + if not isinstance(parsed_chunk, dict): + continue + + event_type = parsed_chunk.get("type") + + if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + response_payload = parsed_chunk.get("response") + if isinstance(response_payload, dict): + response_output = response_payload.get("output") + if isinstance(response_output, list) and len(response_output) > 0: + return cast(List[Dict[str, Any]], response_output) + continue + + if event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + item = parsed_chunk.get("item") + if not isinstance(item, dict): + continue + try: + output_index = int(parsed_chunk.get("output_index")) + except (TypeError, ValueError): + output_index = len(recovered_output_items) + recovered_output_items[output_index] = item + continue + + if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE: + text = parsed_chunk.get("text") + if not isinstance(text, str): + continue + + try: + output_index = int(parsed_chunk.get("output_index")) + except (TypeError, ValueError): + output_index = len(recovered_text_only_items) + + item = recovered_output_items.get( + output_index + ) or recovered_text_only_items.get(output_index) + if item is None: + item = { + "type": "message", + "id": parsed_chunk.get("item_id") or f"msg_{output_index}", + "role": "assistant", + "status": "completed", + "content": [], + } + recovered_text_only_items[output_index] = item + + content = item.setdefault("content", []) + if not isinstance(content, list): + continue + + try: + content_index = int(parsed_chunk.get("content_index")) + except (TypeError, ValueError): + content_index = len(content) + + while len(content) <= content_index: + content.append( + { + "type": "output_text", + "text": "", + "annotations": [], + } + ) + + content_item = content[content_index] + if not isinstance(content_item, dict): + content_item = {} + content[content_index] = content_item + + content_item["type"] = "output_text" + content_item["text"] = text + if parsed_chunk.get("annotations") is not None: + content_item["annotations"] = parsed_chunk["annotations"] + else: + content_item.setdefault("annotations", []) + + if recovered_output_items: + return [item for _, item in sorted(recovered_output_items.items())] + + if recovered_text_only_items: + return [item for _, item in sorted(recovered_text_only_items.items())] + + return [] + + @classmethod + def _recover_output_items_from_logging( + cls, logging_obj: "LiteLLMLoggingObj" + ) -> List[Dict[str, Any]]: + model_call_details = getattr(logging_obj, "model_call_details", {}) or {} + original_response = model_call_details.get("original_response") + return cls._recover_output_items_from_raw_sse(original_response) + def transform_response( # noqa: PLR0915 self, model: str, @@ -625,9 +745,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if raw_response.error is not None: raise ValueError(f"Error in response: {raw_response.error}") + output_items = raw_response.output + if len(output_items) == 0: + recovered_output_items = self._recover_output_items_from_logging( + logging_obj + ) + if recovered_output_items: + output_items = recovered_output_items + raw_response.output = recovered_output_items + verbose_logger.warning( + "Recovered empty Responses API output from raw SSE for model=%s", + model, + ) + # Convert response output to choices using the static helper choices = self._convert_response_output_to_choices( - output_items=raw_response.output, + output_items=output_items, handle_raw_dict_callback=self._handle_raw_dict_response_item, ) @@ -641,7 +774,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) else: raise ValueError( - f"Unknown items in responses API response: {raw_response.output}" + f"Unknown items in responses API response: {output_items}" ) setattr(model_response, "choices", choices) @@ -1237,7 +1370,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): raise ValueError( f"Chat provider: Invalid function argument delta {parsed_chunk}" ) - elif event_type == "response.output_item.done": + elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: # New output item added output_item = parsed_chunk.get("item", {}) if output_item.get("type") == "function_call": diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 66acd93341..4874f08d05 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -1,5 +1,5 @@ import json -from typing import Any, Optional +from typing import Any, Dict, Optional from litellm.constants import STREAM_SSE_DONE_STRING from litellm.exceptions import AuthenticationError @@ -134,6 +134,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): completed_response = None error_message = None + streamed_output_items: Dict[int, dict] = {} for chunk in body_text.splitlines(): stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk) if not stripped_chunk: @@ -150,10 +151,24 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): if not isinstance(parsed_chunk, dict): continue event_type = parsed_chunk.get("type") + if event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + item = parsed_chunk.get("item") + output_index = parsed_chunk.get("output_index") + if isinstance(item, dict): + try: + index = int(output_index) + except (TypeError, ValueError): + index = len(streamed_output_items) + streamed_output_items[index] = item + continue if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED: response_payload = parsed_chunk.get("response") if isinstance(response_payload, dict): response_payload = dict(response_payload) + if not response_payload.get("output") and streamed_output_items: + response_payload["output"] = [ + item for _, item in sorted(streamed_output_items.items()) + ] if "created_at" in response_payload: response_payload["created_at"] = _safe_convert_created_field( response_payload["created_at"] diff --git a/litellm/router.py b/litellm/router.py index 019f565e5c..13047d038d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7320,6 +7320,18 @@ class Router: _shared_model_info = { k: v for k, v in _model_info.items() if k not in _custom_pricing_fields } + _existing_shared_mode = ( + cast(Optional[dict], litellm.model_cost.get(_model_name, {})) or {} + ).get("mode") + if ( + _existing_shared_mode is not None + and _shared_model_info.get("mode") != _existing_shared_mode + ): + # Keep the built-in bridge mode stable for shared backend keys. + # Multiple aliases can point at the same provider/model backend, + # but their deployment-level overrides should not downgrade the + # backend from responses -> chat via last-write-wins registration. + _shared_model_info.pop("mode", None) _backend_alias_cost = {_model_name: _shared_model_info} if "responses/" in _model_name: _stripped_model_name = _model_name.replace("responses/", "") diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 697a9ebc72..d182fad931 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -508,6 +508,240 @@ and I learn to carry this small calm home.""" print("✓ transform_response correctly handled reasoning items and output messages") +def _make_empty_responses_api_response(model: str = "gpt-5.4"): + from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse + + return ResponsesAPIResponse( + id="resp_from_stream", + created_at=1760144904, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model=model, + object="response", + output=[], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + max_output_tokens=None, + previous_response_id=None, + reasoning={"effort": "low", "summary": "detailed"}, + status="completed", + text={"format": {"type": "text"}, "verbosity": "medium"}, + truncation="disabled", + usage=ResponseAPIUsage( + input_tokens=1, + input_tokens_details=None, + output_tokens=1, + output_tokens_details=None, + total_tokens=2, + cost=None, + ), + user=None, + store=True, + background=False, + billing={"payer": "developer"}, + max_tool_calls=None, + prompt_cache_key=None, + safety_identifier=None, + service_tier="default", + top_logprobs=0, + ) + + +def _make_empty_model_response(): + from litellm.types.utils import ModelResponse, Usage + + return ModelResponse( + id="chatcmpl-test-recovered", + created=1760144904, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + + +def test_transform_response_recovers_empty_output_from_raw_sse(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + + raw_sse = "\n".join( + [ + 'data: {"type":"response.output_text.done","output_index":0,"content_index":0,"item_id":"msg_from_stream","text":"Recovered from SSE"}', + 'data: {"type":"response.completed","response":{"id":"resp_from_stream","object":"response","created_at":1760144904,"status":"completed","model":"gpt-5.4","output":[]}}', + "data: [DONE]", + "", + ] + ) + + raw_response = _make_empty_responses_api_response() + model_response = _make_empty_model_response() + logging_obj = Mock() + logging_obj.model_call_details = {"original_response": raw_sse} + + result = handler.transform_response( + model="gpt-5.4", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.4"}, + messages=[{"role": "user", "content": "Reply with exactly: ok"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + assert len(result.choices) == 1 + assert result.choices[0].message.content == "Recovered from SSE" + + +def test_transform_response_recovers_output_item_done_from_raw_sse(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + + raw_sse = "\n".join( + [ + 'data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","id":"msg_from_item","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Recovered from output item","annotations":[]}]}}', + 'data: {"type":"response.completed","response":{"id":"resp_from_stream","object":"response","created_at":1760144904,"status":"completed","model":"gpt-5.4","output":[]}}', + "data: [DONE]", + "", + ] + ) + + raw_response = _make_empty_responses_api_response() + model_response = _make_empty_model_response() + logging_obj = Mock() + logging_obj.model_call_details = {"original_response": raw_sse} + + result = handler.transform_response( + model="gpt-5.4", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.4"}, + messages=[{"role": "user", "content": "Reply with exactly: ok"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + assert len(result.choices) == 1 + assert result.choices[0].message.content == "Recovered from output item" + + +def test_transform_response_recovers_output_item_done_from_whitespace_padded_raw_sse(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + + output_item_event = { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "message", + "id": "msg_from_item", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Recovered from padded output item", + "annotations": [], + } + ], + }, + } + completed_event = { + "type": "response.completed", + "response": { + "id": "resp_from_stream", + "object": "response", + "created_at": 1760144904, + "status": "completed", + "model": "gpt-5.4", + "output": [], + }, + } + raw_sse = "\n".join( + [ + f" data: {json.dumps(output_item_event)} ", + f"\tdata: {json.dumps(completed_event)}", + "data: [DONE]", + "", + ] + ) + + raw_response = _make_empty_responses_api_response() + model_response = _make_empty_model_response() + logging_obj = Mock() + logging_obj.model_call_details = {"original_response": raw_sse} + + result = handler.transform_response( + model="gpt-5.4", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.4"}, + messages=[{"role": "user", "content": "Reply with exactly: ok"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + assert len(result.choices) == 1 + assert result.choices[0].message.content == "Recovered from padded output item" + + +def test_transform_response_prefers_completed_output_from_raw_sse(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + + raw_sse = "\n".join( + [ + 'data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","id":"msg_from_item","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Earlier stream text","annotations":[]}]}}', + 'data: {"type":"response.completed","response":{"id":"resp_from_stream","object":"response","created_at":1760144904,"status":"completed","model":"gpt-5.4","output":[{"type":"message","id":"msg_from_completed","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Authoritative completed text","annotations":[]}]}]}}', + "data: [DONE]", + "", + ] + ) + + raw_response = _make_empty_responses_api_response() + model_response = _make_empty_model_response() + logging_obj = Mock() + logging_obj.model_call_details = {"original_response": raw_sse} + + result = handler.transform_response( + model="gpt-5.4", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.4"}, + messages=[{"role": "user", "content": "Reply with exactly: ok"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + assert len(result.choices) == 1 + assert result.choices[0].message.content == "Authoritative completed text" + + def test_convert_tools_to_responses_format(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 2498946bb5..22f65da21d 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -14,6 +14,7 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../..")) +from litellm.llms.openai.common_utils import OpenAIError from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -201,3 +202,85 @@ class TestChatGPTResponsesAPITransformation: ) assert parsed.output_text == "Hello!" + + @pytest.mark.parametrize( + ("model_name", "response_model"), + [ + ("chatgpt/gpt-5.2-codex", "gpt-5.2-codex"), + ("chatgpt/gpt-5.3-codex", "gpt-5.3-codex"), + ], + ) + def test_chatgpt_non_stream_sse_response_recovers_output_items( + self, model_name: str, response_model: str + ): + config = ChatGPTResponsesAPIConfig() + response_payload = { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": response_model, + "output": [], + } + streamed_output_item = { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello from stream!"}], + } + sse_body = "\n".join( + [ + f"data: {json.dumps({'type': 'response.output_item.done', 'output_index': 0, 'item': streamed_output_item})}", + f"data: {json.dumps({'type': 'response.completed', 'response': response_payload})}", + "data: [DONE]", + "", + ] + ) + raw_response = httpx.Response( + 200, headers={"content-type": "text/event-stream"}, text=sse_body + ) + logging_obj = MagicMock() + + parsed = config.transform_response_api_response( + model=model_name, + raw_response=raw_response, + logging_obj=logging_obj, + ) + + assert parsed.output_text == "Hello from stream!" + + @pytest.mark.parametrize( + "error_chunk", + [ + { + "type": "response.failed", + "response": {"error": {"message": "ChatGPT upstream failed"}}, + }, + { + "type": "error", + "error": {"message": "ChatGPT upstream failed"}, + }, + ], + ) + def test_chatgpt_non_stream_sse_response_raises_openai_error(self, error_chunk): + config = ChatGPTResponsesAPIConfig() + sse_body = "\n".join( + [ + f"data: {json.dumps(error_chunk)}", + "data: [DONE]", + "", + ] + ) + raw_response = httpx.Response( + 502, headers={"content-type": "text/event-stream"}, text=sse_body + ) + logging_obj = MagicMock() + + with pytest.raises(OpenAIError) as exc_info: + config.transform_response_api_response( + model="chatgpt/gpt-5.4", + raw_response=raw_response, + logging_obj=logging_obj, + ) + + assert "ChatGPT upstream failed" in str(exc_info.value) + assert exc_info.value.status_code == 502 diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index c3f9307855..9b01bfa9b2 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -7,8 +7,10 @@ and one has explicit zero-cost pricing in model_info, the other deployment should still use the built-in pricing. """ +import copy import os import sys +from unittest.mock import patch import pytest @@ -19,6 +21,16 @@ sys.path.insert( import litellm from litellm import Router from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +from litellm.utils import _invalidate_model_cost_lowercase_map + + +def _restore_model_cost_entries(original_entries): + for key, value in original_entries.items(): + if value is None: + litellm.model_cost.pop(key, None) + else: + litellm.model_cost[key] = value + _invalidate_model_cost_lowercase_map() def test_should_not_pollute_shared_key_with_zero_cost_pricing(): @@ -323,3 +335,69 @@ def test_responses_prefix_stripped_alias_registered_for_add_deployment(): ) is True ) + + +def test_should_not_downgrade_chatgpt_shared_key_mode_with_alias_override(): + """ + ChatGPT aliases that share the same backend model should not be able to + downgrade the shared backend key from responses -> chat during router setup. + """ + from litellm.main import responses_api_bridge_check + + backend_model = "chatgpt/gpt-5.4" + model_keys = { + backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), + "chatgpt-shared-mode-base": copy.deepcopy( + litellm.model_cost.get("chatgpt-shared-mode-base") + ), + "chatgpt-shared-mode-alias": copy.deepcopy( + litellm.model_cost.get("chatgpt-shared-mode-alias") + ), + } + + try: + backend_entry = copy.deepcopy(model_keys[backend_model]) or {} + backend_entry["litellm_provider"] = "chatgpt" + backend_entry["mode"] = "responses" + litellm.model_cost[backend_model] = backend_entry + _invalidate_model_cost_lowercase_map() + + router = Router(model_list=[]) + with patch.object( + Router, "_add_deployment", lambda self, deployment: deployment + ): + router._create_deployment( + deployment_info={}, + _model_name="chatgpt/gpt-5.4", + _litellm_params={ + "model": "gpt-5.4", + "custom_llm_provider": "chatgpt", + }, + _model_info={ + "id": "chatgpt-shared-mode-base", + "mode": "responses", + }, + ) + router._create_deployment( + deployment_info={}, + _model_name="chatgpt/gpt-5.4-medium", + _litellm_params={ + "model": "gpt-5.4", + "custom_llm_provider": "chatgpt", + }, + _model_info={ + "id": "chatgpt-shared-mode-alias", + "mode": "chat", + }, + ) + + assert litellm.model_cost[backend_model]["mode"] == "responses" + + bridge_model_info, bridge_model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="chatgpt", + ) + assert bridge_model == "gpt-5.4" + assert bridge_model_info["mode"] == "responses" + finally: + _restore_model_cost_entries(model_keys)