From 3564b8d83b095d3ee4d6549bb3bd7924c650822b Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 19 Feb 2026 21:43:19 -0300 Subject: [PATCH 01/10] fix(types): suppress Pydantic serialization warnings on ModelResponse choices Pydantic v2's Union serializer for `List[Union[Choices, StreamingChoices]]` tries both branches when serializing, emitting spurious `PydanticSerializationUnexpectedValue` warnings (field count mismatch on `Message` and type mismatch `Expected StreamingChoices but got Choices`). Add a `WrapSerializer` on the `choices` field that serializes each item individually via its own `model_dump()`, bypassing the Union dispatch entirely while correctly propagating `exclude_none`, `exclude_unset`, and `exclude_defaults` from the parent serialization context. --- litellm/types/utils.py | 38 ++++++++++- .../test_model_response_normalization.py | 66 ++++++++++++++++++- 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9228b25b03..2b64231ca7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -22,8 +22,9 @@ from openai.types.moderation_create_response import Moderation as Moderation from openai.types.moderation_create_response import ( ModerationCreateResponse as ModerationCreateResponse, ) -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator -from typing_extensions import Required, TypedDict +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, SerializationInfo, model_validator +from pydantic.functional_serializers import WrapSerializer +from typing_extensions import Annotated, Required, TypedDict from litellm._uuid import uuid from litellm.types.llms.base import ( @@ -1640,6 +1641,34 @@ class StreamingChatCompletionChunk(OpenAIChatCompletionChunk): super().__init__(**kwargs) +def _serialize_choices_list( + choices: list, handler, info: SerializationInfo +) -> list: + """Serialize each choice individually to avoid Union serializer warnings. + + Pydantic's Union serializer for ``List[Union[Choices, StreamingChoices]]`` + may try the wrong branch first, emitting spurious + ``PydanticSerializationUnexpectedValue`` warnings. By serializing each + item via its own ``model_dump()`` we bypass the Union dispatch entirely. + """ + kwargs: Dict[str, Any] = {} + if info.exclude_none: + kwargs["exclude_none"] = True + if info.exclude_unset: + kwargs["exclude_unset"] = True + if info.exclude_defaults: + kwargs["exclude_defaults"] = True + result = [] + for choice in choices: + if hasattr(choice, "model_dump"): + result.append(choice.model_dump(**kwargs)) + elif isinstance(choice, dict): + result.append(choice) + else: + result.append(choice) + return result + + class ModelResponseBase(OpenAIObject): id: str """A unique identifier for the completion.""" @@ -1748,7 +1777,10 @@ class ModelResponseStream(ModelResponseBase): class ModelResponse(ModelResponseBase): - choices: List[Union[Choices, StreamingChoices]] + choices: Annotated[ + List[Union[Choices, StreamingChoices]], + WrapSerializer(_serialize_choices_list, return_type=list), + ] """The list of completion choices the model generated for the input prompt.""" def __init__( # noqa: PLR0915 diff --git a/tests/test_litellm/test_model_response_normalization.py b/tests/test_litellm/test_model_response_normalization.py index 57281d3c1f..52e6b8bc9e 100644 --- a/tests/test_litellm/test_model_response_normalization.py +++ b/tests/test_litellm/test_model_response_normalization.py @@ -2,7 +2,7 @@ import warnings import pytest -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices def test_modelresponse_normalizes_openai_base_models() -> None: @@ -59,3 +59,67 @@ def test_modelresponse_serialization_avoids_pydantic_warnings() -> None: or "Pydantic serializer warnings" in str(w.message) for w in captured ) + + +def test_modelresponse_model_dump_json_no_pydantic_warnings() -> None: + """model_dump_json() bypasses the Python model_dump() override and uses + Pydantic's C-level serializer directly. The Union[Choices, StreamingChoices] + field previously triggered PydanticSerializationUnexpectedValue warnings via + this path.""" + response = ModelResponse( + model="test-model", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="hello", role="assistant"), + ) + ], + ) + + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + _ = response.model_dump_json() + _ = response.model_dump() + _ = response.model_dump(exclude_none=True) + + pydantic_warnings = [ + w + for w in captured + if "PydanticSerializationUnexpectedValue" in str(w.message) + or "Pydantic serializer warnings" in str(w.message) + ] + assert pydantic_warnings == [], ( + f"Unexpected Pydantic serialization warnings: {pydantic_warnings}" + ) + + +def test_streaming_modelresponse_no_pydantic_warnings() -> None: + """Streaming responses use StreamingChoices in the Union field and should + also serialize without warnings.""" + response = ModelResponse( + model="test-model", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content="hello", role="assistant"), + ) + ], + stream=True, + ) + + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + _ = response.model_dump_json() + _ = response.model_dump() + + pydantic_warnings = [ + w + for w in captured + if "PydanticSerializationUnexpectedValue" in str(w.message) + or "Pydantic serializer warnings" in str(w.message) + ] + assert pydantic_warnings == [], ( + f"Unexpected Pydantic serialization warnings: {pydantic_warnings}" + ) From 0f20976efa8e6fc00b973a4c9801e2e901b0afbb Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 20 Feb 2026 17:47:42 -0300 Subject: [PATCH 02/10] fix(types): remove StreamingChoices from ModelResponse, use ModelResponseStream ModelResponse.choices was typed as List[Union[Choices, StreamingChoices]] which caused Pydantic serialization warnings and false linting errors. Now that ModelResponseStream exists for streaming, narrow ModelResponse.choices to List[Choices] and migrate all ModelResponse(stream=True) call sites to use ModelResponseStream() instead. --- .../litellm_core_utils/streaming_handler.py | 2 +- litellm/llms/bedrock/chat/invoke_handler.py | 4 +- .../codestral/completion/transformation.py | 2 +- .../guardrails/guardrail_hooks/presidio.py | 9 +- litellm/types/utils.py | 94 +++++-------------- litellm/utils.py | 6 +- .../test_stream_chunk_builder_images.py | 6 +- .../test_stream_chunk_builder.py | 6 +- tests/local_testing/test_streaming.py | 12 +-- .../test_model_response_normalization.py | 25 ++--- 10 files changed, 59 insertions(+), 107 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 7a6752fbff..772a7a2894 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1183,7 +1183,7 @@ class CustomStreamWrapper: ], ) _streaming_response = StreamingChoices(delta=_delta_obj) - _model_response = ModelResponse(stream=True) + _model_response = ModelResponseStream() _model_response.choices = [_streaming_response] response_obj = {"original_chunk": _model_response} else: diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 1c58a11eeb..6d8b9c5a16 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -558,7 +558,7 @@ class BedrockLLM(BaseAWSLLM): "INSIDE BEDROCK STREAMING TOOL CALLING CONDITION BLOCK" ) # return an iterator - streaming_model_response = ModelResponse(stream=True) + streaming_model_response = ModelResponseStream() streaming_model_response.choices[0].finish_reason = getattr( model_response.choices[0], "finish_reason", "stop" ) @@ -695,7 +695,7 @@ class BedrockLLM(BaseAWSLLM): ) if stream and provider == "ai21": - streaming_model_response = ModelResponse(stream=True) + streaming_model_response = ModelResponseStream() streaming_model_response.choices[0].finish_reason = model_response.choices[ # type: ignore 0 ].finish_reason diff --git a/litellm/llms/codestral/completion/transformation.py b/litellm/llms/codestral/completion/transformation.py index 646c0e8e56..31d6652f48 100644 --- a/litellm/llms/codestral/completion/transformation.py +++ b/litellm/llms/codestral/completion/transformation.py @@ -102,7 +102,7 @@ class CodestralTextCompletionConfig(OpenAITextCompletionConfig): "finish_reason": finish_reason, } - original_chunk = litellm.ModelResponse(**chunk_data_dict, stream=True) + original_chunk = litellm.ModelResponseStream(**chunk_data_dict) _choices = chunk_data_dict.get("choices", []) or [] if len(_choices) == 0: return { diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 29d57ee473..06e0b38f53 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -808,7 +808,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return response if isinstance(response, ModelResponse) and not isinstance( - response.choices[0], StreamingChoices + response, ModelResponseStream ): # /chat/completions requests if isinstance(response.choices[0].message.content, str): verbose_proxy_logger.debug( @@ -832,7 +832,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return response # skip streaming here; handled in async_post_call_streaming_iterator_hook - if response.choices and isinstance(response.choices[0], StreamingChoices): + if isinstance(response, ModelResponseStream): return response presidio_config = self.get_presidio_settings_from_request_data( @@ -840,10 +840,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) for choice in response.choices: - # Type narrowing: StreamingChoices doesn't have .message attribute - if not hasattr(choice, "message"): - continue - content = getattr(choice.message, "content", None) # type: ignore + content = getattr(choice.message, "content", None) if content is None: continue if isinstance(content, str): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2b64231ca7..7ffd20e827 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -22,9 +22,8 @@ from openai.types.moderation_create_response import Moderation as Moderation from openai.types.moderation_create_response import ( ModerationCreateResponse as ModerationCreateResponse, ) -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, SerializationInfo, model_validator -from pydantic.functional_serializers import WrapSerializer -from typing_extensions import Annotated, Required, TypedDict +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator +from typing_extensions import Required, TypedDict from litellm._uuid import uuid from litellm.types.llms.base import ( @@ -1641,33 +1640,6 @@ class StreamingChatCompletionChunk(OpenAIChatCompletionChunk): super().__init__(**kwargs) -def _serialize_choices_list( - choices: list, handler, info: SerializationInfo -) -> list: - """Serialize each choice individually to avoid Union serializer warnings. - - Pydantic's Union serializer for ``List[Union[Choices, StreamingChoices]]`` - may try the wrong branch first, emitting spurious - ``PydanticSerializationUnexpectedValue`` warnings. By serializing each - item via its own ``model_dump()`` we bypass the Union dispatch entirely. - """ - kwargs: Dict[str, Any] = {} - if info.exclude_none: - kwargs["exclude_none"] = True - if info.exclude_unset: - kwargs["exclude_unset"] = True - if info.exclude_defaults: - kwargs["exclude_defaults"] = True - result = [] - for choice in choices: - if hasattr(choice, "model_dump"): - result.append(choice.model_dump(**kwargs)) - elif isinstance(choice, dict): - result.append(choice) - else: - result.append(choice) - return result - class ModelResponseBase(OpenAIObject): id: str @@ -1777,10 +1749,7 @@ class ModelResponseStream(ModelResponseBase): class ModelResponse(ModelResponseBase): - choices: Annotated[ - List[Union[Choices, StreamingChoices]], - WrapSerializer(_serialize_choices_list, return_type=list), - ] + choices: List[Choices] """The list of completion choices the model generated for the input prompt.""" def __init__( # noqa: PLR0915 @@ -1799,44 +1768,27 @@ class ModelResponse(ModelResponseBase): _response_headers=None, **params, ) -> None: - if stream is not None and stream is True: - object = "chat.completion.chunk" - if choices is not None and isinstance(choices, list): - new_choices = [] - for choice in choices: - _new_choice = None - if isinstance(choice, StreamingChoices): - _new_choice = choice - elif isinstance(choice, dict): - _new_choice = StreamingChoices(**choice) - elif isinstance(choice, BaseModel): - _new_choice = StreamingChoices(**choice.model_dump()) - new_choices.append(_new_choice) - choices = new_choices - else: - choices = [StreamingChoices()] + object = "chat.completion" + if choices is not None and isinstance(choices, list): + new_choices = [] + for choice in choices: + if isinstance(choice, Choices): + _new_choice = choice # type: ignore + elif isinstance(choice, dict): + _new_choice = Choices(**choice) # type: ignore + elif isinstance(choice, BaseModel): + dump = ( + choice.model_dump() + if hasattr(choice, "model_dump") + else choice.dict() + ) + _new_choice = Choices(**dump) # type: ignore + else: + _new_choice = choice + new_choices.append(_new_choice) + choices = new_choices else: - object = "chat.completion" - if choices is not None and isinstance(choices, list): - new_choices = [] - for choice in choices: - if isinstance(choice, Choices): - _new_choice = choice # type: ignore - elif isinstance(choice, dict): - _new_choice = Choices(**choice) # type: ignore - elif isinstance(choice, BaseModel): - dump = ( - choice.model_dump() - if hasattr(choice, "model_dump") - else choice.dict() - ) - _new_choice = Choices(**dump) # type: ignore - else: - _new_choice = choice - new_choices.append(_new_choice) - choices = new_choices - else: - choices = [Choices()] + choices = [Choices()] if id is None: id = _generate_id() else: diff --git a/litellm/utils.py b/litellm/utils.py index 241b9d217b..de8fa9c1e1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7369,9 +7369,9 @@ def _get_base_model_from_metadata(model_call_details=None): class ModelResponseIterator: def __init__(self, model_response: ModelResponse, convert_to_delta: bool = False): if convert_to_delta is True: - self.model_response = ModelResponse(stream=True) - _delta = self.model_response.choices[0].delta # type: ignore - _delta.content = model_response.choices[0].message.content # type: ignore + _stream_response = ModelResponseStream() + _stream_response.choices[0].delta.content = model_response.choices[0].message.content # type: ignore + self.model_response: Union[ModelResponse, ModelResponseStream] = _stream_response else: self.model_response = model_response self.is_done = False diff --git a/tests/litellm/test_stream_chunk_builder_images.py b/tests/litellm/test_stream_chunk_builder_images.py index c51a14ede6..92fb0f93aa 100644 --- a/tests/litellm/test_stream_chunk_builder_images.py +++ b/tests/litellm/test_stream_chunk_builder_images.py @@ -72,7 +72,7 @@ def test_stream_chunk_builder_preserves_images(): chunks = [] for chunk in init_chunks: - chunks.append(litellm.ModelResponse(**chunk, stream=True)) + chunks.append(litellm.ModelResponseStream(**chunk)) response = stream_chunk_builder(chunks=chunks) @@ -163,7 +163,7 @@ def test_stream_chunk_builder_preserves_multiple_images(): chunks = [] for chunk in init_chunks: - chunks.append(litellm.ModelResponse(**chunk, stream=True)) + chunks.append(litellm.ModelResponseStream(**chunk)) response = stream_chunk_builder(chunks=chunks) @@ -230,7 +230,7 @@ def test_stream_chunk_builder_no_images(): chunks = [] for chunk in init_chunks: - chunks.append(litellm.ModelResponse(**chunk, stream=True)) + chunks.append(litellm.ModelResponseStream(**chunk)) response = stream_chunk_builder(chunks=chunks) diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 8224773aa4..ddb1546097 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -542,7 +542,7 @@ def test_stream_chunk_builder_multiple_tool_calls(): chunks = [] for chunk in init_chunks: - chunks.append(litellm.ModelResponse(**chunk, stream=True)) + chunks.append(litellm.ModelResponseStream(**chunk)) response = stream_chunk_builder(chunks=chunks) print(f"Returned response: {response}") @@ -616,7 +616,7 @@ def test_stream_chunk_builder_openai_prompt_caching(): chunks: List[litellm.ModelResponse] = [] usage_obj = None for chunk in chat_completion: - chunks.append(litellm.ModelResponse(**chunk.model_dump(), stream=True)) + chunks.append(litellm.ModelResponseStream(**chunk.model_dump())) print(f"chunks: {chunks}") @@ -661,7 +661,7 @@ def test_stream_chunk_builder_openai_audio_output_usage(): chunks = [] for chunk in completion: - chunks.append(litellm.ModelResponse(**chunk.model_dump(), stream=True)) + chunks.append(litellm.ModelResponseStream(**chunk.model_dump())) usage_obj: Optional[litellm.Usage] = None diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index ee208b5e0e..7f7a43095c 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -393,7 +393,7 @@ def test_completion_azure_stream_content_filter_no_delta(): chunk_list = [] for chunk in chunks: - new_chunk = litellm.ModelResponse(stream=True, id=chunk["id"]) + new_chunk = litellm.ModelResponseStream(id=chunk["id"]) if "choices" in chunk and isinstance(chunk["choices"], list): new_choices = [] for choice in chunk["choices"]: @@ -3026,7 +3026,7 @@ def test_unit_test_custom_stream_wrapper(): {"index": 0, "delta": {"content": "How are you?"}, "finish_reason": "stop"} ], } - chunk = litellm.ModelResponse(**chunk, stream=True) + chunk = litellm.ModelResponseStream(**chunk) completion_stream = ModelResponseIterator(model_response=chunk) @@ -3223,7 +3223,7 @@ def test_unit_test_custom_stream_wrapper_openai(): "system_fingerprint": None, "usage": None, } - chunk = litellm.ModelResponse(**chunk, stream=True) + chunk = litellm.ModelResponseStream(**chunk) completion_stream = ModelResponseIterator(model_response=chunk) @@ -3457,7 +3457,7 @@ def test_aamazing_unit_test_custom_stream_wrapper_n(): chunk_list = [] for chunk in chunks: - new_chunk = litellm.ModelResponse(stream=True, id=chunk["id"]) + new_chunk = litellm.ModelResponseStream(id=chunk["id"]) if "choices" in chunk and isinstance(chunk["choices"], list): print("INSIDE CHUNK CHOICES!") new_choices = [] @@ -3541,7 +3541,7 @@ def test_unit_test_custom_stream_wrapper_function_call(): "system_fingerprint": "fp_44709d6fcb", "choices": [{"index": 0, "delta": delta, "finish_reason": "stop"}], } - chunk = litellm.ModelResponse(**chunk, stream=True) + chunk = litellm.ModelResponseStream(**chunk) completion_stream = ModelResponseIterator(model_response=chunk) @@ -3651,7 +3651,7 @@ def test_unit_test_perplexity_citations_chunk(): } ], } - chunk = litellm.ModelResponse(**chunk, stream=True) + chunk = litellm.ModelResponseStream(**chunk) completion_stream = ModelResponseIterator(model_response=chunk) diff --git a/tests/test_litellm/test_model_response_normalization.py b/tests/test_litellm/test_model_response_normalization.py index 52e6b8bc9e..85b9fc1450 100644 --- a/tests/test_litellm/test_model_response_normalization.py +++ b/tests/test_litellm/test_model_response_normalization.py @@ -2,7 +2,14 @@ import warnings import pytest -from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices +from litellm.types.utils import ( + Choices, + Delta, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) def test_modelresponse_normalizes_openai_base_models() -> None: @@ -62,10 +69,8 @@ def test_modelresponse_serialization_avoids_pydantic_warnings() -> None: def test_modelresponse_model_dump_json_no_pydantic_warnings() -> None: - """model_dump_json() bypasses the Python model_dump() override and uses - Pydantic's C-level serializer directly. The Union[Choices, StreamingChoices] - field previously triggered PydanticSerializationUnexpectedValue warnings via - this path.""" + """model_dump_json() and model_dump() should not trigger any Pydantic + serialization warnings now that choices is List[Choices] (no Union).""" response = ModelResponse( model="test-model", choices=[ @@ -94,11 +99,10 @@ def test_modelresponse_model_dump_json_no_pydantic_warnings() -> None: ) -def test_streaming_modelresponse_no_pydantic_warnings() -> None: - """Streaming responses use StreamingChoices in the Union field and should - also serialize without warnings.""" - response = ModelResponse( - model="test-model", +def test_streaming_modelresponsestream_no_pydantic_warnings() -> None: + """Streaming responses use ModelResponseStream with List[StreamingChoices] + and should serialize without warnings.""" + response = ModelResponseStream( choices=[ StreamingChoices( finish_reason="stop", @@ -106,7 +110,6 @@ def test_streaming_modelresponse_no_pydantic_warnings() -> None: delta=Delta(content="hello", role="assistant"), ) ], - stream=True, ) with warnings.catch_warnings(record=True) as captured: From a2cae0070e352fa8b895bb1ee472992134b59412 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 20 Feb 2026 17:58:06 -0300 Subject: [PATCH 03/10] fix(lint): remove unused StreamingChoices import in presidio guardrail --- litellm/proxy/guardrails/guardrail_hooks/presidio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 06e0b38f53..d3f1fd1781 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -61,7 +61,6 @@ from litellm.utils import ( ImageResponse, ModelResponse, ModelResponseStream, - StreamingChoices, ) From b370fcd8de0993084c219ad6dd0146a80e6095cf Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 20 Feb 2026 17:59:19 -0300 Subject: [PATCH 04/10] fix(presidio): remove redundant isinstance check for ModelResponseStream ModelResponseStream and ModelResponse are sibling classes (both inherit from ModelResponseBase), so the guard was always True. Simplify to just isinstance(response, ModelResponse). --- litellm/proxy/guardrails/guardrail_hooks/presidio.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index d3f1fd1781..b4b25e2d90 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -806,9 +806,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if self.output_parse_pii is False and litellm.output_parse_pii is False: return response - if isinstance(response, ModelResponse) and not isinstance( - response, ModelResponseStream - ): # /chat/completions requests + if isinstance(response, ModelResponse): # /chat/completions requests if isinstance(response.choices[0].message.content, str): verbose_proxy_logger.debug( f"self.pii_tokens: {self.pii_tokens}; initial response: {response.choices[0].message.content}" From 5eeee88ff8e33c506f1d3c36c5e42f92d64d2b2e Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Feb 2026 09:03:12 -0300 Subject: [PATCH 05/10] fix(lint): migrate remaining StreamingChoices callers to ModelResponseStream After narrowing ModelResponse.choices to List[Choices], several files still assigned StreamingChoices into ModelResponse. Migrate streaming call sites to ModelResponseStream and remove dead streaming branches in qwen2/qwen3 transform_response methods. --- .../llms/bedrock/chat/agentcore/transformation.py | 14 +++++++------- .../amazon_qwen2_transformation.py | 9 ++------- .../amazon_qwen3_transformation.py | 9 ++------- litellm/llms/langgraph/chat/sse_iterator.py | 14 +++++++------- .../openai/chat/guardrail_translation/handler.py | 12 ++++++------ litellm/utils.py | 4 +--- 6 files changed, 25 insertions(+), 37 deletions(-) diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 9ae850ad4c..b85a0b70f7 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -26,7 +26,7 @@ from litellm.types.llms.bedrock_agentcore import ( AgentCoreUsage, ) from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices, Usage +from litellm.types.utils import Choices, Delta, Message, ModelResponse, ModelResponseStream, StreamingChoices, Usage if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -481,7 +481,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): text = delta.get("text", "") if text: - chunk = ModelResponse( + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=model, @@ -499,7 +499,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Process metadata/usage metadata = event_payload.get("metadata") if metadata and "usage" in metadata: - chunk = ModelResponse( + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=model, @@ -522,7 +522,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Process final message if "message" in data_obj and isinstance(data_obj["message"], dict): - chunk = ModelResponse( + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=model, @@ -636,7 +636,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): text = delta.get("text", "") if text: - chunk = ModelResponse( + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=model, @@ -654,7 +654,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Process metadata/usage metadata = event_payload.get("metadata") if metadata and "usage" in metadata: - chunk = ModelResponse( + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=model, @@ -677,7 +677,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Process final message if "message" in data_obj and isinstance(data_obj["message"], dict): - chunk = ModelResponse( + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=model, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index c532d8ea27..90adc44a49 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -68,13 +68,8 @@ class AmazonQwen2Config(AmazonQwen3Config): # Set the content in the existing model_response structure if hasattr(model_response, 'choices') and len(model_response.choices) > 0: choice = model_response.choices[0] - if hasattr(choice, 'message'): - choice.message.content = generated_text - choice.finish_reason = "stop" - else: - # Handle streaming choices - choice.delta.content = generated_text - choice.finish_reason = "stop" + choice.message.content = generated_text + choice.finish_reason = "stop" # Set usage information if available in response if "usage" in response_data: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index b3a957ce0f..faf8bb4ac6 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -190,13 +190,8 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): # Set the content in the existing model_response structure if hasattr(model_response, 'choices') and len(model_response.choices) > 0: choice = model_response.choices[0] - if hasattr(choice, 'message'): - choice.message.content = generated_text - choice.finish_reason = "stop" - else: - # Handle streaming choices - choice.delta.content = generated_text - choice.finish_reason = "stop" + choice.message.content = generated_text + choice.finish_reason = "stop" # Set usage information if available in response if "usage" in response_data: diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index bdb32cc0fe..aff1c2f2db 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Optional import httpx from litellm._logging import verbose_logger -from litellm.types.utils import Delta, ModelResponse, StreamingChoices +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices if TYPE_CHECKING: pass @@ -139,9 +139,9 @@ class LangGraphSSEStreamIterator: return self._create_final_chunk() return None - def _create_content_chunk(self, text: str) -> ModelResponse: - """Create a ModelResponse chunk with content.""" - chunk = ModelResponse( + def _create_content_chunk(self, text: str) -> ModelResponseStream: + """Create a ModelResponseStream chunk with content.""" + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=self.model, @@ -158,9 +158,9 @@ class LangGraphSSEStreamIterator: return chunk - def _create_final_chunk(self) -> ModelResponse: - """Create a final ModelResponse chunk with finish_reason.""" - chunk = ModelResponse( + def _create_final_chunk(self) -> ModelResponseStream: + """Create a final ModelResponseStream chunk with finish_reason.""" + chunk = ModelResponseStream( id=f"chatcmpl-{uuid.uuid4()}", created=0, model=self.model, diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 683e165c31..67e9e42bc3 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -542,16 +542,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if len(choice.message.tool_calls) > 0: return True elif isinstance(response, ModelResponseStream): - for choice in response.choices: - if isinstance(choice, litellm.StreamingChoices): + for streaming_choice in response.choices: + if isinstance(streaming_choice, litellm.StreamingChoices): # Check for text content - if choice.delta.content and isinstance(choice.delta.content, str): + if streaming_choice.delta.content and isinstance(streaming_choice.delta.content, str): return True # Check for tool calls - if choice.delta.tool_calls and isinstance( - choice.delta.tool_calls, list + if streaming_choice.delta.tool_calls and isinstance( + streaming_choice.delta.tool_calls, list ): - if len(choice.delta.tool_calls) > 0: + if len(streaming_choice.delta.tool_calls) > 0: return True return False diff --git a/litellm/utils.py b/litellm/utils.py index de8fa9c1e1..eff5bd58ee 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4960,9 +4960,7 @@ def get_response_string(response_obj: Union[ModelResponse, ModelResponseStream]) return delta if isinstance(delta, str) else "" # Handle standard ModelResponse and ModelResponseStream - _choices: Union[List[Union[Choices, StreamingChoices]], List[StreamingChoices]] = ( - response_obj.choices - ) + _choices: Union[List[Choices], List[StreamingChoices]] = response_obj.choices # Use list accumulation to avoid O(n^2) string concatenation across choices response_parts: List[str] = [] From 3e00c833531033aa4749307df45778e99dbd4f1f Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Feb 2026 09:09:49 -0300 Subject: [PATCH 06/10] fix(lint): restore ModelResponse import in langgraph sse_iterator --- litellm/llms/langgraph/chat/sse_iterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index aff1c2f2db..8d946c4e52 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Optional import httpx from litellm._logging import verbose_logger -from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices +from litellm.types.utils import Delta, ModelResponse, ModelResponseStream, StreamingChoices if TYPE_CHECKING: pass From 43319b562a4e2da5979a2b1c35548fa18ae498e7 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Feb 2026 09:17:36 -0300 Subject: [PATCH 07/10] fix(lint): update return/yield types to ModelResponseStream - langgraph sse_iterator: update return types from ModelResponse to ModelResponseStream across all methods - bedrock agentcore: fix async generator yield type annotation - vertex gemini: add type: ignore for MyPy narrowing false positive --- .../llms/bedrock/chat/agentcore/transformation.py | 2 +- litellm/llms/langgraph/chat/sse_iterator.py | 14 +++++++------- .../gemini/vertex_and_google_ai_studio_gemini.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index b85a0b70f7..fe7d4b194a 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -601,7 +601,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): self, response: httpx.Response, model: str, - ) -> AsyncGenerator[ModelResponse, None]: + ) -> AsyncGenerator[ModelResponseStream, None]: """ Internal async generator that parses SSE and yields ModelResponse chunks. """ diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index 8d946c4e52..cf81998055 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Optional import httpx from litellm._logging import verbose_logger -from litellm.types.utils import Delta, ModelResponse, ModelResponseStream, StreamingChoices +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices if TYPE_CHECKING: pass @@ -44,7 +44,7 @@ class LangGraphSSEStreamIterator: self.async_line_iterator = self.response.aiter_lines() return self - def _parse_sse_line(self, line: str) -> Optional[ModelResponse]: + def _parse_sse_line(self, line: str) -> Optional[ModelResponseStream]: """ Parse a single SSE line and return a ModelResponse chunk if applicable. @@ -71,7 +71,7 @@ class LangGraphSSEStreamIterator: return None - def _process_data(self, data) -> Optional[ModelResponse]: + def _process_data(self, data) -> Optional[ModelResponseStream]: """ Process parsed data from SSE stream. @@ -101,7 +101,7 @@ class LangGraphSSEStreamIterator: return None - def _process_messages_event(self, payload) -> Optional[ModelResponse]: + def _process_messages_event(self, payload) -> Optional[ModelResponseStream]: """ Process a messages event from the stream. @@ -128,7 +128,7 @@ class LangGraphSSEStreamIterator: return None - def _process_metadata_event(self, payload) -> Optional[ModelResponse]: + def _process_metadata_event(self, payload) -> Optional[ModelResponseStream]: """ Process a metadata event, which may signal the end of the stream. """ @@ -177,7 +177,7 @@ class LangGraphSSEStreamIterator: return chunk - def __next__(self) -> ModelResponse: + def __next__(self) -> ModelResponseStream: """Sync iteration - parse SSE events and yield ModelResponse chunks.""" try: if self.line_iterator is None: @@ -205,7 +205,7 @@ class LangGraphSSEStreamIterator: verbose_logger.error(f"Error in LangGraph SSE stream: {str(e)}") raise StopIteration - async def __anext__(self) -> ModelResponse: + async def __anext__(self) -> ModelResponseStream: """Async iteration - parse SSE events and yield ModelResponse chunks.""" try: if self.async_line_iterator is None: diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index cf3461a996..1783b5e630 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2093,7 +2093,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_logprobs=chat_completion_logprobs, image_response=image_response, ) - model_response.choices.append(choice) + model_response.choices.append(choice) # type: ignore[arg-type] elif isinstance(model_response, ModelResponse): choice = litellm.Choices( finish_reason=VertexGeminiConfig._check_finish_reason( From eaf38ed3a20df28e7368a9da79fdf145af441418 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Feb 2026 09:22:44 -0300 Subject: [PATCH 08/10] fix(lint): add type: ignore for MyPy narrowing issue in vertex gemini --- .../llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 1783b5e630..377801ea59 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2104,7 +2104,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): logprobs=chat_completion_logprobs, enhancements=None, ) - model_response.choices.append(choice) + model_response.choices.append(choice) # type: ignore[arg-type] return ( grounding_metadata, From 7e4f07c45dcfbdc36df56cae4ab9109f79af1bf5 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Feb 2026 09:37:26 -0300 Subject: [PATCH 09/10] fix(vertex): use module-level import for ModelResponseStream instead of type: ignore Move ModelResponseStream to module-level import so MyPy can properly narrow the Union type after isinstance checks, removing the need for type: ignore suppressions. --- .../vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 377801ea59..28e6919065 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -80,6 +80,7 @@ from litellm.types.utils import ( ChatCompletionTokenLogprob, ChoiceLogprobs, CompletionTokensDetailsWrapper, + ModelResponseStream, PromptTokensDetailsWrapper, TopLogprob, Usage, @@ -1931,7 +1932,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _process_candidates( # noqa: PLR0915 _candidates: List[Candidates], - model_response: Union[ModelResponse, "ModelResponseStream"], + model_response: Union[ModelResponse, ModelResponseStream], standard_optional_params: dict, cumulative_tool_call_index: int = 0, ) -> Tuple[List[dict], List[dict], List, List, int]: @@ -2093,7 +2094,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_logprobs=chat_completion_logprobs, image_response=image_response, ) - model_response.choices.append(choice) # type: ignore[arg-type] + model_response.choices.append(choice) elif isinstance(model_response, ModelResponse): choice = litellm.Choices( finish_reason=VertexGeminiConfig._check_finish_reason( @@ -2104,7 +2105,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): logprobs=chat_completion_logprobs, enhancements=None, ) - model_response.choices.append(choice) # type: ignore[arg-type] + model_response.choices.append(choice) return ( grounding_metadata, From ca5fd1ecec859f8717de36800b9fff68fffa8987 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Feb 2026 09:43:08 -0300 Subject: [PATCH 10/10] Revert "fix(vertex): use module-level import for ModelResponseStream instead of type: ignore" This reverts commit 7e4f07c45dcfbdc36df56cae4ab9109f79af1bf5. --- .../vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 28e6919065..377801ea59 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -80,7 +80,6 @@ from litellm.types.utils import ( ChatCompletionTokenLogprob, ChoiceLogprobs, CompletionTokensDetailsWrapper, - ModelResponseStream, PromptTokensDetailsWrapper, TopLogprob, Usage, @@ -1932,7 +1931,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _process_candidates( # noqa: PLR0915 _candidates: List[Candidates], - model_response: Union[ModelResponse, ModelResponseStream], + model_response: Union[ModelResponse, "ModelResponseStream"], standard_optional_params: dict, cumulative_tool_call_index: int = 0, ) -> Tuple[List[dict], List[dict], List, List, int]: @@ -2094,7 +2093,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_logprobs=chat_completion_logprobs, image_response=image_response, ) - model_response.choices.append(choice) + model_response.choices.append(choice) # type: ignore[arg-type] elif isinstance(model_response, ModelResponse): choice = litellm.Choices( finish_reason=VertexGeminiConfig._check_finish_reason( @@ -2105,7 +2104,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): logprobs=chat_completion_logprobs, enhancements=None, ) - model_response.choices.append(choice) + model_response.choices.append(choice) # type: ignore[arg-type] return ( grounding_metadata,