Merge pull request #21629 from Chesars/fix/pydantic-serialization-warnings

fix(types): remove StreamingChoices from ModelResponse, use ModelResponseStream
This commit is contained in:
Cesar Garcia
2026-02-27 17:48:33 -03:00
committed by GitHub
16 changed files with 146 additions and 113 deletions
@@ -1231,7 +1231,7 @@ class CustomStreamWrapper:
],
)
_streaming_response = StreamingChoices(delta=_delta_obj)
_model_response = ModelResponse(stream=True)
_model_response = ModelResponseStream()
_model_response.choices = [_streaming_response]
response_obj = {"original_chunk": _model_response}
else:
@@ -26,7 +26,7 @@ from litellm.types.llms.bedrock_agentcore import (
AgentCoreUsage,
)
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices, Usage
from litellm.types.utils import Choices, Delta, Message, ModelResponse, ModelResponseStream, StreamingChoices, Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -481,7 +481,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
text = delta.get("text", "")
if text:
chunk = ModelResponse(
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
@@ -499,7 +499,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
# Process metadata/usage
metadata = event_payload.get("metadata")
if metadata and "usage" in metadata:
chunk = ModelResponse(
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
@@ -522,7 +522,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
# Process final message
if "message" in data_obj and isinstance(data_obj["message"], dict):
chunk = ModelResponse(
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
@@ -601,7 +601,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
self,
response: httpx.Response,
model: str,
) -> AsyncGenerator[ModelResponse, None]:
) -> AsyncGenerator[ModelResponseStream, None]:
"""
Internal async generator that parses SSE and yields ModelResponse chunks.
"""
@@ -636,7 +636,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
text = delta.get("text", "")
if text:
chunk = ModelResponse(
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
@@ -654,7 +654,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
# Process metadata/usage
metadata = event_payload.get("metadata")
if metadata and "usage" in metadata:
chunk = ModelResponse(
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
@@ -677,7 +677,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
# Process final message
if "message" in data_obj and isinstance(data_obj["message"], dict):
chunk = ModelResponse(
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
+2 -2
View File
@@ -558,7 +558,7 @@ class BedrockLLM(BaseAWSLLM):
"INSIDE BEDROCK STREAMING TOOL CALLING CONDITION BLOCK"
)
# return an iterator
streaming_model_response = ModelResponse(stream=True)
streaming_model_response = ModelResponseStream()
streaming_model_response.choices[0].finish_reason = getattr(
model_response.choices[0], "finish_reason", "stop"
)
@@ -695,7 +695,7 @@ class BedrockLLM(BaseAWSLLM):
)
if stream and provider == "ai21":
streaming_model_response = ModelResponse(stream=True)
streaming_model_response = ModelResponseStream()
streaming_model_response.choices[0].finish_reason = model_response.choices[ # type: ignore
0
].finish_reason
@@ -68,13 +68,8 @@ class AmazonQwen2Config(AmazonQwen3Config):
# Set the content in the existing model_response structure
if hasattr(model_response, 'choices') and len(model_response.choices) > 0:
choice = model_response.choices[0]
if hasattr(choice, 'message'):
choice.message.content = generated_text
choice.finish_reason = "stop"
else:
# Handle streaming choices
choice.delta.content = generated_text
choice.finish_reason = "stop"
choice.message.content = generated_text
choice.finish_reason = "stop"
# Set usage information if available in response
if "usage" in response_data:
@@ -190,13 +190,8 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
# Set the content in the existing model_response structure
if hasattr(model_response, 'choices') and len(model_response.choices) > 0:
choice = model_response.choices[0]
if hasattr(choice, 'message'):
choice.message.content = generated_text
choice.finish_reason = "stop"
else:
# Handle streaming choices
choice.delta.content = generated_text
choice.finish_reason = "stop"
choice.message.content = generated_text
choice.finish_reason = "stop"
# Set usage information if available in response
if "usage" in response_data:
@@ -102,7 +102,7 @@ class CodestralTextCompletionConfig(OpenAITextCompletionConfig):
"finish_reason": finish_reason,
}
original_chunk = litellm.ModelResponse(**chunk_data_dict, stream=True)
original_chunk = litellm.ModelResponseStream(**chunk_data_dict)
_choices = chunk_data_dict.get("choices", []) or []
if len(_choices) == 0:
return {
+13 -13
View File
@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Optional
import httpx
from litellm._logging import verbose_logger
from litellm.types.utils import Delta, ModelResponse, StreamingChoices
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
if TYPE_CHECKING:
pass
@@ -44,7 +44,7 @@ class LangGraphSSEStreamIterator:
self.async_line_iterator = self.response.aiter_lines()
return self
def _parse_sse_line(self, line: str) -> Optional[ModelResponse]:
def _parse_sse_line(self, line: str) -> Optional[ModelResponseStream]:
"""
Parse a single SSE line and return a ModelResponse chunk if applicable.
@@ -71,7 +71,7 @@ class LangGraphSSEStreamIterator:
return None
def _process_data(self, data) -> Optional[ModelResponse]:
def _process_data(self, data) -> Optional[ModelResponseStream]:
"""
Process parsed data from SSE stream.
@@ -101,7 +101,7 @@ class LangGraphSSEStreamIterator:
return None
def _process_messages_event(self, payload) -> Optional[ModelResponse]:
def _process_messages_event(self, payload) -> Optional[ModelResponseStream]:
"""
Process a messages event from the stream.
@@ -128,7 +128,7 @@ class LangGraphSSEStreamIterator:
return None
def _process_metadata_event(self, payload) -> Optional[ModelResponse]:
def _process_metadata_event(self, payload) -> Optional[ModelResponseStream]:
"""
Process a metadata event, which may signal the end of the stream.
"""
@@ -139,9 +139,9 @@ class LangGraphSSEStreamIterator:
return self._create_final_chunk()
return None
def _create_content_chunk(self, text: str) -> ModelResponse:
"""Create a ModelResponse chunk with content."""
chunk = ModelResponse(
def _create_content_chunk(self, text: str) -> ModelResponseStream:
"""Create a ModelResponseStream chunk with content."""
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=self.model,
@@ -158,9 +158,9 @@ class LangGraphSSEStreamIterator:
return chunk
def _create_final_chunk(self) -> ModelResponse:
"""Create a final ModelResponse chunk with finish_reason."""
chunk = ModelResponse(
def _create_final_chunk(self) -> ModelResponseStream:
"""Create a final ModelResponseStream chunk with finish_reason."""
chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=self.model,
@@ -177,7 +177,7 @@ class LangGraphSSEStreamIterator:
return chunk
def __next__(self) -> ModelResponse:
def __next__(self) -> ModelResponseStream:
"""Sync iteration - parse SSE events and yield ModelResponse chunks."""
try:
if self.line_iterator is None:
@@ -205,7 +205,7 @@ class LangGraphSSEStreamIterator:
verbose_logger.error(f"Error in LangGraph SSE stream: {str(e)}")
raise StopIteration
async def __anext__(self) -> ModelResponse:
async def __anext__(self) -> ModelResponseStream:
"""Async iteration - parse SSE events and yield ModelResponse chunks."""
try:
if self.async_line_iterator is None:
@@ -542,16 +542,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if len(choice.message.tool_calls) > 0:
return True
elif isinstance(response, ModelResponseStream):
for choice in response.choices:
if isinstance(choice, litellm.StreamingChoices):
for streaming_choice in response.choices:
if isinstance(streaming_choice, litellm.StreamingChoices):
# Check for text content
if choice.delta.content and isinstance(choice.delta.content, str):
if streaming_choice.delta.content and isinstance(streaming_choice.delta.content, str):
return True
# Check for tool calls
if choice.delta.tool_calls and isinstance(
choice.delta.tool_calls, list
if streaming_choice.delta.tool_calls and isinstance(
streaming_choice.delta.tool_calls, list
):
if len(choice.delta.tool_calls) > 0:
if len(streaming_choice.delta.tool_calls) > 0:
return True
return False
@@ -2100,7 +2100,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
chat_completion_logprobs=chat_completion_logprobs,
image_response=image_response,
)
model_response.choices.append(choice)
model_response.choices.append(choice) # type: ignore[arg-type]
elif isinstance(model_response, ModelResponse):
choice = litellm.Choices(
finish_reason=VertexGeminiConfig._check_finish_reason(
@@ -2111,7 +2111,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
logprobs=chat_completion_logprobs,
enhancements=None,
)
model_response.choices.append(choice)
model_response.choices.append(choice) # type: ignore[arg-type]
return (
grounding_metadata,
@@ -61,7 +61,6 @@ from litellm.utils import (
ImageResponse,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
@@ -863,9 +862,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
if self.output_parse_pii is False and litellm.output_parse_pii is False:
return response
if isinstance(response, ModelResponse) and not isinstance(
response.choices[0], StreamingChoices
): # /chat/completions requests
if isinstance(response, ModelResponse): # /chat/completions requests
if isinstance(response.choices[0].message.content, str):
verbose_proxy_logger.debug(
f"self.pii_tokens: {self.pii_tokens}; initial response: {response.choices[0].message.content}"
@@ -888,7 +885,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
return response
# skip streaming here; handled in async_post_call_streaming_iterator_hook
if response.choices and isinstance(response.choices[0], StreamingChoices):
if isinstance(response, ModelResponseStream):
return response
presidio_config = self.get_presidio_settings_from_request_data(
@@ -896,10 +893,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
)
for choice in response.choices:
# Type narrowing: StreamingChoices doesn't have .message attribute
if not hasattr(choice, "message"):
continue
content = getattr(choice.message, "content", None) # type: ignore
content = getattr(choice.message, "content", None)
if content is None:
continue
if isinstance(content, str):
+22 -38
View File
@@ -1651,6 +1651,7 @@ class StreamingChatCompletionChunk(OpenAIChatCompletionChunk):
super().__init__(**kwargs)
class ModelResponseBase(OpenAIObject):
id: str
"""A unique identifier for the completion."""
@@ -1759,7 +1760,7 @@ class ModelResponseStream(ModelResponseBase):
class ModelResponse(ModelResponseBase):
choices: List[Union[Choices, StreamingChoices]]
choices: List[Choices]
"""The list of completion choices the model generated for the input prompt."""
def __init__( # noqa: PLR0915
@@ -1778,44 +1779,27 @@ class ModelResponse(ModelResponseBase):
_response_headers=None,
**params,
) -> None:
if stream is not None and stream is True:
object = "chat.completion.chunk"
if choices is not None and isinstance(choices, list):
new_choices = []
for choice in choices:
_new_choice = None
if isinstance(choice, StreamingChoices):
_new_choice = choice
elif isinstance(choice, dict):
_new_choice = StreamingChoices(**choice)
elif isinstance(choice, BaseModel):
_new_choice = StreamingChoices(**choice.model_dump())
new_choices.append(_new_choice)
choices = new_choices
else:
choices = [StreamingChoices()]
object = "chat.completion"
if choices is not None and isinstance(choices, list):
new_choices = []
for choice in choices:
if isinstance(choice, Choices):
_new_choice = choice # type: ignore
elif isinstance(choice, dict):
_new_choice = Choices(**choice) # type: ignore
elif isinstance(choice, BaseModel):
dump = (
choice.model_dump()
if hasattr(choice, "model_dump")
else choice.dict()
)
_new_choice = Choices(**dump) # type: ignore
else:
_new_choice = choice
new_choices.append(_new_choice)
choices = new_choices
else:
object = "chat.completion"
if choices is not None and isinstance(choices, list):
new_choices = []
for choice in choices:
if isinstance(choice, Choices):
_new_choice = choice # type: ignore
elif isinstance(choice, dict):
_new_choice = Choices(**choice) # type: ignore
elif isinstance(choice, BaseModel):
dump = (
choice.model_dump()
if hasattr(choice, "model_dump")
else choice.dict()
)
_new_choice = Choices(**dump) # type: ignore
else:
_new_choice = choice
new_choices.append(_new_choice)
choices = new_choices
else:
choices = [Choices()]
choices = [Choices()]
if id is None:
id = _generate_id()
else:
+4 -6
View File
@@ -4963,9 +4963,7 @@ def get_response_string(response_obj: Union[ModelResponse, ModelResponseStream])
return delta if isinstance(delta, str) else ""
# Handle standard ModelResponse and ModelResponseStream
_choices: Union[List[Union[Choices, StreamingChoices]], List[StreamingChoices]] = (
response_obj.choices
)
_choices: Union[List[Choices], List[StreamingChoices]] = response_obj.choices
# Use list accumulation to avoid O(n^2) string concatenation across choices
response_parts: List[str] = []
@@ -7384,9 +7382,9 @@ def _get_base_model_from_metadata(model_call_details=None):
class ModelResponseIterator:
def __init__(self, model_response: ModelResponse, convert_to_delta: bool = False):
if convert_to_delta is True:
self.model_response = ModelResponse(stream=True)
_delta = self.model_response.choices[0].delta # type: ignore
_delta.content = model_response.choices[0].message.content # type: ignore
_stream_response = ModelResponseStream()
_stream_response.choices[0].delta.content = model_response.choices[0].message.content # type: ignore
self.model_response: Union[ModelResponse, ModelResponseStream] = _stream_response
else:
self.model_response = model_response
self.is_done = False
@@ -72,7 +72,7 @@ def test_stream_chunk_builder_preserves_images():
chunks = []
for chunk in init_chunks:
chunks.append(litellm.ModelResponse(**chunk, stream=True))
chunks.append(litellm.ModelResponseStream(**chunk))
response = stream_chunk_builder(chunks=chunks)
@@ -163,7 +163,7 @@ def test_stream_chunk_builder_preserves_multiple_images():
chunks = []
for chunk in init_chunks:
chunks.append(litellm.ModelResponse(**chunk, stream=True))
chunks.append(litellm.ModelResponseStream(**chunk))
response = stream_chunk_builder(chunks=chunks)
@@ -230,7 +230,7 @@ def test_stream_chunk_builder_no_images():
chunks = []
for chunk in init_chunks:
chunks.append(litellm.ModelResponse(**chunk, stream=True))
chunks.append(litellm.ModelResponseStream(**chunk))
response = stream_chunk_builder(chunks=chunks)
@@ -542,7 +542,7 @@ def test_stream_chunk_builder_multiple_tool_calls():
chunks = []
for chunk in init_chunks:
chunks.append(litellm.ModelResponse(**chunk, stream=True))
chunks.append(litellm.ModelResponseStream(**chunk))
response = stream_chunk_builder(chunks=chunks)
print(f"Returned response: {response}")
@@ -616,7 +616,7 @@ def test_stream_chunk_builder_openai_prompt_caching():
chunks: List[litellm.ModelResponse] = []
usage_obj = None
for chunk in chat_completion:
chunks.append(litellm.ModelResponse(**chunk.model_dump(), stream=True))
chunks.append(litellm.ModelResponseStream(**chunk.model_dump()))
print(f"chunks: {chunks}")
@@ -661,7 +661,7 @@ def test_stream_chunk_builder_openai_audio_output_usage():
chunks = []
for chunk in completion:
chunks.append(litellm.ModelResponse(**chunk.model_dump(), stream=True))
chunks.append(litellm.ModelResponseStream(**chunk.model_dump()))
usage_obj: Optional[litellm.Usage] = None
+6 -6
View File
@@ -393,7 +393,7 @@ def test_completion_azure_stream_content_filter_no_delta():
chunk_list = []
for chunk in chunks:
new_chunk = litellm.ModelResponse(stream=True, id=chunk["id"])
new_chunk = litellm.ModelResponseStream(id=chunk["id"])
if "choices" in chunk and isinstance(chunk["choices"], list):
new_choices = []
for choice in chunk["choices"]:
@@ -3026,7 +3026,7 @@ def test_unit_test_custom_stream_wrapper():
{"index": 0, "delta": {"content": "How are you?"}, "finish_reason": "stop"}
],
}
chunk = litellm.ModelResponse(**chunk, stream=True)
chunk = litellm.ModelResponseStream(**chunk)
completion_stream = ModelResponseIterator(model_response=chunk)
@@ -3223,7 +3223,7 @@ def test_unit_test_custom_stream_wrapper_openai():
"system_fingerprint": None,
"usage": None,
}
chunk = litellm.ModelResponse(**chunk, stream=True)
chunk = litellm.ModelResponseStream(**chunk)
completion_stream = ModelResponseIterator(model_response=chunk)
@@ -3457,7 +3457,7 @@ def test_aamazing_unit_test_custom_stream_wrapper_n():
chunk_list = []
for chunk in chunks:
new_chunk = litellm.ModelResponse(stream=True, id=chunk["id"])
new_chunk = litellm.ModelResponseStream(id=chunk["id"])
if "choices" in chunk and isinstance(chunk["choices"], list):
print("INSIDE CHUNK CHOICES!")
new_choices = []
@@ -3541,7 +3541,7 @@ def test_unit_test_custom_stream_wrapper_function_call():
"system_fingerprint": "fp_44709d6fcb",
"choices": [{"index": 0, "delta": delta, "finish_reason": "stop"}],
}
chunk = litellm.ModelResponse(**chunk, stream=True)
chunk = litellm.ModelResponseStream(**chunk)
completion_stream = ModelResponseIterator(model_response=chunk)
@@ -3651,7 +3651,7 @@ def test_unit_test_perplexity_citations_chunk():
}
],
}
chunk = litellm.ModelResponse(**chunk, stream=True)
chunk = litellm.ModelResponseStream(**chunk)
completion_stream = ModelResponseIterator(model_response=chunk)
@@ -2,7 +2,14 @@ import warnings
import pytest
from litellm.types.utils import Choices, Message, ModelResponse
from litellm.types.utils import (
Choices,
Delta,
Message,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
def test_modelresponse_normalizes_openai_base_models() -> None:
@@ -59,3 +66,63 @@ def test_modelresponse_serialization_avoids_pydantic_warnings() -> None:
or "Pydantic serializer warnings" in str(w.message)
for w in captured
)
def test_modelresponse_model_dump_json_no_pydantic_warnings() -> None:
"""model_dump_json() and model_dump() should not trigger any Pydantic
serialization warnings now that choices is List[Choices] (no Union)."""
response = ModelResponse(
model="test-model",
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(content="hello", role="assistant"),
)
],
)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always")
_ = response.model_dump_json()
_ = response.model_dump()
_ = response.model_dump(exclude_none=True)
pydantic_warnings = [
w
for w in captured
if "PydanticSerializationUnexpectedValue" in str(w.message)
or "Pydantic serializer warnings" in str(w.message)
]
assert pydantic_warnings == [], (
f"Unexpected Pydantic serialization warnings: {pydantic_warnings}"
)
def test_streaming_modelresponsestream_no_pydantic_warnings() -> None:
"""Streaming responses use ModelResponseStream with List[StreamingChoices]
and should serialize without warnings."""
response = ModelResponseStream(
choices=[
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(content="hello", role="assistant"),
)
],
)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always")
_ = response.model_dump_json()
_ = response.model_dump()
pydantic_warnings = [
w
for w in captured
if "PydanticSerializationUnexpectedValue" in str(w.message)
or "Pydantic serializer warnings" in str(w.message)
]
assert pydantic_warnings == [], (
f"Unexpected Pydantic serialization warnings: {pydantic_warnings}"
)