From 078096247b059fdf2c5be255ad9be3bb113eb99f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 28 Jul 2025 17:42:28 -0700 Subject: [PATCH] [Bug Fix] The model gemini-2.5-flash with the merge_reasoning_content_in_choices parameter does not work (#13066) * _optional_combine_thinking_block_in_choices * test_optional_combine_thinking_block_with_none_content --- .../litellm_core_utils/streaming_handler.py | 5 +- litellm/proxy/proxy_config.yaml | 6 ++ .../test_streaming_handler.py | 88 +++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2e3fe091ee..17f5a8d1de 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -922,6 +922,9 @@ class CustomStreamWrapper: ) if reasoning_content: if self.sent_first_thinking_block is False: + # Ensure content is not None before concatenation + if model_response.choices[0].delta.content is None: + model_response.choices[0].delta.content = "" model_response.choices[0].delta.content += ( "" + reasoning_content ) @@ -938,7 +941,7 @@ class CustomStreamWrapper: and model_response.choices[0].delta.content ): model_response.choices[0].delta.content = ( - "" + model_response.choices[0].delta.content + "" + (model_response.choices[0].delta.content or "") ) self.sent_last_thinking_block = True diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index e6f3f7f67e..f1cfe75c7a 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -6,6 +6,12 @@ model_list: litellm_params: model: openai/* + - model_name: vertex_ai/gemini-2.5-pro # Vertex AI Gemini + litellm_params: + model: vertex_ai/gemini-2.5-pro + thinking: {"type": "enabled", "budget_tokens": 1024} + merge_reasoning_content_in_choices: true + litellm_settings: callbacks: ["datadog_llm_observability"] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index fe6fe2cb48..127f3573cb 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -725,3 +725,91 @@ def test_streaming_handler_with_stream_options( mr_dict = mr.model_dump() print(mr_dict) assert "stream_options" not in mr_dict + + +def test_optional_combine_thinking_block_with_none_content( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """Test that reasoning_content is properly combined when delta.content is None""" + # Setup the wrapper to use the merge feature + initialized_custom_stream_wrapper.merge_reasoning_content_in_choices = True + + # First chunk with reasoning_content and None content - should handle None gracefully + first_chunk = { + "id": "chunk1", + "object": "chat.completion.chunk", + "created": 1741037890, + "model": "deepseek-reasoner", + "choices": [ + { + "index": 0, + "delta": { + "content": None, # This is None, not empty string + "reasoning_content": "Let me think about this problem", + }, + "finish_reason": None, + } + ], + } + + # Second chunk with reasoning_content and None content + second_chunk = { + "id": "chunk2", + "object": "chat.completion.chunk", + "created": 1741037891, + "model": "deepseek-reasoner", + "choices": [ + { + "index": 0, + "delta": { + "content": None, # This is None, not empty string + "reasoning_content": " step by step", + }, + "finish_reason": None, + } + ], + } + + # Final chunk with actual content - should add tag + final_chunk = { + "id": "chunk3", + "object": "chat.completion.chunk", + "created": 1741037892, + "model": "deepseek-reasoner", + "choices": [ + { + "index": 0, + "delta": { + "content": "The answer is 42", + "reasoning_content": None + }, + "finish_reason": None, + } + ], + } + + # Process first chunk - should not raise TypeError + first_response = ModelResponseStream(**first_chunk) + initialized_custom_stream_wrapper._optional_combine_thinking_block_in_choices( + first_response + ) + assert first_response.choices[0].delta.content == "Let me think about this problem" + assert not hasattr(first_response.choices[0].delta, "reasoning_content") + assert initialized_custom_stream_wrapper.sent_first_thinking_block is True + + # Process second chunk - should work with continued reasoning + second_response = ModelResponseStream(**second_chunk) + initialized_custom_stream_wrapper._optional_combine_thinking_block_in_choices( + second_response + ) + assert second_response.choices[0].delta.content == " step by step" + assert not hasattr(second_response.choices[0].delta, "reasoning_content") + + # Process final chunk - should add tag + final_response = ModelResponseStream(**final_chunk) + initialized_custom_stream_wrapper._optional_combine_thinking_block_in_choices( + final_response + ) + assert final_response.choices[0].delta.content == "The answer is 42" + assert initialized_custom_stream_wrapper.sent_last_thinking_block is True + assert not hasattr(final_response.choices[0].delta, "reasoning_content")