tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-14 08:26:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23533 from Sameerlite/litellm_fix_responses_bridge_gpr-5.4

Browse Source 
Fix _supports_reasoning_effort_level for responses bridge
This commit is contained in:
yuneng-jiang
2026-03-14 09:38:22 -07:00
committed by GitHub
parent 2405e0d400 6c3e036648
commit fcbfcf7dea
14 changed files with 624 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
+32
View File
@@ -50,6 +50,38 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
# "metadata",
]
def _remove_scope_from_cache_control(
self, anthropic_messages_request: Dict
) -> None:
"""
Remove `scope` field from cache_control blocks.
Some providers (Vertex AI, Azure AI Foundry) do not support the `scope`
field in cache_control (e.g. "global" for cross-request caching).
Processes both `system` and `messages` content blocks.
"""
def _sanitize(cache_control: Any) -> None:
if isinstance(cache_control, dict):
cache_control.pop("scope", None)
def _process_content_list(content: list) -> None:
for item in content:
if isinstance(item, dict) and "cache_control" in item:
_sanitize(item["cache_control"])
if "system" in anthropic_messages_request:
system = anthropic_messages_request["system"]
if isinstance(system, list):
_process_content_list(system)
if "messages" in anthropic_messages_request:
for message in anthropic_messages_request["messages"]:
if isinstance(message, dict) and "content" in message:
content = message["content"]
if isinstance(content, list):
_process_content_list(content)
@staticmethod
def _filter_billing_headers_from_system(system_param):
"""
litellm/llms/moonshot/chat/transformation.py
+54 -16
View File
@@ -2,13 +2,15 @@
Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@@ -17,8 +19,7 @@ class MoonshotChatConfig(OpenAIGPTConfig):
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
) -> Coroutine[Any, Any, List[AllMessageValues]]: ...
@overload
def _transform_messages(
@@ -26,8 +27,7 @@ class MoonshotChatConfig(OpenAIGPTConfig):
messages: List[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> List[AllMessageValues]:
...
) -> List[AllMessageValues]: ...
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
@@ -53,22 +53,14 @@ class MoonshotChatConfig(OpenAIGPTConfig):
messages = handle_messages_with_content_list_to_str_conversion(messages)
if is_async:
return super()._transform_messages(
messages=messages, model=model, is_async=True
)
return super()._transform_messages(messages=messages, model=model, is_async=True)
else:
return super()._transform_messages(
messages=messages, model=model, is_async=False
)
return super()._transform_messages(messages=messages, model=model, is_async=False)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
api_base = (
api_base
or get_secret_str("MOONSHOT_API_BASE")
or "https://api.moonshot.ai/v1"
) # type: ignore
api_base = api_base or get_secret_str("MOONSHOT_API_BASE") or "https://api.moonshot.ai/v1" # type: ignore
dynamic_api_key = api_key or get_secret_str("MOONSHOT_API_KEY")
return api_base, dynamic_api_key
@@ -149,6 +141,48 @@ class MoonshotChatConfig(OpenAIGPTConfig):
optional_params["temperature"] = 0.3
return optional_params
def fill_reasoning_content(self, messages: List[AllMessageValues]) -> List[AllMessageValues]:
"""
Moonshot reasoning models require `reasoning_content` on every assistant
message that contains tool_calls (multi-turn tool-calling flows).
For each such message that is missing the field:
1. Promote provider_specific_fields["reasoning_content"] if present and non-empty
(this is where LiteLLM stores it from a previous response)
2. Otherwise inject a single space — the minimum value the API accepts
Messages that already carry the field, or are not assistant/tool-call messages,
are appended as-is (no copy made).
"""
result: List[AllMessageValues] = []
for msg in messages:
if (
msg.get("role") == "assistant"
and msg.get("tool_calls")
and "reasoning_content" not in msg
):
patched = dict(cast(dict, msg))
provider_fields = patched.get("provider_specific_fields") or {}
stored = provider_fields.get("reasoning_content")
if stored:
patched["reasoning_content"] = stored
# Remove the promoted key from provider_specific_fields to
# avoid sending the value twice in the serialised request body
cleaned_provider_fields = dict(provider_fields)
cleaned_provider_fields.pop("reasoning_content", None)
patched["provider_specific_fields"] = cleaned_provider_fields
else:
litellm.verbose_logger.warning(
"Moonshot reasoning model: assistant tool-call message is missing "
"`reasoning_content`. Injecting a placeholder to satisfy API validation. "
"For best results, preserve `reasoning_content` from the original "
"assistant response when building multi-turn conversation history."
)
patched["reasoning_content"] = " "
result.append(cast(AllMessageValues, patched))
else:
result.append(msg)
return result
def transform_request(
self,
model: str,
@@ -169,6 +203,10 @@ class MoonshotChatConfig(OpenAIGPTConfig):
optional_params=optional_params,
)
# Moonshot reasoning models: fill in reasoning_content before the API call
if supports_reasoning(model=model, custom_llm_provider="moonshot"):
messages = self.fill_reasoning_content(messages)
# Call parent transform_request which handles _transform_messages
return super().transform_request(
model=model,
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
+2
View File
@@ -150,6 +150,8 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
headers=headers,
)
self._remove_scope_from_cache_control(anthropic_messages_request)
anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16"
anthropic_messages_request.pop(
litellm/main.py
+9 -1
View File
@@ -1370,6 +1370,13 @@ def completion( # type: ignore # noqa: PLR0915
api_key=api_key,
)
## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name
responses_api_model_info, model = responses_api_bridge_check(
model=model,
custom_llm_provider=custom_llm_provider,
web_search_options=web_search_options,
)
if not _should_allow_input_examples(
custom_llm_provider=custom_llm_provider, model=model
):
@@ -1614,7 +1621,7 @@ def completion( # type: ignore # noqa: PLR0915
reasoning_effort=reasoning_effort,
)
if model_info.get("mode") == "responses":
if responses_api_model_info.get("mode") == "responses":
from litellm.completion_extras import responses_api_bridge
if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort:
@@ -5122,6 +5129,7 @@ def embedding( # noqa: PLR0915
client=client,
aembedding=aembedding,
litellm_params=litellm_params_dict,
headers=headers,
)
elif custom_llm_provider == "bedrock":
if isinstance(input, str):
litellm/model_prices_and_context_window_backup.json
+3
View File
@@ -22083,6 +22083,7 @@
"output_cost_per_token": 3e-06,
"source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true
@@ -22166,6 +22167,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_web_search": true
},
@@ -22180,6 +22182,7 @@
"output_cost_per_token": 8e-06,
"source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_web_search": true
},
litellm/proxy/response_polling/background_streaming.py
+38 -6
View File
@@ -113,6 +113,16 @@ async def background_streaming_task( # noqa: PLR0915
last_update_time = asyncio.get_event_loop().time()
UPDATE_INTERVAL = 0.150 # 150ms batching interval
# Track the terminal event from the stream (may not be "completed")
terminal_status = None # Will be set by response.completed/failed/incomplete/cancelled
terminal_error = None
_event_to_status = {
"response.completed": "completed",
"response.failed": "failed",
"response.incomplete": "incomplete",
"response.cancelled": "cancelled",
}
async def flush_state_if_needed(force: bool = False) -> None:
"""Flush accumulated state to Redis if interval elapsed or forced"""
nonlocal state_dirty, last_update_time
@@ -131,6 +141,12 @@ async def background_streaming_task( # noqa: PLR0915
last_update_time = current_time
# Handle StreamingResponse
if not hasattr(response, "body_iterator"):
verbose_proxy_logger.warning(
f"background_streaming_task: response for {polling_id} has no "
"body_iterator; this may indicate a misconfiguration or provider error"
)
if hasattr(response, "body_iterator"):
async for chunk in response.body_iterator:
# Parse chunk
@@ -224,10 +240,23 @@ async def background_streaming_task( # noqa: PLR0915
status="in_progress",
)
elif event_type == "response.completed":
# Response completed - extract all ResponsesAPIResponse fields
# https://platform.openai.com/docs/api-reference/responses-streaming/response-completed
elif event_type in (
"response.completed",
"response.failed",
"response.incomplete",
"response.cancelled",
):
# Terminal event - extract all ResponsesAPIResponse fields
# https://platform.openai.com/docs/api-reference/responses-streaming
response_data = event.get("response", {})
terminal_status = response_data.get(
"status",
_event_to_status.get(event_type, "completed"),
)
# Extract error for failed responses
if event_type == "response.failed":
terminal_error = response_data.get("error")
# Core response fields
usage_data = response_data.get("usage")
@@ -278,11 +307,14 @@ async def background_streaming_task( # noqa: PLR0915
# Final flush to ensure all accumulated state is saved
await flush_state_if_needed(force=True)
# Mark as completed with all ResponsesAPIResponse fields
# Use the terminal status from the stream, default to "completed"
final_status = terminal_status or "completed"
await polling_handler.update_state(
polling_id=polling_id,
status="completed",
status=final_status,
usage=usage_data,
error=terminal_error,
reasoning=reasoning_data,
tool_choice=tool_choice_data,
tools=tools_data,
@@ -301,7 +333,7 @@ async def background_streaming_task( # noqa: PLR0915
)
verbose_proxy_logger.info(
f"Completed background streaming for {polling_id}, output_items={len(output_items)}"
f"Finished background streaming for {polling_id}, status={final_status}, output_items={len(output_items)}"
)
except Exception as e:
model_prices_and_context_window.json
+3
View File
@@ -22083,6 +22083,7 @@
"output_cost_per_token": 3e-06,
"source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true
@@ -22166,6 +22167,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_web_search": true
},
@@ -22180,6 +22182,7 @@
"output_cost_per_token": 8e-06,
"source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_web_search": true
},
requirements.txt
+1 -1
View File
@@ -1,7 +1,7 @@
# LITELLM PROXY DEPENDENCIES #
# Security: explicit pins for transitive deps (CVE fixes)
urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441
tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724
tornado>=6.5.5 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724, CVE-2026-31958, GHSA-78cv-mqj4-43f7
filelock>=3.20.1 # CVE-2025-68146
h11>=0.16.0 # CVE-2025-43859, GHSA-vqfr-h8mv-ghfj — HTTP request smuggling
wheel>=0.46.2 # CVE-2026-24049 — path traversal
tests/llm_translation/test_openai.py
+26
View File
@@ -1454,3 +1454,29 @@ def test_gpt_5_web_search():
for chunk in response:
print("chunk: ", chunk)
def test_responses_gpt54_with_xhigh_reasoning():
"""
Ensure chat->responses bridge sends the correct request payload for
openai/responses/gpt-5.4 with reasoning_effort="xhigh".
"""
with patch("litellm.responses") as mock_responses:
# Stop execution right after request generation to avoid external API calls.
mock_responses.side_effect = RuntimeError("stop_after_request_build")
with pytest.raises(Exception):
litellm.completion(
model="openai/responses/gpt-5.4",
messages=[{"role": "user", "content": "What is 2+2?"}],
reasoning_effort="xhigh",
max_tokens=100,
)
mock_responses.assert_called_once()
request_body = mock_responses.call_args.kwargs
# The responses prefix should be stripped before routing.
assert request_body["model"] == "gpt-5.4"
# chat-completions reasoning_effort must map to Responses API reasoning.
assert request_body["reasoning"] == {"effort": "xhigh"}
tests/proxy_unit_tests/test_response_polling_handler.py
+261
View File
@@ -1318,6 +1318,267 @@ class TestStreamingEventParsing:
assert output_items["item_123"]["content"][0]["type"] == "text"
def _make_sse_stream(events: list) -> Mock:
"""Create a mock StreamingResponse with body_iterator from a list of event dicts."""
async def _body_iterator():
for event in events:
yield f"data: {json.dumps(event)}"
yield "data: [DONE]"
mock_response = Mock()
mock_response.body_iterator = _body_iterator()
return mock_response
def _make_background_streaming_kwargs(
polling_id: str,
polling_handler: ResponsePollingHandler,
) -> dict:
"""Build kwargs for background_streaming_task with all required mocks."""
return dict(
polling_id=polling_id,
data={"model": "gpt-4o", "stream": False, "background": True},
polling_handler=polling_handler,
request=Mock(),
fastapi_response=Mock(),
user_api_key_dict=Mock(),
general_settings={},
llm_router=None,
proxy_config=Mock(),
proxy_logging_obj=Mock(),
select_data_generator=Mock(),
user_model=None,
user_temperature=None,
user_request_timeout=None,
user_max_tokens=None,
user_api_base=None,
version=None,
)
@pytest.mark.xdist_group("heavy_imports")
class TestBackgroundStreamingTerminalEvents:
"""
Integration tests that exercise background_streaming_task with mocked
streaming responses, verifying the final update_state call for each
terminal event type.
"""
@pytest.mark.asyncio
async def test_response_failed_sets_failed_status_and_error(self):
"""Test that a response.failed stream event results in failed status with error"""
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
error_payload = {
"type": "server_error",
"message": "The model encountered an error",
"code": "model_error",
}
events = [
{"type": "response.in_progress"},
{
"type": "response.failed",
"response": {
"id": "resp_123",
"status": "failed",
"error": error_payload,
"model": "gpt-4o",
"output": [],
},
},
]
mock_response = _make_sse_stream(events)
handler = AsyncMock(spec=ResponsePollingHandler)
kwargs = _make_background_streaming_kwargs("poll_1", handler)
with patch(
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
# Find the final update_state call (last one)
final_call = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "failed"
assert final_call.kwargs["error"] == error_payload
@pytest.mark.asyncio
async def test_response_incomplete_sets_incomplete_status_and_details(self):
"""Test that a response.incomplete stream event results in incomplete status"""
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
events = [
{"type": "response.in_progress"},
{
"type": "response.incomplete",
"response": {
"id": "resp_123",
"status": "incomplete",
"incomplete_details": {"reason": "max_output_tokens"},
"usage": {"input_tokens": 10, "output_tokens": 4096},
"model": "gpt-4o",
"output": [{"id": "item_1", "type": "message"}],
},
},
]
mock_response = _make_sse_stream(events)
handler = AsyncMock(spec=ResponsePollingHandler)
kwargs = _make_background_streaming_kwargs("poll_2", handler)
with patch(
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "incomplete"
assert final_call.kwargs["incomplete_details"] == {"reason": "max_output_tokens"}
assert final_call.kwargs["usage"] == {"input_tokens": 10, "output_tokens": 4096}
@pytest.mark.asyncio
async def test_response_cancelled_sets_cancelled_status(self):
"""Test that a response.cancelled stream event results in cancelled status"""
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
events = [
{"type": "response.in_progress"},
{
"type": "response.cancelled",
"response": {
"id": "resp_123",
"status": "cancelled",
"model": "gpt-4o",
"output": [],
},
},
]
mock_response = _make_sse_stream(events)
handler = AsyncMock(spec=ResponsePollingHandler)
kwargs = _make_background_streaming_kwargs("poll_3", handler)
with patch(
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "cancelled"
@pytest.mark.asyncio
async def test_response_completed_sets_completed_status(self):
"""Test that a response.completed stream event results in completed status"""
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
events = [
{"type": "response.in_progress"},
{
"type": "response.completed",
"response": {
"id": "resp_123",
"status": "completed",
"usage": {"input_tokens": 10, "output_tokens": 50},
"model": "gpt-4o",
"output": [{"id": "item_1", "type": "message"}],
},
},
]
mock_response = _make_sse_stream(events)
handler = AsyncMock(spec=ResponsePollingHandler)
kwargs = _make_background_streaming_kwargs("poll_4", handler)
with patch(
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "completed"
assert final_call.kwargs["usage"] == {"input_tokens": 10, "output_tokens": 50}
@pytest.mark.asyncio
async def test_fallback_status_derived_from_event_type_when_status_field_missing(self):
"""Test that when the response body lacks a status field, the fallback
is derived from the event type, not hardcoded to 'completed'."""
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
# response.incomplete event with NO status field in the response body
events = [
{"type": "response.in_progress"},
{
"type": "response.incomplete",
"response": {
"id": "resp_123",
# "status" deliberately omitted
"incomplete_details": {"reason": "max_output_tokens"},
"model": "gpt-4o",
"output": [],
},
},
]
mock_response = _make_sse_stream(events)
handler = AsyncMock(spec=ResponsePollingHandler)
kwargs = _make_background_streaming_kwargs("poll_5", handler)
with patch(
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "incomplete"
@pytest.mark.asyncio
async def test_no_terminal_event_defaults_to_completed(self):
"""Test that when no terminal event is received, status defaults to completed"""
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
# Stream with only in_progress, no terminal event
events = [
{"type": "response.in_progress"},
]
mock_response = _make_sse_stream(events)
handler = AsyncMock(spec=ResponsePollingHandler)
kwargs = _make_background_streaming_kwargs("poll_6", handler)
with patch(
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "completed"
class TestEdgeCases:
"""Test edge cases and error scenarios"""
tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py
+147 -1
View File
@@ -7,6 +7,7 @@ Moonshot AI is an OpenAI-compatible provider with minor customizations.
import os
import sys
from unittest.mock import patch
sys.path.insert(
0, os.path.abspath("../../../../..")
@@ -404,4 +405,149 @@ class TestMoonshotConfig:
# Content should be flattened to a plain string
assert isinstance(result["messages"][0]["content"], str)
assert result["messages"][0]["content"] == "Hello, how are you?"
assert result["messages"][0]["content"] == "Hello, how are you?"
# ------------------------------------------------------------------ #
# Tests for fill_reasoning_content #
# ------------------------------------------------------------------ #
def test_reasoning_content_space_injected_when_absent(self):
"""Assistant tool-call message with no reasoning_content gets a space injected."""
config = MoonshotChatConfig()
messages = [
{"role": "user", "content": "What's the weather?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 22°C"},
]
result = config.fill_reasoning_content(messages)
assert result[1].get("reasoning_content") == " "
# Non-assistant messages are untouched
assert "reasoning_content" not in result[0]
assert "reasoning_content" not in result[2]
def test_empty_tool_calls_list_not_injected(self):
"""Assistant message with tool_calls: [] should not get reasoning_content injected."""
config = MoonshotChatConfig()
original_msg = {
"role": "assistant",
"content": "Here is the answer.",
"tool_calls": [],
}
messages = [original_msg]
result = config.fill_reasoning_content(messages)
assert "reasoning_content" not in result[0]
assert result[0] is original_msg
def test_existing_reasoning_content_not_overwritten(self):
"""Message that already has reasoning_content is passed through unchanged."""
config = MoonshotChatConfig()
original_msg = {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}}
],
"reasoning_content": "<actual thinking>",
}
messages = [original_msg]
result = config.fill_reasoning_content(messages)
assert result[0].get("reasoning_content") == "<actual thinking>"
# Same object — no copy was made
assert result[0] is original_msg
def test_provider_specific_fields_reasoning_content_promoted(self):
"""reasoning_content stored in provider_specific_fields is promoted to top level."""
config = MoonshotChatConfig()
messages = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}}
],
"provider_specific_fields": {"reasoning_content": "stored thinking"},
}
]
result = config.fill_reasoning_content(messages)
assert result[0].get("reasoning_content") == "stored thinking"
# The promoted key must be removed from provider_specific_fields to
# avoid sending the value twice in the serialised request body
assert "reasoning_content" not in (result[0].get("provider_specific_fields") or {})
def test_reasoning_model_fill_called_from_transform_request(self):
"""transform_request injects reasoning_content end-to-end for reasoning models."""
config = MoonshotChatConfig()
messages = [
{"role": "user", "content": "Call a tool"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}}
],
},
]
with patch(
"litellm.llms.moonshot.chat.transformation.supports_reasoning",
return_value=True,
):
result = config.transform_request(
model="kimi-k2-thinking",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert result["messages"][1].get("reasoning_content") == " "
def test_non_reasoning_model_messages_untouched(self):
"""For non-reasoning models, transform_request leaves messages unchanged."""
config = MoonshotChatConfig()
messages = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "fn", "arguments": "{}"}}
],
},
]
with patch(
"litellm.llms.moonshot.chat.transformation.supports_reasoning",
return_value=False,
):
result = config.transform_request(
model="moonshot-v1-8k",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
# reasoning_content must not have been injected
for msg in result["messages"]:
assert "reasoning_content" not in msg
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py
+45
View File
@@ -5,6 +5,7 @@ import pytest
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
VertexAIPartnerModelsAnthropicMessagesConfig,
)
from litellm.types.router import GenericLiteLLMParams
def test_validate_environment_uses_vertex_ai_location():
@@ -248,3 +249,47 @@ def test_validate_environment_with_authorization_header_calculates_api_base():
# Verify Authorization header is still present
assert "Authorization" in updated_headers, \
"Authorization header should be preserved"
def test_transform_anthropic_messages_request_removes_scope_from_cache_control():
"""Ensure scope field is removed from cache_control for Vertex AI (not supported)."""
config = VertexAIPartnerModelsAnthropicMessagesConfig()
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Hello",
"cache_control": {"type": "ephemeral", "scope": "global"},
}
],
}
]
anthropic_messages_optional_request_params = {
"max_tokens": 1024,
"system": [
{
"type": "text",
"text": "You are an AI assistant.",
"cache_control": {"type": "ephemeral", "scope": "global"},
}
],
}
result = config.transform_anthropic_messages_request(
model="claude-sonnet-4-6",
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=GenericLiteLLMParams(),
headers={},
)
# scope removed from system
assert "scope" not in result["system"][0]["cache_control"]
assert result["system"][0]["cache_control"]["type"] == "ephemeral"
# scope removed from message content
assert "scope" not in result["messages"][0]["content"][0]["cache_control"]
assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral"
ui/litellm-dashboard/package.json
+1 -1
View File
@@ -88,7 +88,7 @@
"mermaid": ">=11.10.0",
"js-yaml": ">=4.1.1",
"glob": ">=11.1.0",
"tar": ">=7.5.10",
"tar": ">=7.5.11",
"minimatch": ">=10.2.4",
"@isaacs/brace-expansion": ">=5.0.1",
"node-forge": ">=1.3.2",
ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx
+2
View File
@@ -78,6 +78,7 @@ export function FallbackGroupConfig({
value={group.primaryModel}
onChange={handlePrimaryChange}
showSearch
getPopupContainer={(trigger) => trigger.parentElement || document.body}
filterOption={(input, option) =>
(option?.label ?? "").toLowerCase().includes(input.toLowerCase())
}
@@ -125,6 +126,7 @@ export function FallbackGroupConfig({
value={group.fallbackModels}
onChange={handleFallbackSelect}
disabled={!group.primaryModel}
getPopupContainer={(trigger) => trigger.parentElement || document.body}
options={availableFallbackOptions.map((m) => ({
label: m,
value: m,