tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-06 00:23:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14121 from BerriAI/litellm_dev_08_31_2025_p1

Browse Source 
VLLM - handle output parsing responses api output + Ollama - add unified 'thinking' param support (via `reasoning_content`)
This commit is contained in:
Krish Dholakia
2025-09-01 22:43:05 -07:00
committed by GitHub
parent 6d36219a4d 82d6c4d5b1
commit 359780b436
7 changed files with 443 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/ollama/chat/transformation.py
+64 -6
View File
@@ -137,6 +137,7 @@ class OllamaChatConfig(BaseConfig):
"tool_choice",
"functions",
"response_format",
"reasoning_effort",
]
def map_openai_params(
@@ -175,6 +176,8 @@ class OllamaChatConfig(BaseConfig):
if value.get("json_schema") and value["json_schema"].get("schema"):
optional_params["format"] = value["json_schema"]["schema"]
### FUNCTION CALLING LOGIC ###
if param == "reasoning_effort" and value is not None:
optional_params["think"] = True
if param == "tools":
## CHECK IF MODEL SUPPORTS TOOL CALLING ##
try:
@@ -212,9 +215,9 @@ class OllamaChatConfig(BaseConfig):
litellm.add_function_to_prompt = (
True # so that main.py adds the function call to the prompt
)
optional_params[
"functions_unsupported_model"
] = non_default_params.get("functions")
optional_params["functions_unsupported_model"] = (
non_default_params.get("functions")
)
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
non_default_params.pop("functions", None) # causes ollama requests to hang
return optional_params
@@ -346,11 +349,31 @@ class OllamaChatConfig(BaseConfig):
## RESPONSE OBJECT
model_response.choices[0].finish_reason = "stop"
response_json_message = response_json.get("message")
if response_json_message is not None:
if "thinking" in response_json_message:
# remap 'thinking' to 'reasoning_content'
response_json_message["reasoning_content"] = response_json_message[
"thinking"
]
del response_json_message["thinking"]
elif response_json_message.get("content") is not None:
# parse reasoning content from content
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_parse_content_for_reasoning,
)
reasoning_content, content = _parse_content_for_reasoning(
response_json_message["content"]
)
response_json_message["reasoning_content"] = reasoning_content
response_json_message["content"] = content
if (
request_data.get("format", "") == "json"
and litellm_params.get("function_name") is not None
):
function_call = json.loads(response_json["message"]["content"])
function_call = json.loads(response_json_message["content"])
message = litellm.Message(
content=None,
tool_calls=[
@@ -367,11 +390,13 @@ class OllamaChatConfig(BaseConfig):
"type": "function",
}
],
reasoning_content=response_json_message.get("reasoning_content"),
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "tool_calls"
else:
_message = litellm.Message(**response_json["message"])
_message = litellm.Message(**response_json_message)
model_response.choices[0].message = _message # type: ignore
model_response.created = int(time.time())
model_response.model = "ollama_chat/" + model
@@ -412,6 +437,9 @@ class OllamaChatConfig(BaseConfig):
class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
started_reasoning_content: bool = False
finished_reasoning_content: bool = False
def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool:
if isinstance(function_args, dict):
return True
@@ -465,8 +493,38 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
if is_function_call_complete:
tool_call["id"] = str(uuid.uuid4())
# PROCESS REASONING CONTENT
reasoning_content: Optional[str] = None
content: Optional[str] = None
if chunk["message"].get("thinking") is not None:
if self.started_reasoning_content is False:
reasoning_content = chunk["message"].get("thinking")
self.started_reasoning_content = True
elif self.finished_reasoning_content is False:
reasoning_content = chunk["message"].get("thinking")
self.finished_reasoning_content = True
elif chunk["message"].get("content") is not None:
message_content = chunk["message"].get("content")
if "<think>" in message_content:
message_content = message_content.replace("<think>", "")
self.started_reasoning_content = True
if "</think>" in message_content and self.started_reasoning_content:
message_content = message_content.replace("</think>", "")
self.finished_reasoning_content = True
if (
self.started_reasoning_content
and not self.finished_reasoning_content
):
reasoning_content = message_content
else:
content = message_content
delta = Delta(
content=chunk["message"].get("content", ""),
content=content,
reasoning_content=reasoning_content,
tool_calls=tool_calls,
)
litellm/llms/ollama/completion/transformation.py
+83 -19
View File
@@ -19,13 +19,13 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
from litellm.types.utils import (
Delta,
GenericStreamingChunk,
ModelInfoBase,
ModelResponse,
ModelResponseStream,
ProviderField,
StreamingChoices,
Delta,
)
from ..common_utils import OllamaError, _convert_image
@@ -92,9 +92,9 @@ class OllamaConfig(BaseConfig):
repeat_penalty: Optional[float] = None
temperature: Optional[float] = None
seed: Optional[int] = None
stop: Optional[
list
] = None # stop is a list based on this - https://github.com/ollama/ollama/pull/442
stop: Optional[list] = (
None # stop is a list based on this - https://github.com/ollama/ollama/pull/442
)
tfs_z: Optional[float] = None
num_predict: Optional[int] = None
top_k: Optional[int] = None
@@ -154,6 +154,7 @@ class OllamaConfig(BaseConfig):
"stop",
"response_format",
"max_completion_tokens",
"reasoning_effort",
]
def map_openai_params(
@@ -166,19 +167,21 @@ class OllamaConfig(BaseConfig):
for param, value in non_default_params.items():
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["num_predict"] = value
if param == "stream":
elif param == "stream":
optional_params["stream"] = value
if param == "temperature":
elif param == "temperature":
optional_params["temperature"] = value
if param == "seed":
elif param == "seed":
optional_params["seed"] = value
if param == "top_p":
elif param == "top_p":
optional_params["top_p"] = value
if param == "frequency_penalty":
elif param == "frequency_penalty":
optional_params["frequency_penalty"] = value
if param == "stop":
elif param == "stop":
optional_params["stop"] = value
if param == "response_format" and isinstance(value, dict):
elif param == "reasoning_effort" and value is not None:
optional_params["think"] = True
elif param == "response_format" and isinstance(value, dict):
if value["type"] == "json_object":
optional_params["format"] = "json"
elif value["type"] == "json_schema":
@@ -258,12 +261,17 @@ class OllamaConfig(BaseConfig):
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_parse_content_for_reasoning,
)
response_json = raw_response.json()
## RESPONSE OBJECT
model_response.choices[0].finish_reason = "stop"
if request_data.get("format", "") == "json":
# Check if response field exists and is not empty before parsing JSON
response_text = response_json.get("response", "")
if not response_text or not response_text.strip():
# Handle empty response gracefully - set empty content
message = litellm.Message(content="")
@@ -288,7 +296,9 @@ class OllamaConfig(BaseConfig):
"id": f"call_{str(uuid.uuid4())}",
"function": {
"name": function_call["name"],
"arguments": json.dumps(function_call["arguments"]),
"arguments": json.dumps(
function_call["arguments"]
),
},
"type": "function",
}
@@ -305,11 +315,26 @@ class OllamaConfig(BaseConfig):
model_response.choices[0].finish_reason = "stop"
except json.JSONDecodeError:
# If JSON parsing fails, treat as regular text response
message = litellm.Message(content=response_text)
## output parse reasoning content from response_text
reasoning_content: Optional[str] = None
content: Optional[str] = None
if response_text is not None:
reasoning_content, content = _parse_content_for_reasoning(
response_text
)
message = litellm.Message(
content=content, reasoning_content=reasoning_content
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "stop"
else:
model_response.choices[0].message.content = response_json["response"] # type: ignore
response_text = response_json.get("response", "")
content = None
reasoning_content = None
if response_text is not None:
reasoning_content, content = _parse_content_for_reasoning(response_text)
model_response.choices[0].message.content = content # type: ignore
model_response.choices[0].message.reasoning_content = reasoning_content # type: ignore
model_response.created = int(time.time())
model_response.model = "ollama/" + model
_prompt = request_data.get("prompt", "")
@@ -434,12 +459,21 @@ class OllamaConfig(BaseConfig):
class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
def __init__(
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
):
super().__init__(streaming_response, sync_stream, json_mode)
self.started_reasoning_content: bool = False
self.finished_reasoning_content: bool = False
def _handle_string_chunk(
self, str_line: str
) -> Union[GenericStreamingChunk, ModelResponseStream]:
return self.chunk_parser(json.loads(str_line))
def chunk_parser(self, chunk: dict) -> Union[GenericStreamingChunk, ModelResponseStream]:
def chunk_parser(
self, chunk: dict
) -> Union[GenericStreamingChunk, ModelResponseStream]:
try:
if "error" in chunk:
raise Exception(f"Ollama Error - {chunk}")
@@ -469,12 +503,42 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
)
elif chunk["response"]:
text = chunk["response"]
return GenericStreamingChunk(
text=text,
is_finished=is_finished,
finish_reason="stop",
reasoning_content: Optional[str] = None
content: Optional[str] = None
if text is not None:
if "<think>" in text:
text = text.replace("<think>", "")
self.started_reasoning_content = True
elif "</think>" in text:
text = text.replace("</think>", "")
self.finished_reasoning_content = True
if (
self.started_reasoning_content
and not self.finished_reasoning_content
):
reasoning_content = text
else:
content = text
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
reasoning_content=reasoning_content, content=content
),
)
],
finish_reason=finish_reason,
usage=None,
)
# return GenericStreamingChunk(
# text=text,
# is_finished=is_finished,
# finish_reason="stop",
# usage=None,
# )
elif "thinking" in chunk and not chunk["response"]:
# Return reasoning content as ModelResponseStream so UIs can render it
thinking_content = chunk.get("thinking") or ""
litellm/proxy/_experimental/out/model_hub_table.html → litellm/proxy/_experimental/out/model_hub_table/index.html
View File
litellm/proxy/_experimental/out/onboarding.html
-1
View File
File diff suppressed because one or more lines are too long
litellm/proxy/_new_secret_config.yaml
+21 -16
View File
@@ -1,22 +1,27 @@
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
- model_name: gpt-5-mini
litellm_params:
model: azure/gpt-5-mini
api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
stream_timeout: 60
merge_reasoning_content_in_choices: true
model_info:
mode: chat
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
- model_name: gpt-5-mini
litellm_params:
model: azure/gpt-5-mini
api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
stream_timeout: 60
merge_reasoning_content_in_choices: true
model_info:
mode: chat
- model_name: ollama-deepseek-r1
litellm_params:
model: ollama/deepseek-r1:1.5b
model_info:
mode: chat
router_settings:
model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"}
litellm_settings:
callbacks: ["otel"]
success_callback: ["braintrust"]
success_callback: ["braintrust"]
litellm/types/llms/openai.py
+15 -15
View File
@@ -1029,29 +1029,29 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject):
class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
id: str
created_at: int
error: Optional[dict]
incomplete_details: Optional[IncompleteDetails]
instructions: Optional[str]
metadata: Optional[Dict]
model: Optional[str]
object: Optional[str]
error: Optional[dict] = None
incomplete_details: Optional[IncompleteDetails] = None
instructions: Optional[str] = None
metadata: Optional[Dict] = None
model: Optional[str] = None
object: Optional[str] = None
output: Union[
List[Union[ResponseOutputItem, Dict]],
List[Union[GenericResponseOutputItem, OutputFunctionToolCall]],
]
parallel_tool_calls: bool
temperature: Optional[float]
temperature: Optional[float] = None
tool_choice: ToolChoice
tools: Union[List[Tool], List[ResponseFunctionToolCall], List[Dict[str, Any]]]
top_p: Optional[float]
max_output_tokens: Optional[int]
previous_response_id: Optional[str]
reasoning: Optional[Reasoning]
status: Optional[str]
text: Optional[Union["ResponseText", Dict[str, Any]]]
truncation: Optional[Literal["auto", "disabled"]]
usage: Optional[ResponseAPIUsage]
user: Optional[str]
max_output_tokens: Optional[int] = None
previous_response_id: Optional[str] = None
reasoning: Optional[Reasoning] = None
status: Optional[str] = None
text: Optional[Union["ResponseText", Dict[str, Any]]] = None
truncation: Optional[Literal["auto", "disabled"]] = None
usage: Optional[ResponseAPIUsage] = None
user: Optional[str] = None
store: Optional[bool] = None
# Define private attributes using PrivateAttr
_hidden_params: dict = PrivateAttr(default_factory=dict)
tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py
+260 -4
View File
@@ -159,6 +159,261 @@ class TestOllamaConfig:
assert result.choices[0]["finish_reason"] == "stop"
# No usage assertions here as we don't need to test them in every case
def test_transform_response_with_thinking_tags(self):
"""Test that responses with <think>...</think> tags parse reasoning content correctly."""
# Initialize config
config = OllamaConfig()
# Create mock response with thinking tags
raw_response = MagicMock()
raw_response.json.return_value = {
"response": "<think>I need to think about this problem step by step</think>Here is my answer",
"prompt_eval_count": 15,
"eval_count": 8,
}
# Create properly structured model response object
model_response = ModelResponse(
id="test_id",
choices=[{"message": Message(content="")}],
)
# Create mock encoding
mock_encoding = MagicMock()
mock_encoding.encode.return_value = [1, 2, 3]
# Transform response
result = config.transform_response(
model="llama2",
raw_response=raw_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=mock_encoding,
)
# Verify reasoning content is extracted
assert (
result.choices[0]["message"].reasoning_content
== "I need to think about this problem step by step"
)
assert result.choices[0]["message"].content == "Here is my answer"
assert result.choices[0]["finish_reason"] == "stop"
def test_transform_response_with_thinking_tags_alternative(self):
"""Test that responses with <thinking>...</thinking> tags parse reasoning content correctly."""
# Initialize config
config = OllamaConfig()
# Create mock response with thinking tags (alternative format)
raw_response = MagicMock()
raw_response.json.return_value = {
"response": "<thinking>Let me analyze this carefully</thinking>The solution is X",
}
# Create properly structured model response object
model_response = ModelResponse(
id="test_id",
choices=[{"message": Message(content="")}],
)
# Create mock encoding
mock_encoding = MagicMock()
mock_encoding.encode.return_value = [1, 2, 3]
# Transform response
result = config.transform_response(
model="llama2",
raw_response=raw_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=mock_encoding,
)
# Verify reasoning content is extracted
assert (
result.choices[0]["message"].reasoning_content
== "Let me analyze this carefully"
)
assert result.choices[0]["message"].content == "The solution is X"
assert result.choices[0]["finish_reason"] == "stop"
def test_transform_response_with_multiline_thinking_tags(self):
"""Test that responses with multiline thinking content work correctly."""
# Initialize config
config = OllamaConfig()
# Create mock response with multiline thinking content
raw_response = MagicMock()
raw_response.json.return_value = {
"response": "<think>\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\n</think>Based on my analysis, the answer is Y",
}
# Create properly structured model response object
model_response = ModelResponse(
id="test_id",
choices=[{"message": Message(content="")}],
)
# Create mock encoding
mock_encoding = MagicMock()
mock_encoding.encode.return_value = [1, 2, 3]
# Transform response
result = config.transform_response(
model="llama2",
raw_response=raw_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=mock_encoding,
)
# Verify multiline reasoning content is extracted
expected_reasoning = "\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\n"
assert result.choices[0]["message"].reasoning_content == expected_reasoning
assert (
result.choices[0]["message"].content
== "Based on my analysis, the answer is Y"
)
assert result.choices[0]["finish_reason"] == "stop"
def test_transform_response_thinking_only(self):
"""Test response with only thinking content and no additional content."""
# Initialize config
config = OllamaConfig()
# Create mock response with only thinking content
raw_response = MagicMock()
raw_response.json.return_value = {
"response": "<think>Just internal thoughts, no response</think>",
}
# Create properly structured model response object
model_response = ModelResponse(
id="test_id",
choices=[{"message": Message(content="")}],
)
# Create mock encoding
mock_encoding = MagicMock()
mock_encoding.encode.return_value = [1, 2, 3]
# Transform response
result = config.transform_response(
model="llama2",
raw_response=raw_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=mock_encoding,
)
# Verify reasoning content is extracted and content is empty
assert (
result.choices[0]["message"].reasoning_content
== "Just internal thoughts, no response"
)
assert result.choices[0]["message"].content == ""
assert result.choices[0]["finish_reason"] == "stop"
def test_transform_response_json_mode_with_thinking_tags(self):
"""Test JSON mode with thinking tags - should handle as text when JSON parsing fails."""
# Initialize config
config = OllamaConfig()
# Create mock response with thinking tags in JSON mode
raw_response = MagicMock()
raw_response.json.return_value = {
"response": "<think>Planning my JSON response</think>This is not valid JSON",
}
# Create properly structured model response object
model_response = ModelResponse(
id="test_id",
choices=[{"message": Message(content="")}],
)
# Create mock encoding
mock_encoding = MagicMock()
mock_encoding.encode.return_value = [1, 2, 3]
# Transform response
result = config.transform_response(
model="llama2",
raw_response=raw_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data={"format": "json"},
messages=[],
optional_params={},
litellm_params={},
encoding=mock_encoding,
)
# Verify reasoning content is extracted even in JSON mode when JSON parsing fails
assert (
result.choices[0]["message"].reasoning_content
== "Planning my JSON response"
)
assert result.choices[0]["message"].content == "This is not valid JSON"
assert result.choices[0]["finish_reason"] == "stop"
def test_transform_response_no_thinking_tags(self):
"""Test that responses without thinking tags work normally."""
# Initialize config
config = OllamaConfig()
# Create mock response without thinking tags
raw_response = MagicMock()
raw_response.json.return_value = {
"response": "Regular response without any thinking tags",
}
# Create properly structured model response object
model_response = ModelResponse(
id="test_id",
choices=[{"message": Message(content="")}],
)
# Create mock encoding
mock_encoding = MagicMock()
mock_encoding.encode.return_value = [1, 2, 3]
# Transform response
result = config.transform_response(
model="llama2",
raw_response=raw_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=mock_encoding,
)
# Verify no reasoning content is extracted
assert result.choices[0]["message"].reasoning_content is None
assert (
result.choices[0]["message"].content
== "Regular response without any thinking tags"
)
assert result.choices[0]["finish_reason"] == "stop"
class TestOllamaTextCompletionResponseIterator:
def test_chunk_parser_with_thinking_field(self):
@@ -199,10 +454,11 @@ class TestOllamaTextCompletionResponseIterator:
result = iterator.chunk_parser(normal_chunk)
assert result["text"] == "Hello world"
assert result["is_finished"] is False
assert result["finish_reason"] == "stop"
assert result["usage"] is None
# Updated to handle ModelResponseStream return type
assert isinstance(result, ModelResponseStream)
assert result.choices and result.choices[0].delta is not None
assert result.choices[0].delta.content == "Hello world"
assert getattr(result.choices[0].delta, "reasoning_content", None) is None
def test_chunk_parser_done_chunk(self):
"""Test that done chunks work correctly."""