tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-07 04:24:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for fast param

Browse Source
This commit is contained in:
Sameer Kankute
2026-02-09 11:28:00 +05:30
parent 29e6efade9
commit b822e2e0ff
8 changed files with 389 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/anthropic/chat/handler.py
+9 -3
View File
@@ -75,6 +75,7 @@ async def make_call(
logging_obj,
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_aclient
@@ -103,6 +104,7 @@ async def make_call(
streaming_response=response.aiter_lines(),
sync_stream=False,
json_mode=json_mode,
speed=speed,
)
# LOGGING
@@ -126,6 +128,7 @@ def make_sync_call(
logging_obj,
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_client # re-use a module level client
@@ -159,7 +162,7 @@ def make_sync_call(
)
completion_stream = ModelResponseIterator(
streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode
streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode, speed=speed
)
# LOGGING
@@ -213,6 +216,7 @@ class AnthropicChatCompletion(BaseLLM):
logging_obj=logging_obj,
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
)
streamwrapper = CustomStreamWrapper(
completion_stream=completion_stream,
@@ -427,6 +431,7 @@ class AnthropicChatCompletion(BaseLLM):
logging_obj=logging_obj,
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
)
return CustomStreamWrapper(
completion_stream=completion_stream,
@@ -485,13 +490,14 @@ class AnthropicChatCompletion(BaseLLM):
class ModelResponseIterator:
def __init__(
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False, speed: Optional[str] = None
):
self.streaming_response = streaming_response
self.response_iterator = self.streaming_response
self.content_blocks: List[ContentBlockDelta] = []
self.tool_index = -1
self.json_mode = json_mode
self.speed = speed
# Generate response ID once per stream to match OpenAI-compatible behavior
self.response_id = _generate_id()
@@ -541,7 +547,7 @@ class ModelResponseIterator:
def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
return AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None
usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None, speed=self.speed
)
def _content_block_delta_helper(self, chunk: dict) -> Tuple[
litellm/llms/anthropic/chat/transformation.py
+14
View File
@@ -190,6 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"response_format",
"user",
"web_search_options",
"speed",
]
if "claude-3-7-sonnet" in model or supports_reasoning(
@@ -882,6 +883,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
elif param == "context_management" and isinstance(value, dict):
# Pass through Anthropic-specific context_management parameter
optional_params["context_management"] = value
elif param == "speed" and isinstance(value, str):
# Pass through Anthropic-specific speed parameter for fast mode
optional_params["speed"] = value
## handle thinking tokens
self.update_optional_params_with_thinking_tokens(
@@ -1096,6 +1100,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value
)
if optional_params.get("speed") == "fast":
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value
)
return headers
def transform_request(
@@ -1349,6 +1357,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
usage_object: dict,
reasoning_content: Optional[str],
completion_response: Optional[dict] = None,
speed: Optional[str] = None,
) -> Usage:
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
prompt_tokens = usage_object.get("input_tokens", 0) or 0
@@ -1447,6 +1456,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else None
),
inference_geo=inference_geo,
speed=speed,
)
return usage
@@ -1457,6 +1467,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model_response: ModelResponse,
json_mode: Optional[bool] = None,
prefix_prompt: Optional[str] = None,
speed: Optional[str] = None,
):
_hidden_params: Dict = {}
_hidden_params["additional_headers"] = process_anthropic_headers(
@@ -1553,6 +1564,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
usage_object=completion_response["usage"],
reasoning_content=reasoning_content,
completion_response=completion_response,
speed=speed,
)
setattr(model_response, "usage", usage) # type: ignore
@@ -1621,6 +1633,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
prefix_prompt = self.get_prefix_prompt(messages=messages)
speed = optional_params.get("speed")
model_response = self.transform_parsed_response(
completion_response=completion_response,
@@ -1628,6 +1641,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model_response=model_response,
json_mode=json_mode,
prefix_prompt=prefix_prompt,
speed=speed,
)
return model_response
litellm/llms/anthropic/cost_calculation.py
+10 -5
View File
@@ -22,13 +22,18 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
# If usage has inference_geo, prepend it as prefix to model name
model_with_prefix = model
# First, prepend inference_geo if present
if hasattr(usage, "inference_geo") and usage.inference_geo and usage.inference_geo.lower() not in ["global", "not_available"]:
model_with_geo_prefix = f"{usage.inference_geo}/{model}"
else:
model_with_geo_prefix = model
model_with_prefix = f"{usage.inference_geo}/{model_with_prefix}"
# Then, prepend speed if it's "fast"
if hasattr(usage, "speed") and usage.speed == "fast":
model_with_prefix = f"fast/{model_with_prefix}"
prompt_cost, completion_cost = generic_cost_per_token(
model=model_with_geo_prefix, usage=usage, custom_llm_provider="anthropic"
model=model_with_prefix, usage=usage, custom_llm_provider="anthropic"
)
return prompt_cost, completion_cost
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
+7 -1
View File
@@ -47,6 +47,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"context_management",
"output_format",
"inference_geo",
"speed",
# TODO: Add Anthropic `metadata` support
# "metadata",
]
@@ -184,10 +185,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
- context_management: adds 'context-management-2025-06-27'
- tool_search: adds provider-specific tool search header
- output_format: adds 'structured-outputs-2025-11-13'
- speed: adds 'fast-mode-2026-02-01'
Args:
headers: Request headers dict
optional_params: Optional parameters including tools, context_management, output_format
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
"""
beta_values: set = set()
@@ -224,6 +226,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("output_format") is not None:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value)
# Check for fast mode
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
# Check for tool search tools
tools = optional_params.get("tools")
if tools:
litellm/model_prices_and_context_window_backup.json
+93
View File
@@ -7663,6 +7663,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
@@ -7694,6 +7725,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/us/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
@@ -7725,6 +7787,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
litellm/types/llms/anthropic.py
+2
View File
@@ -361,6 +361,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
context_management: Optional[Dict[str, Any]]
container: Optional[Dict[str, Any]] # Container config with skills for code execution
output_format: Optional[AnthropicOutputSchema] # Structured outputs support
speed: Optional[str] # Fast mode support for Opus models
class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False):
@@ -637,6 +638,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
COMPACT_2026_01_12 = "compact-2026-01-12"
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"
# Tool search beta header constant (for Anthropic direct API and Microsoft Foundry)
model_prices_and_context_window.json
+93
View File
@@ -7663,6 +7663,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
@@ -7694,6 +7725,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/us/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
@@ -7725,6 +7787,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
+161
View File
@@ -2506,3 +2506,164 @@ def test_compaction_block_empty_list_not_added():
provider_fields = result.choices[0].message.provider_specific_fields
if provider_fields:
assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None
def test_fast_mode_beta_header():
"""
Test that fast mode correctly adds the fast-mode-2026-02-01 beta header.
"""
config = AnthropicConfig()
headers = {}
optional_params = {"speed": "fast"}
result_headers = config.update_headers_with_optional_anthropic_beta(
headers=headers,
optional_params=optional_params
)
assert "anthropic-beta" in result_headers
assert "fast-mode-2026-02-01" in result_headers["anthropic-beta"]
def test_fast_mode_with_other_beta_headers():
"""
Test that fast mode beta header is combined with other beta headers.
"""
config = AnthropicConfig()
headers = {}
optional_params = {
"speed": "fast",
"output_format": {"type": "json_object"}
}
result_headers = config.update_headers_with_optional_anthropic_beta(
headers=headers,
optional_params=optional_params
)
assert "anthropic-beta" in result_headers
assert "fast-mode-2026-02-01" in result_headers["anthropic-beta"]
assert "structured-outputs-2025-11-13" in result_headers["anthropic-beta"]
def test_fast_mode_usage_calculation():
"""
Test that fast mode speed parameter is passed through to usage object.
"""
config = AnthropicConfig()
usage_object = {
"input_tokens": 1000,
"output_tokens": 500,
}
usage = config.calculate_usage(
usage_object=usage_object,
reasoning_content=None,
speed="fast"
)
assert usage.prompt_tokens == 1000
assert usage.completion_tokens == 500
assert hasattr(usage, "speed")
assert usage.speed == "fast"
def test_fast_mode_cost_calculation():
"""
Test that fast mode correctly prepends 'fast/' to model name for pricing lookup.
"""
from unittest.mock import patch
from litellm.llms.anthropic.cost_calculation import cost_per_token
from litellm.types.utils import Usage
# Mock the generic_cost_per_token to verify correct model name is passed
with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost:
mock_cost.return_value = (0.03, 0.15) # $30 and $150 per MTok
# Test fast mode
usage_fast = Usage(
prompt_tokens=1000,
completion_tokens=1000,
speed="fast"
)
prompt_cost, completion_cost = cost_per_token(
model="claude-opus-4-6",
usage=usage_fast
)
# Verify that generic_cost_per_token was called with "fast/claude-opus-4-6"
mock_cost.assert_called_once()
call_args = mock_cost.call_args
assert call_args[1]['model'] == "fast/claude-opus-4-6"
assert call_args[1]['custom_llm_provider'] == "anthropic"
def test_fast_mode_with_inference_geo():
"""
Test that fast mode works correctly with inference_geo prefix.
Expected format: fast/us/claude-opus-4-6
"""
from unittest.mock import patch
from litellm.llms.anthropic.cost_calculation import cost_per_token
from litellm.types.utils import Usage
# Mock the generic_cost_per_token to verify correct model name is passed
with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost:
mock_cost.return_value = (0.03, 0.15)
# Test with both speed and inference_geo
usage = Usage(
prompt_tokens=1000,
completion_tokens=1000,
speed="fast",
inference_geo="us"
)
# This should look up "fast/us/claude-opus-4-6" in pricing
prompt_cost, completion_cost = cost_per_token(
model="claude-opus-4-6",
usage=usage
)
# Verify that generic_cost_per_token was called with "fast/us/claude-opus-4-6"
mock_cost.assert_called_once()
call_args = mock_cost.call_args
assert call_args[1]['model'] == "fast/us/claude-opus-4-6"
assert call_args[1]['custom_llm_provider'] == "anthropic"
def test_fast_mode_parameter_in_supported_params():
"""
Test that 'speed' is in the list of supported OpenAI params.
"""
config = AnthropicConfig()
supported_params = config.get_supported_openai_params(model="claude-opus-4-6")
assert "speed" in supported_params
def test_fast_mode_parameter_mapping():
"""
Test that speed parameter is correctly mapped in map_openai_params.
"""
config = AnthropicConfig()
non_default_params = {"speed": "fast"}
optional_params = {}
result = config.map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model="claude-opus-4-6",
drop_params=False
)
assert "speed" in result
assert result["speed"] == "fast"