diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 485e95d648..e85c0d0d01 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -75,6 +75,7 @@ async def make_call( logging_obj, timeout: Optional[Union[float, httpx.Timeout]], json_mode: bool, + speed: Optional[str] = None, ) -> Tuple[Any, httpx.Headers]: if client is None: client = litellm.module_level_aclient @@ -103,6 +104,7 @@ async def make_call( streaming_response=response.aiter_lines(), sync_stream=False, json_mode=json_mode, + speed=speed, ) # LOGGING @@ -126,6 +128,7 @@ def make_sync_call( logging_obj, timeout: Optional[Union[float, httpx.Timeout]], json_mode: bool, + speed: Optional[str] = None, ) -> Tuple[Any, httpx.Headers]: if client is None: client = litellm.module_level_client # re-use a module level client @@ -159,7 +162,7 @@ def make_sync_call( ) completion_stream = ModelResponseIterator( - streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode + streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode, speed=speed ) # LOGGING @@ -213,6 +216,7 @@ class AnthropicChatCompletion(BaseLLM): logging_obj=logging_obj, timeout=timeout, json_mode=json_mode, + speed=optional_params.get("speed") if optional_params else None, ) streamwrapper = CustomStreamWrapper( completion_stream=completion_stream, @@ -427,6 +431,7 @@ class AnthropicChatCompletion(BaseLLM): logging_obj=logging_obj, timeout=timeout, json_mode=json_mode, + speed=optional_params.get("speed") if optional_params else None, ) return CustomStreamWrapper( completion_stream=completion_stream, @@ -485,13 +490,14 @@ class AnthropicChatCompletion(BaseLLM): class ModelResponseIterator: def __init__( - self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False + self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False, speed: Optional[str] = None ): self.streaming_response = streaming_response self.response_iterator = self.streaming_response self.content_blocks: List[ContentBlockDelta] = [] self.tool_index = -1 self.json_mode = json_mode + self.speed = speed # Generate response ID once per stream to match OpenAI-compatible behavior self.response_id = _generate_id() @@ -541,7 +547,7 @@ class ModelResponseIterator: def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage: return AnthropicConfig().calculate_usage( - usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None + usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None, speed=self.speed ) def _content_block_delta_helper(self, chunk: dict) -> Tuple[ diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 02b8d95244..82aa739018 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -190,6 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "response_format", "user", "web_search_options", + "speed", ] if "claude-3-7-sonnet" in model or supports_reasoning( @@ -882,6 +883,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): elif param == "context_management" and isinstance(value, dict): # Pass through Anthropic-specific context_management parameter optional_params["context_management"] = value + elif param == "speed" and isinstance(value, str): + # Pass through Anthropic-specific speed parameter for fast mode + optional_params["speed"] = value ## handle thinking tokens self.update_optional_params_with_thinking_tokens( @@ -1096,6 +1100,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self._ensure_beta_header( headers, ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value ) + if optional_params.get("speed") == "fast": + self._ensure_beta_header( + headers, ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value + ) return headers def transform_request( @@ -1349,6 +1357,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None, + speed: Optional[str] = None, ) -> Usage: # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this prompt_tokens = usage_object.get("input_tokens", 0) or 0 @@ -1447,6 +1456,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else None ), inference_geo=inference_geo, + speed=speed, ) return usage @@ -1457,6 +1467,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response: ModelResponse, json_mode: Optional[bool] = None, prefix_prompt: Optional[str] = None, + speed: Optional[str] = None, ): _hidden_params: Dict = {} _hidden_params["additional_headers"] = process_anthropic_headers( @@ -1553,6 +1564,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): usage_object=completion_response["usage"], reasoning_content=reasoning_content, completion_response=completion_response, + speed=speed, ) setattr(model_response, "usage", usage) # type: ignore @@ -1621,6 +1633,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) prefix_prompt = self.get_prefix_prompt(messages=messages) + speed = optional_params.get("speed") model_response = self.transform_parsed_response( completion_response=completion_response, @@ -1628,6 +1641,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response=model_response, json_mode=json_mode, prefix_prompt=prefix_prompt, + speed=speed, ) return model_response diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 11b61cc92f..271406f2f7 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -22,13 +22,18 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - # If usage has inference_geo, prepend it as prefix to model name + model_with_prefix = model + + # First, prepend inference_geo if present if hasattr(usage, "inference_geo") and usage.inference_geo and usage.inference_geo.lower() not in ["global", "not_available"]: - model_with_geo_prefix = f"{usage.inference_geo}/{model}" - else: - model_with_geo_prefix = model + model_with_prefix = f"{usage.inference_geo}/{model_with_prefix}" + + # Then, prepend speed if it's "fast" + if hasattr(usage, "speed") and usage.speed == "fast": + model_with_prefix = f"fast/{model_with_prefix}" + prompt_cost, completion_cost = generic_cost_per_token( - model=model_with_geo_prefix, usage=usage, custom_llm_provider="anthropic" + model=model_with_prefix, usage=usage, custom_llm_provider="anthropic" ) return prompt_cost, completion_cost diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 145a715713..9e28c13968 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -47,6 +47,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "context_management", "output_format", "inference_geo", + "speed", # TODO: Add Anthropic `metadata` support # "metadata", ] @@ -184,10 +185,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): - context_management: adds 'context-management-2025-06-27' - tool_search: adds provider-specific tool search header - output_format: adds 'structured-outputs-2025-11-13' + - speed: adds 'fast-mode-2026-02-01' Args: headers: Request headers dict - optional_params: Optional parameters including tools, context_management, output_format + optional_params: Optional parameters including tools, context_management, output_format, speed custom_llm_provider: Provider name for looking up correct tool search header """ beta_values: set = set() @@ -224,6 +226,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if optional_params.get("output_format") is not None: beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value) + # Check for fast mode + if optional_params.get("speed") == "fast": + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value) + # Check for tool search tools tools = optional_params.get("tools") if tools: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 45dbf14ffc..543f2f14d7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7663,6 +7663,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, @@ -7694,6 +7725,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/us/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1.1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 4.125e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, @@ -7725,6 +7787,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6-20260205": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index d5172c7a26..84ac01a4ec 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -361,6 +361,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): context_management: Optional[Dict[str, Any]] container: Optional[Dict[str, Any]] # Container config with skills for code execution output_format: Optional[AnthropicOutputSchema] # Structured outputs support + speed: Optional[str] # Fast mode support for Opus models class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): @@ -637,6 +638,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): COMPACT_2026_01_12 = "compact-2026-01-12" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" + FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 45dbf14ffc..543f2f14d7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7663,6 +7663,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, @@ -7694,6 +7725,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/us/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1.1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 4.125e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, @@ -7725,6 +7787,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6-20260205": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 49db7367c6..e3bd7d2bb3 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2506,3 +2506,164 @@ def test_compaction_block_empty_list_not_added(): provider_fields = result.choices[0].message.provider_specific_fields if provider_fields: assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None + + +def test_fast_mode_beta_header(): + """ + Test that fast mode correctly adds the fast-mode-2026-02-01 beta header. + """ + config = AnthropicConfig() + + headers = {} + optional_params = {"speed": "fast"} + + result_headers = config.update_headers_with_optional_anthropic_beta( + headers=headers, + optional_params=optional_params + ) + + assert "anthropic-beta" in result_headers + assert "fast-mode-2026-02-01" in result_headers["anthropic-beta"] + + +def test_fast_mode_with_other_beta_headers(): + """ + Test that fast mode beta header is combined with other beta headers. + """ + config = AnthropicConfig() + + headers = {} + optional_params = { + "speed": "fast", + "output_format": {"type": "json_object"} + } + + result_headers = config.update_headers_with_optional_anthropic_beta( + headers=headers, + optional_params=optional_params + ) + + assert "anthropic-beta" in result_headers + assert "fast-mode-2026-02-01" in result_headers["anthropic-beta"] + assert "structured-outputs-2025-11-13" in result_headers["anthropic-beta"] + + +def test_fast_mode_usage_calculation(): + """ + Test that fast mode speed parameter is passed through to usage object. + """ + config = AnthropicConfig() + + usage_object = { + "input_tokens": 1000, + "output_tokens": 500, + } + + usage = config.calculate_usage( + usage_object=usage_object, + reasoning_content=None, + speed="fast" + ) + + assert usage.prompt_tokens == 1000 + assert usage.completion_tokens == 500 + assert hasattr(usage, "speed") + assert usage.speed == "fast" + + +def test_fast_mode_cost_calculation(): + """ + Test that fast mode correctly prepends 'fast/' to model name for pricing lookup. + """ + from unittest.mock import patch + + from litellm.llms.anthropic.cost_calculation import cost_per_token + from litellm.types.utils import Usage + + # Mock the generic_cost_per_token to verify correct model name is passed + with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost: + mock_cost.return_value = (0.03, 0.15) # $30 and $150 per MTok + + # Test fast mode + usage_fast = Usage( + prompt_tokens=1000, + completion_tokens=1000, + speed="fast" + ) + + prompt_cost, completion_cost = cost_per_token( + model="claude-opus-4-6", + usage=usage_fast + ) + + # Verify that generic_cost_per_token was called with "fast/claude-opus-4-6" + mock_cost.assert_called_once() + call_args = mock_cost.call_args + assert call_args[1]['model'] == "fast/claude-opus-4-6" + assert call_args[1]['custom_llm_provider'] == "anthropic" + + +def test_fast_mode_with_inference_geo(): + """ + Test that fast mode works correctly with inference_geo prefix. + Expected format: fast/us/claude-opus-4-6 + """ + from unittest.mock import patch + + from litellm.llms.anthropic.cost_calculation import cost_per_token + from litellm.types.utils import Usage + + # Mock the generic_cost_per_token to verify correct model name is passed + with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost: + mock_cost.return_value = (0.03, 0.15) + + # Test with both speed and inference_geo + usage = Usage( + prompt_tokens=1000, + completion_tokens=1000, + speed="fast", + inference_geo="us" + ) + + # This should look up "fast/us/claude-opus-4-6" in pricing + prompt_cost, completion_cost = cost_per_token( + model="claude-opus-4-6", + usage=usage + ) + + # Verify that generic_cost_per_token was called with "fast/us/claude-opus-4-6" + mock_cost.assert_called_once() + call_args = mock_cost.call_args + assert call_args[1]['model'] == "fast/us/claude-opus-4-6" + assert call_args[1]['custom_llm_provider'] == "anthropic" + + +def test_fast_mode_parameter_in_supported_params(): + """ + Test that 'speed' is in the list of supported OpenAI params. + """ + config = AnthropicConfig() + + supported_params = config.get_supported_openai_params(model="claude-opus-4-6") + + assert "speed" in supported_params + + +def test_fast_mode_parameter_mapping(): + """ + Test that speed parameter is correctly mapped in map_openai_params. + """ + config = AnthropicConfig() + + non_default_params = {"speed": "fast"} + optional_params = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="claude-opus-4-6", + drop_params=False + ) + + assert "speed" in result + assert result["speed"] == "fast"