diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 2f6da4bedc..cc05893522 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -199,6 +199,8 @@ messages=[{"role": "user", "content": [ - `include_usage` *boolean (optional)* - If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value. - `stop`: *string/ array/ null (optional)* - Up to 4 sequences where the API will stop generating further tokens. + + **Note**: OpenAI supports a maximum of 4 stop sequences. If you provide more than 4, LiteLLM will automatically truncate the list to the first 4 elements. To disable this automatic truncation, set `litellm.disable_stop_sequence_limit = True`. - `max_completion_tokens`: *integer (optional)* - An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 67fffc13e4..89e1e2910e 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -178,6 +178,7 @@ router_settings: | turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data [Proxy Logging](logging) | | modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider | | enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.| +| LITELLM_DISABLE_STOP_SEQUENCE_LIMIT | Disable validation for stop sequence limit (default: 4) | | redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) | | mcp_aliases | object | Maps friendly aliases to MCP server names for easier tool access. Only the first alias for each server is used. [MCP Aliases](../mcp#mcp-aliases) | | langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) | diff --git a/litellm/__init__.py b/litellm/__init__.py index 4c59c335ce..9455b32178 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -392,6 +392,9 @@ force_ipv4: bool = ( False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. ) +####### STOP SEQUENCE LIMIT ####### +disable_stop_sequence_limit: bool = False # when True, stop sequence limit is disabled + #### RETRIES #### num_retries: Optional[int] = None # per model endpoint max_fallbacks: Optional[int] = None diff --git a/litellm/main.py b/litellm/main.py index ea41919e19..cff0f88670 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -148,6 +148,7 @@ from litellm.utils import ( validate_and_fix_openai_messages, validate_and_fix_openai_tools, validate_chat_completion_tool_choice, + validate_openai_optional_params ) from ._logging import verbose_logger @@ -1115,6 +1116,9 @@ def completion( # type: ignore # noqa: PLR0915 tools = validate_and_fix_openai_tools(tools=tools) # validate tool_choice tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) + # validate optional params + stop = validate_openai_optional_params(stop=stop) + ######### unpacking kwargs ##################### args = locals() diff --git a/litellm/utils.py b/litellm/utils.py index 11389dd1af..0e80520b8a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7697,6 +7697,27 @@ def validate_chat_completion_tool_choice( f"Invalid tool choice, tool_choice={tool_choice}. Got={type(tool_choice)}. Expecting str, or dict. Please ensure tool_choice follows the OpenAI tool_choice spec" ) +def validate_openai_optional_params( + stop: Optional[Union[str, List[str]]] = None, + **kwargs +) -> Optional[Union[str, List[str]]]: + """ + Validates and fixes OpenAI optional parameters. + + Args: + stop: Stop sequences (string or list of strings) + **kwargs: Additional optional parameters + + Returns: + Validated stop parameter (truncated to 4 elements if needed) + """ + if stop is not None and isinstance(stop, list) and not litellm.disable_stop_sequence_limit: + # Truncate to 4 elements if more are provided as openai only supports up to 4 stop sequences + if len(stop) > 4: + stop = stop[:4] + + return stop + class ProviderConfigManager: # Dictionary mapping for O(1) provider lookup diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 95700eb29b..6386dce54a 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -24,6 +24,7 @@ from litellm.utils import ( get_optional_params_embeddings, get_optional_params_image_gen, get_requester_metadata, + validate_openai_optional_params, ) ## get_optional_params_embeddings @@ -1881,3 +1882,102 @@ def test_optional_params_responses_api_allowed_openai_params(): request_body = mock_post.call_args.kwargs print("request_body: ", request_body) assert "top_logprobs" in request_body["json"] + + +def test_validate_openai_optional_params_stop_truncation(): + """ + Test that validate_openai_optional_params truncates stop sequences to 4 elements + when more than 4 are provided, as OpenAI only supports up to 4 stop sequences. + """ + # Test with more than 4 stop sequences - should truncate to 4 + stop_sequences = ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"] + result = validate_openai_optional_params(stop=stop_sequences) + assert result == ["stop1", "stop2", "stop3", "stop4"] + assert len(result) == 4 + + # Test with exactly 4 stop sequences - should not truncate + stop_sequences_4 = ["stop1", "stop2", "stop3", "stop4"] + result = validate_openai_optional_params(stop=stop_sequences_4) + assert result == ["stop1", "stop2", "stop3", "stop4"] + assert len(result) == 4 + + # Test with less than 4 stop sequences - should not truncate + stop_sequences_2 = ["stop1", "stop2"] + result = validate_openai_optional_params(stop=stop_sequences_2) + assert result == ["stop1", "stop2"] + assert len(result) == 2 + + # Test with single stop sequence as string - should return as is + stop_string = "stop1" + result = validate_openai_optional_params(stop=stop_string) + assert result == "stop1" + + # Test with None - should return None + result = validate_openai_optional_params(stop=None) + assert result is None + + # Test with empty list - should return empty list + result = validate_openai_optional_params(stop=[]) + assert result == [] + + +def test_validate_openai_optional_params_disable_stop_sequence_limit(): + """ + Test that validate_openai_optional_params respects the disable_stop_sequence_limit flag. + When litellm.disable_stop_sequence_limit is True, stop sequences should not be truncated. + """ + # Save original value + original_value = litellm.disable_stop_sequence_limit + + try: + # Test with disable_stop_sequence_limit = True - should NOT truncate + litellm.disable_stop_sequence_limit = True + stop_sequences = ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"] + result = validate_openai_optional_params(stop=stop_sequences) + assert result == ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"] + assert len(result) == 6 + + # Test with disable_stop_sequence_limit = False - should truncate to 4 + litellm.disable_stop_sequence_limit = False + stop_sequences = ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"] + result = validate_openai_optional_params(stop=stop_sequences) + assert result == ["stop1", "stop2", "stop3", "stop4"] + assert len(result) == 4 + finally: + # Restore original value + litellm.disable_stop_sequence_limit = original_value + + +def test_validate_openai_optional_params_integration(): + """ + Test that validate_openai_optional_params is properly integrated in the completion flow. + """ + # Test that completion with more than 4 stop sequences works without error + try: + with patch("litellm.llms.openai.openai.OpenAI") as mock_client: + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.content = "Test response" + mock_response.model = "gpt-3.5-turbo" + mock_response.id = "test-id" + mock_response.created = 1234567890 + mock_response.usage = MagicMock() + mock_response.usage.prompt_tokens = 10 + mock_response.usage.completion_tokens = 5 + mock_response.usage.total_tokens = 15 + + mock_client.return_value.chat.completions.create.return_value = mock_response + + # Call completion with more than 4 stop sequences + response = litellm.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello"}], + stop=["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"], + mock_response="Test response" # This will use mock + ) + + # Verify the call was made (stop sequences should be truncated internally) + assert response is not None + except Exception as e: + # Should not raise an exception + pytest.fail(f"validate_openai_optional_params integration failed: {e}")