diff --git a/docs/my-website/docs/providers/moonshot.md b/docs/my-website/docs/providers/moonshot.md new file mode 100644 index 0000000000..ee4f04bd7a --- /dev/null +++ b/docs/my-website/docs/providers/moonshot.md @@ -0,0 +1,226 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Moonshot AI + +## Overview + +| Property | Details | +|-------|-------| +| Description | Moonshot AI provides large language models including the moonshot-v1 series and kimi models. | +| Provider Route on LiteLLM | `moonshot/` | +| Link to Provider Doc | [Moonshot AI ↗](https://platform.moonshot.ai/) | +| Base URL | `https://api.moonshot.cn/` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +https://platform.moonshot.ai/ + +**We support ALL Moonshot AI models, just set `moonshot/` as a prefix when sending completion requests** + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key +``` + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Moonshot Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Moonshot call +response = completion( + model="moonshot/moonshot-v1-8k", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Moonshot Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Moonshot call with streaming +response = completion( + model="moonshot/moonshot-v1-8k", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +## Usage - LiteLLM Proxy + +Add the following to your LiteLLM Proxy configuration file: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: moonshot-v1-8k + litellm_params: + model: moonshot/moonshot-v1-8k + api_key: os.environ/MOONSHOT_API_KEY + + - model_name: moonshot-v1-32k + litellm_params: + model: moonshot/moonshot-v1-32k + api_key: os.environ/MOONSHOT_API_KEY + + - model_name: moonshot-v1-128k + litellm_params: + model: moonshot/moonshot-v1-128k + api_key: os.environ/MOONSHOT_API_KEY +``` + +Start your LiteLLM Proxy server: + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + + + + +```python showLineNumbers title="Moonshot via Proxy - Non-streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Non-streaming response +response = client.chat.completions.create( + model="moonshot-v1-8k", + messages=[{"role": "user", "content": "hello from litellm"}] +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Moonshot via Proxy - Streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Streaming response +response = client.chat.completions.create( + model="moonshot-v1-8k", + messages=[{"role": "user", "content": "hello from litellm"}], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```python showLineNumbers title="Moonshot via Proxy - LiteLLM SDK" +import litellm + +# Configure LiteLLM to use your proxy +response = litellm.completion( + model="litellm_proxy/moonshot-v1-8k", + messages=[{"role": "user", "content": "hello from litellm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key" +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Moonshot via Proxy - LiteLLM SDK Streaming" +import litellm + +# Configure LiteLLM to use your proxy with streaming +response = litellm.completion( + model="litellm_proxy/moonshot-v1-8k", + messages=[{"role": "user", "content": "hello from litellm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key", + stream=True +) + +for chunk in response: + if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```bash showLineNumbers title="Moonshot via Proxy - cURL" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "moonshot-v1-8k", + "messages": [{"role": "user", "content": "hello from litellm"}] + }' +``` + +```bash showLineNumbers title="Moonshot via Proxy - cURL Streaming" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "moonshot-v1-8k", + "messages": [{"role": "user", "content": "hello from litellm"}], + "stream": true + }' +``` + + + + +For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy). + +## Moonshot AI Limitations & LiteLLM Handling + +LiteLLM automatically handles the following [Moonshot AI limitations](https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-api-compatibility) to provide seamless OpenAI compatibility: + +### Temperature Range Limitation +**Limitation**: Moonshot AI only supports temperature range [0, 1] (vs OpenAI's [0, 2]) +**LiteLLM Handling**: Automatically clamps any temperature > 1 to 1 + +### Temperature + Multiple Outputs Limitation +**Limitation**: If temperature < 0.3 and n > 1, Moonshot AI raises an exception +**LiteLLM Handling**: Automatically sets temperature to 0.3 when this condition is detected + +### Tool Choice "Required" Not Supported +**Limitation**: Moonshot AI doesn't support `tool_choice="required"` +**LiteLLM Handling**: Converts this by: +- Adding message: "Please select a tool to handle the current issue." +- Removing the `tool_choice` parameter from the request diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 9e521126d5..275bdb05d2 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -408,6 +408,7 @@ const sidebars = { "providers/nvidia_nim", { type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" }, "providers/xai", + "providers/moonshot", "providers/lm_studio", "providers/cerebras", "providers/volcano", diff --git a/litellm/__init__.py b/litellm/__init__.py index 0c681d7eb0..6efd04b64d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -497,6 +497,7 @@ nebius_embedding_models: List = [] deepgram_models: List = [] elevenlabs_models: List = [] dashscope_models: List = [] +moonshot_models: List = [] def is_bedrock_pricing_only_model(key: str) -> bool: @@ -674,6 +675,8 @@ def add_known_models(): elevenlabs_models.append(key) elif value.get("litellm_provider") == "dashscope": dashscope_models.append(key) + elif value.get("litellm_provider") == "moonshot": + moonshot_models.append(key) add_known_models() @@ -758,6 +761,7 @@ model_list = ( + deepgram_models + elevenlabs_models + dashscope_models + + moonshot_models ) model_list_set = set(model_list) @@ -824,6 +828,7 @@ models_by_provider: dict = { "deepgram": deepgram_models, "elevenlabs": elevenlabs_models, "dashscope": dashscope_models, + "moonshot": moonshot_models, } # mapping for those models which have larger equivalents @@ -1142,6 +1147,7 @@ from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig from .llms.github_copilot.chat.transformation import GithubCopilotConfig from .llms.nebius.chat.transformation import NebiusConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig +from .llms.moonshot.chat.transformation import MoonshotChatConfig from .main import * # type: ignore from .integrations import * from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients diff --git a/litellm/constants.py b/litellm/constants.py index b27dcbf7cd..924e4869db 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -277,6 +277,7 @@ LITELLM_CHAT_PROVIDERS = [ "nscale", "nebius", "dashscope", + "moonshot", ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ @@ -404,6 +405,7 @@ openai_compatible_endpoints: List = [ "inference.api.nscale.com/v1", "api.studio.nebius.ai/v1", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", + "https://api.moonshot.ai/v1", ] @@ -440,6 +442,7 @@ openai_compatible_providers: List = [ "nscale", "nebius", "dashscope", + "moonshot", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` @@ -451,6 +454,7 @@ openai_text_completion_compatible_providers: List = ( "featherless_ai", "nebius", "dashscope", + "moonshot", ] ) _openai_like_providers: List = [ diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 64eb814597..94c2a859e1 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -234,6 +234,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1": custom_llm_provider = "dashscope" dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY") + elif endpoint == "api.moonshot.ai/v1": + custom_llm_provider = "moonshot" + dynamic_api_key = get_secret_str("MOONSHOT_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception( @@ -670,7 +673,14 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 dynamic_api_key, ) = litellm.DashScopeChatConfig()._get_openai_compatible_provider_info( api_base, api_key - ) + ) + elif custom_llm_provider == "moonshot": + ( + api_base, + dynamic_api_key, + ) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py new file mode 100644 index 0000000000..1b67e315d7 --- /dev/null +++ b/litellm/llms/moonshot/chat/transformation.py @@ -0,0 +1,172 @@ +""" +Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions` +""" + +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload + +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + handle_messages_with_content_list_to_str_conversion, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class MoonshotChatConfig(OpenAIGPTConfig): + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ... + + @overload + def _transform_messages( + self, + messages: List[AllMessageValues], + model: str, + is_async: Literal[False] = False, + ) -> List[AllMessageValues]: + ... + + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: bool = False + ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: + """ + Moonshot AI does not support content in list format. + """ + messages = handle_messages_with_content_list_to_str_conversion(messages) + if is_async: + return super()._transform_messages( + messages=messages, model=model, is_async=True + ) + else: + return super()._transform_messages( + messages=messages, model=model, is_async=False + ) + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("MOONSHOT_API_BASE") + or "https://api.moonshot.ai/v1" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("MOONSHOT_API_KEY") + return api_base, dynamic_api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + If api_base is not provided, use the default Moonshot AI /chat/completions endpoint. + """ + if not api_base: + api_base = "https://api.moonshot.ai/v1" + + if not api_base.endswith("/chat/completions"): + api_base = f"{api_base}/chat/completions" + + return api_base + + def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for Moonshot AI models + + Moonshot AI limitations: + - functions parameter is not supported (use tools instead) + - tool_choice doesn't support "required" value + """ + excluded_params: List[str] = ["functions"] + base_openai_params = super().get_supported_openai_params(model=model) + final_params: List[str] = [] + for param in base_openai_params: + if param not in excluded_params: + final_params.append(param) + + return final_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Moonshot AI parameters + + Handles Moonshot AI specific limitations: + - tool_choice doesn't support "required" value + - Temperature <0.3 limitation for n>1 + """ + supported_openai_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + optional_params[param] = value + + ########################################## + # temperature limitations + # 1. `temperature` on KIMI API is [0, 1] but OpenAI is [0, 2] + # 2. If temperature < 0.3 and n > 1, KIMI will raise an exception. + # If we enter this condition, we set the temperature to 0.3 as suggested by Moonshot AI + ########################################## + if "temperature" in optional_params: + if optional_params["temperature"] > 1: + optional_params["temperature"] = 1 + if optional_params["temperature"] < 0.3 and optional_params.get("n", 1) > 1: + optional_params["temperature"] = 0.3 + return optional_params + + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the overall request to be sent to the API. + Returns: + dict: The transformed request. Sent as the body of the API call. + """ + # Add tool_choice="required" message if needed + if optional_params.get("tool_choice", None) == "required": + messages = self._add_tool_choice_required_message( + messages=messages, + optional_params=optional_params, + ) + + # Call parent transform_request which handles _transform_messages + return super().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + + def _add_tool_choice_required_message(self, messages: List[AllMessageValues], optional_params: dict) -> List[AllMessageValues]: + """ + Add a message to the messages list to indicate that the tool choice is required. + + https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-tool_choice + """ + messages.append({ + "role": "user", + "content": "Please select a tool to handle the current issue.", # Usually, the Kimi large language model understands the intention to invoke a tool and selects one for invocation + }) + optional_params.pop("tool_choice") + return messages diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1f26d4b44e..64e83a7bc8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15963,5 +15963,89 @@ "supports_reasoning": true, "mode": "chat", "source": "https://bailian.console.alibabacloud.com/?spm=a2c63.p38356.0.0.4a615d7bjSUCb4&tab=doc#/doc/?type=model&url=https%3A%2F%2Fwww.alibabacloud.com%2Fhelp%2Fen%2Fdoc-detail%2F2840914.html" + }, + "moonshot/moonshot-v1-8k": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-32k": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-128k": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-auto": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/kimi-k2-0711-preview": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_web_search": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2" + }, + "moonshot/moonshot-v1-32k-0430": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-128k-0430": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-8k-0430": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" } } diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ea5c803681..74608a643a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2272,6 +2272,7 @@ class LlmProviders(str, Enum): CODESTRAL = "codestral" TEXT_COMPLETION_CODESTRAL = "text-completion-codestral" DASHSCOPE = "dashscope" + MOONSHOT = "moonshot" DEEPSEEK = "deepseek" SAMBANOVA = "sambanova" MARITALK = "maritalk" diff --git a/litellm/utils.py b/litellm/utils.py index fd5b6da60e..98fe101b94 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5351,6 +5351,11 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("DASHSCOPE_API_KEY") + elif custom_llm_provider == "moonshot": + if "MOONSHOT_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("MOONSHOT_API_KEY") else: ## openai - chatcompletion + text completion if ( @@ -6812,6 +6817,8 @@ class ProviderConfigManager: return litellm.NebiusConfig() elif litellm.LlmProviders.DASHSCOPE == provider: return litellm.DashScopeChatConfig() + elif litellm.LlmProviders.MOONSHOT == provider: + return litellm.MoonshotChatConfig() elif litellm.LlmProviders.BEDROCK == provider: bedrock_route = BedrockModelInfo.get_bedrock_route(model) bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1f26d4b44e..64e83a7bc8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15963,5 +15963,89 @@ "supports_reasoning": true, "mode": "chat", "source": "https://bailian.console.alibabacloud.com/?spm=a2c63.p38356.0.0.4a615d7bjSUCb4&tab=doc#/doc/?type=model&url=https%3A%2F%2Fwww.alibabacloud.com%2Fhelp%2Fen%2Fdoc-detail%2F2840914.html" + }, + "moonshot/moonshot-v1-8k": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-32k": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-128k": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-auto": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/kimi-k2-0711-preview": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_web_search": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2" + }, + "moonshot/moonshot-v1-32k-0430": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-128k-0430": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" + }, + "moonshot/moonshot-v1-8k-0430": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "litellm_provider": "moonshot", + "supports_function_calling": true, + "supports_tool_choice": true, + "mode": "chat", + "source": "https://platform.moonshot.ai/docs/pricing" } } diff --git a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py new file mode 100644 index 0000000000..62fcec04c1 --- /dev/null +++ b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py @@ -0,0 +1,312 @@ +""" +Unit tests for Moonshot AI configuration. + +These tests validate the MoonshotChatConfig class which extends OpenAIGPTConfig. +Moonshot AI is an OpenAI-compatible provider with minor customizations. +""" + +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +import pytest + +import litellm +import litellm.utils +from litellm import completion +from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig + + +class TestMoonshotConfig: + """Test class for Moonshot AI functionality""" + + def test_default_api_base(self): + """Test that default API base is used when none is provided""" + config = MoonshotChatConfig() + headers = {} + api_key = "fake-moonshot-key" + + # Call validate_environment without specifying api_base + result = config.validate_environment( + headers=headers, + model="moonshot-v1-8k", + messages=[{"role": "user", "content": "Hey"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base=None, # Not providing api_base + ) + + # Verify headers are still set correctly + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + + # We can't directly test the api_base value here since validate_environment + # only returns the headers, but we can verify it doesn't raise an exception + # which would happen if api_base handling was incorrect + + def test_get_supported_openai_params(self): + """Test that get_supported_openai_params returns correct params""" + config = MoonshotChatConfig() + + supported_params = config.get_supported_openai_params("moonshot-v1-8k") + + # Should include these params + assert "tools" in supported_params + assert "tool_choice" in supported_params + assert "temperature" in supported_params + assert "max_tokens" in supported_params + assert "stream" in supported_params + + # Should NOT include functions (not supported by Moonshot AI) + assert "functions" not in supported_params + + def test_map_openai_params_excludes_functions(self): + """Test that functions parameter is not mapped""" + config = MoonshotChatConfig() + + non_default_params = { + "functions": [{"name": "test_function", "description": "Test function"}], + "temperature": 0.7, + "max_tokens": 1000 + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="moonshot-v1-8k", + drop_params=False + ) + + # Functions should not be in result (not in supported params) + assert "functions" not in result + # Other supported params should be included + assert result.get("temperature") == 0.7 + assert result.get("max_tokens") == 1000 + + + + + def test_map_openai_params_allows_other_tool_choice_values(self): + """Test that other tool_choice values are allowed""" + config = MoonshotChatConfig() + + for tool_choice_value in ["auto", "none", {"type": "function", "function": {"name": "test"}}]: + non_default_params = { + "tool_choice": tool_choice_value, + "tools": [{"type": "function", "function": {"name": "test"}}] + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="moonshot-v1-8k", + drop_params=False + ) + + # tool_choice should be included for non-"required" values + assert result.get("tool_choice") == tool_choice_value + + + def test_map_openai_params_max_completion_tokens_mapping(self): + """Test that max_completion_tokens is mapped to max_tokens""" + config = MoonshotChatConfig() + + non_default_params = { + "max_completion_tokens": 1000, + "temperature": 0.7 + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="moonshot-v1-8k", + drop_params=False + ) + + # max_completion_tokens should be mapped to max_tokens + assert result.get("max_tokens") == 1000 + assert "max_completion_tokens" not in result + assert result.get("temperature") == 0.7 + + def test_temperature_handling_clamps_to_max_1(self): + """Test that temperature > 1 is clamped to 1 (Moonshot limitation)""" + config = MoonshotChatConfig() + + non_default_params = { + "temperature": 1.5 # OpenAI allows up to 2, but Moonshot only allows up to 1 + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="moonshot-v1-8k", + drop_params=False + ) + + # Temperature should be clamped to 1 + assert result.get("temperature") == 1 + + def test_temperature_handling_low_temp_with_multiple_n(self): + """Test that temperature < 0.3 with n > 1 is adjusted to 0.3""" + config = MoonshotChatConfig() + + non_default_params = { + "temperature": 0.1, # Less than 0.3 + "n": 3 # Multiple completions + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="moonshot-v1-8k", + drop_params=False + ) + + # Temperature should be adjusted to 0.3 to avoid Moonshot API exceptions + assert result.get("temperature") == 0.3 + assert result.get("n") == 3 + + def test_temperature_handling_low_temp_single_n(self): + """Test that temperature < 0.3 with n = 1 is preserved""" + config = MoonshotChatConfig() + + non_default_params = { + "temperature": 0.1, # Less than 0.3 + "n": 1 # Single completion + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="moonshot-v1-8k", + drop_params=False + ) + + # Temperature should be preserved when n = 1 + assert result.get("temperature") == 0.1 + assert result.get("n") == 1 + + def test_temperature_handling_valid_range(self): + """Test that temperatures in valid range [0.3, 1] are preserved""" + config = MoonshotChatConfig() + + test_temps = [0.3, 0.5, 0.7, 1.0] + + for temp in test_temps: + non_default_params = { + "temperature": temp, + "n": 2 + } + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="moonshot-v1-8k", + drop_params=False + ) + + # Temperature should be preserved + assert result.get("temperature") == temp + + def test_tool_choice_required_adds_message(self): + """Test that tool_choice='required' adds a special message and removes tool_choice""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "What's the weather like?"} + ] + + optional_params = { + "tool_choice": "required", + "tools": [{"type": "function", "function": {"name": "get_weather"}}] + } + + result = config.transform_request( + model="moonshot-v1-8k", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Check that the special message was added + assert len(result["messages"]) == 2 + assert result["messages"][0]["role"] == "user" + assert result["messages"][0]["content"] == "What's the weather like?" + assert result["messages"][1]["role"] == "user" + assert result["messages"][1]["content"] == "Please select a tool to handle the current issue." + + # Check that tool_choice was removed but tools are preserved + assert "tool_choice" not in result + assert "tools" in result + assert len(result["tools"]) == 1 + + def test_tool_choice_required_preserves_other_params(self): + """Test that tool_choice='required' handling preserves other parameters""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "Calculate 2+2"} + ] + + optional_params = { + "tool_choice": "required", + "tools": [{"type": "function", "function": {"name": "calculator"}}], + "temperature": 0.7, + "max_tokens": 1000 + } + + result = config.transform_request( + model="moonshot-v1-8k", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Check that other parameters are preserved + assert result.get("temperature") == 0.7 + assert result.get("max_tokens") == 1000 + assert "tools" in result + + # Check that tool_choice was removed + assert "tool_choice" not in result + + # Check that the message was added + assert len(result["messages"]) == 2 + assert result["messages"][1]["content"] == "Please select a tool to handle the current issue." + + def test_tool_choice_non_required_preserved(self): + """Test that non-'required' tool_choice values are preserved""" + config = MoonshotChatConfig() + + messages = [ + {"role": "user", "content": "What's the weather?"} + ] + + test_values = ["auto", "none", {"type": "function", "function": {"name": "get_weather"}}] + + for tool_choice_value in test_values: + optional_params = { + "tool_choice": tool_choice_value, + "tools": [{"type": "function", "function": {"name": "get_weather"}}] + } + + result = config.transform_request( + model="moonshot-v1-8k", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Check that tool_choice is preserved for non-"required" values + assert result.get("tool_choice") == tool_choice_value + + # Check that no extra message was added + assert len(result["messages"]) == 1 + assert result["messages"][0]["content"] == "What's the weather?" \ No newline at end of file