diff --git a/docs/my-website/docs/providers/hyperbolic.md b/docs/my-website/docs/providers/hyperbolic.md new file mode 100644 index 0000000000..7bad527fcf --- /dev/null +++ b/docs/my-website/docs/providers/hyperbolic.md @@ -0,0 +1,331 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Hyperbolic + +## Overview + +| Property | Details | +|-------|-------| +| Description | Hyperbolic provides access to the latest models at a fraction of legacy cloud costs, with OpenAI-compatible APIs for LLMs, image generation, and more. | +| Provider Route on LiteLLM | `hyperbolic/` | +| Link to Provider Doc | [Hyperbolic Documentation ↗](https://docs.hyperbolic.xyz) | +| Base URL | `https://api.hyperbolic.xyz/v1` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +https://docs.hyperbolic.xyz + +**We support ALL Hyperbolic models, just set `hyperbolic/` as a prefix when sending completion requests** + +## Available Models + +### Language Models + +| Model | Description | Context Window | Pricing per 1M tokens | +|-------|-------------|----------------|----------------------| +| `hyperbolic/deepseek-ai/DeepSeek-V3` | DeepSeek V3 - Fast and efficient | 131,072 tokens | $0.25 | +| `hyperbolic/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 March 2024 version | 131,072 tokens | $0.25 | +| `hyperbolic/deepseek-ai/DeepSeek-R1` | DeepSeek R1 - Reasoning model | 131,072 tokens | $2.00 | +| `hyperbolic/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 May 2028 version | 131,072 tokens | $0.25 | +| `hyperbolic/Qwen/Qwen2.5-72B-Instruct` | Qwen 2.5 72B Instruct | 131,072 tokens | $0.40 | +| `hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct` | Qwen 2.5 Coder 32B for code generation | 131,072 tokens | $0.20 | +| `hyperbolic/Qwen/Qwen3-235B-A22B` | Qwen 3 235B A22B variant | 131,072 tokens | $2.00 | +| `hyperbolic/Qwen/QwQ-32B` | Qwen QwQ 32B | 131,072 tokens | $0.20 | +| `hyperbolic/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B Instruct | 131,072 tokens | $0.80 | +| `hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct` | Llama 3.1 405B Instruct | 131,072 tokens | $5.00 | +| `hyperbolic/moonshotai/Kimi-K2-Instruct` | Kimi K2 Instruct | 131,072 tokens | $2.00 | + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key +``` + +Get your API key from [Hyperbolic dashboard](https://app.hyperbolic.ai). + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Hyperbolic Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key + +messages = [{"content": "What is the capital of France?", "role": "user"}] + +# Hyperbolic call +response = completion( + model="hyperbolic/Qwen/Qwen2.5-72B-Instruct", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Hyperbolic Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key + +messages = [{"content": "Write a short poem about AI", "role": "user"}] + +# Hyperbolic call with streaming +response = completion( + model="hyperbolic/deepseek-ai/DeepSeek-V3", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Function Calling + +```python showLineNumbers title="Hyperbolic Function Calling" +import os +import litellm +from litellm import completion + +os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + } +] + +response = completion( + model="hyperbolic/deepseek-ai/DeepSeek-V3", + messages=[{"role": "user", "content": "What's the weather like in New York?"}], + tools=tools, + tool_choice="auto" +) + +print(response) +``` + +## Usage - LiteLLM Proxy + +Add the following to your LiteLLM Proxy configuration file: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: deepseek-fast + litellm_params: + model: hyperbolic/deepseek-ai/DeepSeek-V3 + api_key: os.environ/HYPERBOLIC_API_KEY + + - model_name: qwen-coder + litellm_params: + model: hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct + api_key: os.environ/HYPERBOLIC_API_KEY + + - model_name: deepseek-reasoning + litellm_params: + model: hyperbolic/deepseek-ai/DeepSeek-R1 + api_key: os.environ/HYPERBOLIC_API_KEY +``` + +Start your LiteLLM Proxy server: + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + + + + +```python showLineNumbers title="Hyperbolic via Proxy - Non-streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Non-streaming response +response = client.chat.completions.create( + model="deepseek-fast", + messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}] +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Hyperbolic via Proxy - Streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Streaming response +response = client.chat.completions.create( + model="qwen-coder", + messages=[{"role": "user", "content": "Write a Python function to sort a list"}], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK" +import litellm + +# Configure LiteLLM to use your proxy +response = litellm.completion( + model="litellm_proxy/deepseek-fast", + messages=[{"role": "user", "content": "What are the benefits of renewable energy?"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key" +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK Streaming" +import litellm + +# Configure LiteLLM to use your proxy with streaming +response = litellm.completion( + model="litellm_proxy/qwen-coder", + messages=[{"role": "user", "content": "Implement a binary search algorithm"}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key", + stream=True +) + +for chunk in response: + if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```bash showLineNumbers title="Hyperbolic via Proxy - cURL" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "deepseek-fast", + "messages": [{"role": "user", "content": "What is machine learning?"}] + }' +``` + +```bash showLineNumbers title="Hyperbolic via Proxy - cURL Streaming" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "qwen-coder", + "messages": [{"role": "user", "content": "Write a REST API in Python"}], + "stream": true + }' +``` + + + + +For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy). + +## Supported OpenAI Parameters + +Hyperbolic supports the following OpenAI-compatible parameters: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `messages` | array | **Required**. Array of message objects with 'role' and 'content' | +| `model` | string | **Required**. Model ID (e.g., deepseek-ai/DeepSeek-V3, Qwen/Qwen2.5-72B-Instruct) | +| `stream` | boolean | Optional. Enable streaming responses | +| `temperature` | float | Optional. Sampling temperature (0.0 to 2.0) | +| `top_p` | float | Optional. Nucleus sampling parameter | +| `max_tokens` | integer | Optional. Maximum tokens to generate | +| `frequency_penalty` | float | Optional. Penalize frequent tokens | +| `presence_penalty` | float | Optional. Penalize tokens based on presence | +| `stop` | string/array | Optional. Stop sequences | +| `n` | integer | Optional. Number of completions to generate | +| `tools` | array | Optional. List of available tools/functions | +| `tool_choice` | string/object | Optional. Control tool/function calling | +| `response_format` | object | Optional. Response format specification | +| `seed` | integer | Optional. Random seed for reproducibility | +| `user` | string | Optional. User identifier | + +## Advanced Usage + +### Custom API Base + +If you're using a custom Hyperbolic deployment: + +```python showLineNumbers title="Custom API Base" +import litellm + +response = litellm.completion( + model="hyperbolic/deepseek-ai/DeepSeek-V3", + messages=[{"role": "user", "content": "Hello"}], + api_base="https://your-custom-hyperbolic-endpoint.com/v1", + api_key="your-api-key" +) +``` + +### Rate Limits + +Hyperbolic offers different tiers: +- **Basic**: 60 requests per minute (RPM) +- **Pro**: 600 RPM +- **Enterprise**: Custom limits + +## Pricing + +Hyperbolic offers competitive pay-as-you-go pricing with no hidden fees or long-term commitments. See the model table above for specific pricing per million tokens. + +### Precision Options +- **BF16**: Best precision and performance, suitable for tasks where accuracy is critical +- **FP8**: Optimized for efficiency and speed, ideal for high-throughput applications at lower cost + +## Additional Resources + +- [Hyperbolic Official Documentation](https://docs.hyperbolic.xyz) +- [Hyperbolic Dashboard](https://app.hyperbolic.ai) +- [API Reference](https://docs.hyperbolic.xyz/docs/rest-api) \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index df7d47678f..1fe19fe979 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -412,6 +412,7 @@ const sidebars = { "providers/huggingface_rerank", ] }, + "providers/hyperbolic", "providers/databricks", "providers/deepgram", "providers/watsonx", diff --git a/litellm/__init__.py b/litellm/__init__.py index 66850ee209..c056e66726 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -144,22 +144,22 @@ prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. -gcs_pub_sub_use_v1: Optional[bool] = ( - False # if you want to use v1 gcs pubsub logged payload -) -generic_api_use_v1: Optional[bool] = ( - False # if you want to use v1 generic api logged payload -) +gcs_pub_sub_use_v1: Optional[ + bool +] = False # if you want to use v1 gcs pubsub logged payload +generic_api_use_v1: Optional[ + bool +] = False # if you want to use v1 generic api logged payload argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[Union[str, Callable, CustomLogger]] = ( - [] -) # internal variable - async custom callbacks are routed here. -_async_success_callback: List[Union[str, Callable, CustomLogger]] = ( - [] -) # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[Union[str, Callable, CustomLogger]] = ( - [] -) # internal variable - async custom callbacks are routed here. +_async_input_callback: List[ + Union[str, Callable, CustomLogger] +] = [] # internal variable - async custom callbacks are routed here. +_async_success_callback: List[ + Union[str, Callable, CustomLogger] +] = [] # internal variable - async custom callbacks are routed here. +_async_failure_callback: List[ + Union[str, Callable, CustomLogger] +] = [] # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False @@ -167,18 +167,18 @@ log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False -add_user_information_to_llm_headers: Optional[bool] = ( - None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers -) +add_user_information_to_llm_headers: Optional[ + bool +] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# -email: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -token: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) +email: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +token: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) @@ -266,15 +266,11 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) -caching: bool = ( - False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -caching_with_models: bool = ( - False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -cache: Optional[Cache] = ( - None # cache object <- use this - https://docs.litellm.ai/docs/caching -) +caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +cache: Optional[ + Cache +] = None # cache object <- use this - https://docs.litellm.ai/docs/caching default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None @@ -282,9 +278,9 @@ model_alias_map: Dict[str, str] = {} model_group_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers -budget_duration: Optional[str] = ( - None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). -) +budget_duration: Optional[ + str +] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) @@ -293,15 +289,11 @@ forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} -add_function_to_prompt: bool = ( - False # if function calling not supported by api, append function call details to system prompt -) +add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' -model_cost_map_url: str = ( - "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" -) +model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" suppress_debug_info = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None @@ -329,9 +321,7 @@ prometheus_metrics_config: Optional[List] = None disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) -disable_copilot_system_to_assistant: bool = ( - False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. -) +disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. public_model_groups: Optional[List[str]] = None public_model_groups_links: Dict[str, str] = {} #### REQUEST PRIORITIZATION ##### @@ -339,17 +329,13 @@ priority_reservation: Optional[Dict[str, float]] = None ######## Networking Settings ######## -use_aiohttp_transport: bool = ( - True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. -) +use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) -force_ipv4: bool = ( - False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. -) +force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. module_level_aclient = AsyncHTTPHandler( timeout=request_timeout, client_alias="module level aclient" ) @@ -363,13 +349,13 @@ fallbacks: Optional[List] = None context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 -num_retries_per_request: Optional[int] = ( - None # for the request overall (incl. fallbacks + model retries) -) +num_retries_per_request: Optional[ + int +] = None # for the request overall (incl. fallbacks + model retries) ####### SECRET MANAGERS ##################### -secret_manager_client: Optional[Any] = ( - None # list of instantiated key management clients - e.g. azure kv, infisical, etc. -) +secret_manager_client: Optional[ + Any +] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc. _google_kms_resource_name: Optional[str] = None _key_management_system: Optional[KeyManagementSystem] = None _key_management_settings: KeyManagementSettings = KeyManagementSettings() @@ -505,6 +491,7 @@ moonshot_models: List = [] v0_models: List = [] morph_models: List = [] lambda_ai_models: List = [] +hyperbolic_models: List = [] recraft_models: List = [] def is_bedrock_pricing_only_model(key: str) -> bool: @@ -690,6 +677,8 @@ def add_known_models(): morph_models.append(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.append(key) + elif value.get("litellm_provider") == "hyperbolic": + hyperbolic_models.append(key) elif value.get("litellm_provider") == "recraft": recraft_models.append(key) @@ -850,6 +839,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "hyperbolic": hyperbolic_models, "recraft": recraft_models, } @@ -1173,6 +1163,7 @@ from .llms.moonshot.chat.transformation import MoonshotChatConfig from .llms.v0.chat.transformation import V0ChatConfig from .llms.morph.chat.transformation import MorphChatConfig from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig +from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig from .main import * # type: ignore from .integrations import * from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients @@ -1231,12 +1222,12 @@ from .types.llms.custom_llm import CustomLLMItem from .types.utils import GenericStreamingChunk custom_provider_map: List[CustomLLMItem] = [] -_custom_providers: List[str] = ( - [] -) # internal helper util, used to track names of custom providers -disable_hf_tokenizer_download: Optional[bool] = ( - None # disable huggingface tokenizer download. Defaults to openai clk100 -) +_custom_providers: List[ + str +] = [] # internal helper util, used to track names of custom providers +disable_hf_tokenizer_download: Optional[ + bool +] = None # disable huggingface tokenizer download. Defaults to openai clk100 global_disable_no_log_param: bool = False ### PASSTHROUGH ### diff --git a/litellm/constants.py b/litellm/constants.py index 3089ae131d..a0dcd80c07 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -412,6 +412,7 @@ openai_compatible_endpoints: List = [ "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", + "https://api.hyperbolic.xyz/v1", ] @@ -452,6 +453,7 @@ openai_compatible_providers: List = [ "v0", "morph", "lambda_ai", + "hyperbolic", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` @@ -466,6 +468,7 @@ openai_text_completion_compatible_providers: List = ( "moonshot", "v0", "lambda_ai", + "hyperbolic", ] ) _openai_like_providers: List = [ diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 32d837c1e0..4e0a2efb0c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -243,6 +243,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.lambda.ai/v1": custom_llm_provider = "lambda_ai" dynamic_api_key = get_secret_str("LAMBDA_API_KEY") + elif endpoint == "https://api.hyperbolic.xyz/v1": + custom_llm_provider = "hyperbolic" + dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception( @@ -533,7 +536,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 # DataRobot is OpenAI compatible. ( api_base, - dynamic_api_key + dynamic_api_key, ) = litellm.DataRobotConfig()._get_openai_compatible_provider_info( api_base, api_key ) @@ -708,6 +711,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "hyperbolic": + ( + api_base, + dynamic_api_key, + ) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/llms/hyperbolic/__init__.py b/litellm/llms/hyperbolic/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/hyperbolic/chat/__init__.py b/litellm/llms/hyperbolic/chat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/hyperbolic/chat/transformation.py b/litellm/llms/hyperbolic/chat/transformation.py new file mode 100644 index 0000000000..48af9fa68a --- /dev/null +++ b/litellm/llms/hyperbolic/chat/transformation.py @@ -0,0 +1,54 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Hyperbolic's `/v1/chat/completions` +""" + +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class HyperbolicChatConfig(OpenAILikeChatConfig): + """ + Hyperbolic is OpenAI-compatible with standard endpoints + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "hyperbolic" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + # Hyperbolic is openai compatible, we just need to set the api_base + api_base = ( + api_base + or get_secret_str("HYPERBOLIC_API_BASE") + or "https://api.hyperbolic.xyz/v1" # Default Hyperbolic API base URL + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("HYPERBOLIC_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + """ + Hyperbolic supports standard OpenAI parameters + Reference: https://docs.hyperbolic.xyz/docs/rest-api + """ + return [ + "messages", # Required + "model", # Required + "stream", # Optional + "temperature", # Optional + "top_p", # Optional + "max_tokens", # Optional + "frequency_penalty", # Optional + "presence_penalty", # Optional + "stop", # Optional + "n", # Optional + "tools", # Optional + "tool_choice", # Optional + "response_format", # Optional + "seed", # Optional + "user", # Optional + ] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3a3c644742..6e9a757c4a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15068,13 +15068,213 @@ "supports_tool_choice": true, "supports_reasoning": true }, - "voyage/voyage-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 0.0, - "litellm_provider": "voyage", - "mode": "embedding" + "hyperbolic/moonshotai/Kimi-K2-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1-0528": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen3-235B-A22B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3-0324": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/QwQ-32B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.2-3B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-72B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "voyage/voyage-lite-01": { "max_tokens": 4096, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a025f387a1..acff566a3f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2315,6 +2315,7 @@ class LlmProviders(str, Enum): LLAMA = "meta_llama" NSCALE = "nscale" PG_VECTOR = "pg_vector" + HYPERBOLIC = "hyperbolic" RECRAFT = "recraft" diff --git a/litellm/utils.py b/litellm/utils.py index 70fc77e467..b2a717814f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6884,6 +6884,8 @@ class ProviderConfigManager: return litellm.OpenAIGPTConfig() elif litellm.LlmProviders.NSCALE == provider: return litellm.NscaleConfig() + elif litellm.LlmProviders.HYPERBOLIC == provider: + return litellm.HyperbolicChatConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3a3c644742..6e9a757c4a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15068,13 +15068,213 @@ "supports_tool_choice": true, "supports_reasoning": true }, - "voyage/voyage-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 0.0, - "litellm_provider": "voyage", - "mode": "embedding" + "hyperbolic/moonshotai/Kimi-K2-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1-0528": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen3-235B-A22B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3-0324": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/QwQ-32B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-R1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/deepseek-ai/DeepSeek-V3": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Llama-3.2-3B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/Qwen/Qwen2.5-72B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "hyperbolic", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "voyage/voyage-lite-01": { "max_tokens": 4096, diff --git a/tests/llm_translation/test_hyperbolic.py b/tests/llm_translation/test_hyperbolic.py new file mode 100644 index 0000000000..38f4dea436 --- /dev/null +++ b/tests/llm_translation/test_hyperbolic.py @@ -0,0 +1,119 @@ +import os +import sys +from datetime import datetime +from unittest.mock import MagicMock + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +import litellm +from litellm import get_llm_provider + + +def test_get_llm_provider_hyperbolic(): + """Test that hyperbolic/ prefix returns the correct provider""" + model, provider, _, _ = get_llm_provider(model="hyperbolic/deepseek-v3") + assert provider == "hyperbolic" + assert model == "deepseek-v3" + + +def test_hyperbolic_completion_call(): + """Test basic completion call structure for Hyperbolic""" + # This is primarily a structure test since we don't have actual API keys + try: + litellm.set_verbose = True + response = litellm.completion( + model="hyperbolic/qwen-2.5-72b", + messages=[{"role": "user", "content": "Hello!"}], + mock_response="Hi there!", + ) + assert response is not None + except Exception as e: + # Expected to fail without valid API key, but should recognize the provider + assert "hyperbolic" in str(e).lower() or "api" in str(e).lower() + + +def test_hyperbolic_config_initialization(): + """Test that HyperbolicChatConfig initializes correctly""" + from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig + + config = HyperbolicChatConfig() + assert config.custom_llm_provider == "hyperbolic" + + +def test_hyperbolic_get_openai_compatible_provider_info(): + """Test API base and key handling""" + from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig + + config = HyperbolicChatConfig() + + # Test default API base + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.hyperbolic.xyz/v1" + # api_key may be set from environment, so we don't test for None + + # Test custom API base + custom_base = "https://custom.hyperbolic.com/v1" + api_base, api_key = config._get_openai_compatible_provider_info(custom_base, "test-key") + assert api_base == custom_base + assert api_key == "test-key" + + +def test_hyperbolic_in_provider_lists(): + """Test that hyperbolic is in all relevant provider lists""" + from litellm.constants import ( + openai_compatible_endpoints, + openai_compatible_providers, + openai_text_completion_compatible_providers, + ) + + assert "hyperbolic" in openai_compatible_providers + assert "hyperbolic" in openai_text_completion_compatible_providers + assert "https://api.hyperbolic.xyz/v1" in openai_compatible_endpoints + + +def test_hyperbolic_models_configuration(): + """Test that Hyperbolic models are properly configured""" + import json + import os + + # Load model configuration directly from the JSON file + json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") + with open(json_path, 'r') as f: + model_data = json.load(f) + + # Test a few key models + test_models = [ + "hyperbolic/deepseek-ai/DeepSeek-V3", + "hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct", + "hyperbolic/deepseek-ai/DeepSeek-R1", + ] + + for model in test_models: + assert model in model_data + model_info = model_data[model] + assert model_info["litellm_provider"] == "hyperbolic" + assert model_info["mode"] == "chat" + assert "max_tokens" in model_info + assert "input_cost_per_token" in model_info + assert "output_cost_per_token" in model_info + + +def test_hyperbolic_supported_params(): + """Test that supported OpenAI parameters are correctly configured""" + from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig + + config = HyperbolicChatConfig() + supported_params = config.get_supported_openai_params("hyperbolic/deepseek-v3") + + # Check for essential parameters + assert "messages" in supported_params + assert "model" in supported_params + assert "stream" in supported_params + assert "temperature" in supported_params + assert "max_tokens" in supported_params + assert "tools" in supported_params + assert "tool_choice" in supported_params \ No newline at end of file