From ae92404d05b8e224b6632edfe5a5ef335061ea42 Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Thu, 18 Sep 2025 13:06:09 -0600 Subject: [PATCH 01/13] Initial addition of Lemonade provider. --- litellm/__init__.py | 7 +++++ litellm/constants.py | 1 + .../get_llm_provider_logic.py | 11 +++++++ litellm/main.py | 31 +++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 10 ++++++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 10 ++++++ 7 files changed, 71 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 02bb773d26..078c434820 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -250,6 +250,7 @@ wandb_key: Optional[str] = None heroku_key: Optional[str] = None cometapi_key: Optional[str] = None ovhcloud_key: Optional[str] = None +lemonade_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], @@ -536,6 +537,7 @@ volcengine_models: Set = set() wandb_models: Set = set(WANDB_MODELS) ovhcloud_models: Set = set() ovhcloud_embedding_models: Set = set() +lemonade_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -756,6 +758,8 @@ def add_known_models(): ovhcloud_models.add(key) elif value.get("litellm_provider") == "ovhcloud-embedding-models": ovhcloud_embedding_models.add(key) + elif value.get("litellm_provider") == "lemonade": + lemonade_models.add(key) add_known_models() @@ -852,6 +856,7 @@ model_list = list( | volcengine_models | wandb_models | ovhcloud_models + | lemonade_models ) model_list_set = set(model_list) @@ -935,6 +940,7 @@ models_by_provider: dict = { "volcengine": volcengine_models, "wandb": wandb_models, "ovhcloud": ovhcloud_models | ovhcloud_embedding_models, + "lemonade": lemonade_models, } # mapping for those models which have larger equivalents @@ -1284,6 +1290,7 @@ from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig +from .llms.lemonade.chat.transformation import LemonadeChatConfig from .main import * # type: ignore from .integrations import * from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients diff --git a/litellm/constants.py b/litellm/constants.py index b839256b78..3ff9a4b6fb 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -315,6 +315,7 @@ LITELLM_CHAT_PROVIDERS = [ "vercel_ai_gateway", "wandb", "ovhcloud", + "lemonade" ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 69c996d813..f209aed483 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -368,6 +368,8 @@ def get_llm_provider( # noqa: PLR0915 # bytez models elif model.startswith("bytez/"): custom_llm_provider = "bytez" + elif model.startswith("lemonade/"): + custom_llm_provider = "lemonade" elif model.startswith("heroku/"): custom_llm_provider = "heroku" # cometapi models @@ -379,6 +381,8 @@ def get_llm_provider( # noqa: PLR0915 custom_llm_provider = "compactifai" elif model.startswith("ovhcloud/"): custom_llm_provider = "ovhcloud" + elif model.startswith("lemonade/"): + custom_llm_provider = "lemonade" if not custom_llm_provider: if litellm.suppress_debug_info is False: print() # noqa @@ -783,6 +787,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "https://api.inference.wandb.ai/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY") + elif custom_llm_provider == "lemonade": + ( + api_base, + dynamic_api_key, + ) = litellm.LemonadeChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/main.py b/litellm/main.py index 351d3d69eb..76b3932986 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -149,6 +149,7 @@ from .llms.bedrock.chat import BedrockConverseLLM, BedrockLLM from .llms.bedrock.embed.embedding import BedrockEmbedding from .llms.bedrock.image.image_handler import BedrockImageGeneration from .llms.bytez.chat.transformation import BytezChatConfig +from .llms.lemonade.chat.transformation import LemonadeChatConfig from .llms.codestral.completion.handler import CodestralTextCompletion from .llms.cohere.embed import handler as cohere_embed from .llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler @@ -267,6 +268,7 @@ bytez_transformation = BytezChatConfig() heroku_transformation = HerokuChatConfig() oci_transformation = OCIChatConfig() ovhcloud_transformation = OVHCloudChatConfig() +lemonade_transformation = LemonadeChatConfig() ####### COMPLETION ENDPOINTS ################ @@ -3545,6 +3547,35 @@ def completion( # type: ignore # noqa: PLR0915 ) pass + elif custom_llm_provider == "lemonade": + api_key = ( + api_key + or litellm.bytez_key + or get_secret_str("LEMONADE_API_KEY") + or litellm.api_key + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=encoding, + stream=stream, + provider_config=lemonade_transformation, + ) + + pass + elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models: api_key = ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f855547a53..e9404f885b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13256,6 +13256,16 @@ ], "supports_tool_choice": false }, + "lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF": { + "input_cost_per_token": 0, + "litellm_provider": "lemonade", + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 0, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "groq/deepseek-r1-distill-llama-70b": { "input_cost_per_token": 7.5e-07, "litellm_provider": "groq", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b0183249ba..e5786e50a5 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2428,6 +2428,7 @@ class LlmProviders(str, Enum): DOTPROMPT = "dotprompt" WANDB = "wandb" OVHCLOUD = "ovhcloud" + LEMONADE = "lemonade" # Create a set of all provider values for quick lookup diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f855547a53..e9404f885b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13256,6 +13256,16 @@ ], "supports_tool_choice": false }, + "lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF": { + "input_cost_per_token": 0, + "litellm_provider": "lemonade", + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 0, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "groq/deepseek-r1-distill-llama-70b": { "input_cost_per_token": 7.5e-07, "litellm_provider": "groq", From eb71611a974298637ea248e509d721b21dad4d6e Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Thu, 18 Sep 2025 13:48:02 -0600 Subject: [PATCH 02/13] Adding lemonade transform --- litellm/llms/lemonade/chat/transformation.py | 110 +++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 litellm/llms/lemonade/chat/transformation.py diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py new file mode 100644 index 0000000000..5e675066b0 --- /dev/null +++ b/litellm/llms/lemonade/chat/transformation.py @@ -0,0 +1,110 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completions` +""" +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload + +import httpx +from pydantic import BaseModel + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, +) +from litellm.types.utils import ModelResponse + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class LemonadeChatConfig(OpenAILikeChatConfig): + repeat_penalty: Optional[float] = None + functions: Optional[list] = None + logit_bias: Optional[dict] = None + max_tokens: Optional[int] = None + max_completion_tokens: Optional[int] = None + n: Optional[int] = None + presence_penalty: Optional[int] = None + stop: Optional[Union[str, list]] = None + temperature: Optional[int] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + response_format: Optional[dict] = None + tools: Optional[list] = None + + def __init__( + self, + repeat_penalty: Optional[float] = None, + functions: Optional[list] = None, + logit_bias: Optional[dict] = None, + max_completion_tokens: Optional[int] = None, + n: Optional[int] = None, + presence_penalty: Optional[int] = None, + stop: Optional[Union[str, list]] = None, + temperature: Optional[int] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + response_format: Optional[dict] = None, + tools: Optional[list] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "lemonade" + + @classmethod + def get_config(cls): + return super().get_config() + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + # lemonade is openai compatible, we just need to set this to custom_openai and have the api_base be lemonade's endpoint + api_base = ( + api_base + or get_secret_str("LEMONADE_API_BASE") + or "http://localhost:8000/api/v1" + ) # type: ignore + # Lemonade doesn't check the key + key = "lemonade" + return api_base, key + + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + model_response = super().transform_response( + model=model, + model_response=model_response, + raw_response=raw_response, + messages=messages, + logging_obj=logging_obj, + request_data=request_data, + encoding=encoding, + optional_params=optional_params, + json_mode=json_mode, + litellm_params=litellm_params, + api_key=api_key, + ) + + return model_response + \ No newline at end of file From 0e045e0bb774e4060073ccb52ab64695f68ec40b Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Thu, 18 Sep 2025 16:24:03 -0600 Subject: [PATCH 03/13] Setting the response model so the cost can be calculated --- litellm/llms/lemonade/chat/transformation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 5e675066b0..a8707cce57 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -106,5 +106,8 @@ class LemonadeChatConfig(OpenAILikeChatConfig): api_key=api_key, ) + # Storing lemonade in the model response for easier cost calculation later + setattr(model_response, "model", "lemonade/" + model) + return model_response \ No newline at end of file From f9e98f75a6165fa8c51abcea6aaffae4fb45482f Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Thu, 18 Sep 2025 21:33:14 -0600 Subject: [PATCH 04/13] Adding max_input_tokens and max_output_tokens --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e9404f885b..a35cac3048 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13260,6 +13260,8 @@ "input_cost_per_token": 0, "litellm_provider": "lemonade", "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 0, "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e9404f885b..a35cac3048 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13260,6 +13260,8 @@ "input_cost_per_token": 0, "litellm_provider": "lemonade", "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 0, "supports_function_calling": true, From 351b63bc67516ecad7cd8e9aa3bd05ad063c8166 Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Tue, 23 Sep 2025 17:07:34 -0600 Subject: [PATCH 05/13] Adding max_tokens to constructor of LemonadeChatConfig --- litellm/llms/lemonade/chat/transformation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index a8707cce57..7a90a693f8 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -42,6 +42,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): functions: Optional[list] = None, logit_bias: Optional[dict] = None, max_completion_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, n: Optional[int] = None, presence_penalty: Optional[int] = None, stop: Optional[Union[str, list]] = None, From 6916f4384316e1a8175249ec33f40cc0fc763484 Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Tue, 23 Sep 2025 20:30:47 -0600 Subject: [PATCH 06/13] Adding functionality for Lemonade to check to see if it is aware of a model and if so use that model --- litellm/llms/lemonade/chat/transformation.py | 40 +++++++++++++++++++- litellm/utils.py | 2 + 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 7a90a693f8..1e372a3a6f 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -16,7 +16,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolParam, ChatCompletionToolParamFunctionChunk, ) -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, ModelInfoBase from ...openai_like.chat.transformation import OpenAILikeChatConfig @@ -65,6 +65,44 @@ class LemonadeChatConfig(OpenAILikeChatConfig): def get_config(cls): return super().get_config() + def get_model_info(self, model: str) -> ModelInfoBase: + if model.startswith("lemonade/"): + model = model.split("/", 1)[1] + api_base = get_secret_str("LEMONADE_API_BASE") or "http://localhost:8000" + + # Getting the list of models from lemonade to verify the model exists + try: + response = litellm.module_level_client.get( + url=f"{api_base}/api/v1/models", + ) + except Exception as e: + raise Exception( + f"LemonadeError: Error getting model info for {model}. Set Lemonade API Base via `LEMONADE_API_BASE` environment variable. Error: {e}" + ) + + # Making sure the model exists in lemonade + model_found = False + model_list = response.json().get("data", []) + for model_iter in model_list: + if model_iter['id'] == model: + model_found = True + break + + if not model_found: + raise ValueError( + f"LemonadeError: Model {model} not found. Available models: {[m['id'] for m in model_list]}" + ) + + # Returning the model if it was found in lemonade. Currently there is no mechanism to report + # if the model supports function calling or the max tokens so we leave those out + return ModelInfoBase( + key=model, + litellm_provider="lemonade", + mode="chat", + input_cost_per_token=0.0, + output_cost_per_token=0.0, + ) + def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: diff --git a/litellm/utils.py b/litellm/utils.py index 9fdde626ae..fbc5d09e57 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4775,6 +4775,8 @@ def _get_model_info_helper( # noqa: PLR0915 custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat" ) and not _is_potential_model_name_in_model_cost(potential_model_names): return litellm.OllamaConfig().get_model_info(model) + elif (custom_llm_provider == "lemonade" and not _is_potential_model_name_in_model_cost(potential_model_names)): + return litellm.LemonadeChatConfig().get_model_info(model) else: """ Check if: (in order of specificity) From 929510ef5d9d329a6993f7cb4afa79903876ffa2 Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Tue, 23 Sep 2025 21:13:52 -0600 Subject: [PATCH 07/13] Adding unit tests and documentation --- docs/my-website/docs/providers/lemonade.md | 188 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + .../llms/lemonade/test_lemonade.py | 180 +++++++++++++++++ 3 files changed, 369 insertions(+) create mode 100644 docs/my-website/docs/providers/lemonade.md create mode 100644 tests/test_litellm/llms/lemonade/test_lemonade.py diff --git a/docs/my-website/docs/providers/lemonade.md b/docs/my-website/docs/providers/lemonade.md new file mode 100644 index 0000000000..87d41902a0 --- /dev/null +++ b/docs/my-website/docs/providers/lemonade.md @@ -0,0 +1,188 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Lemonade + +Lemonade is an OpenAI-compatible AI provider that offers local language model inference on AMD Ryzen AI models. The provider supports standard chat completions with full OpenAI API compatibility. + +| Property | Details | +|-------|-------| +| Description | OpenAI-compatible AI provider for local and cloud-based language model inference | +| Provider Route on LiteLLM | `lemonade/` (add this prefix to the model name - e.g. `lemonade/your-model-name`) | +| API Endpoint for Provider | http://localhost:8000/api/v1 (default) | +| Supported Endpoints | `/chat/completions` | + +## Supported OpenAI Parameters + +Lemonade is fully OpenAI-compatible and supports the following parameters: + +``` +"repeat_penalty" +"functions" +"logit_bias" +"max_tokens" +"max_completion_tokens" +"presence_penalty" +"stop" +"temperature" +"top_p" +"top_k" +"response_format" +"tools" +``` + + +## API Key Setup + +Lemonade can be configured with custom API URLs and doesn't require strict API key validation. Set the `LEMONADE_API_BASE` environment variable to modify the base URL. + +## Usage + + + + +```python +from litellm import completion +import os + +# Optional: Set custom API base. Useful if your lemonade server is on +# a different port +os.environ['LEMONADE_API_BASE'] = "http://localhost:8000/api/v1" + +response = completion( + model="lemonade/your-model-name", + messages=[ + {"role": "user", "content": "Hello from LiteLLM!"} + ], +) +print(response) +``` + +## Streaming + +```python +from litellm import completion +import os + +# Optional: Set custom API base. Useful if your lemonade server is on +# a different port +os.environ['LEMONADE_API_BASE'] = "http://localhost:8000/api/v1" + +response = completion( + model="lemonade/your-model-name", + messages=[ + {"role": "user", "content": "Write a short story"} + ], + stream=True +) + +for chunk in response: + print(chunk.choices[0].delta.content, end='', flush=True) +``` + +## Advanced Usage + +### Custom Parameters + +Lemonade supports additional parameters beyond the standard OpenAI set: + +```python +from litellm import completion + +response = completion( + model="lemonade/your-model-name", + messages=[{"role": "user", "content": "Explain quantum computing"}], + temperature=0.7, + max_tokens=500, + top_p=0.9, + top_k=50, + repeat_penalty=1.1, + stop=["Human:", "AI:"] +) +print(response) +``` + +### Function Calling + +Lemonade supports OpenAI-compatible function calling: + +```python +from litellm import completion + +functions = [ + { + "name": "get_weather", + "description": "Get current weather information", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state" + } + }, + "required": ["location"] + } + } +] + +response = completion( + model="lemonade/your-model-name", + messages=[{"role": "user", "content": "What's the weather in San Francisco?"}], + tools=[{"type": "function", "function": f} for f in functions], + tool_choice="auto" +) +print(response) +``` + +### Response Format + +Lemonade supports structured output with response format: + +```python +from litellm import completion +import json + +# Define schema in response_format +response = completion( + model="lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF", + messages=[{"role": "user", "content": "Generate JSON data for a person with their name, age, and city."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "person", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + "city": {"type": "string"} + }, + "required": ["name", "age"] + } + } + } +) + +print(f"Model: {response.model}") +print(f"JSON Output:") +json_data = json.loads(response.choices[0].message.content) +print(json.dumps(json_data, indent=2)) +``` + +## Available Models + +Lemonade automatically validates available models by querying the `/models` endpoint. You can check available models programmatically: + +```python +import httpx + +api_base = "http://localhost:8000" # or your custom base +response = httpx.get(f"{api_base}/api/v1/models") +models = response.json() +print("Available models:", [model['id'] for model in models.get('data', [])]) +``` + +## Support + +For more information regarding Lemonade please go to to the [Lemonade website](https://lemonade-server.ai/) or [Lemonade repository](https://github.com/lemonade-sdk/lemonade). diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 9ca0201ff7..d450159f93 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -477,6 +477,7 @@ const sidebars = { "providers/fireworks_ai", "providers/clarifai", "providers/compactifai", + "providers/lemonade", "providers/vllm", "providers/llamafile", "providers/infinity", diff --git a/tests/test_litellm/llms/lemonade/test_lemonade.py b/tests/test_litellm/llms/lemonade/test_lemonade.py new file mode 100644 index 0000000000..d3850d6271 --- /dev/null +++ b/tests/test_litellm/llms/lemonade/test_lemonade.py @@ -0,0 +1,180 @@ +import json +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path +from unittest.mock import MagicMock, patch + +from litellm.llms.lemonade.chat.transformation import LemonadeChatConfig +from litellm.types.utils import ModelResponse +import httpx + + +def test_lemonade_config_initialization(): + """Test that LemonadeChatConfig can be initialized with various parameters""" + config = LemonadeChatConfig( + temperature=0.7, + max_tokens=100, + top_p=0.9, + top_k=50, + repeat_penalty=1.1 + ) + + assert config.custom_llm_provider == "lemonade" + assert config.temperature == 0.7 + assert config.max_tokens == 100 + assert config.top_p == 0.9 + assert config.top_k == 50 + assert config.repeat_penalty == 1.1 + + +def test_get_openai_compatible_provider_info(): + """Test the provider info method returns correct API base and key""" + config = LemonadeChatConfig() + + api_base, key = config._get_openai_compatible_provider_info( + api_base=None, + api_key=None + ) + + assert api_base == "http://localhost:8000/api/v1" + assert key == "lemonade" + + +def test_get_openai_compatible_provider_info_with_custom_base(): + """Test the provider info method with custom API base""" + config = LemonadeChatConfig() + + custom_api_base = "https://custom.lemonade.ai/v1" + api_base, key = config._get_openai_compatible_provider_info( + api_base=custom_api_base, + api_key=None + ) + + assert api_base == custom_api_base + assert key == "lemonade" + + +def test_transform_response(): + """Test the response transformation adds lemonade prefix to model name""" + config = LemonadeChatConfig() + + # Mock raw response + raw_response = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {} + + # Create a model response + model_response = ModelResponse() + + # Mock the parent class transform_response method + with patch.object(config.__class__.__bases__[0], 'transform_response') as mock_parent: + mock_parent.return_value = model_response + + result = config.transform_response( + model="test-model", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + api_key="test-key", + json_mode=False, + ) + + # Check that the model name is prefixed with "lemonade/" + assert hasattr(result, 'model') + assert result.model == "lemonade/test-model" + + +def test_config_get_config(): + """Test that get_config method returns the configuration""" + config_dict = LemonadeChatConfig.get_config() + assert isinstance(config_dict, dict) + + +def test_response_format_support(): + """Test that response_format parameter is supported""" + response_format = { + "type": "json_object" + } + + config = LemonadeChatConfig(response_format=response_format) + assert config.response_format == response_format + + +def test_tools_support(): + """Test that tools parameter is supported""" + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information" + } + } + ] + + config = LemonadeChatConfig(tools=tools) + assert config.tools == tools + + +def test_functions_support(): + """Test that functions parameter is supported""" + functions = [ + { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": {} + } + } + ] + + config = LemonadeChatConfig(functions=functions) + assert config.functions == functions + + +def test_stop_parameter_support(): + """Test that stop parameter supports both string and list""" + # Test with string + config1 = LemonadeChatConfig(stop="STOP") + assert config1.stop == "STOP" + + # Test with list + config2 = LemonadeChatConfig(stop=["STOP", "END"]) + assert config2.stop == ["STOP", "END"] + + +def test_logit_bias_support(): + """Test that logit_bias parameter is supported""" + logit_bias = {"50256": -100} + + config = LemonadeChatConfig(logit_bias=logit_bias) + assert config.logit_bias == logit_bias + + +def test_presence_penalty_support(): + """Test that presence_penalty parameter is supported""" + config = LemonadeChatConfig(presence_penalty=0.5) + assert config.presence_penalty == 0.5 + + +def test_n_parameter_support(): + """Test that n parameter (number of completions) is supported""" + config = LemonadeChatConfig(n=3) + assert config.n == 3 + + +def test_max_completion_tokens_support(): + """Test that max_completion_tokens parameter is supported""" + config = LemonadeChatConfig(max_completion_tokens=150) + assert config.max_completion_tokens == 150 \ No newline at end of file From abaf77da4389488ad50d7ab8c68c8bb4afec2e37 Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Tue, 23 Sep 2025 22:01:38 -0600 Subject: [PATCH 08/13] Small updates to documentation --- docs/my-website/docs/providers/lemonade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/lemonade.md b/docs/my-website/docs/providers/lemonade.md index 87d41902a0..8ff7d48b70 100644 --- a/docs/my-website/docs/providers/lemonade.md +++ b/docs/my-website/docs/providers/lemonade.md @@ -3,7 +3,7 @@ import TabItem from '@theme/TabItem'; # Lemonade -Lemonade is an OpenAI-compatible AI provider that offers local language model inference on AMD Ryzen AI models. The provider supports standard chat completions with full OpenAI API compatibility. +[Lemonade Server](https://lemonade-server.ai/) is an OpenAI-compatible local language model inference provider optimized for AMD GPUs and NPUs. The `lemonade` litellm provider supports standard chat completions with full OpenAI API compatibility. | Property | Details | |-------|-------| From 3d62596daa3017f05314428a15018b91fa4daaef Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Tue, 23 Sep 2025 22:06:58 -0600 Subject: [PATCH 09/13] fix lint-ruff --- litellm/llms/lemonade/chat/transformation.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 1e372a3a6f..3fb63dabe0 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -1,20 +1,15 @@ """ Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completions` """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +from typing import Any, List, Optional, Tuple, Union import httpx -from pydantic import BaseModel import litellm -from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, - ChatCompletionAssistantMessage, - ChatCompletionToolParam, - ChatCompletionToolParamFunctionChunk, ) from litellm.types.utils import ModelResponse, ModelInfoBase From bbfa00c61bfbbb287f240501c718386ab089af47 Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Wed, 24 Sep 2025 09:23:20 -0600 Subject: [PATCH 10/13] fixing mypy lint errors --- litellm/llms/lemonade/chat/transformation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 3fb63dabe0..f1dc0e1a37 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -26,7 +26,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): presence_penalty: Optional[int] = None stop: Optional[Union[str, list]] = None temperature: Optional[int] = None - top_p: Optional[float] = None + top_p: Optional[int] = None top_k: Optional[int] = None response_format: Optional[dict] = None tools: Optional[list] = None @@ -42,7 +42,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): presence_penalty: Optional[int] = None, stop: Optional[Union[str, list]] = None, temperature: Optional[int] = None, - top_p: Optional[float] = None, + top_p: Optional[int] = None, top_k: Optional[int] = None, response_format: Optional[dict] = None, tools: Optional[list] = None, @@ -89,13 +89,16 @@ class LemonadeChatConfig(OpenAILikeChatConfig): ) # Returning the model if it was found in lemonade. Currently there is no mechanism to report - # if the model supports function calling or the max tokens so we leave those out + # the models max input or output tokens so leaving those as None return ModelInfoBase( key=model, litellm_provider="lemonade", mode="chat", input_cost_per_token=0.0, output_cost_per_token=0.0, + max_input_tokens=None, + max_output_tokens=None, + max_tokens=None, ) def _get_openai_compatible_provider_info( From 1e1e4c36ac23af9da96d3732948eeff4244ac29b Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Mon, 29 Sep 2025 22:00:55 -0600 Subject: [PATCH 11/13] Fixing key name --- litellm/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 76b3932986..4387e73198 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3550,7 +3550,7 @@ def completion( # type: ignore # noqa: PLR0915 elif custom_llm_provider == "lemonade": api_key = ( api_key - or litellm.bytez_key + or litellm.lemonade_key or get_secret_str("LEMONADE_API_KEY") or litellm.api_key ) From 19e7070b737a25cca3cc9eed0a382b42efee0d8e Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Tue, 30 Sep 2025 09:29:13 -0600 Subject: [PATCH 12/13] Removing get_model_info from Lemonade provider. Implemented get_models which gets hooked into get_valid_models litellm utility. Also, added a simple cost calculator implementation for Lemonade so calling cost_calculator.completion_cost() doesn't return an error when a model is not found in the model_cost json. --- litellm/cost_calculator.py | 5 ++ litellm/llms/lemonade/chat/transformation.py | 63 ++++++++++---------- litellm/llms/lemonade/cost_calculator.py | 35 +++++++++++ litellm/utils.py | 4 +- 4 files changed, 73 insertions(+), 34 deletions(-) create mode 100644 litellm/llms/lemonade/cost_calculator.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 94a9523fac..4bb14eb839 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -58,6 +58,9 @@ from litellm.llms.vertex_ai.cost_calculator import ( ) from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token +from litellm.llms.lemonade.cost_calculator import ( + cost_per_token as lemonade_cost_per_token, +) from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -347,6 +350,8 @@ def cost_per_token( # noqa: PLR0915 return perplexity_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "xai": return xai_cost_per_token(model=model, usage=usage_block) + elif custom_llm_provider == "lemonade": + return lemonade_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "dashscope": from litellm.llms.dashscope.cost_calculator import ( cost_per_token as dashscope_cost_per_token, diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index f1dc0e1a37..094e625fe6 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -60,46 +60,45 @@ class LemonadeChatConfig(OpenAILikeChatConfig): def get_config(cls): return super().get_config() - def get_model_info(self, model: str) -> ModelInfoBase: - if model.startswith("lemonade/"): - model = model.split("/", 1)[1] - api_base = get_secret_str("LEMONADE_API_BASE") or "http://localhost:8000" + def get_models(self, api_key: Optional[str] = None, api_base: Optional[str] = None): + """ + Get available models from Lemonade API. + + This method queries the Lemonade /models endpoint to retrieve the list of available models. + + Args: + api_key: Optional API key (Lemonade doesn't require authentication) + api_base: Optional API base URL (defaults to LEMONADE_API_BASE env var or http://localhost:8000) + + Returns: + List of model names prefixed with "lemonade/" + """ + api_base, api_key = self._get_openai_compatible_provider_info( + api_base=api_base, api_key=api_key + ) + + if api_base is None: + raise ValueError( + "LEMONADE_API_BASE is not set. Please set the environment variable to query Lemonade's /models endpoint." + ) - # Getting the list of models from lemonade to verify the model exists + # Getting the list of models from lemonade try: response = litellm.module_level_client.get( - url=f"{api_base}/api/v1/models", + url=f"{api_base}/models", ) except Exception as e: - raise Exception( - f"LemonadeError: Error getting model info for {model}. Set Lemonade API Base via `LEMONADE_API_BASE` environment variable. Error: {e}" - ) - - # Making sure the model exists in lemonade - model_found = False - model_list = response.json().get("data", []) - for model_iter in model_list: - if model_iter['id'] == model: - model_found = True - break - - if not model_found: raise ValueError( - f"LemonadeError: Model {model} not found. Available models: {[m['id'] for m in model_list]}" + f"Failed to fetch models from Lemonade. Set Lemonade API Base via `LEMONADE_API_BASE` environment variable. Error: {e}" ) - # Returning the model if it was found in lemonade. Currently there is no mechanism to report - # the models max input or output tokens so leaving those as None - return ModelInfoBase( - key=model, - litellm_provider="lemonade", - mode="chat", - input_cost_per_token=0.0, - output_cost_per_token=0.0, - max_input_tokens=None, - max_output_tokens=None, - max_tokens=None, - ) + if response.status_code != 200: + raise ValueError( + f"Failed to fetch models from Lemonade. Status code: {response.status_code}, Response: {response.text}" + ) + + model_list = response.json().get("data", []) + return ["lemonade/" + model["id"] for model in model_list] def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] diff --git a/litellm/llms/lemonade/cost_calculator.py b/litellm/llms/lemonade/cost_calculator.py new file mode 100644 index 0000000000..27e1ca275f --- /dev/null +++ b/litellm/llms/lemonade/cost_calculator.py @@ -0,0 +1,35 @@ +""" +Cost calculation for Lemonade LLM provider. + +Since Lemonade is a local/self-hosted service, all costs default to 0. +This prevents cost calculation errors when using models not in model_prices_and_context_window.json +""" +from typing import Tuple + +from litellm.types.utils import Usage + + +def cost_per_token( + model: str, + usage: Usage, +) -> Tuple[float, float]: + """ + Calculate cost per token for Lemonade models. + + Since Lemonade is a local/self-hosted deployment, there are no per-token costs. + This function returns (0.0, 0.0) for all models to allow cost tracking to work + without errors for any Lemonade model, regardless of whether it's in the + model_prices_and_context_window.json file. + + Args: + model: The model name (with or without "lemonade/" prefix) + usage: Usage object containing token counts + + Returns: + Tuple of (prompt_cost, completion_cost) - always (0.0, 0.0) for Lemonade + """ + # Lemonade is self-hosted/local, so cost is always 0 + prompt_cost = 0.0 + completion_cost = 0.0 + + return prompt_cost, completion_cost diff --git a/litellm/utils.py b/litellm/utils.py index fbc5d09e57..8dfa2416a6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4775,8 +4775,6 @@ def _get_model_info_helper( # noqa: PLR0915 custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat" ) and not _is_potential_model_name_in_model_cost(potential_model_names): return litellm.OllamaConfig().get_model_info(model) - elif (custom_llm_provider == "lemonade" and not _is_potential_model_name_in_model_cost(potential_model_names)): - return litellm.LemonadeChatConfig().get_model_info(model) else: """ Check if: (in order of specificity) @@ -7319,6 +7317,8 @@ class ProviderConfigManager: ) return VLLMModelInfo() + elif LlmProviders.LEMONADE == provider: + return litellm.LemonadeChatConfig() return None @staticmethod From 7da05df534fd6ade36014df16999bb9187542455 Mon Sep 17 00:00:00 2001 From: Eddie Richter Date: Tue, 30 Sep 2025 09:34:00 -0600 Subject: [PATCH 13/13] Removing unecessary import --- litellm/llms/lemonade/chat/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 094e625fe6..8cba844435 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -11,7 +11,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, ) -from litellm.types.utils import ModelResponse, ModelInfoBase +from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig