From de7870cb54e1c29e48b3380ced9701760f3d0efb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 1 May 2025 18:36:55 -0700 Subject: [PATCH] Add `llamafile` as a provider (#10203) (#10482) * Update docs for OpenAI compatible providers, add Llamafile docs, include Llamafile in the sidebar * Add Llamafile as an LlmProviders enum * Add llamafile as a OpenAI compatible provider (in the list of compatible providers) * Add Llamafile chat config and tests * Wire up Llamafile Co-authored-by: Peter Wilson --- docs/my-website/docs/providers/llamafile.md | 158 ++++++++++++++++++ .../docs/providers/openai_compatible.md | 17 +- docs/my-website/sidebars.js | 1 + litellm/__init__.py | 2 +- litellm/constants.py | 3 + .../get_llm_provider_logic.py | 7 +- litellm/llms/llamafile/chat/transformation.py | 46 +++++ litellm/main.py | 1 + litellm/types/utils.py | 1 + litellm/utils.py | 2 + .../test_llamafile_chat_transformation.py | 128 ++++++++++++++ tests/local_testing/test_completion.py | 3 +- tests/local_testing/test_embedding.py | 22 +++ tests/local_testing/test_get_llm_provider.py | 10 ++ 14 files changed, 396 insertions(+), 5 deletions(-) create mode 100644 docs/my-website/docs/providers/llamafile.md create mode 100644 litellm/llms/llamafile/chat/transformation.py create mode 100644 tests/litellm/llms/llamafile/chat/test_llamafile_chat_transformation.py diff --git a/docs/my-website/docs/providers/llamafile.md b/docs/my-website/docs/providers/llamafile.md new file mode 100644 index 0000000000..3539bc2eb4 --- /dev/null +++ b/docs/my-website/docs/providers/llamafile.md @@ -0,0 +1,158 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Llamafile + +LiteLLM supports all models on Llamafile. + +| Property | Details | +|---------------------------|--------------------------------------------------------------------------------------------------------------------------------------| +| Description | llamafile lets you distribute and run LLMs with a single file. [Docs](https://github.com/Mozilla-Ocho/llamafile/blob/main/README.md) | +| Provider Route on LiteLLM | `llamafile/` (for OpenAI compatible server) | +| Provider Doc | [llamafile ↗](https://github.com/Mozilla-Ocho/llamafile/blob/main/llama.cpp/server/README.md#api-endpoints) | +| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions` | + + +# Quick Start + +## Usage - litellm.completion (calling OpenAI compatible endpoint) +llamafile Provides an OpenAI compatible endpoint for chat completions - here's how to call it with LiteLLM + +To use litellm to call llamafile add the following to your completion call + +* `model="llamafile/"` +* `api_base = "your-hosted-llamafile"` + +```python +import litellm + +response = litellm.completion( + model="llamafile/mistralai/mistral-7b-instruct-v0.2", # pass the llamafile model name for completeness + messages=messages, + api_base="http://localhost:8080/v1", + temperature=0.2, + max_tokens=80) + +print(response) +``` + + +## Usage - LiteLLM Proxy Server (calling OpenAI compatible endpoint) + +Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server + +1. Modify the config.yaml + + ```yaml + model_list: + - model_name: my-model + litellm_params: + model: llamafile/mistralai/mistral-7b-instruct-v0.2 # add llamafile/ prefix to route as OpenAI provider + api_base: http://localhost:8080/v1 # add api base for OpenAI compatible provider + ``` + +1. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml + ``` + +1. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + client = openai.OpenAI( + api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="my-model", + messages = [ + { + "role": "user", + "content": "what llm are you" + } + ], + ) + + print(response) + ``` + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "my-model", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' + ``` + + + + + +## Embeddings + + + + +```python +from litellm import embedding +import os + +os.environ["LLAMAFILE_API_BASE"] = "http://localhost:8080/v1" + + +embedding = embedding(model="llamafile/sentence-transformers/all-MiniLM-L6-v2", input=["Hello world"]) + +print(embedding) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: my-model + litellm_params: + model: llamafile/sentence-transformers/all-MiniLM-L6-v2 # add llamafile/ prefix to route as OpenAI provider + api_base: http://localhost:8080/v1 # add api base for OpenAI compatible provider +``` + +1. Start the proxy + +```bash +$ litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +1. Test it! + +```bash +curl -L -X POST 'http://0.0.0.0:4000/embeddings' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{"input": ["hello world"], "model": "my-model"}' +``` + +[See OpenAI SDK/Langchain/etc. examples](../proxy/user_keys.md#embeddings) + + + \ No newline at end of file diff --git a/docs/my-website/docs/providers/openai_compatible.md b/docs/my-website/docs/providers/openai_compatible.md index c7f9bf6f40..2f11379a8d 100644 --- a/docs/my-website/docs/providers/openai_compatible.md +++ b/docs/my-website/docs/providers/openai_compatible.md @@ -3,13 +3,26 @@ import TabItem from '@theme/TabItem'; # OpenAI-Compatible Endpoints +:::info + +Selecting `openai` as the provider routes your request to an OpenAI-compatible endpoint using the upstream +[official OpenAI Python API library](https://github.com/openai/openai-python/blob/main/README.md). + +This library **requires** an API key for all requests, either through the `api_key` parameter +or the `OPENAI_API_KEY` environment variable. + +If you don’t want to provide a fake API key in each request, consider using a provider that directly matches your +OpenAI-compatible endpoint, such as [`hosted_vllm`](/docs/providers/vllm) or [`llamafile`](/docs/providers/llamafile). + +::: + To call models hosted behind an openai proxy, make 2 changes: 1. For `/chat/completions`: Put `openai/` in front of your model name, so litellm knows you're trying to call an openai `/chat/completions` endpoint. -2. For `/completions`: Put `text-completion-openai/` in front of your model name, so litellm knows you're trying to call an openai `/completions` endpoint. [NOT REQUIRED for `openai/` endpoints called via `/v1/completions` route]. +1. For `/completions`: Put `text-completion-openai/` in front of your model name, so litellm knows you're trying to call an openai `/completions` endpoint. [NOT REQUIRED for `openai/` endpoints called via `/v1/completions` route]. -2. **Do NOT** add anything additional to the base url e.g. `/v1/embedding`. LiteLLM uses the openai-client to make these calls, and that automatically adds the relevant endpoints. +1. **Do NOT** add anything additional to the base url e.g. `/v1/embedding`. LiteLLM uses the openai-client to make these calls, and that automatically adds the relevant endpoints. ## Usage - completion diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index c1573db350..e278b36ed7 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -236,6 +236,7 @@ const sidebars = { "providers/fireworks_ai", "providers/clarifai", "providers/vllm", + "providers/llamafile", "providers/infinity", "providers/xinference", "providers/cloudflare_workers", diff --git a/litellm/__init__.py b/litellm/__init__.py index 904e2bda8b..a015eceb0a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -72,7 +72,6 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager import httpx import dotenv -from enum import Enum litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" if litellm_mode == "DEV": @@ -1015,6 +1014,7 @@ from .llms.azure.azure import ( from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig from .llms.azure.completion.transformation import AzureOpenAITextConfig from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig +from .llms.llamafile.chat.transformation import LlamafileChatConfig from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig from .llms.vllm.completion.transformation import VLLMConfig from .llms.deepseek.chat.transformation import DeepSeekChatConfig diff --git a/litellm/constants.py b/litellm/constants.py index c04a018260..aaaddd39cd 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -156,6 +156,7 @@ LITELLM_CHAT_PROVIDERS = [ "custom", "litellm_proxy", "hosted_vllm", + "llamafile", "lm_studio", "galadriel", ] @@ -245,6 +246,7 @@ openai_compatible_providers: List = [ "github", "litellm_proxy", "hosted_vllm", + "llamafile", "lm_studio", "galadriel", ] @@ -253,6 +255,7 @@ openai_text_completion_compatible_providers: List = ( "together_ai", "fireworks_ai", "hosted_vllm", + "llamafile", ] ) _openai_like_providers: List = [ diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 13103c85a0..331087e02e 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -101,7 +101,6 @@ def get_llm_provider( # noqa: PLR0915 Return model, custom_llm_provider, dynamic_api_key, api_base """ - try: ## IF LITELLM PARAMS GIVEN ## if litellm_params is not None: @@ -477,6 +476,12 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.HostedVLLMChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "llamafile": + # llamafile is OpenAI compatible. + (api_base, dynamic_api_key) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info( + api_base, + api_key + ) elif custom_llm_provider == "lm_studio": # lm_studio is openai compatible, we just need to set this to custom_openai ( diff --git a/litellm/llms/llamafile/chat/transformation.py b/litellm/llms/llamafile/chat/transformation.py new file mode 100644 index 0000000000..b0f8cd3fc3 --- /dev/null +++ b/litellm/llms/llamafile/chat/transformation.py @@ -0,0 +1,46 @@ +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class LlamafileChatConfig(OpenAIGPTConfig): + """LlamafileChatConfig is used to provide configuration for the LlamaFile's chat API.""" + + @staticmethod + def _resolve_api_key(api_key: Optional[str] = None) -> str: + """Attempt to ensure that the API key is set, preferring the user-provided key + over the secret manager key (``LLAMAFILE_API_KEY``). + + If both are None, a fake API key is returned. + """ + return api_key or get_secret_str("LLAMAFILE_API_KEY") or "fake-api-key" # llamafile does not require an API key + + @staticmethod + def _resolve_api_base(api_base: Optional[str] = None) -> Optional[str]: + """Attempt to ensure that the API base is set, preferring the user-provided key + over the secret manager key (``LLAMAFILE_API_BASE``). + + If both are None, a default Llamafile server URL is returned. + See: https://github.com/Mozilla-Ocho/llamafile/blob/bd1bbe9aabb1ee12dbdcafa8936db443c571eb9d/README.md#L61 + """ + return api_base or get_secret_str("LLAMAFILE_API_BASE") or "http://127.0.0.1:8080/v1" # type: ignore + + + def _get_openai_compatible_provider_info( + self, + api_base: Optional[str], + api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + """Attempts to ensure that the API base and key are set, preferring user-provided values, + before falling back to secret manager values (``LLAMAFILE_API_BASE`` and ``LLAMAFILE_API_KEY`` + respectively). + + If an API key cannot be resolved via either method, a fake key is returned. Llamafile + does not require an API key, but the underlying OpenAI library may expect one anyway. + """ + api_base = LlamafileChatConfig._resolve_api_base(api_base) + dynamic_api_key = LlamafileChatConfig._resolve_api_key(api_key) + + return api_base, dynamic_api_key diff --git a/litellm/main.py b/litellm/main.py index 28c2ddce4d..52e6609f37 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3610,6 +3610,7 @@ def embedding( # noqa: PLR0915 custom_llm_provider == "openai_like" or custom_llm_provider == "jina_ai" or custom_llm_provider == "hosted_vllm" + or custom_llm_provider == "llamafile" or custom_llm_provider == "lm_studio" ): api_base = ( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4be152c6e4..5bd8dfc50d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2093,6 +2093,7 @@ class LlmProviders(str, Enum): CUSTOM = "custom" LITELLM_PROXY = "litellm_proxy" HOSTED_VLLM = "hosted_vllm" + LLAMAFILE = "llamafile" LM_STUDIO = "lm_studio" GALADRIEL = "galadriel" INFINITY = "infinity" diff --git a/litellm/utils.py b/litellm/utils.py index 1bcfbef169..87012d5eb5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6464,6 +6464,8 @@ class ProviderConfigManager: return litellm.AiohttpOpenAIChatConfig() elif litellm.LlmProviders.HOSTED_VLLM == provider: return litellm.HostedVLLMChatConfig() + elif litellm.LlmProviders.LLAMAFILE == provider: + return litellm.LlamafileChatConfig() elif litellm.LlmProviders.LM_STUDIO == provider: return litellm.LMStudioChatConfig() elif litellm.LlmProviders.GALADRIEL == provider: diff --git a/tests/litellm/llms/llamafile/chat/test_llamafile_chat_transformation.py b/tests/litellm/llms/llamafile/chat/test_llamafile_chat_transformation.py new file mode 100644 index 0000000000..3dfd0d6628 --- /dev/null +++ b/tests/litellm/llms/llamafile/chat/test_llamafile_chat_transformation.py @@ -0,0 +1,128 @@ +from typing import Optional + +from unittest.mock import patch +import pytest + +import litellm +from litellm.llms.llamafile.chat.transformation import LlamafileChatConfig + + +@pytest.mark.parametrize( + "input_api_key, api_key_from_secret_manager, expected_api_key, secret_manager_called", + [ + ("user-provided-key", "secret-key", "user-provided-key", False), + (None, "secret-key", "secret-key", True), + (None, None, "fake-api-key", True), + ("", "secret-key", "secret-key", True), # Empty string should fall back to secret + ("", None, "fake-api-key", True), # Empty string with no secret should use the fake key + ] +) +def test_resolve_api_key(input_api_key, api_key_from_secret_manager, expected_api_key, secret_manager_called): + with patch("litellm.llms.llamafile.chat.transformation.get_secret_str") as mock_get_secret: + mock_get_secret.return_value = api_key_from_secret_manager + + result = LlamafileChatConfig._resolve_api_key(input_api_key) + + if secret_manager_called: + mock_get_secret.assert_called_once_with("LLAMAFILE_API_KEY") + else: + mock_get_secret.assert_not_called() + + assert result == expected_api_key + + +@pytest.mark.parametrize( + "input_api_base, api_base_from_secret_manager, expected_api_base, secret_manager_called", + [ + ("https://user-api.example.com", "https://secret-api.example.com", "https://user-api.example.com", False), + (None, "https://secret-api.example.com", "https://secret-api.example.com", True), + (None, None, "http://127.0.0.1:8080/v1", True), + ("", "https://secret-api.example.com", "https://secret-api.example.com", True), # Empty string should fall back + ] +) +def test_resolve_api_base(input_api_base, api_base_from_secret_manager, expected_api_base, secret_manager_called): + with patch("litellm.llms.llamafile.chat.transformation.get_secret_str") as mock_get_secret: + mock_get_secret.return_value = api_base_from_secret_manager + + result = LlamafileChatConfig._resolve_api_base(input_api_base) + + if secret_manager_called: + mock_get_secret.assert_called_once_with("LLAMAFILE_API_BASE") + else: + mock_get_secret.assert_not_called() + + assert result == expected_api_base + + +@pytest.mark.parametrize( + "api_base, api_key, secret_base, secret_key, expected_base, expected_key", + [ + # User-provided values + ("https://user-api.example.com", "user-key", "https://secret-api.example.com", "secret-key", "https://user-api.example.com", "user-key"), + # Fallback to secrets + (None, None, "https://secret-api.example.com", "secret-key", "https://secret-api.example.com", "secret-key"), + # Nothing provided, use defaults + (None, None, None, None, "http://127.0.0.1:8080/v1", "fake-api-key"), + # Mixed scenarios + ("https://user-api.example.com", None, None, "secret-key", "https://user-api.example.com", "secret-key"), + (None, "user-key", "https://secret-api.example.com", None, "https://secret-api.example.com", "user-key"), + ] +) +def test_get_openai_compatible_provider_info(api_base, api_key, secret_base, secret_key, expected_base, expected_key): + config = LlamafileChatConfig() + + def fake_get_secret(key: str) -> Optional[str]: + return { + "LLAMAFILE_API_BASE": secret_base, + "LLAMAFILE_API_KEY": secret_key + }.get(key) + + patch_secret = patch("litellm.llms.llamafile.chat.transformation.get_secret_str", side_effect=fake_get_secret) + patch_base = patch.object(LlamafileChatConfig, "_resolve_api_base", wraps=LlamafileChatConfig._resolve_api_base) + patch_key = patch.object(LlamafileChatConfig, "_resolve_api_key", wraps=LlamafileChatConfig._resolve_api_key) + + with patch_secret as mock_secret, patch_base as mock_base, patch_key as mock_key: + result_base, result_key = config._get_openai_compatible_provider_info(api_base, api_key) + + assert result_base == expected_base + assert result_key == expected_key + + mock_base.assert_called_once_with(api_base) + mock_key.assert_called_once_with(api_key) + + # Ensure get_secret_str was used as expected within the methods + if api_base and api_key: + mock_secret.assert_not_called() + elif api_base or api_key: + mock_secret.assert_called_once() + else: + assert mock_secret.call_count == 2 + + +def test_completion_with_custom_llamafile_model(): + with patch("litellm.main.openai_chat_completions.completion") as mock_llamafile_completion_func: + mock_llamafile_completion_func.return_value = {} # Return an empty dictionary for the mocked response + + provider = "llamafile" + model_name = "my-custom-test-model" + model = f"{provider}/{model_name}" + messages = [{"role": "user", "content": "Hey, how's it going?"}] + + _ = litellm.completion( + model=model, + messages=messages, + max_retries=2, + max_tokens=100, + ) + + mock_llamafile_completion_func.assert_called_once() + _, call_kwargs = mock_llamafile_completion_func.call_args + assert call_kwargs.get("custom_llm_provider") == provider + assert call_kwargs.get("model") == model_name + assert call_kwargs.get("messages") == messages + assert call_kwargs.get("api_base") == "http://127.0.0.1:8080/v1" + assert call_kwargs.get("api_key") == "fake-api-key" + optional_params = call_kwargs.get("optional_params") + assert optional_params + assert optional_params.get("max_retries") == 2 + assert optional_params.get("max_tokens") == 100 diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 6bad4a46b7..c024b43e3c 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1477,7 +1477,7 @@ HF Tests we should pass @pytest.mark.parametrize( - "provider", ["openai", "hosted_vllm", "lm_studio"] + "provider", ["openai", "hosted_vllm", "lm_studio", "llamafile"] ) # "vertex_ai", @pytest.mark.asyncio async def test_openai_compatible_custom_api_base(provider): @@ -1519,6 +1519,7 @@ async def test_openai_compatible_custom_api_base(provider): [ "openai", "hosted_vllm", + "llamafile", ], ) # "vertex_ai", @pytest.mark.asyncio diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index 9146944aa2..75e0c35283 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -1029,6 +1029,28 @@ def test_hosted_vllm_embedding(monkeypatch): assert json_data["model"] == "jina-embeddings-v3" +def test_llamafile_embedding(monkeypatch): + monkeypatch.setenv("LLAMAFILE_API_BASE", "http://localhost:8080/v1") + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + with patch.object(client, "post") as mock_post: + try: + embedding( + model="llamafile/jina-embeddings-v3", + input=["Hello world"], + client=client, + ) + except Exception as e: + print(e) + + mock_post.assert_called_once() + + json_data = json.loads(mock_post.call_args.kwargs["data"]) + assert json_data["input"] == ["Hello world"] + assert json_data["model"] == "jina-embeddings-v3" + + @pytest.mark.asyncio @pytest.mark.parametrize("sync_mode", [True, False]) async def test_lm_studio_embedding(monkeypatch, sync_mode): diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index fa27a8378c..b5ea128971 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -185,6 +185,16 @@ def test_get_llm_provider_hosted_vllm(): assert dynamic_api_key == "fake-api-key" +def test_get_llm_provider_llamafile(): + model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( + model="llamafile/mistralai/mistral-7b-instruct-v0.2", + ) + assert custom_llm_provider == "llamafile" + assert model == "mistralai/mistral-7b-instruct-v0.2" + assert dynamic_api_key == "fake-api-key" + assert api_base == "http://127.0.0.1:8080/v1" + + def test_get_llm_provider_watson_text(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( model="watsonx_text/watson-text-to-speech",