mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-14 14:26:43 +00:00
* Update docs for OpenAI compatible providers, add Llamafile docs, include Llamafile in the sidebar * Add Llamafile as an LlmProviders enum * Add llamafile as a OpenAI compatible provider (in the list of compatible providers) * Add Llamafile chat config and tests * Wire up Llamafile Co-authored-by: Peter Wilson <peter@mozilla.ai>
This commit is contained in:
Ishaan Jaff
co-authored by
Peter Wilson
Peter Wilson
parent
66cf75cd5d
commit
de7870cb54
@@ -0,0 +1,158 @@
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Llamafile
|
||||
|
||||
LiteLLM supports all models on Llamafile.
|
||||
|
||||
| Property | Details |
|
||||
|---------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| Description | llamafile lets you distribute and run LLMs with a single file. [Docs](https://github.com/Mozilla-Ocho/llamafile/blob/main/README.md) |
|
||||
| Provider Route on LiteLLM | `llamafile/` (for OpenAI compatible server) |
|
||||
| Provider Doc | [llamafile ↗](https://github.com/Mozilla-Ocho/llamafile/blob/main/llama.cpp/server/README.md#api-endpoints) |
|
||||
| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions` |
|
||||
|
||||
|
||||
# Quick Start
|
||||
|
||||
## Usage - litellm.completion (calling OpenAI compatible endpoint)
|
||||
llamafile Provides an OpenAI compatible endpoint for chat completions - here's how to call it with LiteLLM
|
||||
|
||||
To use litellm to call llamafile add the following to your completion call
|
||||
|
||||
* `model="llamafile/<your-llamafile-model-name>"`
|
||||
* `api_base = "your-hosted-llamafile"`
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="llamafile/mistralai/mistral-7b-instruct-v0.2", # pass the llamafile model name for completeness
|
||||
messages=messages,
|
||||
api_base="http://localhost:8080/v1",
|
||||
temperature=0.2,
|
||||
max_tokens=80)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
|
||||
## Usage - LiteLLM Proxy Server (calling OpenAI compatible endpoint)
|
||||
|
||||
Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
|
||||
|
||||
1. Modify the config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: my-model
|
||||
litellm_params:
|
||||
model: llamafile/mistralai/mistral-7b-instruct-v0.2 # add llamafile/ prefix to route as OpenAI provider
|
||||
api_base: http://localhost:8080/v1 # add api base for OpenAI compatible provider
|
||||
```
|
||||
|
||||
1. Start the proxy
|
||||
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
1. Send Request to LiteLLM Proxy Server
|
||||
|
||||
<Tabs>
|
||||
|
||||
<TabItem value="openai" label="OpenAI Python v1.0.0+">
|
||||
|
||||
```python
|
||||
import openai
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
|
||||
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="my-model",
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="curl">
|
||||
|
||||
```shell
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"model": "my-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
],
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Embeddings
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import embedding
|
||||
import os
|
||||
|
||||
os.environ["LLAMAFILE_API_BASE"] = "http://localhost:8080/v1"
|
||||
|
||||
|
||||
embedding = embedding(model="llamafile/sentence-transformers/all-MiniLM-L6-v2", input=["Hello world"])
|
||||
|
||||
print(embedding)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: my-model
|
||||
litellm_params:
|
||||
model: llamafile/sentence-transformers/all-MiniLM-L6-v2 # add llamafile/ prefix to route as OpenAI provider
|
||||
api_base: http://localhost:8080/v1 # add api base for OpenAI compatible provider
|
||||
```
|
||||
|
||||
1. Start the proxy
|
||||
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
1. Test it!
|
||||
|
||||
```bash
|
||||
curl -L -X POST 'http://0.0.0.0:4000/embeddings' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"input": ["hello world"], "model": "my-model"}'
|
||||
```
|
||||
|
||||
[See OpenAI SDK/Langchain/etc. examples](../proxy/user_keys.md#embeddings)
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
@@ -3,13 +3,26 @@ import TabItem from '@theme/TabItem';
|
||||
|
||||
# OpenAI-Compatible Endpoints
|
||||
|
||||
:::info
|
||||
|
||||
Selecting `openai` as the provider routes your request to an OpenAI-compatible endpoint using the upstream
|
||||
[official OpenAI Python API library](https://github.com/openai/openai-python/blob/main/README.md).
|
||||
|
||||
This library **requires** an API key for all requests, either through the `api_key` parameter
|
||||
or the `OPENAI_API_KEY` environment variable.
|
||||
|
||||
If you don’t want to provide a fake API key in each request, consider using a provider that directly matches your
|
||||
OpenAI-compatible endpoint, such as [`hosted_vllm`](/docs/providers/vllm) or [`llamafile`](/docs/providers/llamafile).
|
||||
|
||||
:::
|
||||
|
||||
To call models hosted behind an openai proxy, make 2 changes:
|
||||
|
||||
1. For `/chat/completions`: Put `openai/` in front of your model name, so litellm knows you're trying to call an openai `/chat/completions` endpoint.
|
||||
|
||||
2. For `/completions`: Put `text-completion-openai/` in front of your model name, so litellm knows you're trying to call an openai `/completions` endpoint. [NOT REQUIRED for `openai/` endpoints called via `/v1/completions` route].
|
||||
1. For `/completions`: Put `text-completion-openai/` in front of your model name, so litellm knows you're trying to call an openai `/completions` endpoint. [NOT REQUIRED for `openai/` endpoints called via `/v1/completions` route].
|
||||
|
||||
2. **Do NOT** add anything additional to the base url e.g. `/v1/embedding`. LiteLLM uses the openai-client to make these calls, and that automatically adds the relevant endpoints.
|
||||
1. **Do NOT** add anything additional to the base url e.g. `/v1/embedding`. LiteLLM uses the openai-client to make these calls, and that automatically adds the relevant endpoints.
|
||||
|
||||
|
||||
## Usage - completion
|
||||
|
||||
@@ -236,6 +236,7 @@ const sidebars = {
|
||||
"providers/fireworks_ai",
|
||||
"providers/clarifai",
|
||||
"providers/vllm",
|
||||
"providers/llamafile",
|
||||
"providers/infinity",
|
||||
"providers/xinference",
|
||||
"providers/cloudflare_workers",
|
||||
|
||||
+1
-1
@@ -72,7 +72,6 @@ from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
|
||||
import httpx
|
||||
import dotenv
|
||||
from enum import Enum
|
||||
|
||||
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
|
||||
if litellm_mode == "DEV":
|
||||
@@ -1015,6 +1014,7 @@ from .llms.azure.azure import (
|
||||
from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig
|
||||
from .llms.azure.completion.transformation import AzureOpenAITextConfig
|
||||
from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig
|
||||
from .llms.llamafile.chat.transformation import LlamafileChatConfig
|
||||
from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig
|
||||
from .llms.vllm.completion.transformation import VLLMConfig
|
||||
from .llms.deepseek.chat.transformation import DeepSeekChatConfig
|
||||
|
||||
@@ -156,6 +156,7 @@ LITELLM_CHAT_PROVIDERS = [
|
||||
"custom",
|
||||
"litellm_proxy",
|
||||
"hosted_vllm",
|
||||
"llamafile",
|
||||
"lm_studio",
|
||||
"galadriel",
|
||||
]
|
||||
@@ -245,6 +246,7 @@ openai_compatible_providers: List = [
|
||||
"github",
|
||||
"litellm_proxy",
|
||||
"hosted_vllm",
|
||||
"llamafile",
|
||||
"lm_studio",
|
||||
"galadriel",
|
||||
]
|
||||
@@ -253,6 +255,7 @@ openai_text_completion_compatible_providers: List = (
|
||||
"together_ai",
|
||||
"fireworks_ai",
|
||||
"hosted_vllm",
|
||||
"llamafile",
|
||||
]
|
||||
)
|
||||
_openai_like_providers: List = [
|
||||
|
||||
@@ -101,7 +101,6 @@ def get_llm_provider( # noqa: PLR0915
|
||||
|
||||
Return model, custom_llm_provider, dynamic_api_key, api_base
|
||||
"""
|
||||
|
||||
try:
|
||||
## IF LITELLM PARAMS GIVEN ##
|
||||
if litellm_params is not None:
|
||||
@@ -477,6 +476,12 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
|
||||
) = litellm.HostedVLLMChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
elif custom_llm_provider == "llamafile":
|
||||
# llamafile is OpenAI compatible.
|
||||
(api_base, dynamic_api_key) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base,
|
||||
api_key
|
||||
)
|
||||
elif custom_llm_provider == "lm_studio":
|
||||
# lm_studio is openai compatible, we just need to set this to custom_openai
|
||||
(
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
||||
class LlamafileChatConfig(OpenAIGPTConfig):
|
||||
"""LlamafileChatConfig is used to provide configuration for the LlamaFile's chat API."""
|
||||
|
||||
@staticmethod
|
||||
def _resolve_api_key(api_key: Optional[str] = None) -> str:
|
||||
"""Attempt to ensure that the API key is set, preferring the user-provided key
|
||||
over the secret manager key (``LLAMAFILE_API_KEY``).
|
||||
|
||||
If both are None, a fake API key is returned.
|
||||
"""
|
||||
return api_key or get_secret_str("LLAMAFILE_API_KEY") or "fake-api-key" # llamafile does not require an API key
|
||||
|
||||
@staticmethod
|
||||
def _resolve_api_base(api_base: Optional[str] = None) -> Optional[str]:
|
||||
"""Attempt to ensure that the API base is set, preferring the user-provided key
|
||||
over the secret manager key (``LLAMAFILE_API_BASE``).
|
||||
|
||||
If both are None, a default Llamafile server URL is returned.
|
||||
See: https://github.com/Mozilla-Ocho/llamafile/blob/bd1bbe9aabb1ee12dbdcafa8936db443c571eb9d/README.md#L61
|
||||
"""
|
||||
return api_base or get_secret_str("LLAMAFILE_API_BASE") or "http://127.0.0.1:8080/v1" # type: ignore
|
||||
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Attempts to ensure that the API base and key are set, preferring user-provided values,
|
||||
before falling back to secret manager values (``LLAMAFILE_API_BASE`` and ``LLAMAFILE_API_KEY``
|
||||
respectively).
|
||||
|
||||
If an API key cannot be resolved via either method, a fake key is returned. Llamafile
|
||||
does not require an API key, but the underlying OpenAI library may expect one anyway.
|
||||
"""
|
||||
api_base = LlamafileChatConfig._resolve_api_base(api_base)
|
||||
dynamic_api_key = LlamafileChatConfig._resolve_api_key(api_key)
|
||||
|
||||
return api_base, dynamic_api_key
|
||||
@@ -3610,6 +3610,7 @@ def embedding( # noqa: PLR0915
|
||||
custom_llm_provider == "openai_like"
|
||||
or custom_llm_provider == "jina_ai"
|
||||
or custom_llm_provider == "hosted_vllm"
|
||||
or custom_llm_provider == "llamafile"
|
||||
or custom_llm_provider == "lm_studio"
|
||||
):
|
||||
api_base = (
|
||||
|
||||
@@ -2093,6 +2093,7 @@ class LlmProviders(str, Enum):
|
||||
CUSTOM = "custom"
|
||||
LITELLM_PROXY = "litellm_proxy"
|
||||
HOSTED_VLLM = "hosted_vllm"
|
||||
LLAMAFILE = "llamafile"
|
||||
LM_STUDIO = "lm_studio"
|
||||
GALADRIEL = "galadriel"
|
||||
INFINITY = "infinity"
|
||||
|
||||
@@ -6464,6 +6464,8 @@ class ProviderConfigManager:
|
||||
return litellm.AiohttpOpenAIChatConfig()
|
||||
elif litellm.LlmProviders.HOSTED_VLLM == provider:
|
||||
return litellm.HostedVLLMChatConfig()
|
||||
elif litellm.LlmProviders.LLAMAFILE == provider:
|
||||
return litellm.LlamafileChatConfig()
|
||||
elif litellm.LlmProviders.LM_STUDIO == provider:
|
||||
return litellm.LMStudioChatConfig()
|
||||
elif litellm.LlmProviders.GALADRIEL == provider:
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
from typing import Optional
|
||||
|
||||
from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.llamafile.chat.transformation import LlamafileChatConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_api_key, api_key_from_secret_manager, expected_api_key, secret_manager_called",
|
||||
[
|
||||
("user-provided-key", "secret-key", "user-provided-key", False),
|
||||
(None, "secret-key", "secret-key", True),
|
||||
(None, None, "fake-api-key", True),
|
||||
("", "secret-key", "secret-key", True), # Empty string should fall back to secret
|
||||
("", None, "fake-api-key", True), # Empty string with no secret should use the fake key
|
||||
]
|
||||
)
|
||||
def test_resolve_api_key(input_api_key, api_key_from_secret_manager, expected_api_key, secret_manager_called):
|
||||
with patch("litellm.llms.llamafile.chat.transformation.get_secret_str") as mock_get_secret:
|
||||
mock_get_secret.return_value = api_key_from_secret_manager
|
||||
|
||||
result = LlamafileChatConfig._resolve_api_key(input_api_key)
|
||||
|
||||
if secret_manager_called:
|
||||
mock_get_secret.assert_called_once_with("LLAMAFILE_API_KEY")
|
||||
else:
|
||||
mock_get_secret.assert_not_called()
|
||||
|
||||
assert result == expected_api_key
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_api_base, api_base_from_secret_manager, expected_api_base, secret_manager_called",
|
||||
[
|
||||
("https://user-api.example.com", "https://secret-api.example.com", "https://user-api.example.com", False),
|
||||
(None, "https://secret-api.example.com", "https://secret-api.example.com", True),
|
||||
(None, None, "http://127.0.0.1:8080/v1", True),
|
||||
("", "https://secret-api.example.com", "https://secret-api.example.com", True), # Empty string should fall back
|
||||
]
|
||||
)
|
||||
def test_resolve_api_base(input_api_base, api_base_from_secret_manager, expected_api_base, secret_manager_called):
|
||||
with patch("litellm.llms.llamafile.chat.transformation.get_secret_str") as mock_get_secret:
|
||||
mock_get_secret.return_value = api_base_from_secret_manager
|
||||
|
||||
result = LlamafileChatConfig._resolve_api_base(input_api_base)
|
||||
|
||||
if secret_manager_called:
|
||||
mock_get_secret.assert_called_once_with("LLAMAFILE_API_BASE")
|
||||
else:
|
||||
mock_get_secret.assert_not_called()
|
||||
|
||||
assert result == expected_api_base
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base, api_key, secret_base, secret_key, expected_base, expected_key",
|
||||
[
|
||||
# User-provided values
|
||||
("https://user-api.example.com", "user-key", "https://secret-api.example.com", "secret-key", "https://user-api.example.com", "user-key"),
|
||||
# Fallback to secrets
|
||||
(None, None, "https://secret-api.example.com", "secret-key", "https://secret-api.example.com", "secret-key"),
|
||||
# Nothing provided, use defaults
|
||||
(None, None, None, None, "http://127.0.0.1:8080/v1", "fake-api-key"),
|
||||
# Mixed scenarios
|
||||
("https://user-api.example.com", None, None, "secret-key", "https://user-api.example.com", "secret-key"),
|
||||
(None, "user-key", "https://secret-api.example.com", None, "https://secret-api.example.com", "user-key"),
|
||||
]
|
||||
)
|
||||
def test_get_openai_compatible_provider_info(api_base, api_key, secret_base, secret_key, expected_base, expected_key):
|
||||
config = LlamafileChatConfig()
|
||||
|
||||
def fake_get_secret(key: str) -> Optional[str]:
|
||||
return {
|
||||
"LLAMAFILE_API_BASE": secret_base,
|
||||
"LLAMAFILE_API_KEY": secret_key
|
||||
}.get(key)
|
||||
|
||||
patch_secret = patch("litellm.llms.llamafile.chat.transformation.get_secret_str", side_effect=fake_get_secret)
|
||||
patch_base = patch.object(LlamafileChatConfig, "_resolve_api_base", wraps=LlamafileChatConfig._resolve_api_base)
|
||||
patch_key = patch.object(LlamafileChatConfig, "_resolve_api_key", wraps=LlamafileChatConfig._resolve_api_key)
|
||||
|
||||
with patch_secret as mock_secret, patch_base as mock_base, patch_key as mock_key:
|
||||
result_base, result_key = config._get_openai_compatible_provider_info(api_base, api_key)
|
||||
|
||||
assert result_base == expected_base
|
||||
assert result_key == expected_key
|
||||
|
||||
mock_base.assert_called_once_with(api_base)
|
||||
mock_key.assert_called_once_with(api_key)
|
||||
|
||||
# Ensure get_secret_str was used as expected within the methods
|
||||
if api_base and api_key:
|
||||
mock_secret.assert_not_called()
|
||||
elif api_base or api_key:
|
||||
mock_secret.assert_called_once()
|
||||
else:
|
||||
assert mock_secret.call_count == 2
|
||||
|
||||
|
||||
def test_completion_with_custom_llamafile_model():
|
||||
with patch("litellm.main.openai_chat_completions.completion") as mock_llamafile_completion_func:
|
||||
mock_llamafile_completion_func.return_value = {} # Return an empty dictionary for the mocked response
|
||||
|
||||
provider = "llamafile"
|
||||
model_name = "my-custom-test-model"
|
||||
model = f"{provider}/{model_name}"
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
|
||||
_ = litellm.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
max_retries=2,
|
||||
max_tokens=100,
|
||||
)
|
||||
|
||||
mock_llamafile_completion_func.assert_called_once()
|
||||
_, call_kwargs = mock_llamafile_completion_func.call_args
|
||||
assert call_kwargs.get("custom_llm_provider") == provider
|
||||
assert call_kwargs.get("model") == model_name
|
||||
assert call_kwargs.get("messages") == messages
|
||||
assert call_kwargs.get("api_base") == "http://127.0.0.1:8080/v1"
|
||||
assert call_kwargs.get("api_key") == "fake-api-key"
|
||||
optional_params = call_kwargs.get("optional_params")
|
||||
assert optional_params
|
||||
assert optional_params.get("max_retries") == 2
|
||||
assert optional_params.get("max_tokens") == 100
|
||||
@@ -1477,7 +1477,7 @@ HF Tests we should pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider", ["openai", "hosted_vllm", "lm_studio"]
|
||||
"provider", ["openai", "hosted_vllm", "lm_studio", "llamafile"]
|
||||
) # "vertex_ai",
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_compatible_custom_api_base(provider):
|
||||
@@ -1519,6 +1519,7 @@ async def test_openai_compatible_custom_api_base(provider):
|
||||
[
|
||||
"openai",
|
||||
"hosted_vllm",
|
||||
"llamafile",
|
||||
],
|
||||
) # "vertex_ai",
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -1029,6 +1029,28 @@ def test_hosted_vllm_embedding(monkeypatch):
|
||||
assert json_data["model"] == "jina-embeddings-v3"
|
||||
|
||||
|
||||
def test_llamafile_embedding(monkeypatch):
|
||||
monkeypatch.setenv("LLAMAFILE_API_BASE", "http://localhost:8080/v1")
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
client = HTTPHandler()
|
||||
with patch.object(client, "post") as mock_post:
|
||||
try:
|
||||
embedding(
|
||||
model="llamafile/jina-embeddings-v3",
|
||||
input=["Hello world"],
|
||||
client=client,
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
|
||||
json_data = json.loads(mock_post.call_args.kwargs["data"])
|
||||
assert json_data["input"] == ["Hello world"]
|
||||
assert json_data["model"] == "jina-embeddings-v3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
async def test_lm_studio_embedding(monkeypatch, sync_mode):
|
||||
|
||||
@@ -185,6 +185,16 @@ def test_get_llm_provider_hosted_vllm():
|
||||
assert dynamic_api_key == "fake-api-key"
|
||||
|
||||
|
||||
def test_get_llm_provider_llamafile():
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
|
||||
model="llamafile/mistralai/mistral-7b-instruct-v0.2",
|
||||
)
|
||||
assert custom_llm_provider == "llamafile"
|
||||
assert model == "mistralai/mistral-7b-instruct-v0.2"
|
||||
assert dynamic_api_key == "fake-api-key"
|
||||
assert api_base == "http://127.0.0.1:8080/v1"
|
||||
|
||||
|
||||
def test_get_llm_provider_watson_text():
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
|
||||
model="watsonx_text/watson-text-to-speech",
|
||||
|
||||
Reference in New Issue
Block a user