tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-09 18:25:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix inference endpoints (#11630) (#11631)

Browse Source 
Co-authored-by: célina <hanouticelina@gmail.com>
This commit is contained in:
Ishaan Jaff
2025-06-11 10:45:54 -07:00
committed by GitHub
co-authored by célina
parent ec52600f98
commit aeaf2d185a
2 changed files with 429 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/llms/huggingface/chat/transformation.py
+26 -3
View File
@@ -23,6 +23,21 @@ logger = logging.getLogger(__name__)
BASE_URL = "https://router.huggingface.co"
def _build_chat_completion_url(model_url: str) -> str:
# Strip trailing /
model_url = model_url.rstrip("/")
# Append /chat/completions if not already present
if model_url.endswith("/v1"):
model_url += "/chat/completions"
# Append /v1/chat/completions if not already present
if not model_url.endswith("/chat/completions"):
model_url += "/v1/chat/completions"
return model_url
class HuggingFaceChatConfig(OpenAIGPTConfig):
"""
Reference: https://huggingface.co/docs/huggingface_hub/guides/inference
@@ -80,16 +95,18 @@ class HuggingFaceChatConfig(OpenAIGPTConfig):
Get the complete URL for the API call.
For provider-specific routing through huggingface
"""
# 1. Check if api_base is provided
# Check if api_base is provided
if api_base is not None:
complete_url = api_base
complete_url = _build_chat_completion_url(complete_url)
elif os.getenv("HF_API_BASE") or os.getenv("HUGGINGFACE_API_BASE"):
complete_url = str(os.getenv("HF_API_BASE")) or str(
os.getenv("HUGGINGFACE_API_BASE")
)
elif model.startswith(("http://", "https://")):
complete_url = model
# 4. Default construction with provider
complete_url = _build_chat_completion_url(complete_url)
# Default construction with provider
else:
# Parse provider and model
first_part, remaining = model.split("/", 1)
@@ -101,7 +118,9 @@ class HuggingFaceChatConfig(OpenAIGPTConfig):
if provider == "hf-inference":
route = f"{provider}/models/{model}/v1/chat/completions"
elif provider == "novita":
route = f"{provider}/chat/completions"
route = f"{provider}/v3/openai/chat/completions"
elif provider == "fireworks-ai":
route = f"{provider}/inference/v1/chat/completions"
else:
route = f"{provider}/v1/chat/completions"
complete_url = f"{BASE_URL}/{route}"
@@ -118,6 +137,10 @@ class HuggingFaceChatConfig(OpenAIGPTConfig):
litellm_params: dict,
headers: dict,
) -> dict:
if litellm_params.get("api_base"):
return dict(
ChatCompletionRequest(model=model, messages=messages, **optional_params)
)
if "max_retries" in optional_params:
logger.warning("`max_retries` is not supported. It will be ignored.")
optional_params.pop("max_retries", None)
tests/llm_translation/test_huggingface_chat_completion.py
+403 -165
View File
@@ -2,9 +2,6 @@
Test HuggingFace LLM
"""
from re import M
import httpx
from base_llm_unit_tests import BaseLLMChatTest
import json
import os
@@ -18,82 +15,149 @@ sys.path.insert(
import litellm
import pytest
from litellm.types.utils import ModelResponseStream, ModelResponse
from respx import MockRouter
MOCK_COMPLETION_RESPONSE = {
"id": "9115d3daeab10608",
"object": "chat.completion",
"created": 11111,
"model": "meta-llama/Meta-Llama-3-8B-Instruct",
"prompt": [],
"choices": [
{
"finish_reason": "stop",
"seed": 3629048360264764400,
"logprobs": None,
"index": 0,
"message": {
"role": "assistant",
"content": "This is a test response from the mocked HuggingFace API.",
"tool_calls": []
}
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30
"id": "9115d3daeab10608",
"object": "chat.completion",
"created": 11111,
"model": "meta-llama/Meta-Llama-3-8B-Instruct",
"prompt": [],
"choices": [
{
"finish_reason": "stop",
"seed": 3629048360264764400,
"logprobs": None,
"index": 0,
"message": {
"role": "assistant",
"content": "This is a test response from the mocked HuggingFace API.",
"tool_calls": [],
},
}
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
}
MOCK_STREAMING_CHUNKS = [
{"id": "id1", "object": "chat.completion.chunk", "created": 1111,
"choices": [{"index": 0, "text": "Deep", "logprobs": None, "finish_reason": None, "seed": None,
"delta": {"token_id": 34564, "role": "assistant", "content": "Deep", "tool_calls": None}}],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo", "usage": None},
{"id": "id2", "object": "chat.completion.chunk", "created": 1111,
"choices": [{"index": 0, "text": " learning", "logprobs": None, "finish_reason": None, "seed": None,
"delta": {"token_id": 6975, "role": "assistant", "content": " learning", "tool_calls": None}}],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo", "usage": None},
{"id": "id3", "object": "chat.completion.chunk", "created": 1111,
"choices": [{"index": 0, "text": " is", "logprobs": None, "finish_reason": None, "seed": None,
"delta": {"token_id": 374, "role": "assistant", "content": " is", "tool_calls": None}}],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo", "usage": None},
{"id": "sid4", "object": "chat.completion.chunk", "created": 1111,
"choices": [{"index": 0, "text": " response", "logprobs": None, "finish_reason": "length", "seed": 2853637492034609700,
"delta": {"token_id": 323, "role": "assistant", "content": " response", "tool_calls": None}}],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"usage": {"prompt_tokens": 26, "completion_tokens": 20, "total_tokens": 46}}
]
{
"id": "id1",
"object": "chat.completion.chunk",
"created": 1111,
"choices": [
{
"index": 0,
"text": "Deep",
"logprobs": None,
"finish_reason": None,
"seed": None,
"delta": {
"token_id": 34564,
"role": "assistant",
"content": "Deep",
"tool_calls": None,
},
}
],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"usage": None,
},
{
"id": "id2",
"object": "chat.completion.chunk",
"created": 1111,
"choices": [
{
"index": 0,
"text": " learning",
"logprobs": None,
"finish_reason": None,
"seed": None,
"delta": {
"token_id": 6975,
"role": "assistant",
"content": " learning",
"tool_calls": None,
},
}
],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"usage": None,
},
{
"id": "id3",
"object": "chat.completion.chunk",
"created": 1111,
"choices": [
{
"index": 0,
"text": " is",
"logprobs": None,
"finish_reason": None,
"seed": None,
"delta": {
"token_id": 374,
"role": "assistant",
"content": " is",
"tool_calls": None,
},
}
],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"usage": None,
},
{
"id": "sid4",
"object": "chat.completion.chunk",
"created": 1111,
"choices": [
{
"index": 0,
"text": " response",
"logprobs": None,
"finish_reason": "length",
"seed": 2853637492034609700,
"delta": {
"token_id": 323,
"role": "assistant",
"content": " response",
"tool_calls": None,
},
}
],
"model": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"usage": {"prompt_tokens": 26, "completion_tokens": 20, "total_tokens": 46},
},
]
PROVIDER_MAPPING_RESPONSE = {
"fireworks-ai": {
"status": "live",
"providerId": "accounts/fireworks/models/llama-v3-8b-instruct",
"task": "conversational"
"status": "live",
"providerId": "accounts/fireworks/models/llama-v3-8b-instruct",
"task": "conversational",
},
"together": {
"status": "live",
"providerId": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"task": "conversational"
"status": "live",
"providerId": "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"task": "conversational",
},
"hf-inference": {
"status": "live",
"providerId": "meta-llama/Meta-Llama-3-8B-Instruct",
"task": "conversational"
"status": "live",
"providerId": "meta-llama/Meta-Llama-3-8B-Instruct",
"task": "conversational",
},
}
@pytest.fixture
def mock_provider_mapping():
with patch("litellm.llms.huggingface.chat.transformation._fetch_inference_provider_mapping") as mock:
with patch(
"litellm.llms.huggingface.chat.transformation._fetch_inference_provider_mapping"
) as mock:
mock.return_value = PROVIDER_MAPPING_RESPONSE
yield mock
@pytest.fixture(autouse=True)
def clear_lru_cache():
from litellm.llms.huggingface.common_utils import _fetch_inference_provider_mapping
@@ -101,123 +165,120 @@ def clear_lru_cache():
_fetch_inference_provider_mapping.cache_clear()
yield
_fetch_inference_provider_mapping.cache_clear()
@pytest.fixture
def mock_http_handler():
"""Fixture to mock the HTTP handler"""
with patch(
"litellm.llms.custom_httpx.http_handler.HTTPHandler.post"
) as mock:
print(f"Creating mock HTTP handler: {mock}")
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock:
print(f"Creating mock HTTP handler: {mock}") # noqa: T201
mock_response = MagicMock()
mock_response.raise_for_status.return_value = None
mock_response.status_code = 200
def mock_side_effect(*args, **kwargs):
if kwargs.get("stream", True):
mock_response.iter_lines.return_value = iter([
f"data: {json.dumps(chunk)}".encode('utf-8')
for chunk in MOCK_STREAMING_CHUNKS
] + [b'data: [DONE]'])
mock_response.iter_lines.return_value = iter(
[
f"data: {json.dumps(chunk)}".encode("utf-8")
for chunk in MOCK_STREAMING_CHUNKS
]
+ [b"data: [DONE]"]
)
else:
mock_response.json.return_value = MOCK_COMPLETION_RESPONSE
return mock_response
mock.side_effect = mock_side_effect
yield mock
@pytest.fixture
def mock_http_async_handler():
"""Fixture to mock the async HTTP handler"""
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new_callable=AsyncMock
new_callable=AsyncMock,
) as mock:
print(f"Creating mock async HTTP handler: {mock}")
mock_response = MagicMock()
print(f"Creating mock async HTTP handler: {mock}") # noqa: T201
mock_response = MagicMock()
mock_response.raise_for_status.return_value = None
mock_response.status_code = 200
mock_response.headers = {"content-type": "application/json"}
mock_response.json.return_value = MOCK_COMPLETION_RESPONSE
mock_response.text = json.dumps(MOCK_COMPLETION_RESPONSE)
async def mock_side_effect(*args, **kwargs):
if kwargs.get("stream", True):
async def mock_aiter():
for chunk in MOCK_STREAMING_CHUNKS:
yield f"data: {json.dumps(chunk)}".encode('utf-8')
yield f"data: {json.dumps(chunk)}".encode("utf-8")
yield b"data: [DONE]"
mock_response.aiter_lines = mock_aiter
return mock_response
mock.side_effect = mock_side_effect
yield mock
yield mock
class TestHuggingFace(BaseLLMChatTest):
@pytest.fixture(autouse=True)
def setup(self, mock_provider_mapping, mock_http_handler, mock_http_async_handler):
self.mock_provider_mapping = mock_provider_mapping
self.mock_http = mock_http_handler
self.mock_http_async = mock_http_async_handler
self.model = "huggingface/together/meta-llama/Meta-Llama-3-8B-Instruct"
litellm.set_verbose = False
def get_base_completion_call_args(self) -> dict:
"""Implementation of abstract method from BaseLLMChatTest"""
return {"model": self.model}
def test_completion_non_streaming(self):
messages = [{"role": "user", "content": "This is a dummy message"}]
response = litellm.completion(
model=self.model,
messages=messages,
stream=False
)
response = litellm.completion(model=self.model, messages=messages, stream=False)
assert isinstance(response, ModelResponse)
assert response.choices[0].message.content == "This is a test response from the mocked HuggingFace API."
assert (
response.choices[0].message.content
== "This is a test response from the mocked HuggingFace API."
)
assert response.usage is not None
assert response.model == self.model.split("/",2)[2]
assert response.model == self.model.split("/", 2)[2]
def test_completion_streaming(self):
messages = [{"role": "user", "content": "This is a dummy message"}]
response = litellm.completion(
model=self.model,
messages=messages,
stream=True
)
response = litellm.completion(model=self.model, messages=messages, stream=True)
chunks = list(response)
assert len(chunks) > 0
assert self.mock_http.called
call_args = self.mock_http.call_args
assert call_args is not None
kwargs = call_args[1]
data = json.loads(kwargs["data"])
assert data["stream"] is True
assert data["messages"] == messages
assert isinstance(chunks, list)
assert isinstance(chunks[0], ModelResponseStream)
assert isinstance(chunks[0].id, str)
assert chunks[0].model == self.model.split("/",1)[1]
assert chunks[0].model == self.model.split("/", 1)[1]
@pytest.mark.asyncio
async def test_async_completion_streaming(self):
"""Test async streaming completion"""
messages = [{"role": "user", "content": "This is a dummy message"}]
response = await litellm.acompletion(
model=self.model,
messages=messages,
stream=True
model=self.model, messages=messages, stream=True
)
chunks = []
@@ -228,73 +289,100 @@ class TestHuggingFace(BaseLLMChatTest):
assert len(chunks) > 0
assert isinstance(chunks[0], ModelResponseStream)
assert isinstance(chunks[0].id, str)
assert chunks[0].model == self.model.split("/",1)[1]
assert chunks[0].model == self.model.split("/", 1)[1]
@pytest.mark.asyncio
async def test_async_completion_non_streaming(self):
"""Test async non-streaming completion"""
messages = [{"role": "user", "content": "This is a dummy message"}]
response = await litellm.acompletion(
model=self.model,
messages=messages,
stream=False
model=self.model, messages=messages, stream=False
)
assert self.mock_http_async.called
assert isinstance(response, ModelResponse)
assert response.choices[0].message.content == "This is a test response from the mocked HuggingFace API."
assert (
response.choices[0].message.content
== "This is a test response from the mocked HuggingFace API."
)
assert response.usage is not None
assert response.model == self.model.split("/",2)[2]
assert response.model == self.model.split("/", 2)[2]
def test_tool_call_no_arguments(self, tool_call_no_arguments):
mock_tool_response = {
**MOCK_COMPLETION_RESPONSE,
"choices": [{
"finish_reason": "tool_calls",
"index": 0,
"message": tool_call_no_arguments
}]
}
with patch.object(self.mock_http, "side_effect", lambda *args, **kwargs: MagicMock(
status_code=200,
json=lambda: mock_tool_response,
raise_for_status=lambda: None
)):
messages = [{"role": "user", "content": "Get the FAQ"}]
tools = [{
"type": "function",
"function": {
"name": "Get-FAQ",
"description": "Get FAQ information",
"parameters": {
"type": "object",
"properties": {},
"required": []
}
"choices": [
{
"finish_reason": "tool_calls",
"index": 0,
"message": tool_call_no_arguments,
}
}]
],
}
with patch.object(
self.mock_http,
"side_effect",
lambda *args, **kwargs: MagicMock(
status_code=200,
json=lambda: mock_tool_response,
raise_for_status=lambda: None,
),
):
messages = [{"role": "user", "content": "Get the FAQ"}]
tools = [
{
"type": "function",
"function": {
"name": "Get-FAQ",
"description": "Get FAQ information",
"parameters": {
"type": "object",
"properties": {},
"required": [],
},
},
}
]
response = litellm.completion(
model=self.model,
messages=messages,
tools=tools,
tool_choice="auto"
model=self.model, messages=messages, tools=tools, tool_choice="auto"
)
assert response.choices[0].message.tool_calls is not None
assert len(response.choices[0].message.tool_calls) == 1
assert response.choices[0].message.tool_calls[0].function.name == tool_call_no_arguments["tool_calls"][0]["function"]["name"]
assert response.choices[0].message.tool_calls[0].function.arguments == tool_call_no_arguments["tool_calls"][0]["function"]["arguments"]
assert (
response.choices[0].message.tool_calls[0].function.name
== tool_call_no_arguments["tool_calls"][0]["function"]["name"]
)
assert (
response.choices[0].message.tool_calls[0].function.arguments
== tool_call_no_arguments["tool_calls"][0]["function"]["arguments"]
)
@pytest.mark.parametrize(
"model, provider, expected_url",
[
("meta-llama/Llama-3-8B-Instruct", None, "https://router.huggingface.co/hf-inference/models/meta-llama/Llama-3-8B-Instruct/v1/chat/completions"),
("together/meta-llama/Llama-3-8B-Instruct", None, "https://router.huggingface.co/together/v1/chat/completions"),
("novita/meta-llama/Llama-3-8B-Instruct", None, "https://router.huggingface.co/novita/chat/completions"),
("http://custom-endpoint.com/v1/chat/completions", None, "http://custom-endpoint.com/v1/chat/completions"),
(
"meta-llama/Llama-3-8B-Instruct",
None,
"https://router.huggingface.co/hf-inference/models/meta-llama/Llama-3-8B-Instruct/v1/chat/completions",
),
(
"together/meta-llama/Llama-3-8B-Instruct",
None,
"https://router.huggingface.co/together/v1/chat/completions",
),
(
"novita/meta-llama/Llama-3-8B-Instruct",
None,
"https://router.huggingface.co/novita/v3/openai/chat/completions",
),
(
"http://custom-endpoint.com/v1/chat/completions",
None,
"http://custom-endpoint.com/v1/chat/completions",
),
],
)
def test_get_complete_url(self, model, provider, expected_url):
@@ -308,52 +396,202 @@ class TestHuggingFace(BaseLLMChatTest):
optional_params={},
stream=False,
api_key="test_api_key",
litellm_params={}
litellm_params={},
)
assert url == expected_url
@pytest.mark.parametrize(
"api_base, model, expected_url",
[
(
"https://abcd123.us-east-1.aws.endpoints.huggingface.cloud",
"huggingface/tgi",
"https://abcd123.us-east-1.aws.endpoints.huggingface.cloud/v1/chat/completions",
),
(
"https://abcd123.us-east-1.aws.endpoints.huggingface.cloud/",
"huggingface/tgi",
"https://abcd123.us-east-1.aws.endpoints.huggingface.cloud/v1/chat/completions",
),
(
"https://abcd123.us-east-1.aws.endpoints.huggingface.cloud/v1/chat/completions",
"huggingface/tgi",
"https://abcd123.us-east-1.aws.endpoints.huggingface.cloud/v1/chat/completions",
),
(
"https://example.com/custom/path",
"huggingface/tgi",
"https://example.com/custom/path/v1/chat/completions",
),
(
"https://example.com/custom/path/v1/chat/completions",
"huggingface/tgi",
"https://example.com/custom/path/v1/chat/completions",
),
(
"https://example.com/v1",
"huggingface/tgi",
"https://example.com/v1/chat/completions",
),
],
)
def test_get_complete_url_inference_endpoints(self, api_base, model, expected_url):
from litellm.llms.huggingface.chat.transformation import HuggingFaceChatConfig
config = HuggingFaceChatConfig()
url = config.get_complete_url(
api_base=api_base,
model=model,
optional_params={},
stream=False,
api_key="test_api_key",
litellm_params={},
)
assert url == expected_url
def test_completion_with_api_base(self):
messages = [{"role": "user", "content": "This is a test message"}]
api_base = "https://abcd123.us-east-1.aws.endpoints.huggingface.cloud"
response = litellm.completion(
model="huggingface/tgi", messages=messages, api_base=api_base, stream=False
)
assert isinstance(response, ModelResponse)
assert (
response.choices[0].message.content
== "This is a test response from the mocked HuggingFace API."
)
assert self.mock_http.called
call_args = self.mock_http.call_args
assert call_args is not None
called_url = call_args[1]["url"]
assert called_url == f"{api_base}/v1/chat/completions"
@pytest.mark.asyncio
async def test_async_completion_with_api_base(self):
messages = [{"role": "user", "content": "This is a test message"}]
api_base = "https://abcd123.us-east-1.aws.endpoints.huggingface.cloud"
response = await litellm.acompletion(
model="huggingface/tgi", messages=messages, api_base=api_base, stream=False
)
assert isinstance(response, ModelResponse)
assert (
response.choices[0].message.content
== "This is a test response from the mocked HuggingFace API."
)
assert self.mock_http_async.called
call_args = self.mock_http_async.call_args
assert call_args is not None
called_url = call_args[1]["url"]
assert called_url == f"{api_base}/v1/chat/completions"
def test_completion_streaming_with_api_base(self):
"""Test streaming completion with api_base parameter"""
messages = [{"role": "user", "content": "This is a test message"}]
api_base = "https://abcd123.us-east-1.aws.endpoints.huggingface.cloud"
response = litellm.completion(
model="huggingface/tgi", messages=messages, api_base=api_base, stream=True
)
chunks = list(response)
assert len(chunks) > 0
assert isinstance(chunks[0], ModelResponseStream)
# Check that the correct URL was called
assert self.mock_http.called
call_args = self.mock_http.call_args
assert call_args is not None
called_url = call_args[1]["url"]
assert called_url == f"{api_base}/v1/chat/completions"
def test_build_chat_completion_url_function(self):
"""Test the _build_chat_completion_url helper function"""
from litellm.llms.huggingface.chat.transformation import (
_build_chat_completion_url,
)
test_cases = [
("https://example.com", "https://example.com/v1/chat/completions"),
("https://example.com/", "https://example.com/v1/chat/completions"),
("https://example.com/v1", "https://example.com/v1/chat/completions"),
("https://example.com/v1/", "https://example.com/v1/chat/completions"),
(
"https://example.com/v1/chat/completions",
"https://example.com/v1/chat/completions",
),
(
"https://example.com/custom/path",
"https://example.com/custom/path/v1/chat/completions",
),
(
"https://example.com/custom/path/",
"https://example.com/custom/path/v1/chat/completions",
),
]
for input_url, expected_url in test_cases:
result = _build_chat_completion_url(input_url)
assert (
result == expected_url
), f"Failed for input: {input_url}, expected: {expected_url}, got: {result}"
def test_validate_environment(self):
"""Test that the environment is validated correctly"""
from litellm.llms.huggingface.chat.transformation import HuggingFaceChatConfig
config = HuggingFaceChatConfig()
headers = config.validate_environment(
headers={},
model="huggingface/fireworks-ai/meta-llama/Meta-Llama-3-8B-Instruct",
messages=[{"role": "user", "content": "Hello"}],
optional_params={},
api_key="test_api_key",
litellm_params={}
litellm_params={},
)
assert headers["Authorization"] == "Bearer test_api_key"
assert headers["content-type"] == "application/json"
@pytest.mark.parametrize(
"model, expected_model",
[
("together/meta-llama/Llama-3-8B-Instruct", "meta-llama/Meta-Llama-3-8B-Instruct-Turbo"),
("meta-llama/Meta-Llama-3-8B-Instruct", "meta-llama/Meta-Llama-3-8B-Instruct"),
(
"together/meta-llama/Llama-3-8B-Instruct",
"meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
),
(
"meta-llama/Meta-Llama-3-8B-Instruct",
"meta-llama/Meta-Llama-3-8B-Instruct",
),
],
)
def test_transform_request(self, model, expected_model):
from litellm.llms.huggingface.chat.transformation import HuggingFaceChatConfig
config = HuggingFaceChatConfig()
messages = [{"role": "user", "content": "Hello"}]
transformed_request = config.transform_request(
model=model,
messages=messages,
optional_params={},
litellm_params={},
headers={}
headers={},
)
assert transformed_request["model"] == expected_model
assert transformed_request["messages"] == messages
@pytest.mark.asyncio
async def test_completion_cost(self):
pass
pass