mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-15 18:24:07 +00:00
Add vllm e2e test for embedding
This commit is contained in:
Sameer Kankute
@@ -17112,6 +17112,19 @@
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/claude-opus-4.6-fast": {
|
||||
"litellm_provider": "github_copilot",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 16000,
|
||||
"max_tokens": 16000,
|
||||
"mode": "chat",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/claude-opus-41": {
|
||||
"litellm_provider": "github_copilot",
|
||||
"max_input_tokens": 80000,
|
||||
@@ -17363,6 +17376,20 @@
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/gpt-5.3-codex": {
|
||||
"litellm_provider": "github_copilot",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
"supported_endpoints": [
|
||||
"/v1/responses"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"github_copilot/text-embedding-3-small": {
|
||||
"litellm_provider": "github_copilot",
|
||||
"max_input_tokens": 8191,
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
E2E test for hosted_vllm embeddings with real API calls.
|
||||
|
||||
This test verifies that the hosted_vllm provider works correctly with real API endpoints.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
class TestHostedVLLMEmbeddingE2E:
|
||||
"""E2E test suite for hosted_vllm provider embeddings."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
async def test_hosted_vllm_embedding_basic(self, sync_mode):
|
||||
"""Test basic embedding call with hosted_vllm provider."""
|
||||
# Skip if API base is not configured
|
||||
api_base = os.getenv("HOSTED_VLLM_API_BASE")
|
||||
if not api_base:
|
||||
pytest.skip("HOSTED_VLLM_API_BASE environment variable not set")
|
||||
|
||||
model = "hosted_vllm/nomic-ai/nomic-embed-text-v1.5"
|
||||
input_text = "Hello, this is a test embedding"
|
||||
|
||||
if sync_mode:
|
||||
response = litellm.embedding(
|
||||
model=model,
|
||||
input=input_text,
|
||||
api_base=api_base,
|
||||
)
|
||||
else:
|
||||
response = await litellm.aembedding(
|
||||
model=model,
|
||||
input=input_text,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
assert response is not None
|
||||
assert hasattr(response, "data")
|
||||
assert len(response.data) == 1
|
||||
# Adapt for response data as a dict (legacy or OpenAI compat)
|
||||
item = response.data[0]
|
||||
# If data is a dict, use key lookup; if it's an object, fallback to attribute
|
||||
if isinstance(item, dict):
|
||||
assert "embedding" in item
|
||||
assert len(item["embedding"]) > 0
|
||||
else:
|
||||
assert hasattr(item, "embedding")
|
||||
assert len(item.embedding) > 0
|
||||
assert hasattr(response, "usage")
|
||||
assert getattr(response.usage, "total_tokens", 0) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
async def test_hosted_vllm_embedding_multiple_inputs(self, sync_mode):
|
||||
"""Test embedding with multiple inputs."""
|
||||
api_base = os.getenv("HOSTED_VLLM_API_BASE")
|
||||
if not api_base:
|
||||
pytest.skip("HOSTED_VLLM_API_BASE environment variable not set")
|
||||
|
||||
model = "hosted_vllm/nomic-ai/nomic-embed-text-v1.5"
|
||||
inputs = [
|
||||
"First test sentence",
|
||||
"Second test sentence",
|
||||
"Third test sentence",
|
||||
]
|
||||
|
||||
if sync_mode:
|
||||
response = litellm.embedding(
|
||||
model=model,
|
||||
input=inputs,
|
||||
api_base=api_base,
|
||||
)
|
||||
else:
|
||||
response = await litellm.aembedding(
|
||||
model=model,
|
||||
input=inputs,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
assert response is not None
|
||||
assert len(response.data) == 3
|
||||
for i, emb_data in enumerate(response.data):
|
||||
assert emb_data["index"] == i
|
||||
assert len(emb_data["embedding"]) > 0
|
||||
|
||||
def test_hosted_vllm_embedding_with_api_key(self):
|
||||
"""Test embedding with API key authentication."""
|
||||
api_base = os.getenv("HOSTED_VLLM_API_BASE")
|
||||
api_key = os.getenv("HOSTED_VLLM_API_KEY")
|
||||
|
||||
if not api_base:
|
||||
pytest.skip("HOSTED_VLLM_API_BASE environment variable not set")
|
||||
|
||||
if not api_key:
|
||||
pytest.skip("HOSTED_VLLM_API_KEY environment variable not set")
|
||||
|
||||
model = "hosted_vllm/nomic-ai/nomic-embed-text-v1.5"
|
||||
input_text = "Test with API key"
|
||||
|
||||
response = litellm.embedding(
|
||||
model=model,
|
||||
input=input_text,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
# Verify response
|
||||
assert response is not None
|
||||
assert len(response.data) == 1
|
||||
assert len(response.data[0]["embedding"]) > 0
|
||||
|
||||
def test_hosted_vllm_embedding_deterministic(self):
|
||||
"""Test that same input produces same embedding (deterministic)."""
|
||||
api_base = os.getenv("HOSTED_VLLM_API_BASE")
|
||||
if not api_base:
|
||||
pytest.skip("HOSTED_VLLM_API_BASE environment variable not set")
|
||||
|
||||
model = "hosted_vllm/nomic-ai/nomic-embed-text-v1.5"
|
||||
input_text = "This should produce the same embedding every time"
|
||||
|
||||
response1 = litellm.embedding(
|
||||
model=model,
|
||||
input=input_text,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
response2 = litellm.embedding(
|
||||
model=model,
|
||||
input=input_text,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
# Verify embeddings are identical
|
||||
emb1 = response1.data[0]["embedding"]
|
||||
emb2 = response2.data[0]["embedding"]
|
||||
assert emb1 == emb2, "Embeddings should be deterministic"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "-s"])
|
||||
+366
@@ -0,0 +1,366 @@
|
||||
"""
|
||||
Test transformation logic for hosted_vllm embeddings.
|
||||
|
||||
This test verifies that the transformation layer correctly handles parameters,
|
||||
especially ensuring that encoding_format is not included when not provided.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
import litellm
|
||||
from litellm.llms.hosted_vllm.embedding.transformation import (
|
||||
HostedVLLMEmbeddingConfig,
|
||||
)
|
||||
|
||||
|
||||
class TestHostedVLLMEmbeddingTransformation:
|
||||
"""Test suite for hosted_vllm embedding transformation logic."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.config = HostedVLLMEmbeddingConfig()
|
||||
self.model = "hosted_vllm/BAAI/bge-small-en-v1.5"
|
||||
|
||||
def test_transform_embedding_request_basic(self):
|
||||
"""Test basic embedding request transformation."""
|
||||
input_data = ["hello world"]
|
||||
result = self.config.transform_embedding_request(
|
||||
model=self.model,
|
||||
input=input_data,
|
||||
optional_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
expected_result = {
|
||||
"model": "BAAI/bge-small-en-v1.5", # prefix should be stripped
|
||||
"input": input_data,
|
||||
}
|
||||
assert result == expected_result
|
||||
|
||||
def test_transform_embedding_request_string_input(self):
|
||||
"""Test that string input is converted to list."""
|
||||
input_data = "hello world"
|
||||
result = self.config.transform_embedding_request(
|
||||
model=self.model,
|
||||
input=input_data,
|
||||
optional_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["input"] == ["hello world"]
|
||||
assert result["model"] == "BAAI/bge-small-en-v1.5"
|
||||
|
||||
def test_transform_embedding_request_with_dimensions(self):
|
||||
"""Test embedding request with dimensions parameter."""
|
||||
input_data = ["hello world"]
|
||||
optional_params = {"dimensions": 384}
|
||||
|
||||
result = self.config.transform_embedding_request(
|
||||
model=self.model,
|
||||
input=input_data,
|
||||
optional_params=optional_params,
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["model"] == "BAAI/bge-small-en-v1.5"
|
||||
assert result["input"] == input_data
|
||||
assert result["dimensions"] == 384
|
||||
|
||||
def test_encoding_format_not_included_when_not_provided(self):
|
||||
"""
|
||||
Test that encoding_format is NOT included in the request when not provided.
|
||||
|
||||
This is critical because vLLM rejects requests with encoding_format=None or
|
||||
encoding_format="" with error: "unknown variant ``, expected float or base64"
|
||||
"""
|
||||
input_data = ["hello world"]
|
||||
|
||||
# Test with no encoding_format in optional_params
|
||||
result = self.config.transform_embedding_request(
|
||||
model=self.model,
|
||||
input=input_data,
|
||||
optional_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "encoding_format" not in result, (
|
||||
"encoding_format should not be in request when not provided"
|
||||
)
|
||||
|
||||
def test_encoding_format_not_included_when_none(self):
|
||||
"""
|
||||
Test that encoding_format is NOT included when explicitly set to None.
|
||||
"""
|
||||
input_data = ["hello world"]
|
||||
optional_params = {"encoding_format": None}
|
||||
|
||||
result = self.config.transform_embedding_request(
|
||||
model=self.model,
|
||||
input=input_data,
|
||||
optional_params=optional_params,
|
||||
headers={},
|
||||
)
|
||||
|
||||
# encoding_format=None should be passed through, but filtered later
|
||||
# by the HTTP handler
|
||||
assert result.get("encoding_format") is None
|
||||
|
||||
def test_encoding_format_included_when_float(self):
|
||||
"""Test that encoding_format is included when set to 'float'."""
|
||||
input_data = ["hello world"]
|
||||
optional_params = {"encoding_format": "float"}
|
||||
|
||||
result = self.config.transform_embedding_request(
|
||||
model=self.model,
|
||||
input=input_data,
|
||||
optional_params=optional_params,
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["encoding_format"] == "float"
|
||||
|
||||
def test_encoding_format_included_when_base64(self):
|
||||
"""Test that encoding_format is included when set to 'base64'."""
|
||||
input_data = ["hello world"]
|
||||
optional_params = {"encoding_format": "base64"}
|
||||
|
||||
result = self.config.transform_embedding_request(
|
||||
model=self.model,
|
||||
input=input_data,
|
||||
optional_params=optional_params,
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["encoding_format"] == "base64"
|
||||
|
||||
def test_get_supported_openai_params(self):
|
||||
"""Test that supported OpenAI parameters are correctly listed."""
|
||||
supported = self.config.get_supported_openai_params(self.model)
|
||||
|
||||
assert "timeout" in supported
|
||||
assert "dimensions" in supported
|
||||
assert "encoding_format" in supported
|
||||
assert "user" in supported
|
||||
|
||||
def test_map_openai_params(self):
|
||||
"""Test mapping of OpenAI parameters."""
|
||||
non_default_params = {
|
||||
"dimensions": 512,
|
||||
"encoding_format": "float",
|
||||
"user": "test-user",
|
||||
}
|
||||
|
||||
result = self.config.map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params={},
|
||||
model=self.model,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["dimensions"] == 512
|
||||
assert result["encoding_format"] == "float"
|
||||
assert result["user"] == "test-user"
|
||||
|
||||
def test_map_openai_params_filters_unsupported(self):
|
||||
"""Test that unsupported parameters are not mapped."""
|
||||
non_default_params = {
|
||||
"dimensions": 512,
|
||||
"unsupported_param": "value",
|
||||
}
|
||||
|
||||
result = self.config.map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params={},
|
||||
model=self.model,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["dimensions"] == 512
|
||||
assert "unsupported_param" not in result
|
||||
|
||||
def test_get_complete_url(self):
|
||||
"""Test URL construction for embeddings endpoint."""
|
||||
api_base = "https://test-vllm.example.com/v1"
|
||||
|
||||
url = self.config.get_complete_url(
|
||||
api_base=api_base,
|
||||
api_key="test-key",
|
||||
model=self.model,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert url == "https://test-vllm.example.com/v1/embeddings"
|
||||
|
||||
def test_get_complete_url_adds_embeddings_suffix(self):
|
||||
"""Test that /embeddings is added if not present."""
|
||||
api_base = "https://test-vllm.example.com"
|
||||
|
||||
url = self.config.get_complete_url(
|
||||
api_base=api_base,
|
||||
api_key="test-key",
|
||||
model=self.model,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert url == "https://test-vllm.example.com/embeddings"
|
||||
|
||||
def test_validate_environment_with_api_key(self):
|
||||
"""Test environment validation with API key."""
|
||||
headers = {}
|
||||
|
||||
result = self.config.validate_environment(
|
||||
headers=headers,
|
||||
model=self.model,
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key="test-api-key",
|
||||
)
|
||||
|
||||
assert "Authorization" in result
|
||||
assert result["Authorization"] == "Bearer test-api-key"
|
||||
assert result["Content-Type"] == "application/json"
|
||||
|
||||
def test_validate_environment_without_api_key(self):
|
||||
"""Test environment validation without API key (uses fake-api-key)."""
|
||||
headers = {}
|
||||
|
||||
result = self.config.validate_environment(
|
||||
headers=headers,
|
||||
model=self.model,
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key=None,
|
||||
)
|
||||
|
||||
# Should not include Authorization header with fake-api-key
|
||||
assert "Authorization" not in result
|
||||
assert result["Content-Type"] == "application/json"
|
||||
|
||||
def test_encoding_format_not_sent_in_actual_request(self):
|
||||
"""
|
||||
E2E test that encoding_format is not sent when not provided.
|
||||
|
||||
This test mocks the HTTP client to verify the actual request payload.
|
||||
"""
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
client = HTTPHandler()
|
||||
|
||||
with patch.object(client, "post") as mock_post:
|
||||
# Mock response
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {"content-type": "application/json"}
|
||||
mock_response.json.return_value = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"object": "embedding",
|
||||
"index": 0,
|
||||
"embedding": [0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
}
|
||||
],
|
||||
"model": "BAAI/bge-small-en-v1.5",
|
||||
"usage": {
|
||||
"prompt_tokens": 5,
|
||||
"total_tokens": 5,
|
||||
},
|
||||
}
|
||||
mock_response.text = json.dumps(mock_response.json.return_value)
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
try:
|
||||
litellm.embedding(
|
||||
model=self.model,
|
||||
input=["Hello world"],
|
||||
api_base="https://test-vllm.example.com/v1",
|
||||
client=client,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Verify the request was made
|
||||
mock_post.assert_called_once()
|
||||
|
||||
# Get the data that was sent
|
||||
call_kwargs = mock_post.call_args[1]
|
||||
sent_data = json.loads(call_kwargs["data"])
|
||||
|
||||
# Assert that encoding_format is NOT in the sent data
|
||||
assert "encoding_format" not in sent_data, (
|
||||
"encoding_format should not be in request when not provided"
|
||||
)
|
||||
assert sent_data["model"] == "BAAI/bge-small-en-v1.5"
|
||||
assert sent_data["input"] == ["Hello world"]
|
||||
|
||||
def test_encoding_format_float_sent_in_actual_request(self):
|
||||
"""
|
||||
Test that encoding_format='float' is sent when explicitly provided.
|
||||
"""
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
client = HTTPHandler()
|
||||
|
||||
with patch.object(client, "post") as mock_post:
|
||||
# Mock response
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {"content-type": "application/json"}
|
||||
mock_response.json.return_value = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"object": "embedding",
|
||||
"index": 0,
|
||||
"embedding": [0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
}
|
||||
],
|
||||
"model": "BAAI/bge-small-en-v1.5",
|
||||
"usage": {
|
||||
"prompt_tokens": 5,
|
||||
"total_tokens": 5,
|
||||
},
|
||||
}
|
||||
mock_response.text = json.dumps(mock_response.json.return_value)
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
try:
|
||||
litellm.embedding(
|
||||
model=self.model,
|
||||
input=["Hello world"],
|
||||
api_base="https://test-vllm.example.com/v1",
|
||||
encoding_format="float",
|
||||
client=client,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Verify the request was made
|
||||
mock_post.assert_called_once()
|
||||
|
||||
# Get the data that was sent
|
||||
call_kwargs = mock_post.call_args[1]
|
||||
sent_data = json.loads(call_kwargs["data"])
|
||||
|
||||
# Assert that encoding_format IS in the sent data
|
||||
assert "encoding_format" in sent_data, (
|
||||
"encoding_format='float' should be in request when provided"
|
||||
)
|
||||
assert sent_data["encoding_format"] == "float"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "-s"])
|
||||
Reference in New Issue
Block a user