mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-13 17:10:08 +00:00
Merge pull request #14532 from timelfrink/feat/issue-14476-compactifai-provider
Add CompactifAI provider support
This commit is contained in:
Krish Dholakia
@@ -316,6 +316,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
|
||||
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | |
|
||||
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
||||
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | |
|
||||
| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | ✅ | | |
|
||||
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
||||
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | |
|
||||
| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ |
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# CompactifAI
|
||||
https://docs.compactif.ai/
|
||||
|
||||
CompactifAI offers highly compressed versions of leading language models, delivering up to **70% lower inference costs**, **4x throughput gains**, and **low-latency inference** with minimal quality loss (<5%). CompactifAI's OpenAI-compatible API makes integration straightforward, enabling developers to build ultra-efficient, scalable AI applications with superior concurrency and resource efficiency.
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | CompactifAI offers compressed versions of leading language models with up to 70% cost reduction and 4x throughput gains |
|
||||
| Provider Route on LiteLLM | `compactifai/` (add this prefix to the model name - e.g. `compactifai/cai-llama-3-1-8b-slim`) |
|
||||
| Provider Doc | [CompactifAI ↗](https://docs.compactif.ai/) |
|
||||
| API Endpoint for Provider | https://api.compactif.ai/v1 |
|
||||
| Supported Endpoints | `/chat/completions`, `/completions` |
|
||||
|
||||
## Supported OpenAI Parameters
|
||||
|
||||
CompactifAI is fully OpenAI-compatible and supports the following parameters:
|
||||
|
||||
```
|
||||
"stream",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"max_tokens",
|
||||
"presence_penalty",
|
||||
"frequency_penalty",
|
||||
"logit_bias",
|
||||
"user",
|
||||
"response_format",
|
||||
"seed",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"parallel_tool_calls",
|
||||
"extra_headers"
|
||||
```
|
||||
|
||||
## API Key Setup
|
||||
|
||||
CompactifAI API keys are available through AWS Marketplace subscription:
|
||||
|
||||
1. Subscribe via [AWS Marketplace](https://aws.amazon.com/marketplace)
|
||||
2. Complete subscription verification (24-hour review process)
|
||||
3. Access MultiverseIAM dashboard with provided credentials
|
||||
4. Retrieve your API key from the dashboard
|
||||
|
||||
```python
|
||||
import os
|
||||
|
||||
os.environ["COMPACTIFAI_API_KEY"] = "your-api-key"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['COMPACTIFAI_API_KEY'] = "your-api-key"
|
||||
|
||||
response = completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[
|
||||
{"role": "user", "content": "Hello from LiteLLM!"}
|
||||
],
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="Proxy">
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: llama-2-compressed
|
||||
litellm_params:
|
||||
model: compactifai/cai-llama-3-1-8b-slim
|
||||
api_key: os.environ/COMPACTIFAI_API_KEY
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Streaming
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['COMPACTIFAI_API_KEY'] = "your-api-key"
|
||||
|
||||
response = completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[
|
||||
{"role": "user", "content": "Write a short story"}
|
||||
],
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Custom Parameters
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Explain quantum computing"}],
|
||||
temperature=0.7,
|
||||
max_tokens=500,
|
||||
top_p=0.9,
|
||||
stop=["Human:", "AI:"]
|
||||
)
|
||||
```
|
||||
|
||||
### Function Calling
|
||||
|
||||
CompactifAI supports OpenAI-compatible function calling:
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
functions = [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get current weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
response = completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "What's the weather in San Francisco?"}],
|
||||
tools=[{"type": "function", "function": f} for f in functions],
|
||||
tool_choice="auto"
|
||||
)
|
||||
```
|
||||
|
||||
### Async Usage
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from litellm import acompletion
|
||||
|
||||
async def async_call():
|
||||
response = await acompletion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Hello async world!"}]
|
||||
)
|
||||
return response
|
||||
|
||||
# Run async function
|
||||
response = asyncio.run(async_call())
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Available Models
|
||||
|
||||
CompactifAI offers compressed versions of popular models. Use the `/models` endpoint to get the latest list:
|
||||
|
||||
```python
|
||||
import httpx
|
||||
|
||||
headers = {"Authorization": f"Bearer {your_api_key}"}
|
||||
response = httpx.get("https://api.compactif.ai/v1/models", headers=headers)
|
||||
models = response.json()
|
||||
```
|
||||
|
||||
Common model formats:
|
||||
- `compactifai/cai-llama-3-1-8b-slim`
|
||||
- `compactifai/mistral-7b-compressed`
|
||||
- `compactifai/codellama-7b-compressed`
|
||||
|
||||
## Benefits
|
||||
|
||||
- **Cost Efficient**: Up to 70% lower inference costs compared to standard models
|
||||
- **High Performance**: 4x throughput gains with minimal quality loss (<5%)
|
||||
- **Low Latency**: Optimized for fast response times
|
||||
- **Drop-in Replacement**: Full OpenAI API compatibility
|
||||
- **Scalable**: Superior concurrency and resource efficiency
|
||||
|
||||
## Error Handling
|
||||
|
||||
CompactifAI returns standard OpenAI-compatible error responses:
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
from litellm.exceptions import AuthenticationError, RateLimitError
|
||||
|
||||
try:
|
||||
response = completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Hello"}]
|
||||
)
|
||||
except AuthenticationError:
|
||||
print("Invalid API key")
|
||||
except RateLimitError:
|
||||
print("Rate limit exceeded")
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
- Documentation: https://docs.compactif.ai/
|
||||
- LinkedIn: [MultiverseComputing](https://www.linkedin.com/company/multiversecomputing)
|
||||
- Analysis: [Artificial Analysis Provider Comparison](https://artificialanalysis.ai/providers/compactifai)
|
||||
@@ -453,6 +453,7 @@ const sidebars = {
|
||||
"providers/elevenlabs",
|
||||
"providers/fireworks_ai",
|
||||
"providers/clarifai",
|
||||
"providers/compactifai",
|
||||
"providers/vllm",
|
||||
"providers/llamafile",
|
||||
"providers/infinity",
|
||||
|
||||
@@ -1023,6 +1023,7 @@ from .llms.openai_like.chat.handler import OpenAILikeChatConfig
|
||||
from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig
|
||||
from .llms.galadriel.chat.transformation import GaladrielChatConfig
|
||||
from .llms.github.chat.transformation import GithubChatConfig
|
||||
from .llms.compactifai.chat.transformation import CompactifAIChatConfig
|
||||
from .llms.empower.chat.transformation import EmpowerChatConfig
|
||||
from .llms.huggingface.chat.transformation import HuggingFaceChatConfig
|
||||
from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig
|
||||
|
||||
@@ -372,6 +372,8 @@ def get_llm_provider( # noqa: PLR0915
|
||||
custom_llm_provider = "cometapi"
|
||||
elif model.startswith("oci/"):
|
||||
custom_llm_provider = "oci"
|
||||
elif model.startswith("compactifai/"):
|
||||
custom_llm_provider = "compactifai"
|
||||
elif model.startswith("ovhcloud/"):
|
||||
custom_llm_provider = "ovhcloud"
|
||||
if not custom_llm_provider:
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CompactifAI provider for LiteLLM
|
||||
@@ -0,0 +1 @@
|
||||
# CompactifAI chat completions
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
CompactifAI chat completion transformation
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.llms.openai.common_utils import OpenAIError
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class CompactifAIChatConfig(OpenAIGPTConfig):
|
||||
"""
|
||||
Configuration class for CompactifAI chat completions.
|
||||
Since CompactifAI is OpenAI-compatible, we extend OpenAIGPTConfig.
|
||||
"""
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Get API base and key for CompactifAI provider.
|
||||
"""
|
||||
api_base = api_base or "https://api.compactif.ai/v1"
|
||||
dynamic_api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or ""
|
||||
return api_base, dynamic_api_key
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
messages: List,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ModelResponse:
|
||||
"""
|
||||
Transform CompactifAI response to LiteLLM format.
|
||||
Since CompactifAI is OpenAI-compatible, we can use the standard OpenAI transformation.
|
||||
"""
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=raw_response.text,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
)
|
||||
|
||||
## RESPONSE OBJECT
|
||||
response_json = raw_response.json()
|
||||
|
||||
# Handle JSON mode if needed
|
||||
if json_mode:
|
||||
for choice in response_json["choices"]:
|
||||
message = choice.get("message")
|
||||
if message and message.get("tool_calls"):
|
||||
# Convert tool calls to content for JSON mode
|
||||
tool_calls = message.get("tool_calls", [])
|
||||
if len(tool_calls) == 1:
|
||||
message["content"] = tool_calls[0]["function"].get("arguments", "")
|
||||
message["tool_calls"] = None
|
||||
|
||||
returned_response = ModelResponse(**response_json)
|
||||
|
||||
# Set model name with provider prefix
|
||||
returned_response.model = f"compactifai/{model}"
|
||||
|
||||
return returned_response
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
"""
|
||||
Get the appropriate error class for CompactifAI errors.
|
||||
Since CompactifAI is OpenAI-compatible, we use OpenAI error handling.
|
||||
"""
|
||||
return OpenAIError(
|
||||
status_code=status_code,
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
)
|
||||
@@ -2549,6 +2549,37 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
encoding=encoding,
|
||||
stream=stream,
|
||||
)
|
||||
elif custom_llm_provider == "compactifai":
|
||||
api_key = (
|
||||
api_key
|
||||
or get_secret_str("COMPACTIFAI_API_KEY")
|
||||
or litellm.api_key
|
||||
)
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or "https://api.compactif.ai/v1"
|
||||
)
|
||||
|
||||
## COMPLETION CALL
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=encoding,
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
elif custom_llm_provider == "oobabooga":
|
||||
custom_llm_provider = "oobabooga"
|
||||
model_response = oobabooga.completion(
|
||||
|
||||
@@ -2327,6 +2327,7 @@ class LlmProviders(str, Enum):
|
||||
DATABRICKS = "databricks"
|
||||
EMPOWER = "empower"
|
||||
GITHUB = "github"
|
||||
COMPACTIFAI = "compactifai"
|
||||
CUSTOM = "custom"
|
||||
LITELLM_PROXY = "litellm_proxy"
|
||||
HOSTED_VLLM = "hosted_vllm"
|
||||
|
||||
@@ -6954,6 +6954,8 @@ class ProviderConfigManager:
|
||||
return litellm.EmpowerChatConfig()
|
||||
elif litellm.LlmProviders.GITHUB == provider:
|
||||
return litellm.GithubChatConfig()
|
||||
elif litellm.LlmProviders.COMPACTIFAI == provider:
|
||||
return litellm.CompactifAIChatConfig()
|
||||
elif litellm.LlmProviders.GITHUB_COPILOT == provider:
|
||||
return litellm.GithubCopilotConfig()
|
||||
elif (
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from respx import MockRouter
|
||||
|
||||
import litellm
|
||||
from litellm import Choices, Message, ModelResponse
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_compactifai_completion_basic(respx_mock):
|
||||
"""Test basic CompactifAI completion functionality"""
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_response = {
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "cai-llama-3-1-8b-slim",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Hello! How can I help you today?"
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 9,
|
||||
"completion_tokens": 12,
|
||||
"total_tokens": 21
|
||||
}
|
||||
}
|
||||
|
||||
respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond(
|
||||
json=mock_response, status_code=200
|
||||
)
|
||||
|
||||
response = litellm.completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_key="test-key"
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "Hello! How can I help you today?"
|
||||
assert response.model == "compactifai/cai-llama-3-1-8b-slim"
|
||||
assert response.usage.total_tokens == 21
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_compactifai_completion_streaming(respx_mock):
|
||||
"""Test CompactifAI streaming completion"""
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_chunks = [
|
||||
"data: " + json.dumps({
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "cai-llama-3-1-8b-slim",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "Hello"},
|
||||
"finish_reason": None
|
||||
}
|
||||
]
|
||||
}) + "\n\n",
|
||||
"data: " + json.dumps({
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "cai-llama-3-1-8b-slim",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "!"},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
]
|
||||
}) + "\n\n",
|
||||
"data: [DONE]\n\n"
|
||||
]
|
||||
|
||||
respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond(
|
||||
status_code=200,
|
||||
headers={"content-type": "text/plain"},
|
||||
content="".join(mock_chunks)
|
||||
)
|
||||
|
||||
response = litellm.completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_key="test-key",
|
||||
stream=True
|
||||
)
|
||||
|
||||
chunks = list(response)
|
||||
assert len(chunks) >= 2
|
||||
assert chunks[0].choices[0].delta.content == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_compactifai_models_endpoint(respx_mock):
|
||||
"""Test CompactifAI models listing"""
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_response = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "cai-llama-3-1-8b-slim",
|
||||
"object": "model",
|
||||
"created": 1677610602,
|
||||
"owned_by": "compactifai"
|
||||
},
|
||||
{
|
||||
"id": "mistral-7b-compressed",
|
||||
"object": "model",
|
||||
"created": 1677610602,
|
||||
"owned_by": "compactifai"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond(
|
||||
json={
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "cai-llama-3-1-8b-slim",
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Test response"
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": 10,
|
||||
"total_tokens": 15
|
||||
}
|
||||
},
|
||||
status_code=200
|
||||
)
|
||||
|
||||
# This would be tested if litellm had a models() function
|
||||
# For now, we'll test that the provider is properly configured
|
||||
response = litellm.completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
api_key="test-key"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_compactifai_authentication_error(respx_mock):
|
||||
"""Test CompactifAI authentication error handling"""
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_error = {
|
||||
"error": {
|
||||
"message": "Invalid API key provided",
|
||||
"type": "invalid_request_error",
|
||||
"param": None,
|
||||
"code": "invalid_api_key"
|
||||
}
|
||||
}
|
||||
|
||||
respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond(
|
||||
json=mock_error, status_code=401
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError) as exc_info:
|
||||
litellm.completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
api_key="invalid-key"
|
||||
)
|
||||
|
||||
# Verify the error contains the expected authentication error message
|
||||
assert "Invalid API key provided" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_compactifai_provider_detection(respx_mock):
|
||||
"""Test that CompactifAI provider is properly detected from model name"""
|
||||
from litellm.utils import get_llm_provider
|
||||
|
||||
model, provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model="compactifai/cai-llama-3-1-8b-slim"
|
||||
)
|
||||
|
||||
assert provider == "compactifai"
|
||||
assert model == "cai-llama-3-1-8b-slim"
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_compactifai_with_optional_params(respx_mock):
|
||||
"""Test CompactifAI with optional parameters like temperature, max_tokens"""
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_response = {
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "cai-llama-3-1-8b-slim",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "This is a test response with custom parameters."
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 15,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 35
|
||||
}
|
||||
}
|
||||
|
||||
request_mock = respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond(
|
||||
json=mock_response, status_code=200
|
||||
)
|
||||
|
||||
response = litellm.completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Hello with params"}],
|
||||
api_key="test-key",
|
||||
temperature=0.7,
|
||||
max_tokens=100,
|
||||
top_p=0.9
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "This is a test response with custom parameters."
|
||||
|
||||
# Verify the request was made with correct parameters
|
||||
assert request_mock.called
|
||||
request_data = request_mock.calls[0].request.content
|
||||
parsed_data = json.loads(request_data)
|
||||
assert parsed_data["temperature"] == 0.7
|
||||
assert parsed_data["max_tokens"] == 100
|
||||
assert parsed_data["top_p"] == 0.9
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_compactifai_headers_authentication(respx_mock):
|
||||
"""Test that CompactifAI request includes proper authorization headers"""
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_response = {
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "cai-llama-3-1-8b-slim",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Test response"
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": 10,
|
||||
"total_tokens": 15
|
||||
}
|
||||
}
|
||||
|
||||
request_mock = respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond(
|
||||
json=mock_response, status_code=200
|
||||
)
|
||||
|
||||
response = litellm.completion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Test auth"}],
|
||||
api_key="test-api-key-123"
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "Test response"
|
||||
|
||||
# Verify authorization header was set correctly
|
||||
assert request_mock.called
|
||||
request_headers = request_mock.calls[0].request.headers
|
||||
assert "authorization" in request_headers
|
||||
assert request_headers["authorization"] == "Bearer test-api-key-123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.respx()
|
||||
async def test_compactifai_async_completion(respx_mock):
|
||||
"""Test CompactifAI async completion"""
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_response = {
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "cai-llama-3-1-8b-slim",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Async response from CompactifAI"
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 8,
|
||||
"completion_tokens": 15,
|
||||
"total_tokens": 23
|
||||
}
|
||||
}
|
||||
|
||||
respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond(
|
||||
json=mock_response, status_code=200
|
||||
)
|
||||
|
||||
response = await litellm.acompletion(
|
||||
model="compactifai/cai-llama-3-1-8b-slim",
|
||||
messages=[{"role": "user", "content": "Async test"}],
|
||||
api_key="test-key"
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "Async response from CompactifAI"
|
||||
assert response.usage.total_tokens == 23
|
||||
Reference in New Issue
Block a user