diff --git a/README.md b/README.md index f1e13c00d1..46acf6cef0 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | | | [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | | | [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | | +| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | ✅ | | | | [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | | | [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | | | [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ | diff --git a/docs/my-website/docs/providers/compactifai.md b/docs/my-website/docs/providers/compactifai.md new file mode 100644 index 0000000000..0e6e8f4ed3 --- /dev/null +++ b/docs/my-website/docs/providers/compactifai.md @@ -0,0 +1,223 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# CompactifAI +https://docs.compactif.ai/ + +CompactifAI offers highly compressed versions of leading language models, delivering up to **70% lower inference costs**, **4x throughput gains**, and **low-latency inference** with minimal quality loss (<5%). CompactifAI's OpenAI-compatible API makes integration straightforward, enabling developers to build ultra-efficient, scalable AI applications with superior concurrency and resource efficiency. + +| Property | Details | +|-------|-------| +| Description | CompactifAI offers compressed versions of leading language models with up to 70% cost reduction and 4x throughput gains | +| Provider Route on LiteLLM | `compactifai/` (add this prefix to the model name - e.g. `compactifai/cai-llama-3-1-8b-slim`) | +| Provider Doc | [CompactifAI ↗](https://docs.compactif.ai/) | +| API Endpoint for Provider | https://api.compactif.ai/v1 | +| Supported Endpoints | `/chat/completions`, `/completions` | + +## Supported OpenAI Parameters + +CompactifAI is fully OpenAI-compatible and supports the following parameters: + +``` +"stream", +"stop", +"temperature", +"top_p", +"max_tokens", +"presence_penalty", +"frequency_penalty", +"logit_bias", +"user", +"response_format", +"seed", +"tools", +"tool_choice", +"parallel_tool_calls", +"extra_headers" +``` + +## API Key Setup + +CompactifAI API keys are available through AWS Marketplace subscription: + +1. Subscribe via [AWS Marketplace](https://aws.amazon.com/marketplace) +2. Complete subscription verification (24-hour review process) +3. Access MultiverseIAM dashboard with provided credentials +4. Retrieve your API key from the dashboard + +```python +import os + +os.environ["COMPACTIFAI_API_KEY"] = "your-api-key" +``` + +## Usage + + + + +```python +from litellm import completion +import os + +os.environ['COMPACTIFAI_API_KEY'] = "your-api-key" + +response = completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[ + {"role": "user", "content": "Hello from LiteLLM!"} + ], +) +print(response) +``` + + + + +```yaml +model_list: + - model_name: llama-2-compressed + litellm_params: + model: compactifai/cai-llama-3-1-8b-slim + api_key: os.environ/COMPACTIFAI_API_KEY +``` + + + + +## Streaming + +```python +from litellm import completion +import os + +os.environ['COMPACTIFAI_API_KEY'] = "your-api-key" + +response = completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[ + {"role": "user", "content": "Write a short story"} + ], + stream=True +) + +for chunk in response: + print(chunk) +``` + +## Advanced Usage + +### Custom Parameters + +```python +from litellm import completion + +response = completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Explain quantum computing"}], + temperature=0.7, + max_tokens=500, + top_p=0.9, + stop=["Human:", "AI:"] +) +``` + +### Function Calling + +CompactifAI supports OpenAI-compatible function calling: + +```python +from litellm import completion + +functions = [ + { + "name": "get_weather", + "description": "Get current weather information", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state" + } + }, + "required": ["location"] + } + } +] + +response = completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "What's the weather in San Francisco?"}], + tools=[{"type": "function", "function": f} for f in functions], + tool_choice="auto" +) +``` + +### Async Usage + +```python +import asyncio +from litellm import acompletion + +async def async_call(): + response = await acompletion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Hello async world!"}] + ) + return response + +# Run async function +response = asyncio.run(async_call()) +print(response) +``` + +## Available Models + +CompactifAI offers compressed versions of popular models. Use the `/models` endpoint to get the latest list: + +```python +import httpx + +headers = {"Authorization": f"Bearer {your_api_key}"} +response = httpx.get("https://api.compactif.ai/v1/models", headers=headers) +models = response.json() +``` + +Common model formats: +- `compactifai/cai-llama-3-1-8b-slim` +- `compactifai/mistral-7b-compressed` +- `compactifai/codellama-7b-compressed` + +## Benefits + +- **Cost Efficient**: Up to 70% lower inference costs compared to standard models +- **High Performance**: 4x throughput gains with minimal quality loss (<5%) +- **Low Latency**: Optimized for fast response times +- **Drop-in Replacement**: Full OpenAI API compatibility +- **Scalable**: Superior concurrency and resource efficiency + +## Error Handling + +CompactifAI returns standard OpenAI-compatible error responses: + +```python +from litellm import completion +from litellm.exceptions import AuthenticationError, RateLimitError + +try: + response = completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Hello"}] + ) +except AuthenticationError: + print("Invalid API key") +except RateLimitError: + print("Rate limit exceeded") +``` + +## Support + +- Documentation: https://docs.compactif.ai/ +- LinkedIn: [MultiverseComputing](https://www.linkedin.com/company/multiversecomputing) +- Analysis: [Artificial Analysis Provider Comparison](https://artificialanalysis.ai/providers/compactifai) \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 64de5c2927..f3bab0219f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -453,6 +453,7 @@ const sidebars = { "providers/elevenlabs", "providers/fireworks_ai", "providers/clarifai", + "providers/compactifai", "providers/vllm", "providers/llamafile", "providers/infinity", diff --git a/litellm/__init__.py b/litellm/__init__.py index 736cd2c2fe..f5628d5486 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1023,6 +1023,7 @@ from .llms.openai_like.chat.handler import OpenAILikeChatConfig from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig from .llms.galadriel.chat.transformation import GaladrielChatConfig from .llms.github.chat.transformation import GithubChatConfig +from .llms.compactifai.chat.transformation import CompactifAIChatConfig from .llms.empower.chat.transformation import EmpowerChatConfig from .llms.huggingface.chat.transformation import HuggingFaceChatConfig from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 7cbb2afcb2..414ccb7ab8 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -372,6 +372,8 @@ def get_llm_provider( # noqa: PLR0915 custom_llm_provider = "cometapi" elif model.startswith("oci/"): custom_llm_provider = "oci" + elif model.startswith("compactifai/"): + custom_llm_provider = "compactifai" elif model.startswith("ovhcloud/"): custom_llm_provider = "ovhcloud" if not custom_llm_provider: diff --git a/litellm/llms/compactifai/__init__.py b/litellm/llms/compactifai/__init__.py new file mode 100644 index 0000000000..16b0c04cda --- /dev/null +++ b/litellm/llms/compactifai/__init__.py @@ -0,0 +1 @@ +# CompactifAI provider for LiteLLM \ No newline at end of file diff --git a/litellm/llms/compactifai/chat/__init__.py b/litellm/llms/compactifai/chat/__init__.py new file mode 100644 index 0000000000..d1a4463166 --- /dev/null +++ b/litellm/llms/compactifai/chat/__init__.py @@ -0,0 +1 @@ +# CompactifAI chat completions \ No newline at end of file diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py new file mode 100644 index 0000000000..5cb8cd9a4a --- /dev/null +++ b/litellm/llms/compactifai/chat/transformation.py @@ -0,0 +1,100 @@ +""" +CompactifAI chat completion transformation +""" + +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union + +import httpx + +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import ModelResponse +from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.base_llm.chat.transformation import BaseLLMException + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class CompactifAIChatConfig(OpenAIGPTConfig): + """ + Configuration class for CompactifAI chat completions. + Since CompactifAI is OpenAI-compatible, we extend OpenAIGPTConfig. + """ + + def _get_openai_compatible_provider_info( + self, + api_base: Optional[str], + api_key: Optional[str], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Get API base and key for CompactifAI provider. + """ + api_base = api_base or "https://api.compactif.ai/v1" + dynamic_api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or "" + return api_base, dynamic_api_key + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform CompactifAI response to LiteLLM format. + Since CompactifAI is OpenAI-compatible, we can use the standard OpenAI transformation. + """ + ## LOGGING + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=raw_response.text, + additional_args={"complete_input_dict": request_data}, + ) + + ## RESPONSE OBJECT + response_json = raw_response.json() + + # Handle JSON mode if needed + if json_mode: + for choice in response_json["choices"]: + message = choice.get("message") + if message and message.get("tool_calls"): + # Convert tool calls to content for JSON mode + tool_calls = message.get("tool_calls", []) + if len(tool_calls) == 1: + message["content"] = tool_calls[0]["function"].get("arguments", "") + message["tool_calls"] = None + + returned_response = ModelResponse(**response_json) + + # Set model name with provider prefix + returned_response.model = f"compactifai/{model}" + + return returned_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + """ + Get the appropriate error class for CompactifAI errors. + Since CompactifAI is OpenAI-compatible, we use OpenAI error handling. + """ + return OpenAIError( + status_code=status_code, + message=error_message, + headers=headers, + ) \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index bfae912cca..2f24f8b3be 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2549,6 +2549,37 @@ def completion( # type: ignore # noqa: PLR0915 encoding=encoding, stream=stream, ) + elif custom_llm_provider == "compactifai": + api_key = ( + api_key + or get_secret_str("COMPACTIFAI_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or "https://api.compactif.ai/v1" + ) + + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=encoding, + stream=stream, + provider_config=provider_config, + ) elif custom_llm_provider == "oobabooga": custom_llm_provider = "oobabooga" model_response = oobabooga.completion( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 289291bde0..9ed66003e7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2327,6 +2327,7 @@ class LlmProviders(str, Enum): DATABRICKS = "databricks" EMPOWER = "empower" GITHUB = "github" + COMPACTIFAI = "compactifai" CUSTOM = "custom" LITELLM_PROXY = "litellm_proxy" HOSTED_VLLM = "hosted_vllm" diff --git a/litellm/utils.py b/litellm/utils.py index d279ce5c14..e3dba0c7e4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6954,6 +6954,8 @@ class ProviderConfigManager: return litellm.EmpowerChatConfig() elif litellm.LlmProviders.GITHUB == provider: return litellm.GithubChatConfig() + elif litellm.LlmProviders.COMPACTIFAI == provider: + return litellm.CompactifAIChatConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotConfig() elif ( diff --git a/tests/test_litellm/llms/compactifai/test_compactifai.py b/tests/test_litellm/llms/compactifai/test_compactifai.py new file mode 100644 index 0000000000..99b8acc3dc --- /dev/null +++ b/tests/test_litellm/llms/compactifai/test_compactifai.py @@ -0,0 +1,344 @@ +import json +import os +import sys +from unittest.mock import AsyncMock, patch +from typing import Optional + +import httpx +import pytest +import respx +from respx import MockRouter + +import litellm +from litellm import Choices, Message, ModelResponse + + +@pytest.mark.respx() +def test_compactifai_completion_basic(respx_mock): + """Test basic CompactifAI completion functionality""" + litellm.disable_aiohttp_transport = True + + mock_response = { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "cai-llama-3-1-8b-slim", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I help you today?" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + + respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( + json=mock_response, status_code=200 + ) + + response = litellm.completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key" + ) + + assert response.choices[0].message.content == "Hello! How can I help you today?" + assert response.model == "compactifai/cai-llama-3-1-8b-slim" + assert response.usage.total_tokens == 21 + + +@pytest.mark.respx() +def test_compactifai_completion_streaming(respx_mock): + """Test CompactifAI streaming completion""" + litellm.disable_aiohttp_transport = True + + mock_chunks = [ + "data: " + json.dumps({ + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "cai-llama-3-1-8b-slim", + "choices": [ + { + "index": 0, + "delta": {"content": "Hello"}, + "finish_reason": None + } + ] + }) + "\n\n", + "data: " + json.dumps({ + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "cai-llama-3-1-8b-slim", + "choices": [ + { + "index": 0, + "delta": {"content": "!"}, + "finish_reason": "stop" + } + ] + }) + "\n\n", + "data: [DONE]\n\n" + ] + + respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( + status_code=200, + headers={"content-type": "text/plain"}, + content="".join(mock_chunks) + ) + + response = litellm.completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + stream=True + ) + + chunks = list(response) + assert len(chunks) >= 2 + assert chunks[0].choices[0].delta.content == "Hello" + + +@pytest.mark.respx() +def test_compactifai_models_endpoint(respx_mock): + """Test CompactifAI models listing""" + litellm.disable_aiohttp_transport = True + + mock_response = { + "object": "list", + "data": [ + { + "id": "cai-llama-3-1-8b-slim", + "object": "model", + "created": 1677610602, + "owned_by": "compactifai" + }, + { + "id": "mistral-7b-compressed", + "object": "model", + "created": 1677610602, + "owned_by": "compactifai" + } + ] + } + + respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "cai-llama-3-1-8b-slim", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "Test response" + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 10, + "total_tokens": 15 + } + }, + status_code=200 + ) + + # This would be tested if litellm had a models() function + # For now, we'll test that the provider is properly configured + response = litellm.completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "test"}], + api_key="test-key" + ) + + +@pytest.mark.respx() +def test_compactifai_authentication_error(respx_mock): + """Test CompactifAI authentication error handling""" + litellm.disable_aiohttp_transport = True + + mock_error = { + "error": { + "message": "Invalid API key provided", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key" + } + } + + respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( + json=mock_error, status_code=401 + ) + + with pytest.raises(litellm.APIConnectionError) as exc_info: + litellm.completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "test"}], + api_key="invalid-key" + ) + + # Verify the error contains the expected authentication error message + assert "Invalid API key provided" in str(exc_info.value) + + +@pytest.mark.respx() +def test_compactifai_provider_detection(respx_mock): + """Test that CompactifAI provider is properly detected from model name""" + from litellm.utils import get_llm_provider + + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="compactifai/cai-llama-3-1-8b-slim" + ) + + assert provider == "compactifai" + assert model == "cai-llama-3-1-8b-slim" + + +@pytest.mark.respx() +def test_compactifai_with_optional_params(respx_mock): + """Test CompactifAI with optional parameters like temperature, max_tokens""" + litellm.disable_aiohttp_transport = True + + mock_response = { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "cai-llama-3-1-8b-slim", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a test response with custom parameters." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 15, + "completion_tokens": 20, + "total_tokens": 35 + } + } + + request_mock = respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( + json=mock_response, status_code=200 + ) + + response = litellm.completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Hello with params"}], + api_key="test-key", + temperature=0.7, + max_tokens=100, + top_p=0.9 + ) + + assert response.choices[0].message.content == "This is a test response with custom parameters." + + # Verify the request was made with correct parameters + assert request_mock.called + request_data = request_mock.calls[0].request.content + parsed_data = json.loads(request_data) + assert parsed_data["temperature"] == 0.7 + assert parsed_data["max_tokens"] == 100 + assert parsed_data["top_p"] == 0.9 + + +@pytest.mark.respx() +def test_compactifai_headers_authentication(respx_mock): + """Test that CompactifAI request includes proper authorization headers""" + litellm.disable_aiohttp_transport = True + + mock_response = { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "cai-llama-3-1-8b-slim", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Test response" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 10, + "total_tokens": 15 + } + } + + request_mock = respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( + json=mock_response, status_code=200 + ) + + response = litellm.completion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Test auth"}], + api_key="test-api-key-123" + ) + + assert response.choices[0].message.content == "Test response" + + # Verify authorization header was set correctly + assert request_mock.called + request_headers = request_mock.calls[0].request.headers + assert "authorization" in request_headers + assert request_headers["authorization"] == "Bearer test-api-key-123" + + +@pytest.mark.asyncio +@pytest.mark.respx() +async def test_compactifai_async_completion(respx_mock): + """Test CompactifAI async completion""" + litellm.disable_aiohttp_transport = True + + mock_response = { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "cai-llama-3-1-8b-slim", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Async response from CompactifAI" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 8, + "completion_tokens": 15, + "total_tokens": 23 + } + } + + respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( + json=mock_response, status_code=200 + ) + + response = await litellm.acompletion( + model="compactifai/cai-llama-3-1-8b-slim", + messages=[{"role": "user", "content": "Async test"}], + api_key="test-key" + ) + + assert response.choices[0].message.content == "Async response from CompactifAI" + assert response.usage.total_tokens == 23 \ No newline at end of file