diff --git a/docs/my-website/docs/providers/meta_llama.md b/docs/my-website/docs/providers/meta_llama.md new file mode 100644 index 0000000000..5eb3850be9 --- /dev/null +++ b/docs/my-website/docs/providers/meta_llama.md @@ -0,0 +1,189 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Meta Llama + +| Property | Details | +|-------|-------| +| Description | Meta's Llama API provides access to Meta's family of large language models. | +| Provider Route on LiteLLM | `meta_llama/` | +| Supported Endpoints | `/chat/completions`, `/completions` | +| API Reference | [Llama API Reference ↗](https://www.llama.com/products/llama-api/) | + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key +``` +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Meta Llama Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Meta Llama call +response = completion(model="meta_llama/Llama-3.3-70B-Instruct", messages=messages) +``` + +### Streaming + +```python showLineNumbers title="Meta Llama Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Meta Llama call with streaming +response = completion( + model="meta_llama/Llama-3.3-70B-Instruct", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + + +## Usage - LiteLLM Proxy + + +Add the following to your LiteLLM Proxy configuration file: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: meta_llama/Llama-3.3-70B-Instruct + litellm_params: + model: meta_llama/Llama-3.3-70B-Instruct + api_key: os.environ/LLAMA_API_KEY + + - model_name: meta_llama/Llama-3.3-8B-Instruct + litellm_params: + model: meta_llama/Llama-3.3-8B-Instruct + api_key: os.environ/LLAMA_API_KEY +``` + +Start your LiteLLM Proxy server: + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + + + + +```python showLineNumbers title="Meta Llama via Proxy - Non-streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Non-streaming response +response = client.chat.completions.create( + model="meta_llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "Write a short poem about AI."}] +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Meta Llama via Proxy - Streaming" +from openai import OpenAI + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-proxy-api-key" # Your proxy API key +) + +# Streaming response +response = client.chat.completions.create( + model="meta_llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "Write a short poem about AI."}], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```python showLineNumbers title="Meta Llama via Proxy - LiteLLM SDK" +import litellm + +# Configure LiteLLM to use your proxy +response = litellm.completion( + model="litellm_proxy/meta_llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "Write a short poem about AI."}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key" +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Meta Llama via Proxy - LiteLLM SDK Streaming" +import litellm + +# Configure LiteLLM to use your proxy with streaming +response = litellm.completion( + model="litellm_proxy/meta_llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "Write a short poem about AI."}], + api_base="http://localhost:4000", + api_key="your-proxy-api-key", + stream=True +) + +for chunk in response: + if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + + +```bash showLineNumbers title="Meta Llama via Proxy - cURL" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "meta_llama/Llama-3.3-70B-Instruct", + "messages": [{"role": "user", "content": "Write a short poem about AI."}] + }' +``` + +```bash showLineNumbers title="Meta Llama via Proxy - cURL Streaming" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "meta_llama/Llama-3.3-70B-Instruct", + "messages": [{"role": "user", "content": "Write a short poem about AI."}], + "stream": true + }' +``` + + + + +For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy). diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 9f4c392d41..e59b8aae5c 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -226,6 +226,7 @@ const sidebars = { ] }, "providers/litellm_proxy", + "providers/meta_llama", "providers/mistral", "providers/codestral", "providers/cohere", diff --git a/litellm/__init__.py b/litellm/__init__.py index b507e52bdb..2a7894c233 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -190,6 +190,7 @@ predibase_tenant_id: Optional[str] = None togetherai_api_key: Optional[str] = None cloudflare_api_key: Optional[str] = None baseten_key: Optional[str] = None +llama_api_key: Optional[str] = None aleph_alpha_key: Optional[str] = None nlp_cloud_key: Optional[str] = None snowflake_key: Optional[str] = None @@ -432,6 +433,7 @@ galadriel_models: List = [] sambanova_models: List = [] assemblyai_models: List = [] snowflake_models: List = [] +llama_models: List = [] def is_bedrock_pricing_only_model(key: str) -> bool: @@ -555,6 +557,8 @@ def add_known_models(): xai_models.append(key) elif value.get("litellm_provider") == "deepseek": deepseek_models.append(key) + elif value.get("litellm_provider") == "meta_llama": + llama_models.append(key) elif value.get("litellm_provider") == "azure_ai": azure_ai_models.append(key) elif value.get("litellm_provider") == "voyage": @@ -665,6 +669,7 @@ model_list = ( + assemblyai_models + jina_ai_models + snowflake_models + + llama_models ) model_list_set = set(model_list) @@ -722,6 +727,7 @@ models_by_provider: dict = { "assemblyai": assemblyai_models, "jina_ai": jina_ai_models, "snowflake": snowflake_models, + "meta_llama": llama_models, } # mapping for those models which have larger equivalents @@ -848,6 +854,7 @@ from .llms.infinity.rerank.transformation import InfinityRerankConfig from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig from .llms.clarifai.chat.transformation import ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config +from .llms.meta_llama.chat.transformation import LlamaAPIConfig from .llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) diff --git a/litellm/constants.py b/litellm/constants.py index 9526ae141a..fa944c0dfa 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -161,6 +161,7 @@ LITELLM_CHAT_PROVIDERS = [ "llamafile", "lm_studio", "galadriel", + "meta_llama", ] @@ -221,6 +222,7 @@ openai_compatible_endpoints: List = [ "api.sambanova.ai/v1", "api.x.ai/v1", "api.galadriel.ai/v1", + "api.llama.com/compat/v1/", ] @@ -251,12 +253,14 @@ openai_compatible_providers: List = [ "llamafile", "lm_studio", "galadriel", + "meta_llama", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` "together_ai", "fireworks_ai", "hosted_vllm", + "meta_llama", "llamafile", ] ) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 331087e02e..bdfe0e9025 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -215,6 +215,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "api.galadriel.com/v1": custom_llm_provider = "galadriel" dynamic_api_key = get_secret_str("GALADRIEL_API_KEY") + elif endpoint == "https://api.llama.com/compat/v1": + custom_llm_provider = "meta_llama" + dynamic_api_key = api_key or get_secret_str("LLAMA_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception( @@ -444,6 +447,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "https://api.sambanova.ai/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("SAMBANOVA_API_KEY") + elif custom_llm_provider == "meta_llama": + api_base = ( + api_base + or get_secret("LLAMA_API_BASE") + or "https://api.llama.com/compat/v1" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("LLAMA_API_KEY") elif (custom_llm_provider == "ai21_chat") or ( custom_llm_provider == "ai21" and model in litellm.ai21_chat_models ): @@ -478,10 +488,12 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) elif custom_llm_provider == "llamafile": # llamafile is OpenAI compatible. - (api_base, dynamic_api_key) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info( - api_base, - api_key - ) + ( + api_base, + dynamic_api_key, + ) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "lm_studio": # lm_studio is openai compatible, we just need to set this to custom_openai ( diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index c0f638ddc2..2cb8daa4c5 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -46,6 +46,12 @@ def get_supported_openai_params( # noqa: PLR0915 if custom_llm_provider == "bedrock": return litellm.AmazonConverseConfig().get_supported_openai_params(model=model) + elif custom_llm_provider == "meta_llama": + provider_config = litellm.ProviderConfigManager.get_provider_chat_config( + model=model, provider=LlmProviders.LLAMA + ) + if provider_config: + return provider_config.get_supported_openai_params(model=model) elif custom_llm_provider == "ollama": return litellm.OllamaConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "ollama_chat": @@ -222,7 +228,9 @@ def get_supported_openai_params( # noqa: PLR0915 elif custom_llm_provider == "voyage": return litellm.VoyageEmbeddingConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "infinity": - return litellm.InfinityEmbeddingConfig().get_supported_openai_params(model=model) + return litellm.InfinityEmbeddingConfig().get_supported_openai_params( + model=model + ) elif custom_llm_provider == "triton": if request_type == "embeddings": return litellm.TritonEmbeddingConfig().get_supported_openai_params( diff --git a/litellm/llms/meta_llama/chat/transformation.py b/litellm/llms/meta_llama/chat/transformation.py new file mode 100644 index 0000000000..aa09e33091 --- /dev/null +++ b/litellm/llms/meta_llama/chat/transformation.py @@ -0,0 +1,60 @@ +""" +Support for Llama API's `https://api.llama.com/compat/v1` endpoint. + +Calls done in OpenAI/openai.py as Llama API is openai-compatible. + +Docs: https://llama.developer.meta.com/docs/features/compatibility/ +""" + +from typing import Optional + +from litellm import get_model_info, verbose_logger +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + +class LlamaAPIConfig(OpenAIGPTConfig): + def get_supported_openai_params(self, model: str) -> list: + """ + Llama API has limited support for OpenAI parameters + + Tool calling, Functional Calling, tool choice are not working right now + response_format: only json_schema is working + """ + supports_function_calling: Optional[bool] = None + supports_tool_choice: Optional[bool] = None + try: + model_info = get_model_info(model, custom_llm_provider="meta_llama") + supports_function_calling = model_info.get( + "supports_function_calling", False + ) + supports_tool_choice = model_info.get("supports_tool_choice", False) + except Exception as e: + verbose_logger.debug(f"Error getting supported openai params: {e}") + pass + + optional_params = super().get_supported_openai_params(model) + if not supports_function_calling: + optional_params.remove("function_call") + if not supports_tool_choice: + optional_params.remove("tools") + optional_params.remove("tool_choice") + return optional_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + mapped_openai_params = super().map_openai_params( + non_default_params, optional_params, model, drop_params + ) + + # Only json_schema is working for response_format + if ( + "response_format" in mapped_openai_params + and mapped_openai_params["response_format"].get("type") != "json_schema" + ): + mapped_openai_params.pop("response_format") + return mapped_openai_params diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9c1fa7240a..e81ff3c57d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4860,6 +4860,54 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", "supports_tool_choice": true }, + "meta_llama/Llama-4-Scout-17B-16E-Instruct-FP8": { + "max_tokens": 128000, + "max_input_tokens": 10000000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"] + }, + "meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { + "max_tokens": 128000, + "max_input_tokens": 1000000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"] + }, + "meta_llama/Llama-3.3-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text"], + "supported_output_modalities": ["text"] + }, + "meta_llama/Llama-3.3-8B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text"], + "supported_output_modalities": ["text"] + }, "gemini-pro": { "max_tokens": 8192, "max_input_tokens": 32760, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 95e51e0231..50fd5cd9c1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2156,6 +2156,7 @@ class LlmProviders(str, Enum): TOPAZ = "topaz" ASSEMBLYAI = "assemblyai" SNOWFLAKE = "snowflake" + LLAMA = "meta_llama" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index e8f40f7205..90f7b14151 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6177,6 +6177,8 @@ class ProviderConfigManager: return litellm.DatabricksConfig() elif litellm.LlmProviders.XAI == provider: return litellm.XAIChatConfig() + elif litellm.LlmProviders.LLAMA == provider: + return litellm.LlamaAPIConfig() elif litellm.LlmProviders.TEXT_COMPLETION_OPENAI == provider: return litellm.OpenAITextCompletionConfig() elif litellm.LlmProviders.COHERE_CHAT == provider: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9c1fa7240a..e81ff3c57d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4860,6 +4860,54 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", "supports_tool_choice": true }, + "meta_llama/Llama-4-Scout-17B-16E-Instruct-FP8": { + "max_tokens": 128000, + "max_input_tokens": 10000000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"] + }, + "meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { + "max_tokens": 128000, + "max_input_tokens": 1000000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"] + }, + "meta_llama/Llama-3.3-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text"], + "supported_output_modalities": ["text"] + }, + "meta_llama/Llama-3.3-8B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 4028, + "litellm_provider": "meta_llama", + "mode": "chat", + "supports_function_calling": false, + "source": "https://llama.developer.meta.com/docs/models", + "supports_tool_choice": false, + "supported_modalities": ["text"], + "supported_output_modalities": ["text"] + }, "gemini-pro": { "max_tokens": 8192, "max_input_tokens": 32760, diff --git a/tests/litellm/llms/meta_llama/test_meta_llama_chat_transformation.py b/tests/litellm/llms/meta_llama/test_meta_llama_chat_transformation.py new file mode 100644 index 0000000000..7e9c097ded --- /dev/null +++ b/tests/litellm/llms/meta_llama/test_meta_llama_chat_transformation.py @@ -0,0 +1,58 @@ +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +from litellm.llms.meta_llama.chat.transformation import LlamaAPIConfig + + +def test_get_supported_openai_params(): + """Test that LlamaAPIConfig correctly filters unsupported parameters""" + config = LlamaAPIConfig() + + # Test error handling + with patch("litellm.get_model_info", side_effect=Exception("Test error")): + params = config.get_supported_openai_params("llama-3.3-8B-instruct") + assert "function_call" not in params + assert "tools" not in params + assert "tool_choice" not in params + + +def test_map_openai_params(): + """Test that LlamaAPIConfig correctly maps OpenAI parameters""" + config = LlamaAPIConfig() + + # Test response_format handling - json_schema is allowed + non_default_params = {"response_format": {"type": "json_schema"}} + optional_params = {"response_format": True} + result = config.map_openai_params( + non_default_params, optional_params, "llama-3.3-8B-instruct", False + ) + assert "response_format" in result + assert result["response_format"]["type"] == "json_schema" + + # Test response_format handling - other types are removed + non_default_params = {"response_format": {"type": "text"}} + optional_params = {"response_format": True} + result = config.map_openai_params( + non_default_params, optional_params, "llama-3.3-8B-instruct", False + ) + assert "response_format" not in result + + # Test that other parameters are passed through + non_default_params = { + "temperature": 0.7, + "response_format": {"type": "json_schema"}, + } + optional_params = {"temperature": True, "response_format": True} + result = config.map_openai_params( + non_default_params, optional_params, "llama-3.3-8B-instruct", False + ) + assert "temperature" in result + assert result["temperature"] == 0.7 + assert "response_format" in result