mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-17 04:26:20 +00:00
* Support Llama-api as an LLM provider (#10451) * init: support llama-api as a llm provider * docs: fix endpoint url * fix: rename meta dir to meta-llama * docs: add meta-llama info * fix: mv LlamaAPIConfig under chat directory * feat: add LlamaAPIConfig in ProviderConfigManager * fix: provider_config from ProviderConfigManager * feat: add supports_tool_choice param * fix: remove optional_params using model_info * fix: rename meta-llama to meta_llama * init: test for meta_llama * fix: model names --------- Co-authored-by: Krish Dholakia <krrishdholakia@gmail.com> * fix file naming convention * fix file naming convention for meta_llama * docs meta llama api litellm --------- Co-authored-by: Young Han <110819238+seyeong-han@users.noreply.github.com> Co-authored-by: Krish Dholakia <krrishdholakia@gmail.com>
This commit is contained in:
Ishaan Jaff
co-authored by
Young Han
Krish Dholakia
Young Han
Krish Dholakia
parent
299f8f18a0
commit
f52593486c
@@ -0,0 +1,189 @@
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Meta Llama
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Meta's Llama API provides access to Meta's family of large language models. |
|
||||
| Provider Route on LiteLLM | `meta_llama/` |
|
||||
| Supported Endpoints | `/chat/completions`, `/completions` |
|
||||
| API Reference | [Llama API Reference ↗](https://www.llama.com/products/llama-api/) |
|
||||
|
||||
## Required Variables
|
||||
|
||||
```python showLineNumbers title="Environment Variables"
|
||||
os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key
|
||||
```
|
||||
## Usage - LiteLLM Python SDK
|
||||
|
||||
### Non-streaming
|
||||
|
||||
```python showLineNumbers title="Meta Llama Non-streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key
|
||||
|
||||
messages = [{"content": "Hello, how are you?", "role": "user"}]
|
||||
|
||||
# Meta Llama call
|
||||
response = completion(model="meta_llama/Llama-3.3-70B-Instruct", messages=messages)
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
```python showLineNumbers title="Meta Llama Streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key
|
||||
|
||||
messages = [{"content": "Hello, how are you?", "role": "user"}]
|
||||
|
||||
# Meta Llama call with streaming
|
||||
response = completion(
|
||||
model="meta_llama/Llama-3.3-70B-Instruct",
|
||||
messages=messages,
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
|
||||
## Usage - LiteLLM Proxy
|
||||
|
||||
|
||||
Add the following to your LiteLLM Proxy configuration file:
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: meta_llama/Llama-3.3-70B-Instruct
|
||||
litellm_params:
|
||||
model: meta_llama/Llama-3.3-70B-Instruct
|
||||
api_key: os.environ/LLAMA_API_KEY
|
||||
|
||||
- model_name: meta_llama/Llama-3.3-8B-Instruct
|
||||
litellm_params:
|
||||
model: meta_llama/Llama-3.3-8B-Instruct
|
||||
api_key: os.environ/LLAMA_API_KEY
|
||||
```
|
||||
|
||||
Start your LiteLLM Proxy server:
|
||||
|
||||
```bash showLineNumbers title="Start LiteLLM Proxy"
|
||||
litellm --config config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai-sdk" label="OpenAI SDK">
|
||||
|
||||
```python showLineNumbers title="Meta Llama via Proxy - Non-streaming"
|
||||
from openai import OpenAI
|
||||
|
||||
# Initialize client with your proxy URL
|
||||
client = OpenAI(
|
||||
base_url="http://localhost:4000", # Your proxy URL
|
||||
api_key="your-proxy-api-key" # Your proxy API key
|
||||
)
|
||||
|
||||
# Non-streaming response
|
||||
response = client.chat.completions.create(
|
||||
model="meta_llama/Llama-3.3-70B-Instruct",
|
||||
messages=[{"role": "user", "content": "Write a short poem about AI."}]
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
```python showLineNumbers title="Meta Llama via Proxy - Streaming"
|
||||
from openai import OpenAI
|
||||
|
||||
# Initialize client with your proxy URL
|
||||
client = OpenAI(
|
||||
base_url="http://localhost:4000", # Your proxy URL
|
||||
api_key="your-proxy-api-key" # Your proxy API key
|
||||
)
|
||||
|
||||
# Streaming response
|
||||
response = client.chat.completions.create(
|
||||
model="meta_llama/Llama-3.3-70B-Instruct",
|
||||
messages=[{"role": "user", "content": "Write a short poem about AI."}],
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
if chunk.choices[0].delta.content is not None:
|
||||
print(chunk.choices[0].delta.content, end="")
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="litellm-sdk" label="LiteLLM SDK">
|
||||
|
||||
```python showLineNumbers title="Meta Llama via Proxy - LiteLLM SDK"
|
||||
import litellm
|
||||
|
||||
# Configure LiteLLM to use your proxy
|
||||
response = litellm.completion(
|
||||
model="litellm_proxy/meta_llama/Llama-3.3-70B-Instruct",
|
||||
messages=[{"role": "user", "content": "Write a short poem about AI."}],
|
||||
api_base="http://localhost:4000",
|
||||
api_key="your-proxy-api-key"
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
```python showLineNumbers title="Meta Llama via Proxy - LiteLLM SDK Streaming"
|
||||
import litellm
|
||||
|
||||
# Configure LiteLLM to use your proxy with streaming
|
||||
response = litellm.completion(
|
||||
model="litellm_proxy/meta_llama/Llama-3.3-70B-Instruct",
|
||||
messages=[{"role": "user", "content": "Write a short poem about AI."}],
|
||||
api_base="http://localhost:4000",
|
||||
api_key="your-proxy-api-key",
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
|
||||
print(chunk.choices[0].delta.content, end="")
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="cURL">
|
||||
|
||||
```bash showLineNumbers title="Meta Llama via Proxy - cURL"
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer your-proxy-api-key" \
|
||||
-d '{
|
||||
"model": "meta_llama/Llama-3.3-70B-Instruct",
|
||||
"messages": [{"role": "user", "content": "Write a short poem about AI."}]
|
||||
}'
|
||||
```
|
||||
|
||||
```bash showLineNumbers title="Meta Llama via Proxy - cURL Streaming"
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer your-proxy-api-key" \
|
||||
-d '{
|
||||
"model": "meta_llama/Llama-3.3-70B-Instruct",
|
||||
"messages": [{"role": "user", "content": "Write a short poem about AI."}],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
|
||||
@@ -226,6 +226,7 @@ const sidebars = {
|
||||
]
|
||||
},
|
||||
"providers/litellm_proxy",
|
||||
"providers/meta_llama",
|
||||
"providers/mistral",
|
||||
"providers/codestral",
|
||||
"providers/cohere",
|
||||
|
||||
Reference in New Issue
Block a user