tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-07 22:21:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'BerriAI:main' into ollama-special-symbol

Browse Source
This commit is contained in:
frob
2024-04-20 00:49:57 +02:00
committed by GitHub
parent 3df7231fa5 62a8bf134f
commit 5fb5bce780
17 changed files with 166 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/my-website/docs/providers/groq.md
+2
View File
@@ -48,6 +48,8 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` |
| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` |
| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` |
| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` |
| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
litellm/__init__.py
+1
View File
@@ -51,6 +51,7 @@ replicate_key: Optional[str] = None
cohere_key: Optional[str] = None
maritalk_key: Optional[str] = None
ai21_key: Optional[str] = None
ollama_key: Optional[str] = None
openrouter_key: Optional[str] = None
huggingface_key: Optional[str] = None
vertex_project: Optional[str] = None
litellm/integrations/langfuse.py
+2 -2
View File
@@ -282,9 +282,9 @@ class LangFuseLogger:
for key, value in metadata.items():
# generate langfuse tags
if key in [
"user_api_key",
"user_api_key_alias",
"user_api_key_user_id",
"user_api_key_team_id",
"user_api_key_team_alias",
"semantic-similarity",
]:
tags.append(f"{key}:{value}")
litellm/llms/anthropic.py
+2 -1
View File
@@ -258,8 +258,9 @@ class AnthropicChatCompletion(BaseLLM):
self.async_handler = AsyncHTTPHandler(
timeout=httpx.Timeout(timeout=600.0, connect=5.0)
)
data["stream"] = True
response = await self.async_handler.post(
api_base, headers=headers, data=json.dumps(data)
api_base, headers=headers, data=json.dumps(data), stream=True
)
if response.status_code != 200:
litellm/llms/custom_httpx/http_handler.py
+4 -5
View File
@@ -41,13 +41,12 @@ class AsyncHTTPHandler:
data: Optional[Union[dict, str]] = None, # type: ignore
params: Optional[dict] = None,
headers: Optional[dict] = None,
stream: bool = False,
):
response = await self.client.post(
url,
data=data, # type: ignore
params=params,
headers=headers,
req = self.client.build_request(
"POST", url, data=data, params=params, headers=headers # type: ignore
)
response = await self.client.send(req, stream=stream)
return response
def __del__(self) -> None:
litellm/llms/ollama_chat.py
+49 -15
View File
@@ -184,6 +184,7 @@ class OllamaChatConfig:
# ollama implementation
def get_ollama_response(
api_base="http://localhost:11434",
api_key: Optional[str] = None,
model="llama2",
messages=None,
optional_params=None,
@@ -236,6 +237,7 @@ def get_ollama_response(
if stream == True:
response = ollama_async_streaming(
url=url,
api_key=api_key,
data=data,
model_response=model_response,
encoding=encoding,
@@ -244,6 +246,7 @@ def get_ollama_response(
else:
response = ollama_acompletion(
url=url,
api_key=api_key,
data=data,
model_response=model_response,
encoding=encoding,
@@ -252,12 +255,17 @@ def get_ollama_response(
)
return response
elif stream == True:
return ollama_completion_stream(url=url, data=data, logging_obj=logging_obj)
return ollama_completion_stream(
url=url, api_key=api_key, data=data, logging_obj=logging_obj
)
response = requests.post(
url=f"{url}",
json=data,
)
_request = {
"url": f"{url}",
"json": data,
}
if api_key is not None:
_request["headers"] = "Bearer {}".format(api_key)
response = requests.post(**_request) # type: ignore
if response.status_code != 200:
raise OllamaError(status_code=response.status_code, message=response.text)
@@ -307,10 +315,16 @@ def get_ollama_response(
return model_response
def ollama_completion_stream(url, data, logging_obj):
with httpx.stream(
url=url, json=data, method="POST", timeout=litellm.request_timeout
) as response:
def ollama_completion_stream(url, api_key, data, logging_obj):
_request = {
"url": f"{url}",
"json": data,
"method": "POST",
"timeout": litellm.request_timeout,
}
if api_key is not None:
_request["headers"] = "Bearer {}".format(api_key)
with httpx.stream(**_request) as response:
try:
if response.status_code != 200:
raise OllamaError(
@@ -329,12 +343,20 @@ def ollama_completion_stream(url, data, logging_obj):
raise e
async def ollama_async_streaming(url, data, model_response, encoding, logging_obj):
async def ollama_async_streaming(
url, api_key, data, model_response, encoding, logging_obj
):
try:
client = httpx.AsyncClient()
async with client.stream(
url=f"{url}", json=data, method="POST", timeout=litellm.request_timeout
) as response:
_request = {
"url": f"{url}",
"json": data,
"method": "POST",
"timeout": litellm.request_timeout,
}
if api_key is not None:
_request["headers"] = "Bearer {}".format(api_key)
async with client.stream(**_request) as response:
if response.status_code != 200:
raise OllamaError(
status_code=response.status_code, message=response.text
@@ -353,13 +375,25 @@ async def ollama_async_streaming(url, data, model_response, encoding, logging_ob
async def ollama_acompletion(
url, data, model_response, encoding, logging_obj, function_name
url,
api_key: Optional[str],
data,
model_response,
encoding,
logging_obj,
function_name,
):
data["stream"] = False
try:
timeout = aiohttp.ClientTimeout(total=litellm.request_timeout) # 10 minutes
async with aiohttp.ClientSession(timeout=timeout) as session:
resp = await session.post(url, json=data)
_request = {
"url": f"{url}",
"json": data,
}
if api_key is not None:
_request["headers"] = "Bearer {}".format(api_key)
resp = await session.post(**_request)
if resp.status != 200:
text = await resp.text()
litellm/main.py
+7
View File
@@ -1941,9 +1941,16 @@ def completion(
or "http://localhost:11434"
)
api_key = (
api_key
or litellm.ollama_key
or os.environ.get("OLLAMA_API_KEY")
or litellm.api_key
)
## LOGGING
generator = ollama_chat.get_ollama_response(
api_base,
api_key,
model,
messages,
optional_params,
litellm/model_prices_and_context_window_backup.json
+20
View File
@@ -735,6 +735,26 @@
"mode": "chat",
"supports_function_calling": true
},
"groq/llama3-8b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 0.00000010,
"output_cost_per_token": 0.00000010,
"litellm_provider": "groq",
"mode": "chat",
"supports_function_calling": true
},
"groq/llama3-70b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 0.00000064,
"output_cost_per_token": 0.00000080,
"litellm_provider": "groq",
"mode": "chat",
"supports_function_calling": true
},
"groq/mixtral-8x7b-32768": {
"max_tokens": 32768,
"max_input_tokens": 32768,
litellm/proxy/_new_secret_config.yaml
+6 -6
View File
@@ -31,12 +31,12 @@ litellm_settings:
upperbound_key_generate_params:
max_budget: os.environ/LITELLM_UPPERBOUND_KEYS_MAX_BUDGET
# router_settings:
# routing_strategy: usage-based-routing-v2
# redis_host: os.environ/REDIS_HOST
# redis_password: os.environ/REDIS_PASSWORD
# redis_port: os.environ/REDIS_PORT
# enable_pre_call_checks: True
router_settings:
routing_strategy: usage-based-routing-v2
redis_host: os.environ/REDIS_HOST
redis_password: os.environ/REDIS_PASSWORD
redis_port: os.environ/REDIS_PORT
enable_pre_call_checks: True
general_settings:
master_key: sk-1234
litellm/proxy/_types.py
+1
View File
@@ -792,6 +792,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
"""
team_spend: Optional[float] = None
team_alias: Optional[str] = None
team_tpm_limit: Optional[int] = None
team_rpm_limit: Optional[int] = None
team_max_budget: Optional[float] = None
litellm/proxy/proxy_server.py
+18
View File
@@ -3361,6 +3361,9 @@ async def completion(
data["metadata"]["user_api_key_team_id"] = getattr(
user_api_key_dict, "team_id", None
)
data["metadata"]["user_api_key_team_alias"] = getattr(
user_api_key_dict, "team_alias", None
)
_headers = dict(request.headers)
_headers.pop(
"authorization", None
@@ -3562,6 +3565,9 @@ async def chat_completion(
data["metadata"]["user_api_key_team_id"] = getattr(
user_api_key_dict, "team_id", None
)
data["metadata"]["user_api_key_team_alias"] = getattr(
user_api_key_dict, "team_alias", None
)
data["metadata"]["user_api_key_metadata"] = user_api_key_dict.metadata
_headers = dict(request.headers)
_headers.pop(
@@ -3793,6 +3799,9 @@ async def embeddings(
data["metadata"]["user_api_key_team_id"] = getattr(
user_api_key_dict, "team_id", None
)
data["metadata"]["user_api_key_team_alias"] = getattr(
user_api_key_dict, "team_alias", None
)
data["metadata"]["endpoint"] = str(request.url)
### TEAM-SPECIFIC PARAMS ###
@@ -3971,6 +3980,9 @@ async def image_generation(
data["metadata"]["user_api_key_team_id"] = getattr(
user_api_key_dict, "team_id", None
)
data["metadata"]["user_api_key_team_alias"] = getattr(
user_api_key_dict, "team_alias", None
)
data["metadata"]["endpoint"] = str(request.url)
### TEAM-SPECIFIC PARAMS ###
@@ -4127,6 +4139,9 @@ async def audio_transcriptions(
data["metadata"]["user_api_key_team_id"] = getattr(
user_api_key_dict, "team_id", None
)
data["metadata"]["user_api_key_team_alias"] = getattr(
user_api_key_dict, "team_alias", None
)
data["metadata"]["endpoint"] = str(request.url)
data["metadata"]["file_name"] = file.filename
@@ -4302,6 +4317,9 @@ async def moderations(
data["metadata"]["user_api_key_team_id"] = getattr(
user_api_key_dict, "team_id", None
)
data["metadata"]["user_api_key_team_alias"] = getattr(
user_api_key_dict, "team_alias", None
)
data["metadata"]["endpoint"] = str(request.url)
### TEAM-SPECIFIC PARAMS ###
litellm/proxy/utils.py
+1
View File
@@ -1186,6 +1186,7 @@ class PrismaClient:
t.rpm_limit AS team_rpm_limit,
t.models AS team_models,
t.blocked AS team_blocked,
t.team_alias AS team_alias,
m.aliases as team_model_aliases
FROM "LiteLLM_VerificationToken" AS v
LEFT JOIN "LiteLLM_TeamTable" AS t ON v.team_id = t.team_id
model_prices_and_context_window.json
+20
View File
@@ -735,6 +735,26 @@
"mode": "chat",
"supports_function_calling": true
},
"groq/llama3-8b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 0.00000010,
"output_cost_per_token": 0.00000010,
"litellm_provider": "groq",
"mode": "chat",
"supports_function_calling": true
},
"groq/llama3-70b-8192": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 0.00000064,
"output_cost_per_token": 0.00000080,
"litellm_provider": "groq",
"mode": "chat",
"supports_function_calling": true
},
"groq/mixtral-8x7b-32768": {
"max_tokens": 32768,
"max_input_tokens": 32768,
ui/litellm-dashboard/src/app/page.tsx
+1
View File
@@ -145,6 +145,7 @@ const CreateKeyPage = () => {
userRole={userRole}
token={token}
keys={keys}
teams={teams}
accessToken={accessToken}
setKeys={setKeys}
/>
ui/litellm-dashboard/src/components/create_user_button.tsx
+28 -40
View File
@@ -1,15 +1,16 @@
import React, { useState, useEffect } from "react";
import { Button, Modal, Form, Input, message, Select, InputNumber } from "antd";
import { Button as Button2 } from "@tremor/react";
import { Button as Button2, Text } from "@tremor/react";
import { userCreateCall, modelAvailableCall } from "./networking";
const { Option } = Select;
interface CreateuserProps {
userID: string;
accessToken: string;
teams: any[] | null;
}
const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken, teams }) => {
const [form] = Form.useForm();
const [isModalVisible, setIsModalVisible] = useState(false);
const [apiuser, setApiuser] = useState<string | null>(null);
@@ -59,7 +60,7 @@ const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
message.info("Making API Call");
setIsModalVisible(true);
console.log("formValues in create user:", formValues);
const response = await userCreateCall(accessToken, userID, formValues);
const response = await userCreateCall(accessToken, null, formValues);
console.log("user create Response:", response);
setApiuser(response["key"]);
message.success("API user Created");
@@ -73,16 +74,18 @@ const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
return (
<div>
<Button2 className="mx-auto" onClick={() => setIsModalVisible(true)}>
+ Create New User
+ Invite User
</Button2>
<Modal
title="Create User"
title="Invite User"
visible={isModalVisible}
width={800}
footer={null}
onOk={handleOk}
onCancel={handleCancel}
>
<Text className="mb-1">Invite a user to login to the Admin UI and create Keys</Text>
<Text className="mb-6"><b>Note: SSO Setup Required for this</b></Text>
<Form
form={form}
onFinish={handleCreate}
@@ -90,38 +93,27 @@ const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
wrapperCol={{ span: 16 }}
labelAlign="left"
>
<Form.Item label="User ID" name="user_id">
<Input placeholder="Enter User ID" />
<Form.Item label="User Email" name="user_email">
<Input placeholder="Enter User Email" />
</Form.Item>
<Form.Item label="Team ID" name="team_id">
<Input placeholder="ai_team" />
</Form.Item>
<Form.Item label="Models" name="models">
<Select
mode="multiple"
placeholder="Select models"
<Select
placeholder="Select Team ID"
style={{ width: "100%" }}
>
{userModels.map((model) => (
<Option key={model} value={model}>
{model}
{teams ? (
teams.map((team: any) => (
<Option key={team.team_id} value={team.team_id}>
{team.team_alias}
</Option>
))
) : (
<Option key="default" value={null}>
Default Team
</Option>
))}
)}
</Select>
</Form.Item>
<Form.Item label="Max Budget (USD)" name="max_budget">
<InputNumber step={0.01} precision={2} width={200} />
</Form.Item>
<Form.Item label="Tokens per minute Limit (TPM)" name="tpm_limit">
<InputNumber step={1} width={400} />
</Form.Item>
<Form.Item label="Requests per minute Limit (RPM)" name="rpm_limit">
<InputNumber step={1} width={400} />
</Form.Item>
<Form.Item label="Duration (eg: 30s, 30h, 30d)" name="duration">
<Input />
</Form.Item>
<Form.Item label="Metadata" name="metadata">
<Input.TextArea rows={4} placeholder="Enter metadata as JSON" />
</Form.Item>
@@ -132,23 +124,19 @@ const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
</Modal>
{apiuser && (
<Modal
title="Save Your User"
title="User Created Successfully"
visible={isModalVisible}
onOk={handleOk}
onCancel={handleCancel}
footer={null}
>
<p>
Please save this secret user somewhere safe and accessible. For
security reasons, <b>you will not be able to view it again</b>{" "}
through your LiteLLM account. If you lose this secret user, you will
need to generate a new one.
</p>
<p>
{apiuser != null
? `API user: ${apiuser}`
: "User being created, this might take 30s"}
User has been created to access your proxy. Please Ask them to Log In.
</p>
<br></br>
<p><b>Note: This Feature is only supported through SSO on the Admin UI</b></p>
</Modal>
)}
</div>
ui/litellm-dashboard/src/components/networking.tsx
+1 -1
View File
@@ -158,7 +158,7 @@ export const keyCreateCall = async (
export const userCreateCall = async (
accessToken: string,
userID: string,
userID: string | null,
formValues: Record<string, any> // Assuming formValues is an object
) => {
try {
ui/litellm-dashboard/src/components/view_users.tsx
+3 -1
View File
@@ -36,6 +36,7 @@ interface ViewUserDashboardProps {
keys: any[] | null;
userRole: string | null;
userID: string | null;
teams: any[] | null;
setKeys: React.Dispatch<React.SetStateAction<Object[] | null>>;
}
@@ -45,6 +46,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
keys,
userRole,
userID,
teams,
setKeys,
}) => {
const [userData, setUserData] = useState<null | any[]>(null);
@@ -151,7 +153,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
return (
<div style={{ width: "100%" }}>
<Grid className="gap-2 p-2 h-[75vh] w-full mt-8">
<CreateUser userID={userID} accessToken={accessToken} />
<CreateUser userID={userID} accessToken={accessToken} teams={teams}/>
<Card className="w-full mx-auto flex-auto overflow-y-auto max-h-[50vh] mb-4">
<TabGroup>
<TabList variant="line" defaultValue="1">