diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index 8443387a5f..da453c3ceb 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -48,6 +48,8 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion | Model Name | Function Call | |--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` | +| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` | | llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` | | mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` | | gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` | diff --git a/litellm/__init__.py b/litellm/__init__.py index 5ef78dce4d..21f98e8b36 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -51,6 +51,7 @@ replicate_key: Optional[str] = None cohere_key: Optional[str] = None maritalk_key: Optional[str] = None ai21_key: Optional[str] = None +ollama_key: Optional[str] = None openrouter_key: Optional[str] = None huggingface_key: Optional[str] = None vertex_project: Optional[str] = None diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index cef5900c42..3b13446a6a 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -282,9 +282,9 @@ class LangFuseLogger: for key, value in metadata.items(): # generate langfuse tags if key in [ - "user_api_key", + "user_api_key_alias", "user_api_key_user_id", - "user_api_key_team_id", + "user_api_key_team_alias", "semantic-similarity", ]: tags.append(f"{key}:{value}") diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index d836ed8db5..24d889b0f4 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -258,8 +258,9 @@ class AnthropicChatCompletion(BaseLLM): self.async_handler = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) + data["stream"] = True response = await self.async_handler.post( - api_base, headers=headers, data=json.dumps(data) + api_base, headers=headers, data=json.dumps(data), stream=True ) if response.status_code != 200: diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index dd03e7dbec..7c7d4938a4 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -41,13 +41,12 @@ class AsyncHTTPHandler: data: Optional[Union[dict, str]] = None, # type: ignore params: Optional[dict] = None, headers: Optional[dict] = None, + stream: bool = False, ): - response = await self.client.post( - url, - data=data, # type: ignore - params=params, - headers=headers, + req = self.client.build_request( + "POST", url, data=data, params=params, headers=headers # type: ignore ) + response = await self.client.send(req, stream=stream) return response def __del__(self) -> None: diff --git a/litellm/llms/ollama_chat.py b/litellm/llms/ollama_chat.py index d442ba5aae..aea00a303f 100644 --- a/litellm/llms/ollama_chat.py +++ b/litellm/llms/ollama_chat.py @@ -184,6 +184,7 @@ class OllamaChatConfig: # ollama implementation def get_ollama_response( api_base="http://localhost:11434", + api_key: Optional[str] = None, model="llama2", messages=None, optional_params=None, @@ -236,6 +237,7 @@ def get_ollama_response( if stream == True: response = ollama_async_streaming( url=url, + api_key=api_key, data=data, model_response=model_response, encoding=encoding, @@ -244,6 +246,7 @@ def get_ollama_response( else: response = ollama_acompletion( url=url, + api_key=api_key, data=data, model_response=model_response, encoding=encoding, @@ -252,12 +255,17 @@ def get_ollama_response( ) return response elif stream == True: - return ollama_completion_stream(url=url, data=data, logging_obj=logging_obj) + return ollama_completion_stream( + url=url, api_key=api_key, data=data, logging_obj=logging_obj + ) - response = requests.post( - url=f"{url}", - json=data, - ) + _request = { + "url": f"{url}", + "json": data, + } + if api_key is not None: + _request["headers"] = "Bearer {}".format(api_key) + response = requests.post(**_request) # type: ignore if response.status_code != 200: raise OllamaError(status_code=response.status_code, message=response.text) @@ -307,10 +315,16 @@ def get_ollama_response( return model_response -def ollama_completion_stream(url, data, logging_obj): - with httpx.stream( - url=url, json=data, method="POST", timeout=litellm.request_timeout - ) as response: +def ollama_completion_stream(url, api_key, data, logging_obj): + _request = { + "url": f"{url}", + "json": data, + "method": "POST", + "timeout": litellm.request_timeout, + } + if api_key is not None: + _request["headers"] = "Bearer {}".format(api_key) + with httpx.stream(**_request) as response: try: if response.status_code != 200: raise OllamaError( @@ -329,12 +343,20 @@ def ollama_completion_stream(url, data, logging_obj): raise e -async def ollama_async_streaming(url, data, model_response, encoding, logging_obj): +async def ollama_async_streaming( + url, api_key, data, model_response, encoding, logging_obj +): try: client = httpx.AsyncClient() - async with client.stream( - url=f"{url}", json=data, method="POST", timeout=litellm.request_timeout - ) as response: + _request = { + "url": f"{url}", + "json": data, + "method": "POST", + "timeout": litellm.request_timeout, + } + if api_key is not None: + _request["headers"] = "Bearer {}".format(api_key) + async with client.stream(**_request) as response: if response.status_code != 200: raise OllamaError( status_code=response.status_code, message=response.text @@ -353,13 +375,25 @@ async def ollama_async_streaming(url, data, model_response, encoding, logging_ob async def ollama_acompletion( - url, data, model_response, encoding, logging_obj, function_name + url, + api_key: Optional[str], + data, + model_response, + encoding, + logging_obj, + function_name, ): data["stream"] = False try: timeout = aiohttp.ClientTimeout(total=litellm.request_timeout) # 10 minutes async with aiohttp.ClientSession(timeout=timeout) as session: - resp = await session.post(url, json=data) + _request = { + "url": f"{url}", + "json": data, + } + if api_key is not None: + _request["headers"] = "Bearer {}".format(api_key) + resp = await session.post(**_request) if resp.status != 200: text = await resp.text() diff --git a/litellm/main.py b/litellm/main.py index b1e75f744c..65696b3c0c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1941,9 +1941,16 @@ def completion( or "http://localhost:11434" ) + api_key = ( + api_key + or litellm.ollama_key + or os.environ.get("OLLAMA_API_KEY") + or litellm.api_key + ) ## LOGGING generator = ollama_chat.get_ollama_response( api_base, + api_key, model, messages, optional_params, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index eedcaa4932..113f9413fd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -735,6 +735,26 @@ "mode": "chat", "supports_function_calling": true }, + "groq/llama3-8b-8192": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000010, + "output_cost_per_token": 0.00000010, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true + }, + "groq/llama3-70b-8192": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000064, + "output_cost_per_token": 0.00000080, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true + }, "groq/mixtral-8x7b-32768": { "max_tokens": 32768, "max_input_tokens": 32768, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index ca8b4c5393..0f7c24576e 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -31,12 +31,12 @@ litellm_settings: upperbound_key_generate_params: max_budget: os.environ/LITELLM_UPPERBOUND_KEYS_MAX_BUDGET -# router_settings: -# routing_strategy: usage-based-routing-v2 -# redis_host: os.environ/REDIS_HOST -# redis_password: os.environ/REDIS_PASSWORD -# redis_port: os.environ/REDIS_PORT -# enable_pre_call_checks: True +router_settings: + routing_strategy: usage-based-routing-v2 + redis_host: os.environ/REDIS_HOST + redis_password: os.environ/REDIS_PASSWORD + redis_port: os.environ/REDIS_PORT + enable_pre_call_checks: True general_settings: master_key: sk-1234 diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6dbeed4753..b697b6e976 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -792,6 +792,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): """ team_spend: Optional[float] = None + team_alias: Optional[str] = None team_tpm_limit: Optional[int] = None team_rpm_limit: Optional[int] = None team_max_budget: Optional[float] = None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c7415dea46..db85b7ba10 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3361,6 +3361,9 @@ async def completion( data["metadata"]["user_api_key_team_id"] = getattr( user_api_key_dict, "team_id", None ) + data["metadata"]["user_api_key_team_alias"] = getattr( + user_api_key_dict, "team_alias", None + ) _headers = dict(request.headers) _headers.pop( "authorization", None @@ -3562,6 +3565,9 @@ async def chat_completion( data["metadata"]["user_api_key_team_id"] = getattr( user_api_key_dict, "team_id", None ) + data["metadata"]["user_api_key_team_alias"] = getattr( + user_api_key_dict, "team_alias", None + ) data["metadata"]["user_api_key_metadata"] = user_api_key_dict.metadata _headers = dict(request.headers) _headers.pop( @@ -3793,6 +3799,9 @@ async def embeddings( data["metadata"]["user_api_key_team_id"] = getattr( user_api_key_dict, "team_id", None ) + data["metadata"]["user_api_key_team_alias"] = getattr( + user_api_key_dict, "team_alias", None + ) data["metadata"]["endpoint"] = str(request.url) ### TEAM-SPECIFIC PARAMS ### @@ -3971,6 +3980,9 @@ async def image_generation( data["metadata"]["user_api_key_team_id"] = getattr( user_api_key_dict, "team_id", None ) + data["metadata"]["user_api_key_team_alias"] = getattr( + user_api_key_dict, "team_alias", None + ) data["metadata"]["endpoint"] = str(request.url) ### TEAM-SPECIFIC PARAMS ### @@ -4127,6 +4139,9 @@ async def audio_transcriptions( data["metadata"]["user_api_key_team_id"] = getattr( user_api_key_dict, "team_id", None ) + data["metadata"]["user_api_key_team_alias"] = getattr( + user_api_key_dict, "team_alias", None + ) data["metadata"]["endpoint"] = str(request.url) data["metadata"]["file_name"] = file.filename @@ -4302,6 +4317,9 @@ async def moderations( data["metadata"]["user_api_key_team_id"] = getattr( user_api_key_dict, "team_id", None ) + data["metadata"]["user_api_key_team_alias"] = getattr( + user_api_key_dict, "team_alias", None + ) data["metadata"]["endpoint"] = str(request.url) ### TEAM-SPECIFIC PARAMS ### diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 954d2496ea..02e8a41668 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1186,6 +1186,7 @@ class PrismaClient: t.rpm_limit AS team_rpm_limit, t.models AS team_models, t.blocked AS team_blocked, + t.team_alias AS team_alias, m.aliases as team_model_aliases FROM "LiteLLM_VerificationToken" AS v LEFT JOIN "LiteLLM_TeamTable" AS t ON v.team_id = t.team_id diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index eedcaa4932..113f9413fd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -735,6 +735,26 @@ "mode": "chat", "supports_function_calling": true }, + "groq/llama3-8b-8192": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000010, + "output_cost_per_token": 0.00000010, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true + }, + "groq/llama3-70b-8192": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000064, + "output_cost_per_token": 0.00000080, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true + }, "groq/mixtral-8x7b-32768": { "max_tokens": 32768, "max_input_tokens": 32768, diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 7bd0656236..0a7cc6403f 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -145,6 +145,7 @@ const CreateKeyPage = () => { userRole={userRole} token={token} keys={keys} + teams={teams} accessToken={accessToken} setKeys={setKeys} /> diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/create_user_button.tsx index 94ddcabb04..b3fbfd5d77 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/create_user_button.tsx @@ -1,15 +1,16 @@ import React, { useState, useEffect } from "react"; import { Button, Modal, Form, Input, message, Select, InputNumber } from "antd"; -import { Button as Button2 } from "@tremor/react"; +import { Button as Button2, Text } from "@tremor/react"; import { userCreateCall, modelAvailableCall } from "./networking"; const { Option } = Select; interface CreateuserProps { userID: string; accessToken: string; + teams: any[] | null; } -const Createuser: React.FC = ({ userID, accessToken }) => { +const Createuser: React.FC = ({ userID, accessToken, teams }) => { const [form] = Form.useForm(); const [isModalVisible, setIsModalVisible] = useState(false); const [apiuser, setApiuser] = useState(null); @@ -59,7 +60,7 @@ const Createuser: React.FC = ({ userID, accessToken }) => { message.info("Making API Call"); setIsModalVisible(true); console.log("formValues in create user:", formValues); - const response = await userCreateCall(accessToken, userID, formValues); + const response = await userCreateCall(accessToken, null, formValues); console.log("user create Response:", response); setApiuser(response["key"]); message.success("API user Created"); @@ -73,16 +74,18 @@ const Createuser: React.FC = ({ userID, accessToken }) => { return (
setIsModalVisible(true)}> - + Create New User + + Invite User + Invite a user to login to the Admin UI and create Keys + Note: SSO Setup Required for this
= ({ userID, accessToken }) => { wrapperCol={{ span: 16 }} labelAlign="left" > - - + + - - - - - - - - - - - - - - - - - @@ -132,23 +124,19 @@ const Createuser: React.FC = ({ userID, accessToken }) => { {apiuser && (

- Please save this secret user somewhere safe and accessible. For - security reasons, you will not be able to view it again{" "} - through your LiteLLM account. If you lose this secret user, you will - need to generate a new one. -

-

- {apiuser != null - ? `API user: ${apiuser}` - : "User being created, this might take 30s"} + User has been created to access your proxy. Please Ask them to Log In.

+

+ +

Note: This Feature is only supported through SSO on the Admin UI

+ )}
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 46ec87e399..4b961ca347 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -158,7 +158,7 @@ export const keyCreateCall = async ( export const userCreateCall = async ( accessToken: string, - userID: string, + userID: string | null, formValues: Record // Assuming formValues is an object ) => { try { diff --git a/ui/litellm-dashboard/src/components/view_users.tsx b/ui/litellm-dashboard/src/components/view_users.tsx index 7270b0d22d..d9fa5f7844 100644 --- a/ui/litellm-dashboard/src/components/view_users.tsx +++ b/ui/litellm-dashboard/src/components/view_users.tsx @@ -36,6 +36,7 @@ interface ViewUserDashboardProps { keys: any[] | null; userRole: string | null; userID: string | null; + teams: any[] | null; setKeys: React.Dispatch>; } @@ -45,6 +46,7 @@ const ViewUserDashboard: React.FC = ({ keys, userRole, userID, + teams, setKeys, }) => { const [userData, setUserData] = useState(null); @@ -151,7 +153,7 @@ const ViewUserDashboard: React.FC = ({ return (
- +