Merge pull request #3198 from BerriAI/litellm_fix_create_key

ui - fix create key flow / cleanup non admin flow
This commit is contained in:
Ishaan Jaff
2024-04-20 19:31:58 -07:00
committed by GitHub
9 changed files with 353 additions and 143 deletions
+3
View File
@@ -9,6 +9,7 @@ import Teams from "@/components/teams";
import AdminPanel from "@/components/admins";
import Settings from "@/components/settings";
import GeneralSettings from "@/components/general_settings";
import APIRef from "@/components/api_ref";
import ChatUI from "@/components/chat_ui";
import Sidebar from "../components/leftnav";
import Usage from "../components/usage";
@@ -165,6 +166,8 @@ const CreateKeyPage = () => {
accessToken={accessToken}
showSSOBanner={showSSOBanner}
/>
) : page == "api_ref" ? (
<APIRef/>
) : page == "settings" ? (
<Settings
userID={userID}
@@ -0,0 +1,152 @@
"use client";
import React, { useEffect, useState } from "react";
import {
Badge,
Card,
Table,
Metric,
TableBody,
TableCell,
TableHead,
TableHeaderCell,
TableRow,
Text,
Title,
Icon,
Accordion,
AccordionBody,
AccordionHeader,
List,
ListItem,
Tab,
TabGroup,
TabList,
TabPanel,
TabPanels,
Grid,
} from "@tremor/react";
import { Statistic } from "antd"
import { modelAvailableCall } from "./networking";
import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
const APIRef = ({}) => {
return (
<>
<Grid className="gap-2 p-8 h-[80vh] w-full mt-2">
<div className="mb-5">
<p className="text-2xl text-tremor-content-strong dark:text-dark-tremor-content-strong font-semibold">OpenAI Compatible Proxy: API Reference</p>
<Text className="mt-2 mb-2">LiteLLM is OpenAI Compatible. This means your API Key works with the OpenAI SDK. Just replace the base_url to point to your litellm proxy. Example Below </Text>
<TabGroup>
<TabList>
<Tab>OpenAI Python SDK</Tab>
<Tab>LlamaIndex</Tab>
<Tab>Langchain Py</Tab>
</TabList>
<TabPanels>
<TabPanel>
<SyntaxHighlighter language="python">
{`
import openai
client = openai.OpenAI(
api_key="your_api_key",
base_url="http://0.0.0.0:4000" # LiteLLM Proxy is OpenAI compatible, Read More: https://docs.litellm.ai/docs/proxy/user_keys
)
response = client.chat.completions.create(
model="gpt-3.5-turbo", # model to send to the proxy
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
]
)
print(response)
`}
</SyntaxHighlighter>
</TabPanel>
<TabPanel>
<SyntaxHighlighter language="python">
{`
import os, dotenv
from llama_index.llms import AzureOpenAI
from llama_index.embeddings import AzureOpenAIEmbedding
from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext
llm = AzureOpenAI(
engine="azure-gpt-3.5", # model_name on litellm proxy
temperature=0.0,
azure_endpoint="http://0.0.0.0:4000", # litellm proxy endpoint
api_key="sk-1234", # litellm proxy API Key
api_version="2023-07-01-preview",
)
embed_model = AzureOpenAIEmbedding(
deployment_name="azure-embedding-model",
azure_endpoint="http://0.0.0.0:4000",
api_key="sk-1234",
api_version="2023-07-01-preview",
)
documents = SimpleDirectoryReader("llama_index_data").load_data()
service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)
index = VectorStoreIndex.from_documents(documents, service_context=service_context)
query_engine = index.as_query_engine()
response = query_engine.query("What did the author do growing up?")
print(response)
`}
</SyntaxHighlighter>
</TabPanel>
<TabPanel>
<SyntaxHighlighter language="python">
{`
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model = "gpt-3.5-turbo",
temperature=0.1
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
`}
</SyntaxHighlighter>
</TabPanel>
</TabPanels>
</TabGroup>
</div>
</Grid>
</>
)
}
export default APIRef;
+2 -120
View File
@@ -13,12 +13,12 @@ import {
TabGroup,
TabList,
TabPanel,
TabPanels,
Metric,
Col,
Text,
SelectItem,
TextInput,
TabPanels,
Button,
} from "@tremor/react";
@@ -201,7 +201,6 @@ const ChatUI: React.FC<ChatUIProps> = ({
<TabGroup>
<TabList>
<Tab>Chat</Tab>
<Tab>API Reference</Tab>
</TabList>
<TabPanels>
@@ -272,124 +271,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
</div>
</div>
</TabPanel>
<TabPanel>
<TabGroup>
<TabList>
<Tab>OpenAI Python SDK</Tab>
<Tab>LlamaIndex</Tab>
<Tab>Langchain Py</Tab>
</TabList>
<TabPanels>
<TabPanel>
<SyntaxHighlighter language="python">
{`
import openai
client = openai.OpenAI(
api_key="your_api_key",
base_url="http://0.0.0.0:4000" # proxy base url
)
response = client.chat.completions.create(
model="gpt-3.5-turbo", # model to use from Models Tab
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_body={
"metadata": {
"generation_name": "ishaan-generation-openai-client",
"generation_id": "openai-client-gen-id22",
"trace_id": "openai-client-trace-id22",
"trace_user_id": "openai-client-user-id2"
}
}
)
print(response)
`}
</SyntaxHighlighter>
</TabPanel>
<TabPanel>
<SyntaxHighlighter language="python">
{`
import os, dotenv
from llama_index.llms import AzureOpenAI
from llama_index.embeddings import AzureOpenAIEmbedding
from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext
llm = AzureOpenAI(
engine="azure-gpt-3.5", # model_name on litellm proxy
temperature=0.0,
azure_endpoint="http://0.0.0.0:4000", # litellm proxy endpoint
api_key="sk-1234", # litellm proxy API Key
api_version="2023-07-01-preview",
)
embed_model = AzureOpenAIEmbedding(
deployment_name="azure-embedding-model",
azure_endpoint="http://0.0.0.0:4000",
api_key="sk-1234",
api_version="2023-07-01-preview",
)
documents = SimpleDirectoryReader("llama_index_data").load_data()
service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)
index = VectorStoreIndex.from_documents(documents, service_context=service_context)
query_engine = index.as_query_engine()
response = query_engine.query("What did the author do growing up?")
print(response)
`}
</SyntaxHighlighter>
</TabPanel>
<TabPanel>
<SyntaxHighlighter language="python">
{`
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:8000",
model = "gpt-3.5-turbo",
temperature=0.1,
extra_body={
"metadata": {
"generation_name": "ishaan-generation-langchain-client",
"generation_id": "langchain-client-gen-id22",
"trace_id": "langchain-client-trace-id22",
"trace_user_id": "langchain-client-user-id2"
}
}
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
`}
</SyntaxHighlighter>
</TabPanel>
</TabPanels>
</TabGroup>
</TabPanel>
</TabPanels>
</TabGroup>
</Card>
@@ -147,6 +147,17 @@ const CreateKey: React.FC<CreateKeyProps> = ({
mode="multiple"
placeholder="Select models"
style={{ width: "100%" }}
onChange={(values) => {
// Check if "All Team Models" is selected
const isAllTeamModelsSelected = values.includes("all-team-models");
// If "All Team Models" is selected, deselect all other models
if (isAllTeamModelsSelected) {
const newValues = ["all-team-models"];
// You can call the form's setFieldsValue method to update the value
form.setFieldsValue({ models: newValues });
}
}}
>
<Option key="all-team-models" value="all-team-models">
All Team Models
@@ -270,6 +281,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({
<Form.Item
label="Models"
name="models"
className="mb-12"
rules={[{ required: true, message: 'Please select a model' }]}
help="required"
>
@@ -277,6 +289,15 @@ const CreateKey: React.FC<CreateKeyProps> = ({
mode="multiple"
placeholder="Select models"
style={{ width: "100%" }}
onChange={(values) => {
const isAllTeamModelsSelected = values.includes("all-team-models");
if (isAllTeamModelsSelected) {
const newValues = ["all-team-models"];
form.setFieldsValue({ models: newValues });
}
}}
>
<Option key="all-team-models" value="all-team-models">
All Team Models
@@ -308,7 +329,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({
</Select>
</Form.Item>
<Accordion className="mt-8">
<Accordion className="mt-20 mb-8" >
<AccordionHeader>
<b>Optional Settings</b>
</AccordionHeader>
@@ -34,20 +34,19 @@ const DashboardTeam: React.FC<DashboardTeamProps> = ({
} else {
updatedTeams = teams ? [...teams, defaultTeam] : [defaultTeam];
}
if (userRole === 'App User') return null;
return (
<div className="mt-5 mb-5">
<Title>Select Team</Title>
{userRole !== "App User" && (
<>
<Text>
If you belong to multiple teams, this setting controls which team is used by default when creating new API Keys.
</Text>
<Text className="mt-3 mb-3">
<b>Default Team:</b> If no team_id is set for a key, it will be grouped under here.
</Text>
</>
)}
<Text>
If you belong to multiple teams, this setting controls which team is used by default when creating new API Keys.
</Text>
<Text className="mt-3 mb-3">
<b>Default Team:</b> If no team_id is set for a key, it will be grouped under here.
</Text>
{updatedTeams && updatedTeams.length > 0 ? (
<Select defaultValue="0">
{updatedTeams.map((team: any, index) => (
@@ -46,8 +46,8 @@ const Sidebar: React.FC<SidebarProps> = ({
);
}
return (
<Layout style={{ minHeight: "100vh", maxWidth: "120px" }}>
<Sider width={120}>
<Layout style={{ minHeight: "100vh", maxWidth: "130px" }}>
<Sider width={130}>
<Menu
mode="inline"
defaultSelectedKeys={defaultSelectedKey ? defaultSelectedKey : ["1"]}
@@ -63,6 +63,13 @@ const Sidebar: React.FC<SidebarProps> = ({
Test Key
</Text>
</Menu.Item>
<Menu.Item key="11" onClick={() => setPage("api_ref")}>
<Text>
API Reference
</Text>
</Menu.Item>
{
userRole == "Admin" ? (
<Menu.Item key="2" onClick={() => setPage("models")}>
@@ -5,6 +5,7 @@ import { Grid, Col, Card, Text, Title } from "@tremor/react";
import CreateKey from "./create_key_button";
import ViewKeyTable from "./view_key_table";
import ViewUserSpend from "./view_user_spend";
import ViewUserTeam from "./view_user_team";
import DashboardTeam from "./dashboard_default_team";
import { useSearchParams, useRouter } from "next/navigation";
import { jwtDecode } from "jwt-decode";
@@ -232,11 +233,19 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
<div className="w-full mx-4">
<Grid numItems={1} className="gap-2 p-8 h-[75vh] w-full mt-2">
<Col numColSpan={1}>
<ViewUserTeam
userID={userID}
userRole={userRole}
selectedTeam={selectedTeam ? selectedTeam : null}
accessToken={accessToken}
/>
<ViewUserSpend
userID={userID}
userRole={userRole}
accessToken={accessToken}
userSpend={teamSpend}
selectedTeam = {selectedTeam ? selectedTeam : null}
/>
<ViewKeyTable
@@ -2,7 +2,7 @@
import React, { useEffect, useState } from "react";
import { keyDeleteCall, getTotalSpendCall } from "./networking";
import { StatusOnlineIcon, TrashIcon } from "@heroicons/react/outline";
import { DonutChart } from "@tremor/react";
import { Accordion, AccordionHeader, AccordionList, DonutChart } from "@tremor/react";
import {
Badge,
Card,
@@ -16,9 +16,13 @@ import {
Text,
Title,
Icon,
AccordionBody,
List,
ListItem,
} from "@tremor/react";
import { Statistic } from "antd"
import { spendUsersCall } from "./networking";
import { spendUsersCall, modelAvailableCall } from "./networking";
// Define the props type
@@ -32,11 +36,13 @@ interface ViewUserSpendProps {
userRole: string | null;
accessToken: string | null;
userSpend: number | null;
selectedTeam: any | null;
}
const ViewUserSpend: React.FC<ViewUserSpendProps> = ({ userID, userRole, accessToken, userSpend }) => {
const ViewUserSpend: React.FC<ViewUserSpendProps> = ({ userID, userRole, accessToken, userSpend, selectedTeam }) => {
console.log(`userSpend: ${userSpend}`)
let [spend, setSpend] = useState(userSpend !== null ? userSpend : 0.0);
const [maxBudget, setMaxBudget] = useState(0.0);
const [userModels, setUserModels] = useState([]);
useEffect(() => {
const fetchData = async () => {
if (!accessToken || !userID || !userRole) {
@@ -62,9 +68,30 @@ const ViewUserSpend: React.FC<ViewUserSpendProps> = ({ userID, userRole, accessT
}
}
};
const fetchUserModels = async () => {
try {
if (userID === null || userRole === null) {
return;
}
if (accessToken !== null) {
const model_available = await modelAvailableCall(accessToken, userID, userRole);
let available_model_names = model_available["data"].map(
(element: { id: string }) => element.id
);
console.log("available_model_names:", available_model_names);
setUserModels(available_model_names);
}
} catch (error) {
console.error("Error fetching user models:", error);
}
};
fetchUserModels();
fetchData();
}, [userRole, accessToken]);
}, [userRole, accessToken, userID]);
useEffect(() => {
if (userSpend !== null) {
@@ -72,18 +99,50 @@ const ViewUserSpend: React.FC<ViewUserSpendProps> = ({ userID, userRole, accessT
}
}, [userSpend])
// logic to decide what models to display
let modelsToDisplay = [];
if (selectedTeam && selectedTeam.models) {
modelsToDisplay = selectedTeam.models;
}
// check if "all-proxy-models" is in modelsToDisplay
if (modelsToDisplay && modelsToDisplay.includes("all-proxy-models")) {
console.log("user models:", userModels);
modelsToDisplay = userModels;
}
const displayMaxBudget = maxBudget !== null ? `$${maxBudget} limit` : "No limit";
const roundedSpend = spend !== undefined ? spend.toFixed(4) : null;
console.log(`spend in view user spend: ${spend}`)
return (
<>
<p className="text-tremor-default text-tremor-content dark:text-dark-tremor-content">Total Spend </p>
<p className="text-3xl text-tremor-content-strong dark:text-dark-tremor-content-strong font-semibold">${roundedSpend}</p>
</>
)
<div className="flex items-center">
<div>
<p className="text-tremor-default text-tremor-content dark:text-dark-tremor-content">
Total Spend{" "}
</p>
<p className="text-2xl text-tremor-content-strong dark:text-dark-tremor-content-strong font-semibold">
${roundedSpend}
</p>
</div>
<div className="ml-auto">
<Accordion>
<AccordionHeader>Models</AccordionHeader>
<AccordionBody className="absolute right-0 z-10 bg-white p-2 shadow-lg max-w-xs">
<List>
{modelsToDisplay.map((model: string) => (
<ListItem key={model}>
<Text>{model}</Text>
</ListItem>
))}
</List>
</AccordionBody>
</Accordion>
</div>
</div>
);
}
export default ViewUserSpend;
@@ -0,0 +1,78 @@
"use client";
import React, { useEffect, useState } from "react";
import {
Badge,
Card,
Table,
Metric,
TableBody,
TableCell,
TableHead,
TableHeaderCell,
TableRow,
Text,
Title,
Icon,
Accordion,
AccordionBody,
AccordionHeader,
List,
ListItem,
} from "@tremor/react";
import { Statistic } from "antd"
import { modelAvailableCall } from "./networking";
interface ViewUserTeamProps {
userID: string | null;
userRole: string | null;
selectedTeam: any | null;
accessToken: string | null;
}
const ViewUserTeam: React.FC<ViewUserTeamProps> = ({ userID, userRole, selectedTeam, accessToken}) => {
const [userModels, setUserModels] = useState([]);
useEffect(() => {
const fetchUserModels = async () => {
try {
if (userID === null || userRole === null) {
return;
}
if (accessToken !== null) {
const model_available = await modelAvailableCall(accessToken, userID, userRole);
let available_model_names = model_available["data"].map(
(element: { id: string }) => element.id
);
console.log("available_model_names:", available_model_names);
setUserModels(available_model_names);
}
} catch (error) {
console.error("Error fetching user models:", error);
}
};
fetchUserModels();
}, [accessToken, userID, userRole]);
// logic to decide what models to display
let modelsToDisplay = [];
if (selectedTeam && selectedTeam.models) {
modelsToDisplay = selectedTeam.models;
}
// check if "all-proxy-models" is in modelsToDisplay
if (modelsToDisplay && modelsToDisplay.includes("all-proxy-models")) {
console.log("user models:", userModels);
modelsToDisplay = userModels;
}
return (
<>
<div className="mb-5">
<p className="text-3xl text-tremor-content-strong dark:text-dark-tremor-content-strong font-semibold">{selectedTeam?.team_alias}</p>
</div>
</>
)
}
export default ViewUserTeam;