feat(ui): for adding pass-through endpoints

litellm/proxy/config_management_endpoints/pass_through_endpoints.py

@@ -0,0 +1,47 @@
+"""
+What is this? 
+
+CRUD endpoints for managing pass-through endpoints
+"""
+
+import asyncio
+import traceback
+from datetime import datetime, timedelta, timezone
+from typing import List, Optional
+
+import fastapi
+import httpx
+from fastapi import (
+    APIRouter,
+    Depends,
+    File,
+    Form,
+    Header,
+    HTTPException,
+    Request,
+    Response,
+    UploadFile,
+    status,
+)
+
+import litellm
+from litellm._logging import verbose_proxy_logger
+from litellm.batches.main import FileObject
+from litellm.proxy._types import *
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+router = APIRouter()
+
+
+@router.get(
+    "/config/pass_through_endpoints/settings",
+    dependencies=[Depends(user_api_key_auth)],
+    tags=["pass-through-endpoints"],
+    summary="Create pass-through endpoints for provider specific endpoints - https://docs.litellm.ai/docs/proxy/pass_through",
+)
+async def create_fine_tuning_job(
+    request: Request,
+    fastapi_response: Response,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    pass

litellm/proxy/proxy_server.py

@@ -9473,11 +9473,10 @@ async def get_config_list(
             typed_dict_type = allowed_args[field_name]["type"]
 
             if typed_dict_type == "PydanticModel":
-                pydantic_class_list: Optional[Any] = _resolve_pydantic_type(
-                    field_info.annotation
-                )
-                if pydantic_class_list is None:
-                    continue
+                if field_name == "pass_through_endpoints":
+                    pydantic_class_list = [PassThroughGenericEndpoint]
+                else:
+                    pydantic_class_list = []
 
                 for pydantic_class in pydantic_class_list:
                     # Get type hints from the TypedDict to create FieldDetail objects

ui/litellm-dashboard/src/app/page.tsx

@@ -9,6 +9,7 @@ import Teams from "@/components/teams";
 import AdminPanel from "@/components/admins";
 import Settings from "@/components/settings";
 import GeneralSettings from "@/components/general_settings";
+import PassThroughSettings from "@/components/pass_through_settings";
 import BudgetPanel from "@/components/budgets/budget_panel";
 import ModelHub from "@/components/model_hub";
 import APIRef from "@/components/api_ref";
@@ -263,6 +264,13 @@ const CreateKeyPage = () => {
               accessToken={accessToken}
               premiumUser={premiumUser}
             />
+          ) : page == "pass-through-settings" ? (
+            <PassThroughSettings
+              userID={userID}
+              userRole={userRole}
+              accessToken={accessToken}
+              modelData={modelData}
+            />
           ) : (
             <Usage
               userID={userID}

ui/litellm-dashboard/src/components/add_pass_through.tsx

@@ -0,0 +1,149 @@
+/**
+ * Modal to add fallbacks to the proxy router config
+ */
+
+
+
+import React, { useState, useEffect, useRef } from "react";
+import { Button, TextInput, Grid, Col } from "@tremor/react";
+import { Select, SelectItem, MultiSelect, MultiSelectItem, Card, Metric, Text, Title, Subtitle, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react";
+import { CopyToClipboard } from 'react-copy-to-clipboard';
+import { createPassThroughEndpoint } from "./networking";
+import {
+  Button as Button2,
+  Modal,
+  Form,
+  Input,
+  InputNumber,
+  Select as Select2,
+  message,
+} from "antd";
+import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking";
+import { list } from "postcss";
+import KeyValueInput from "./key_value_input";
+import { passThroughItem } from "./pass_through_settings";
+const { Option } = Select2;
+
+interface AddFallbacksProps {
+//   models: string[] | undefined; 
+  accessToken: string;
+  passThroughItems: passThroughItem[];
+  setPassThroughItems: React.Dispatch<React.SetStateAction<passThroughItem[]>>;
+}
+
+const AddPassThroughEndpoint: React.FC<AddFallbacksProps> = ({
+    accessToken, setPassThroughItems, passThroughItems
+}) => {
+  const [form] = Form.useForm();
+  const [isModalVisible, setIsModalVisible] = useState(false);
+  const [selectedModel, setSelectedModel] = useState("");
+  const handleOk = () => {
+    setIsModalVisible(false);
+    form.resetFields();
+  };
+
+  const handleCancel = () => {
+    setIsModalVisible(false);
+    form.resetFields();
+  };
+
+  const addPassThrough = (formValues: Record<string, any>) => {
+    // Print the received value
+    console.log(formValues);
+
+    // // Extract model_name and models from formValues
+    // const { model_name, models } = formValues;
+
+    // // Create new fallback
+    // const newFallback = { [model_name]: models };
+
+    // // Get current fallbacks, or an empty array if it's null
+    // const currentFallbacks = routerSettings.fallbacks || [];
+
+    // // Add new fallback to the current fallbacks
+    // const updatedFallbacks = [...currentFallbacks, newFallback];
+
+    // // Create a new routerSettings object with updated fallbacks
+    // const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks };
+
+    const newPassThroughItem: passThroughItem = {
+        "headers": formValues["headers"],
+        "path": formValues["path"],
+        "target": formValues["target"]
+    }
+    const updatedPassThroughSettings = [...passThroughItems, newPassThroughItem]
+
+
+    try {
+        createPassThroughEndpoint(accessToken, formValues);
+        setPassThroughItems(updatedPassThroughSettings)
+    } catch (error) {
+        message.error("Failed to update router settings: " + error, 20);
+    }
+
+    message.success("Pass through endpoint successfully added");
+
+    setIsModalVisible(false)
+    form.resetFields();
+  };
+
+
+  return (
+    <div>
+      <Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
+        + Add Pass-Through Endpoint
+      </Button>
+      <Modal
+        title="Add Pass-Through Endpoint"
+        visible={isModalVisible}
+        width={800}
+        footer={null}
+        onOk={handleOk}
+        onCancel={handleCancel}
+      >
+        <Form
+          form={form}
+          onFinish={addPassThrough}
+          labelCol={{ span: 8 }}
+          wrapperCol={{ span: 16 }}
+          labelAlign="left"
+        >
+            <>
+              <Form.Item 
+                label="Path" 
+                name="path"
+                rules={[{ required: true, message: 'The route to be added to the LiteLLM Proxy Server.' }]}
+                help="required"
+              >
+                <TextInput/>
+              </Form.Item>
+
+              <Form.Item 
+                label="Target" 
+                name="target"
+                rules={[{ required: true, message: 'The URL to which requests for this path should be forwarded.' }]}
+                help="required"
+              >
+                <TextInput/>
+              </Form.Item>
+              <Form.Item 
+                label="Headers" 
+                name="headers"
+                rules={[{ required: true, message: 'Key-value pairs of headers to be forwarded with the request. You can set any key value pair here and it will be forwarded to your target endpoint' }]}
+                help="required"
+              >
+                <KeyValueInput/>
+              </Form.Item>
+            </>
+          
+          <div style={{ textAlign: "right", marginTop: "10px" }}>
+            <Button2 htmlType="submit">Add Pass-Through Endpoint</Button2>
+          </div>
+        </Form>
+      </Modal>
+
+    </div>
+  );
+};
+
+export default AddPassThroughEndpoint;

ui/litellm-dashboard/src/components/general_settings.tsx

@@ -597,7 +597,7 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
                   </TableRow>
                 </TableHead>
                 <TableBody>
-                  {generalSettings.map((value, index) => (
+                  {generalSettings.filter((value) => value.field_type !== "TypedDictionary").map((value, index) => (
                     <TableRow key={index}>
                       <TableCell>
                         <Text>{value.field_name}</Text>

ui/litellm-dashboard/src/components/key_value_input.tsx

@@ -0,0 +1,56 @@
+import React, { useState } from 'react';
+import { Form, Input, Button, Space } from 'antd';
+import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons';
+import { TextInput, Grid, Col } from "@tremor/react";
+import { TrashIcon } from "@heroicons/react/outline";
+
+interface KeyValueInputProps {
+  value?: Record<string, string>;
+  onChange?: (value: Record<string, string>) => void;
+}
+
+const KeyValueInput: React.FC<KeyValueInputProps> = ({ value = {}, onChange }) => {
+  const [pairs, setPairs] = useState<[string, string][]>(Object.entries(value));
+
+  const handleAdd = () => {
+    setPairs([...pairs, ['', '']]);
+  };
+
+  const handleRemove = (index: number) => {
+    const newPairs = pairs.filter((_, i) => i !== index);
+    setPairs(newPairs);
+    onChange?.(Object.fromEntries(newPairs));
+  };
+
+  const handleChange = (index: number, key: string, val: string) => {
+    const newPairs = [...pairs];
+    newPairs[index] = [key, val];
+    setPairs(newPairs);
+    onChange?.(Object.fromEntries(newPairs));
+  };
+
+  return (
+    <div>
+      {pairs.map(([key, val], index) => (
+        <Space key={index} style={{ display: 'flex', marginBottom: 8 }} align="start">
+          <TextInput
+            placeholder="Header Name"
+            value={key}
+            onChange={(e) => handleChange(index, e.target.value, val)}
+          />
+          <TextInput
+            placeholder="Header Value"
+            value={val}
+            onChange={(e) => handleChange(index, key, e.target.value)}
+          />
+          <MinusCircleOutlined onClick={() => handleRemove(index)} />
+        </Space>
+      ))}
+      <Button type="dashed" onClick={handleAdd} icon={<PlusOutlined />}>
+        Add Header
+      </Button>
+    </div>
+  );
+};
+
+export default KeyValueInput;

ui/litellm-dashboard/src/components/leftnav.tsx

@@ -102,15 +102,21 @@ const Sidebar: React.FC<SidebarProps> = ({
               <Text>Router Settings</Text>
             </Menu.Item>
           ) : null}
+          
           {userRole == "Admin" ? (
-            <Menu.Item key="12" onClick={() => setPage("admin-panel")}>
+            <Menu.Item key="12" onClick={() => setPage("pass-through-settings")}>
+              <Text>Pass-Through</Text>
+            </Menu.Item>
+          ) : null}
+          {userRole == "Admin" ? (
+            <Menu.Item key="13" onClick={() => setPage("admin-panel")}>
               <Text>Admin Settings</Text>
             </Menu.Item>
           ) : null}
-          <Menu.Item key="13" onClick={() => setPage("api_ref")}>
+          <Menu.Item key="14" onClick={() => setPage("api_ref")}>
             <Text>API Reference</Text>
           </Menu.Item>
-          <Menu.Item key="15" onClick={() => setPage("model-hub")}>
+          <Menu.Item key="16" onClick={() => setPage("model-hub")}>
             <Text>Model Hub</Text>
           </Menu.Item>
         </Menu>

ui/litellm-dashboard/src/components/networking.tsx

@@ -2388,6 +2388,38 @@ export const getGeneralSettingsCall = async (accessToken: String) => {
   }
 };
 
+
+export const getPassThroughEndpointsCall = async (accessToken: String) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/pass_through_endpoint`
+      : `/config/pass_through_endpoint`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to get callbacks:", error);
+    throw error;
+  }
+};
+
 export const getConfigFieldSetting = async (
   accessToken: String,
   fieldName: string
@@ -2420,6 +2452,85 @@ export const getConfigFieldSetting = async (
   }
 };
 
+export const updatePassThroughFieldSetting = async (
+  accessToken: String,
+  fieldName: string,
+  fieldValue: any
+) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/pass_through_endpoint`
+      : `/config/pass_through_endpoint`;
+
+    let formData = {
+      field_name: fieldName,
+      field_value: fieldValue,
+    };
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(formData),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    message.success("Successfully updated value!");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to set callbacks:", error);
+    throw error;
+  }
+};
+
+export const createPassThroughEndpoint = async (
+  accessToken: String,
+  formValues: Record<string, any>
+) => {
+  /**
+   * Set callbacks on proxy
+   */
+  try {
+    let url = proxyBaseUrl ? `${proxyBaseUrl}/config/pass_through_endpoint` : `/config/pass_through_endpoint`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        ...formValues, // Include formValues in the request body
+      }),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to set callbacks:", error);
+    throw error;
+  }
+};
+
 export const updateConfigFieldSetting = async (
   accessToken: String,
   fieldName: string,
@@ -2500,6 +2611,38 @@ export const deleteConfigFieldSetting = async (
     throw error;
   }
 };
+
+export const deletePassThroughEndpointsCall = async (accessToken: String, endpointId: string) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/pass_through_endpoint?endpoint_id=${endpointId}`
+      : `/config/pass_through_endpoint${endpointId}`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "DELETE",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to get callbacks:", error);
+    throw error;
+  }
+};
+
 export const setCallbacksCall = async (
   accessToken: String,
   formValues: Record<string, any>

ui/litellm-dashboard/src/components/pass_through_settings.tsx

@@ -0,0 +1,196 @@
+import React, { useState, useEffect } from "react";
+import {
+  Card,
+  Title,
+  Subtitle,
+  Table,
+  TableHead,
+  TableRow,
+  Badge,
+  TableHeaderCell,
+  TableCell,
+  TableBody,
+  Metric,
+  Text,
+  Grid,
+  Button,
+  TextInput,
+  Select as Select2,
+  SelectItem,
+  Col,
+  Accordion,
+  AccordionBody,
+  AccordionHeader,
+  AccordionList,
+} from "@tremor/react";
+import {
+  TabPanel,
+  TabPanels,
+  TabGroup,
+  TabList,
+  Tab,
+  Icon,
+} from "@tremor/react";
+import {
+  getCallbacksCall,
+  setCallbacksCall,
+  getGeneralSettingsCall,
+  deletePassThroughEndpointsCall,
+  getPassThroughEndpointsCall,
+  serviceHealthCheck,
+  updateConfigFieldSetting,
+  deleteConfigFieldSetting,
+} from "./networking";
+import {
+  Modal,
+  Form,
+  Input,
+  Select,
+  Button as Button2,
+  message,
+  InputNumber,
+} from "antd";
+import {
+  InformationCircleIcon,
+  PencilAltIcon,
+  PencilIcon,
+  StatusOnlineIcon,
+  TrashIcon,
+  RefreshIcon,
+  CheckCircleIcon,
+  XCircleIcon,
+  QuestionMarkCircleIcon,
+} from "@heroicons/react/outline";
+import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
+import AddFallbacks from "./add_fallbacks";
+import AddPassThroughEndpoint from "./add_pass_through";
+import openai from "openai";
+import Paragraph from "antd/es/skeleton/Paragraph";
+interface GeneralSettingsPageProps {
+  accessToken: string | null;
+  userRole: string | null;
+  userID: string | null;
+  modelData: any;
+}
+
+
+interface routingStrategyArgs {
+  ttl?: number;
+  lowest_latency_buffer?: number;
+}
+
+interface nestedFieldItem {
+  field_name: string;
+  field_type: string; 
+  field_value: any; 
+  field_description: string; 
+  stored_in_db: boolean | null;
+}
+
+export interface passThroughItem {
+  path: string
+  target: string
+  headers: object
+}
+
+
+
+
+const PassThroughSettings: React.FC<GeneralSettingsPageProps> = ({
+  accessToken,
+  userRole,
+  userID,
+  modelData,
+}) => {
+  const [generalSettings, setGeneralSettings] = useState<passThroughItem[]>(
+    []
+  );
+  useEffect(() => {
+    if (!accessToken || !userRole || !userID) {
+      return;
+    }
+    getPassThroughEndpointsCall(accessToken).then((data) => {
+      let general_settings = data["endpoints"];
+      setGeneralSettings(general_settings);
+    });
+  }, [accessToken, userRole, userID]);
+
+
+  const handleResetField = (fieldName: string, idx: number) => {
+    if (!accessToken) {
+      return;
+    }
+
+    try {
+      deletePassThroughEndpointsCall(accessToken, fieldName);
+      // update value in state
+
+      const updatedSettings = generalSettings.filter((setting) => setting.path !== fieldName);
+      setGeneralSettings(updatedSettings);
+
+      message.success("Endpoint deleted successfully.");
+
+    } catch (error) {
+      // do something
+    }
+  };
+
+
+  if (!accessToken) {
+    return null;
+  }
+
+
+
+  return (
+    <div className="w-full mx-4">
+      <TabGroup className="gap-2 p-8 h-[75vh] w-full mt-2">
+        <Card>
+          <Table>
+            <TableHead>
+              <TableRow>
+                <TableHeaderCell>Path</TableHeaderCell>
+                <TableHeaderCell>Target</TableHeaderCell>
+                <TableHeaderCell>Headers</TableHeaderCell>
+                <TableHeaderCell>Action</TableHeaderCell>
+              </TableRow>
+            </TableHead>
+            <TableBody>
+            {generalSettings.map((value, index) => (
+                    <TableRow key={index}>
+                      <TableCell>
+                        <Text>{value.path}</Text>
+                      </TableCell>
+                      <TableCell>
+                        {
+                          value.target
+                        }
+                      </TableCell>
+                      <TableCell>
+                        {
+                          JSON.stringify(value.headers)
+                        }
+                      </TableCell>
+                      <TableCell>
+                        <Icon
+                          icon={TrashIcon}
+                          color="red"
+                          onClick={() =>
+                            handleResetField(value.path, index)
+                          }
+                        >
+                          Reset
+                        </Icon>
+                      </TableCell>
+                    </TableRow>
+                  ))}
+            </TableBody>
+          </Table>
+          <AddPassThroughEndpoint accessToken={accessToken} setPassThroughItems={setGeneralSettings} passThroughItems={generalSettings}/>
+        </Card>
+      </TabGroup>
+    </div>
+  );
+};
+
+export default PassThroughSettings;