Merge pull request #5215 from BerriAI/litellm_pass_through_endpoints_api

CRUD Endpoints for Pass-Through endpoints

litellm/proxy/_types.py

@@ -5,10 +5,10 @@ import sys
 import uuid
 from dataclasses import fields
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, TypedDict, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Extra, Field, Json, model_validator
-from typing_extensions import Annotated
+from typing_extensions import Annotated, TypedDict
 
 from litellm.types.router import UpdateRouterConfig
 from litellm.types.utils import ProviderField
@@ -1082,6 +1082,20 @@ class DynamoDBArgs(LiteLLMBase):
     assume_role_aws_session_name: Optional[str] = None
 
 
+class PassThroughGenericEndpoint(LiteLLMBase):
+    path: str = Field(description="The route to be added to the LiteLLM Proxy Server.")
+    target: str = Field(
+        description="The URL to which requests for this path should be forwarded."
+    )
+    headers: dict = Field(
+        description="Key-value pairs of headers to be forwarded with the request. You can set any key value pair here and it will be forwarded to your target endpoint"
+    )
+
+
+class PassThroughEndpointResponse(LiteLLMBase):
+    endpoints: List[PassThroughGenericEndpoint]
+
+
 class ConfigFieldUpdate(LiteLLMBase):
     field_name: str
     field_value: Any
@@ -1093,6 +1107,14 @@ class ConfigFieldDelete(LiteLLMBase):
     field_name: str
 
 
+class FieldDetail(BaseModel):
+    field_name: str
+    field_type: str
+    field_description: str
+    field_default_value: Any = None
+    stored_in_db: Optional[bool]
+
+
 class ConfigList(LiteLLMBase):
     field_name: str
     field_type: str
@@ -1101,6 +1123,9 @@ class ConfigList(LiteLLMBase):
     stored_in_db: Optional[bool]
     field_default_value: Any
     premium_field: bool = False
+    nested_fields: Optional[List[FieldDetail]] = (
+        None  # For nested dictionary or Pydantic fields
+    )
 
 
 class ConfigGeneralSettings(LiteLLMBase):
@@ -1203,6 +1228,10 @@ class ConfigGeneralSettings(LiteLLMBase):
         default=False,
         description="Public model hub for users to see what models they have access to, supported openai params, etc.",
     )
+    pass_through_endpoints: Optional[List[PassThroughGenericEndpoint]] = Field(
+        default=None,
+        description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through",
+    )
 
 
 class ConfigYAML(LiteLLMBase):
@@ -1761,3 +1790,9 @@ class VirtualKeyEvent(LiteLLMBase):
     created_by_user_role: str
     created_by_key_alias: Optional[str]
     request_kwargs: dict
+
+
+class CreatePassThroughEndpoint(LiteLLMBase):
+    path: str
+    target: str
+    headers: dict

litellm/proxy/config_management_endpoints/pass_through_endpoints.py

@@ -0,0 +1,47 @@
+"""
+What is this? 
+
+CRUD endpoints for managing pass-through endpoints
+"""
+
+import asyncio
+import traceback
+from datetime import datetime, timedelta, timezone
+from typing import List, Optional
+
+import fastapi
+import httpx
+from fastapi import (
+    APIRouter,
+    Depends,
+    File,
+    Form,
+    Header,
+    HTTPException,
+    Request,
+    Response,
+    UploadFile,
+    status,
+)
+
+import litellm
+from litellm._logging import verbose_proxy_logger
+from litellm.batches.main import FileObject
+from litellm.proxy._types import *
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+router = APIRouter()
+
+
+@router.get(
+    "/config/pass_through_endpoints/settings",
+    dependencies=[Depends(user_api_key_auth)],
+    tags=["pass-through-endpoints"],
+    summary="Create pass-through endpoints for provider specific endpoints - https://docs.litellm.ai/docs/proxy/pass_through",
+)
+async def create_fine_tuning_job(
+    request: Request,
+    fastapi_response: Response,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    pass

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -20,9 +20,18 @@ from fastapi.responses import StreamingResponse
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.proxy._types import (
+    ConfigFieldInfo,
+    ConfigFieldUpdate,
+    PassThroughEndpointResponse,
+    PassThroughGenericEndpoint,
+    ProxyException,
+    UserAPIKeyAuth,
+)
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 
+router = APIRouter()
+
 
 async def set_env_variables_in_header(custom_headers: dict):
     """
@@ -476,3 +485,188 @@ async def initialize_pass_through_endpoints(pass_through_endpoints: list):
         )
 
         verbose_proxy_logger.debug("Added new pass through endpoint: %s", _path)
+
+
+@router.get(
+    "/config/pass_through_endpoint",
+    tags=["Internal User management"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=PassThroughEndpointResponse,
+)
+async def get_pass_through_endpoints(
+    endpoint_id: Optional[str] = None,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    GET configured pass through endpoint.
+
+    If no endpoint_id given, return all configured endpoints.
+    """
+    from litellm.proxy.proxy_server import get_config_general_settings
+
+    ## Get existing pass-through endpoint field value
+    try:
+        response: ConfigFieldInfo = await get_config_general_settings(
+            field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict
+        )
+    except Exception:
+        return PassThroughEndpointResponse(endpoints=[])
+
+    pass_through_endpoint_data: Optional[List] = response.field_value
+    if pass_through_endpoint_data is None:
+        return PassThroughEndpointResponse(endpoints=[])
+
+    returned_endpoints = []
+    if endpoint_id is None:
+        for endpoint in pass_through_endpoint_data:
+            if isinstance(endpoint, dict):
+                returned_endpoints.append(PassThroughGenericEndpoint(**endpoint))
+            elif isinstance(endpoint, PassThroughGenericEndpoint):
+                returned_endpoints.append(endpoint)
+    elif endpoint_id is not None:
+        for endpoint in pass_through_endpoint_data:
+            _endpoint: Optional[PassThroughGenericEndpoint] = None
+            if isinstance(endpoint, dict):
+                _endpoint = PassThroughGenericEndpoint(**endpoint)
+            elif isinstance(endpoint, PassThroughGenericEndpoint):
+                _endpoint = endpoint
+
+            if _endpoint is not None and _endpoint.path == endpoint_id:
+                returned_endpoints.append(_endpoint)
+
+    return PassThroughEndpointResponse(endpoints=returned_endpoints)
+
+
+@router.post(
+    "/config/pass_through_endpoint/{endpoint_id}",
+    tags=["Internal User management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def update_pass_through_endpoints(request: Request, endpoint_id: str):
+    """
+    Update a pass-through endpoint
+    """
+    pass
+
+
+@router.post(
+    "/config/pass_through_endpoint",
+    tags=["Internal User management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def create_pass_through_endpoints(
+    data: PassThroughGenericEndpoint,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Create new pass-through endpoint
+    """
+    from litellm.proxy.proxy_server import (
+        get_config_general_settings,
+        update_config_general_settings,
+    )
+
+    ## Get existing pass-through endpoint field value
+
+    try:
+        response: ConfigFieldInfo = await get_config_general_settings(
+            field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict
+        )
+    except Exception:
+        response = ConfigFieldInfo(
+            field_name="pass_through_endpoints", field_value=None
+        )
+
+    ## Update field with new endpoint
+    data_dict = data.model_dump()
+    if response.field_value is None:
+        response.field_value = [data_dict]
+    elif isinstance(response.field_value, List):
+        response.field_value.append(data_dict)
+
+    ## Update db
+    updated_data = ConfigFieldUpdate(
+        field_name="pass_through_endpoints",
+        field_value=response.field_value,
+        config_type="general_settings",
+    )
+    await update_config_general_settings(
+        data=updated_data, user_api_key_dict=user_api_key_dict
+    )
+
+
+@router.delete(
+    "/config/pass_through_endpoint",
+    tags=["Internal User management"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=PassThroughEndpointResponse,
+)
+async def delete_pass_through_endpoints(
+    endpoint_id: str,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Delete a pass-through endpoint
+
+    Returns - the deleted endpoint
+    """
+    from litellm.proxy.proxy_server import (
+        get_config_general_settings,
+        update_config_general_settings,
+    )
+
+    ## Get existing pass-through endpoint field value
+
+    try:
+        response: ConfigFieldInfo = await get_config_general_settings(
+            field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict
+        )
+    except Exception:
+        response = ConfigFieldInfo(
+            field_name="pass_through_endpoints", field_value=None
+        )
+
+    ## Update field by removing endpoint
+    pass_through_endpoint_data: Optional[List] = response.field_value
+    response_obj: Optional[PassThroughGenericEndpoint] = None
+    if response.field_value is None or pass_through_endpoint_data is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": "There are no pass-through endpoints setup."},
+        )
+    elif isinstance(response.field_value, List):
+        invalid_idx: Optional[int] = None
+        for idx, endpoint in enumerate(pass_through_endpoint_data):
+            _endpoint: Optional[PassThroughGenericEndpoint] = None
+            if isinstance(endpoint, dict):
+                _endpoint = PassThroughGenericEndpoint(**endpoint)
+            elif isinstance(endpoint, PassThroughGenericEndpoint):
+                _endpoint = endpoint
+
+            if _endpoint is not None and _endpoint.path == endpoint_id:
+                invalid_idx = idx
+                response_obj = _endpoint
+
+        if invalid_idx is not None:
+            pass_through_endpoint_data.pop(invalid_idx)
+
+    ## Update db
+    updated_data = ConfigFieldUpdate(
+        field_name="pass_through_endpoints",
+        field_value=pass_through_endpoint_data,
+        config_type="general_settings",
+    )
+    await update_config_general_settings(
+        data=updated_data, user_api_key_dict=user_api_key_dict
+    )
+
+    if response_obj is None:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": "Endpoint={} was not found in pass-through endpoint list.".format(
+                    endpoint_id
+                )
+            },
+        )
+    return PassThroughEndpointResponse(endpoints=[response_obj])

litellm/proxy/proxy_server.py

@@ -13,7 +13,15 @@ import traceback
 import uuid
 import warnings
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    List,
+    Optional,
+    get_args,
+    get_origin,
+    get_type_hints,
+)
 
 import requests
 
@@ -187,7 +195,11 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_confi
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     initialize_pass_through_endpoints,
 )
+from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+    router as pass_through_router,
+)
 from litellm.proxy.route_llm_request import route_request
+
 from litellm.proxy.secret_managers.aws_secret_manager import (
     load_aws_kms,
     load_aws_secret_manager,
@@ -550,6 +562,37 @@ async def check_request_disconnection(request: Request, llm_api_call_task):
             )
 
 
+def _resolve_typed_dict_type(typ):
+    """Resolve the actual TypedDict class from a potentially wrapped type."""
+    from typing_extensions import _TypedDictMeta  # type: ignore
+
+    origin = get_origin(typ)
+    if origin is Union:  # Check if it's a Union (like Optional)
+        for arg in get_args(typ):
+            if isinstance(arg, _TypedDictMeta):
+                return arg
+    elif isinstance(typ, type) and isinstance(typ, dict):
+        return typ
+    return None
+
+
+def _resolve_pydantic_type(typ) -> List:
+    """Resolve the actual TypedDict class from a potentially wrapped type."""
+    origin = get_origin(typ)
+    typs = []
+    if origin is Union:  # Check if it's a Union (like Optional)
+        for arg in get_args(typ):
+            if (
+                arg is not None
+                and not isinstance(arg, type(None))
+                and "NoneType" not in str(arg)
+            ):
+                typs.append(arg)
+    elif isinstance(typ, type) and isinstance(typ, BaseModel):
+        return [typ]
+    return typs
+
+
 def prisma_setup(database_url: Optional[str]):
     global prisma_client, proxy_logging_obj, user_api_key_cache
 
@@ -2192,6 +2235,15 @@ class ProxyConfig:
                 alerting_args=general_settings["alerting_args"],
             )
 
+        ## PASS-THROUGH ENDPOINTS ##
+        if "pass_through_endpoints" in _general_settings:
+            general_settings["pass_through_endpoints"] = _general_settings[
+                "pass_through_endpoints"
+            ]
+            await initialize_pass_through_endpoints(
+                pass_through_endpoints=general_settings["pass_through_endpoints"]
+            )
+
     async def add_deployment(
         self,
         prisma_client: PrismaClient,
@@ -9162,6 +9214,7 @@ async def get_config_list(
         "global_max_parallel_requests": {"type": "Integer"},
         "max_request_size_mb": {"type": "Integer"},
         "max_response_size_mb": {"type": "Integer"},
+        "pass_through_endpoints": {"type": "PydanticModel"},
     }
 
     return_val = []
@@ -9169,21 +9222,79 @@ async def get_config_list(
     for field_name, field_info in ConfigGeneralSettings.model_fields.items():
         if field_name in allowed_args:
 
-            _stored_in_db = None
-            if field_name in db_general_settings_dict:
-                _stored_in_db = True
-            elif field_name in general_settings:
-                _stored_in_db = False
+            ## HANDLE TYPED DICT
 
-            _response_obj = ConfigList(
-                field_name=field_name,
-                field_type=allowed_args[field_name]["type"],
-                field_description=field_info.description or "",
-                field_value=general_settings.get(field_name, None),
-                stored_in_db=_stored_in_db,
-                field_default_value=field_info.default,
-            )
-            return_val.append(_response_obj)
+            typed_dict_type = allowed_args[field_name]["type"]
+
+            if typed_dict_type == "PydanticModel":
+                if field_name == "pass_through_endpoints":
+                    pydantic_class_list = [PassThroughGenericEndpoint]
+                else:
+                    pydantic_class_list = []
+
+                for pydantic_class in pydantic_class_list:
+                    # Get type hints from the TypedDict to create FieldDetail objects
+                    nested_fields = [
+                        FieldDetail(
+                            field_name=sub_field,
+                            field_type=sub_field_type.__name__,
+                            field_description="",  # Add custom logic if descriptions are available
+                            field_default_value=general_settings.get(sub_field, None),
+                            stored_in_db=None,
+                        )
+                        for sub_field, sub_field_type in pydantic_class.__annotations__.items()
+                    ]
+
+                    idx = 0
+                    for (
+                        sub_field,
+                        sub_field_info,
+                    ) in pydantic_class.model_fields.items():
+                        if (
+                            hasattr(sub_field_info, "description")
+                            and sub_field_info.description is not None
+                        ):
+                            nested_fields[idx].field_description = (
+                                sub_field_info.description
+                            )
+                        idx += 1
+
+                    _stored_in_db = None
+                    if field_name in db_general_settings_dict:
+                        _stored_in_db = True
+                    elif field_name in general_settings:
+                        _stored_in_db = False
+
+                    _response_obj = ConfigList(
+                        field_name=field_name,
+                        field_type=allowed_args[field_name]["type"],
+                        field_description=field_info.description or "",
+                        field_value=general_settings.get(field_name, None),
+                        stored_in_db=_stored_in_db,
+                        field_default_value=field_info.default,
+                        nested_fields=nested_fields,
+                    )
+                    return_val.append(_response_obj)
+
+            else:
+                nested_fields = None
+
+                _stored_in_db = None
+                if field_name in db_general_settings_dict:
+                    _stored_in_db = True
+                elif field_name in general_settings:
+                    _stored_in_db = False
+
+                _response_obj = ConfigList(
+                    field_name=field_name,
+                    field_type=allowed_args[field_name]["type"],
+                    field_description=field_info.description or "",
+                    field_value=general_settings.get(field_name, None),
+                    stored_in_db=_stored_in_db,
+                    field_default_value=field_info.default,
+                    nested_fields=nested_fields,
+                )
+                return_val.append(_response_obj)
 
     return return_val
 
@@ -9597,6 +9708,7 @@ def cleanup_router_config_variables():
 app.include_router(router)
 app.include_router(fine_tuning_router)
 app.include_router(vertex_router)
+app.include_router(pass_through_router)
 app.include_router(health_router)
 app.include_router(key_management_router)
 app.include_router(internal_user_router)

ui/litellm-dashboard/src/app/page.tsx

@@ -9,6 +9,7 @@ import Teams from "@/components/teams";
 import AdminPanel from "@/components/admins";
 import Settings from "@/components/settings";
 import GeneralSettings from "@/components/general_settings";
+import PassThroughSettings from "@/components/pass_through_settings";
 import BudgetPanel from "@/components/budgets/budget_panel";
 import ModelHub from "@/components/model_hub";
 import APIRef from "@/components/api_ref";
@@ -263,6 +264,13 @@ const CreateKeyPage = () => {
               accessToken={accessToken}
               premiumUser={premiumUser}
             />
+          ) : page == "pass-through-settings" ? (
+            <PassThroughSettings
+              userID={userID}
+              userRole={userRole}
+              accessToken={accessToken}
+              modelData={modelData}
+            />
           ) : (
             <Usage
               userID={userID}

ui/litellm-dashboard/src/components/add_pass_through.tsx

@@ -0,0 +1,149 @@
+/**
+ * Modal to add fallbacks to the proxy router config
+ */
+
+
+
+import React, { useState, useEffect, useRef } from "react";
+import { Button, TextInput, Grid, Col } from "@tremor/react";
+import { Select, SelectItem, MultiSelect, MultiSelectItem, Card, Metric, Text, Title, Subtitle, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react";
+import { CopyToClipboard } from 'react-copy-to-clipboard';
+import { createPassThroughEndpoint } from "./networking";
+import {
+  Button as Button2,
+  Modal,
+  Form,
+  Input,
+  InputNumber,
+  Select as Select2,
+  message,
+} from "antd";
+import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking";
+import { list } from "postcss";
+import KeyValueInput from "./key_value_input";
+import { passThroughItem } from "./pass_through_settings";
+const { Option } = Select2;
+
+interface AddFallbacksProps {
+//   models: string[] | undefined; 
+  accessToken: string;
+  passThroughItems: passThroughItem[];
+  setPassThroughItems: React.Dispatch<React.SetStateAction<passThroughItem[]>>;
+}
+
+const AddPassThroughEndpoint: React.FC<AddFallbacksProps> = ({
+    accessToken, setPassThroughItems, passThroughItems
+}) => {
+  const [form] = Form.useForm();
+  const [isModalVisible, setIsModalVisible] = useState(false);
+  const [selectedModel, setSelectedModel] = useState("");
+  const handleOk = () => {
+    setIsModalVisible(false);
+    form.resetFields();
+  };
+
+  const handleCancel = () => {
+    setIsModalVisible(false);
+    form.resetFields();
+  };
+
+  const addPassThrough = (formValues: Record<string, any>) => {
+    // Print the received value
+    console.log(formValues);
+
+    // // Extract model_name and models from formValues
+    // const { model_name, models } = formValues;
+
+    // // Create new fallback
+    // const newFallback = { [model_name]: models };
+
+    // // Get current fallbacks, or an empty array if it's null
+    // const currentFallbacks = routerSettings.fallbacks || [];
+
+    // // Add new fallback to the current fallbacks
+    // const updatedFallbacks = [...currentFallbacks, newFallback];
+
+    // // Create a new routerSettings object with updated fallbacks
+    // const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks };
+
+    const newPassThroughItem: passThroughItem = {
+        "headers": formValues["headers"],
+        "path": formValues["path"],
+        "target": formValues["target"]
+    }
+    const updatedPassThroughSettings = [...passThroughItems, newPassThroughItem]
+
+
+    try {
+        createPassThroughEndpoint(accessToken, formValues);
+        setPassThroughItems(updatedPassThroughSettings)
+    } catch (error) {
+        message.error("Failed to update router settings: " + error, 20);
+    }
+
+    message.success("Pass through endpoint successfully added");
+
+    setIsModalVisible(false)
+    form.resetFields();
+  };
+
+
+  return (
+    <div>
+      <Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
+        + Add Pass-Through Endpoint
+      </Button>
+      <Modal
+        title="Add Pass-Through Endpoint"
+        visible={isModalVisible}
+        width={800}
+        footer={null}
+        onOk={handleOk}
+        onCancel={handleCancel}
+      >
+        <Form
+          form={form}
+          onFinish={addPassThrough}
+          labelCol={{ span: 8 }}
+          wrapperCol={{ span: 16 }}
+          labelAlign="left"
+        >
+            <>
+              <Form.Item 
+                label="Path" 
+                name="path"
+                rules={[{ required: true, message: 'The route to be added to the LiteLLM Proxy Server.' }]}
+                help="required"
+              >
+                <TextInput/>
+              </Form.Item>
+
+              <Form.Item 
+                label="Target" 
+                name="target"
+                rules={[{ required: true, message: 'The URL to which requests for this path should be forwarded.' }]}
+                help="required"
+              >
+                <TextInput/>
+              </Form.Item>
+              <Form.Item 
+                label="Headers" 
+                name="headers"
+                rules={[{ required: true, message: 'Key-value pairs of headers to be forwarded with the request. You can set any key value pair here and it will be forwarded to your target endpoint' }]}
+                help="required"
+              >
+                <KeyValueInput/>
+              </Form.Item>
+            </>
+          
+          <div style={{ textAlign: "right", marginTop: "10px" }}>
+            <Button2 htmlType="submit">Add Pass-Through Endpoint</Button2>
+          </div>
+        </Form>
+      </Modal>
+
+    </div>
+  );
+};
+
+export default AddPassThroughEndpoint;

ui/litellm-dashboard/src/components/general_settings.tsx

@@ -597,7 +597,7 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
                   </TableRow>
                 </TableHead>
                 <TableBody>
-                  {generalSettings.map((value, index) => (
+                  {generalSettings.filter((value) => value.field_type !== "TypedDictionary").map((value, index) => (
                     <TableRow key={index}>
                       <TableCell>
                         <Text>{value.field_name}</Text>

ui/litellm-dashboard/src/components/key_value_input.tsx

@@ -0,0 +1,56 @@
+import React, { useState } from 'react';
+import { Form, Input, Button, Space } from 'antd';
+import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons';
+import { TextInput, Grid, Col } from "@tremor/react";
+import { TrashIcon } from "@heroicons/react/outline";
+
+interface KeyValueInputProps {
+  value?: Record<string, string>;
+  onChange?: (value: Record<string, string>) => void;
+}
+
+const KeyValueInput: React.FC<KeyValueInputProps> = ({ value = {}, onChange }) => {
+  const [pairs, setPairs] = useState<[string, string][]>(Object.entries(value));
+
+  const handleAdd = () => {
+    setPairs([...pairs, ['', '']]);
+  };
+
+  const handleRemove = (index: number) => {
+    const newPairs = pairs.filter((_, i) => i !== index);
+    setPairs(newPairs);
+    onChange?.(Object.fromEntries(newPairs));
+  };
+
+  const handleChange = (index: number, key: string, val: string) => {
+    const newPairs = [...pairs];
+    newPairs[index] = [key, val];
+    setPairs(newPairs);
+    onChange?.(Object.fromEntries(newPairs));
+  };
+
+  return (
+    <div>
+      {pairs.map(([key, val], index) => (
+        <Space key={index} style={{ display: 'flex', marginBottom: 8 }} align="start">
+          <TextInput
+            placeholder="Header Name"
+            value={key}
+            onChange={(e) => handleChange(index, e.target.value, val)}
+          />
+          <TextInput
+            placeholder="Header Value"
+            value={val}
+            onChange={(e) => handleChange(index, key, e.target.value)}
+          />
+          <MinusCircleOutlined onClick={() => handleRemove(index)} />
+        </Space>
+      ))}
+      <Button type="dashed" onClick={handleAdd} icon={<PlusOutlined />}>
+        Add Header
+      </Button>
+    </div>
+  );
+};
+
+export default KeyValueInput;

ui/litellm-dashboard/src/components/leftnav.tsx

@@ -102,15 +102,21 @@ const Sidebar: React.FC<SidebarProps> = ({
               <Text>Router Settings</Text>
             </Menu.Item>
           ) : null}
+          
           {userRole == "Admin" ? (
-            <Menu.Item key="12" onClick={() => setPage("admin-panel")}>
+            <Menu.Item key="12" onClick={() => setPage("pass-through-settings")}>
+              <Text>Pass-Through</Text>
+            </Menu.Item>
+          ) : null}
+          {userRole == "Admin" ? (
+            <Menu.Item key="13" onClick={() => setPage("admin-panel")}>
               <Text>Admin Settings</Text>
             </Menu.Item>
           ) : null}
-          <Menu.Item key="13" onClick={() => setPage("api_ref")}>
+          <Menu.Item key="14" onClick={() => setPage("api_ref")}>
             <Text>API Reference</Text>
           </Menu.Item>
-          <Menu.Item key="15" onClick={() => setPage("model-hub")}>
+          <Menu.Item key="16" onClick={() => setPage("model-hub")}>
             <Text>Model Hub</Text>
           </Menu.Item>
         </Menu>

ui/litellm-dashboard/src/components/networking.tsx

@@ -2388,6 +2388,38 @@ export const getGeneralSettingsCall = async (accessToken: String) => {
   }
 };
 
+
+export const getPassThroughEndpointsCall = async (accessToken: String) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/pass_through_endpoint`
+      : `/config/pass_through_endpoint`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to get callbacks:", error);
+    throw error;
+  }
+};
+
 export const getConfigFieldSetting = async (
   accessToken: String,
   fieldName: string
@@ -2420,6 +2452,85 @@ export const getConfigFieldSetting = async (
   }
 };
 
+export const updatePassThroughFieldSetting = async (
+  accessToken: String,
+  fieldName: string,
+  fieldValue: any
+) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/pass_through_endpoint`
+      : `/config/pass_through_endpoint`;
+
+    let formData = {
+      field_name: fieldName,
+      field_value: fieldValue,
+    };
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(formData),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    message.success("Successfully updated value!");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to set callbacks:", error);
+    throw error;
+  }
+};
+
+export const createPassThroughEndpoint = async (
+  accessToken: String,
+  formValues: Record<string, any>
+) => {
+  /**
+   * Set callbacks on proxy
+   */
+  try {
+    let url = proxyBaseUrl ? `${proxyBaseUrl}/config/pass_through_endpoint` : `/config/pass_through_endpoint`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        ...formValues, // Include formValues in the request body
+      }),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to set callbacks:", error);
+    throw error;
+  }
+};
+
 export const updateConfigFieldSetting = async (
   accessToken: String,
   fieldName: string,
@@ -2500,6 +2611,38 @@ export const deleteConfigFieldSetting = async (
     throw error;
   }
 };
+
+export const deletePassThroughEndpointsCall = async (accessToken: String, endpointId: string) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/pass_through_endpoint?endpoint_id=${endpointId}`
+      : `/config/pass_through_endpoint${endpointId}`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "DELETE",
+      headers: {
+        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      handleError(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    //message.info("Received model data");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to get callbacks:", error);
+    throw error;
+  }
+};
+
 export const setCallbacksCall = async (
   accessToken: String,
   formValues: Record<string, any>

ui/litellm-dashboard/src/components/pass_through_settings.tsx

@@ -0,0 +1,196 @@
+import React, { useState, useEffect } from "react";
+import {
+  Card,
+  Title,
+  Subtitle,
+  Table,
+  TableHead,
+  TableRow,
+  Badge,
+  TableHeaderCell,
+  TableCell,
+  TableBody,
+  Metric,
+  Text,
+  Grid,
+  Button,
+  TextInput,
+  Select as Select2,
+  SelectItem,
+  Col,
+  Accordion,
+  AccordionBody,
+  AccordionHeader,
+  AccordionList,
+} from "@tremor/react";
+import {
+  TabPanel,
+  TabPanels,
+  TabGroup,
+  TabList,
+  Tab,
+  Icon,
+} from "@tremor/react";
+import {
+  getCallbacksCall,
+  setCallbacksCall,
+  getGeneralSettingsCall,
+  deletePassThroughEndpointsCall,
+  getPassThroughEndpointsCall,
+  serviceHealthCheck,
+  updateConfigFieldSetting,
+  deleteConfigFieldSetting,
+} from "./networking";
+import {
+  Modal,
+  Form,
+  Input,
+  Select,
+  Button as Button2,
+  message,
+  InputNumber,
+} from "antd";
+import {
+  InformationCircleIcon,
+  PencilAltIcon,
+  PencilIcon,
+  StatusOnlineIcon,
+  TrashIcon,
+  RefreshIcon,
+  CheckCircleIcon,
+  XCircleIcon,
+  QuestionMarkCircleIcon,
+} from "@heroicons/react/outline";
+import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
+import AddFallbacks from "./add_fallbacks";
+import AddPassThroughEndpoint from "./add_pass_through";
+import openai from "openai";
+import Paragraph from "antd/es/skeleton/Paragraph";
+interface GeneralSettingsPageProps {
+  accessToken: string | null;
+  userRole: string | null;
+  userID: string | null;
+  modelData: any;
+}
+
+
+interface routingStrategyArgs {
+  ttl?: number;
+  lowest_latency_buffer?: number;
+}
+
+interface nestedFieldItem {
+  field_name: string;
+  field_type: string; 
+  field_value: any; 
+  field_description: string; 
+  stored_in_db: boolean | null;
+}
+
+export interface passThroughItem {
+  path: string
+  target: string
+  headers: object
+}
+
+
+
+
+const PassThroughSettings: React.FC<GeneralSettingsPageProps> = ({
+  accessToken,
+  userRole,
+  userID,
+  modelData,
+}) => {
+  const [generalSettings, setGeneralSettings] = useState<passThroughItem[]>(
+    []
+  );
+  useEffect(() => {
+    if (!accessToken || !userRole || !userID) {
+      return;
+    }
+    getPassThroughEndpointsCall(accessToken).then((data) => {
+      let general_settings = data["endpoints"];
+      setGeneralSettings(general_settings);
+    });
+  }, [accessToken, userRole, userID]);
+
+
+  const handleResetField = (fieldName: string, idx: number) => {
+    if (!accessToken) {
+      return;
+    }
+
+    try {
+      deletePassThroughEndpointsCall(accessToken, fieldName);
+      // update value in state
+
+      const updatedSettings = generalSettings.filter((setting) => setting.path !== fieldName);
+      setGeneralSettings(updatedSettings);
+
+      message.success("Endpoint deleted successfully.");
+
+    } catch (error) {
+      // do something
+    }
+  };
+
+
+  if (!accessToken) {
+    return null;
+  }
+
+
+
+  return (
+    <div className="w-full mx-4">
+      <TabGroup className="gap-2 p-8 h-[75vh] w-full mt-2">
+        <Card>
+          <Table>
+            <TableHead>
+              <TableRow>
+                <TableHeaderCell>Path</TableHeaderCell>
+                <TableHeaderCell>Target</TableHeaderCell>
+                <TableHeaderCell>Headers</TableHeaderCell>
+                <TableHeaderCell>Action</TableHeaderCell>
+              </TableRow>
+            </TableHead>
+            <TableBody>
+            {generalSettings.map((value, index) => (
+                    <TableRow key={index}>
+                      <TableCell>
+                        <Text>{value.path}</Text>
+                      </TableCell>
+                      <TableCell>
+                        {
+                          value.target
+                        }
+                      </TableCell>
+                      <TableCell>
+                        {
+                          JSON.stringify(value.headers)
+                        }
+                      </TableCell>
+                      <TableCell>
+                        <Icon
+                          icon={TrashIcon}
+                          color="red"
+                          onClick={() =>
+                            handleResetField(value.path, index)
+                          }
+                        >
+                          Reset
+                        </Icon>
+                      </TableCell>
+                    </TableRow>
+                  ))}
+            </TableBody>
+          </Table>
+          <AddPassThroughEndpoint accessToken={accessToken} setPassThroughItems={setGeneralSettings} passThroughItems={generalSettings}/>
+        </Card>
+      </TabGroup>
+    </div>
+  );
+};
+
+export default PassThroughSettings;