tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-05 12:23:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Feat] Use dedicated Rest endpoints for list, calling MCP tools (#11684)

Browse Source 
* fix: (fix) use specific rest endpoints for MCP

* ui - use rest mcp endpoints

* fix imports

* docs DISABLE_AIOHTTP_TRUST_ENV
This commit is contained in:
Ishaan Jaff
2025-06-12 19:02:20 -07:00
committed by GitHub
parent eb0d0c2f72
commit 61838bbbfc
6 changed files with 138 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/my-website/docs/proxy/config_settings.md
+1
View File
@@ -416,6 +416,7 @@ router_settings:
| DIRECT_URL | Direct URL for service endpoint
| DISABLE_ADMIN_UI | Toggle to disable the admin UI
| DISABLE_AIOHTTP_TRANSPORT | Flag to disable aiohttp transport. When this is set to True, litellm will use httpx instead of aiohttp. **Default is False**
| DISABLE_AIOHTTP_TRUST_ENV | Flag to disable aiohttp trust environment. When this is set to True, litellm will not trust the environment for aiohttp eg. `HTTP_PROXY` and `HTTPS_PROXY` environment variables will not be used when this is set to True. **Default is False**
| DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates
| DOCS_DESCRIPTION | Description text for documentation pages
| DOCS_FILTERED | Flag indicating filtered documentation
litellm/model_prices_and_context_window_backup.json
+4 -2
View File
@@ -4274,7 +4274,8 @@
"source": "https://mistral.ai/news/magistral",
"supports_function_calling": true,
"supports_assistant_prefill": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_reasoning": true
},
"mistral/magistral-small-2506": {
"max_tokens": 40000,
@@ -4287,7 +4288,8 @@
"source": "https://mistral.ai/news/magistral",
"supports_function_calling": true,
"supports_assistant_prefill": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_reasoning": true
},
"mistral/mistral-embed": {
"max_tokens": 8192,
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
+105
View File
@@ -0,0 +1,105 @@
import importlib
from typing import List, Optional
from fastapi import APIRouter, Depends, Query, Request
from litellm._logging import verbose_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
MCP_AVAILABLE: bool = True
try:
importlib.import_module("mcp")
except ImportError as e:
verbose_logger.debug(f"MCP module not found: {e}")
MCP_AVAILABLE = False
router = APIRouter(
prefix="/mcp-rest",
tags=["mcp"],
)
if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.server import (
ListMCPToolsRestAPIResponseObject,
call_mcp_tool,
)
########################################################
############ MCP Server REST API Routes #################
########################################################
@router.get("/tools/list", dependencies=[Depends(user_api_key_auth)])
async def list_tool_rest_api(
server_id: Optional[str] = Query(
None, description="The server id to list tools for"
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> List[ListMCPToolsRestAPIResponseObject]:
"""
List all available tools with information about the server they belong to.
Example response:
Tools:
[
{
"name": "create_zap",
"description": "Create a new zap",
"inputSchema": "tool_input_schema",
"mcp_info": {
"server_name": "zapier",
"logo_url": "https://www.zapier.com/logo.png",
}
},
{
"name": "fetch_data",
"description": "Fetch data from a URL",
"inputSchema": "tool_input_schema",
"mcp_info": {
"server_name": "fetch",
"logo_url": "https://www.fetch.com/logo.png",
}
}
]
"""
list_tools_result: List[ListMCPToolsRestAPIResponseObject] = []
for server in global_mcp_server_manager.get_registry().values():
if server_id and server.server_id != server_id:
continue
try:
tools = await global_mcp_server_manager._get_tools_from_server(server)
for tool in tools:
list_tools_result.append(
ListMCPToolsRestAPIResponseObject(
name=tool.name,
description=tool.description,
inputSchema=tool.inputSchema,
mcp_info=server.mcp_info,
)
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
continue
return list_tools_result
@router.post("/tools/call", dependencies=[Depends(user_api_key_auth)])
async def call_tool_rest_api(
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
REST API to call a specific MCP tool with the provided arguments
"""
from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config
data = await request.json()
data = await add_litellm_data_to_request(
data=data,
request=request,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config,
)
return await call_mcp_tool(**data)
litellm/proxy/_experimental/mcp_server/server.py
+20 -106
View File
@@ -6,24 +6,17 @@ import asyncio
import contextlib
from typing import Any, AsyncIterator, Dict, List, Optional, Union
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query, Request
from fastapi import FastAPI, HTTPException
from pydantic import ConfigDict
from starlette.types import Receive, Scope, Send
from litellm._logging import verbose_logger
from litellm.constants import MCP_TOOL_NAME_PREFIX
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.mcp_server.mcp_server_manager import MCPInfo
from litellm.types.utils import StandardLoggingMCPToolCall
from litellm.utils import client
router = APIRouter(
prefix="/mcp",
tags=["mcp"],
)
LITELLM_MCP_SERVER_NAME = "litellm-mcp-server"
LITELLM_MCP_SERVER_VERSION = "1.0.0"
LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM"
@@ -40,18 +33,6 @@ except ImportError as e:
MCP_AVAILABLE = False
# Routes
@router.get(
"/enabled",
description="Returns if the MCP server is enabled",
)
def get_mcp_server_enabled() -> Dict[str, bool]:
"""
Returns if the MCP server is enabled
"""
return {"enabled": MCP_AVAILABLE}
# Global variables to track initialization
_SESSION_MANAGERS_INITIALIZED = False
_SESSION_MANAGER_TASK = None
@@ -240,15 +221,15 @@ if MCP_AVAILABLE:
"litellm_logging_obj", None
)
if litellm_logging_obj:
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = (
standard_logging_mcp_tool_call
)
litellm_logging_obj.model_call_details["model"] = (
f"{MCP_TOOL_NAME_PREFIX}: {standard_logging_mcp_tool_call.get('name') or ''}"
)
litellm_logging_obj.model_call_details["custom_llm_provider"] = (
standard_logging_mcp_tool_call.get("mcp_server_name")
)
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model_call_details[
"model"
] = f"{MCP_TOOL_NAME_PREFIX}: {standard_logging_mcp_tool_call.get('name') or ''}"
litellm_logging_obj.model_call_details[
"custom_llm_provider"
] = standard_logging_mcp_tool_call.get("mcp_server_name")
# Try managed server tool first
if name in global_mcp_server_manager.tool_name_to_mcp_server_name_mapping:
@@ -331,81 +312,6 @@ if MCP_AVAILABLE:
verbose_logger.exception(f"Error handling MCP request: {e}")
raise e
########################################################
############ MCP Server REST API Routes #################
########################################################
@router.get("/tools/list", dependencies=[Depends(user_api_key_auth)])
async def list_tool_rest_api(
server_id: Optional[str] = Query(
None, description="The server id to list tools for"
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> List[ListMCPToolsRestAPIResponseObject]:
"""
List all available tools with information about the server they belong to.
Example response:
Tools:
[
{
"name": "create_zap",
"description": "Create a new zap",
"inputSchema": "tool_input_schema",
"mcp_info": {
"server_name": "zapier",
"logo_url": "https://www.zapier.com/logo.png",
}
},
{
"name": "fetch_data",
"description": "Fetch data from a URL",
"inputSchema": "tool_input_schema",
"mcp_info": {
"server_name": "fetch",
"logo_url": "https://www.fetch.com/logo.png",
}
}
]
"""
list_tools_result: List[ListMCPToolsRestAPIResponseObject] = []
for server in global_mcp_server_manager.get_registry().values():
if server_id and server.server_id != server_id:
continue
try:
tools = await global_mcp_server_manager._get_tools_from_server(server)
for tool in tools:
list_tools_result.append(
ListMCPToolsRestAPIResponseObject(
name=tool.name,
description=tool.description,
inputSchema=tool.inputSchema,
mcp_info=server.mcp_info,
)
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
continue
return list_tools_result
@router.post("/tools/call", dependencies=[Depends(user_api_key_auth)])
async def call_tool_rest_api(
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
REST API to call a specific MCP tool with the provided arguments
"""
from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config
data = await request.json()
data = await add_litellm_data_to_request(
data=data,
request=request,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config,
)
return await call_mcp_tool(**data)
app = FastAPI(
title=LITELLM_MCP_SERVER_NAME,
description=LITELLM_MCP_SERVER_DESCRIPTION,
@@ -413,8 +319,16 @@ if MCP_AVAILABLE:
lifespan=lifespan,
)
# Include the MCP router
app.include_router(router)
# Routes
@app.get(
"/enabled",
description="Returns if the MCP server is enabled",
)
def get_mcp_server_enabled() -> Dict[str, bool]:
"""
Returns if the MCP server is enabled
"""
return {"enabled": MCP_AVAILABLE}
# Mount the MCP handlers
app.mount("/", handle_streamable_http_mcp)
litellm/proxy/proxy_server.py
+4
View File
@@ -144,6 +144,9 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.proxy._experimental.mcp_server.rest_endpoints import (
router as mcp_rest_endpoints_router,
)
from litellm.proxy._experimental.mcp_server.server import app as mcp_app
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
@@ -8301,3 +8304,4 @@ app.include_router(ui_discovery_endpoints_router)
# MCP Server
########################################################
app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
app.include_router(mcp_rest_endpoints_router)
ui/litellm-dashboard/src/components/networking.tsx
+4 -4
View File
@@ -4609,8 +4609,8 @@ export const listMCPTools = async (accessToken: string, serverId: string) => {
try {
// Construct base URL
let url = proxyBaseUrl
? `${proxyBaseUrl}/mcp/tools/list?server_id=${serverId}`
: `/mcp/tools/list?server_id=${serverId}`;
? `${proxyBaseUrl}/mcp-rest/tools/list?server_id=${serverId}`
: `/mcp-rest/tools/list?server_id=${serverId}`;
console.log("Fetching MCP tools from:", url);
@@ -4642,8 +4642,8 @@ export const callMCPTool = async (accessToken: string, toolName: string, toolArg
try {
// Construct base URL
let url = proxyBaseUrl
? `${proxyBaseUrl}/mcp/tools/call`
: `/mcp/tools/call`;
? `${proxyBaseUrl}/mcp-rest/tools/call`
: `/mcp-rest/tools/call`;
console.log("Calling MCP tool:", toolName, "with arguments:", toolArguments);