diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index e8db12e51f..cebb5fa712 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -416,6 +416,7 @@ router_settings: | DIRECT_URL | Direct URL for service endpoint | DISABLE_ADMIN_UI | Toggle to disable the admin UI | DISABLE_AIOHTTP_TRANSPORT | Flag to disable aiohttp transport. When this is set to True, litellm will use httpx instead of aiohttp. **Default is False** +| DISABLE_AIOHTTP_TRUST_ENV | Flag to disable aiohttp trust environment. When this is set to True, litellm will not trust the environment for aiohttp eg. `HTTP_PROXY` and `HTTPS_PROXY` environment variables will not be used when this is set to True. **Default is False** | DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates | DOCS_DESCRIPTION | Description text for documentation pages | DOCS_FILTERED | Flag indicating filtered documentation diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 294e7b4e75..18b0326248 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4274,7 +4274,8 @@ "source": "https://mistral.ai/news/magistral", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_reasoning": true }, "mistral/magistral-small-2506": { "max_tokens": 40000, @@ -4287,7 +4288,8 @@ "source": "https://mistral.ai/news/magistral", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_reasoning": true }, "mistral/mistral-embed": { "max_tokens": 8192, diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py new file mode 100644 index 0000000000..2b983bd7e8 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -0,0 +1,105 @@ +import importlib +from typing import List, Optional + +from fastapi import APIRouter, Depends, Query, Request + +from litellm._logging import verbose_logger +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +MCP_AVAILABLE: bool = True +try: + importlib.import_module("mcp") +except ImportError as e: + verbose_logger.debug(f"MCP module not found: {e}") + MCP_AVAILABLE = False + + +router = APIRouter( + prefix="/mcp-rest", + tags=["mcp"], +) + +if MCP_AVAILABLE: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._experimental.mcp_server.server import ( + ListMCPToolsRestAPIResponseObject, + call_mcp_tool, + ) + + ######################################################## + ############ MCP Server REST API Routes ################# + ######################################################## + @router.get("/tools/list", dependencies=[Depends(user_api_key_auth)]) + async def list_tool_rest_api( + server_id: Optional[str] = Query( + None, description="The server id to list tools for" + ), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ) -> List[ListMCPToolsRestAPIResponseObject]: + """ + List all available tools with information about the server they belong to. + + Example response: + Tools: + [ + { + "name": "create_zap", + "description": "Create a new zap", + "inputSchema": "tool_input_schema", + "mcp_info": { + "server_name": "zapier", + "logo_url": "https://www.zapier.com/logo.png", + } + }, + { + "name": "fetch_data", + "description": "Fetch data from a URL", + "inputSchema": "tool_input_schema", + "mcp_info": { + "server_name": "fetch", + "logo_url": "https://www.fetch.com/logo.png", + } + } + ] + """ + list_tools_result: List[ListMCPToolsRestAPIResponseObject] = [] + for server in global_mcp_server_manager.get_registry().values(): + if server_id and server.server_id != server_id: + continue + try: + tools = await global_mcp_server_manager._get_tools_from_server(server) + for tool in tools: + list_tools_result.append( + ListMCPToolsRestAPIResponseObject( + name=tool.name, + description=tool.description, + inputSchema=tool.inputSchema, + mcp_info=server.mcp_info, + ) + ) + except Exception as e: + verbose_logger.exception(f"Error getting tools from {server.name}: {e}") + continue + return list_tools_result + + @router.post("/tools/call", dependencies=[Depends(user_api_key_auth)]) + async def call_tool_rest_api( + request: Request, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + REST API to call a specific MCP tool with the provided arguments + """ + from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config + + data = await request.json() + data = await add_litellm_data_to_request( + data=data, + request=request, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + ) + return await call_mcp_tool(**data) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index d61ae75716..caceec0658 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -6,24 +6,17 @@ import asyncio import contextlib from typing import Any, AsyncIterator, Dict, List, Optional, Union -from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query, Request +from fastapi import FastAPI, HTTPException from pydantic import ConfigDict from starlette.types import Receive, Scope, Send from litellm._logging import verbose_logger from litellm.constants import MCP_TOOL_NAME_PREFIX from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.mcp_server.mcp_server_manager import MCPInfo from litellm.types.utils import StandardLoggingMCPToolCall from litellm.utils import client -router = APIRouter( - prefix="/mcp", - tags=["mcp"], -) - LITELLM_MCP_SERVER_NAME = "litellm-mcp-server" LITELLM_MCP_SERVER_VERSION = "1.0.0" LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM" @@ -40,18 +33,6 @@ except ImportError as e: MCP_AVAILABLE = False -# Routes -@router.get( - "/enabled", - description="Returns if the MCP server is enabled", -) -def get_mcp_server_enabled() -> Dict[str, bool]: - """ - Returns if the MCP server is enabled - """ - return {"enabled": MCP_AVAILABLE} - - # Global variables to track initialization _SESSION_MANAGERS_INITIALIZED = False _SESSION_MANAGER_TASK = None @@ -240,15 +221,15 @@ if MCP_AVAILABLE: "litellm_logging_obj", None ) if litellm_logging_obj: - litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = ( - standard_logging_mcp_tool_call - ) - litellm_logging_obj.model_call_details["model"] = ( - f"{MCP_TOOL_NAME_PREFIX}: {standard_logging_mcp_tool_call.get('name') or ''}" - ) - litellm_logging_obj.model_call_details["custom_llm_provider"] = ( - standard_logging_mcp_tool_call.get("mcp_server_name") - ) + litellm_logging_obj.model_call_details[ + "mcp_tool_call_metadata" + ] = standard_logging_mcp_tool_call + litellm_logging_obj.model_call_details[ + "model" + ] = f"{MCP_TOOL_NAME_PREFIX}: {standard_logging_mcp_tool_call.get('name') or ''}" + litellm_logging_obj.model_call_details[ + "custom_llm_provider" + ] = standard_logging_mcp_tool_call.get("mcp_server_name") # Try managed server tool first if name in global_mcp_server_manager.tool_name_to_mcp_server_name_mapping: @@ -331,81 +312,6 @@ if MCP_AVAILABLE: verbose_logger.exception(f"Error handling MCP request: {e}") raise e - ######################################################## - ############ MCP Server REST API Routes ################# - ######################################################## - @router.get("/tools/list", dependencies=[Depends(user_api_key_auth)]) - async def list_tool_rest_api( - server_id: Optional[str] = Query( - None, description="The server id to list tools for" - ), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - ) -> List[ListMCPToolsRestAPIResponseObject]: - """ - List all available tools with information about the server they belong to. - - Example response: - Tools: - [ - { - "name": "create_zap", - "description": "Create a new zap", - "inputSchema": "tool_input_schema", - "mcp_info": { - "server_name": "zapier", - "logo_url": "https://www.zapier.com/logo.png", - } - }, - { - "name": "fetch_data", - "description": "Fetch data from a URL", - "inputSchema": "tool_input_schema", - "mcp_info": { - "server_name": "fetch", - "logo_url": "https://www.fetch.com/logo.png", - } - } - ] - """ - list_tools_result: List[ListMCPToolsRestAPIResponseObject] = [] - for server in global_mcp_server_manager.get_registry().values(): - if server_id and server.server_id != server_id: - continue - try: - tools = await global_mcp_server_manager._get_tools_from_server(server) - for tool in tools: - list_tools_result.append( - ListMCPToolsRestAPIResponseObject( - name=tool.name, - description=tool.description, - inputSchema=tool.inputSchema, - mcp_info=server.mcp_info, - ) - ) - except Exception as e: - verbose_logger.exception(f"Error getting tools from {server.name}: {e}") - continue - return list_tools_result - - @router.post("/tools/call", dependencies=[Depends(user_api_key_auth)]) - async def call_tool_rest_api( - request: Request, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - ): - """ - REST API to call a specific MCP tool with the provided arguments - """ - from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config - - data = await request.json() - data = await add_litellm_data_to_request( - data=data, - request=request, - user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config, - ) - return await call_mcp_tool(**data) - app = FastAPI( title=LITELLM_MCP_SERVER_NAME, description=LITELLM_MCP_SERVER_DESCRIPTION, @@ -413,8 +319,16 @@ if MCP_AVAILABLE: lifespan=lifespan, ) - # Include the MCP router - app.include_router(router) + # Routes + @app.get( + "/enabled", + description="Returns if the MCP server is enabled", + ) + def get_mcp_server_enabled() -> Dict[str, bool]: + """ + Returns if the MCP server is enabled + """ + return {"enabled": MCP_AVAILABLE} # Mount the MCP handlers app.mount("/", handle_streamable_http_mcp) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 87cfda32a3..2fe494e406 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -144,6 +144,9 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + router as mcp_rest_endpoints_router, +) from litellm.proxy._experimental.mcp_server.server import app as mcp_app from litellm.proxy._experimental.mcp_server.tool_registry import ( global_mcp_tool_registry, @@ -8301,3 +8304,4 @@ app.include_router(ui_discovery_endpoints_router) # MCP Server ######################################################## app.mount(path=BASE_MCP_ROUTE, app=mcp_app) +app.include_router(mcp_rest_endpoints_router) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 7e8edc9f1c..3988627b12 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4609,8 +4609,8 @@ export const listMCPTools = async (accessToken: string, serverId: string) => { try { // Construct base URL let url = proxyBaseUrl - ? `${proxyBaseUrl}/mcp/tools/list?server_id=${serverId}` - : `/mcp/tools/list?server_id=${serverId}`; + ? `${proxyBaseUrl}/mcp-rest/tools/list?server_id=${serverId}` + : `/mcp-rest/tools/list?server_id=${serverId}`; console.log("Fetching MCP tools from:", url); @@ -4642,8 +4642,8 @@ export const callMCPTool = async (accessToken: string, toolName: string, toolArg try { // Construct base URL let url = proxyBaseUrl - ? `${proxyBaseUrl}/mcp/tools/call` - : `/mcp/tools/call`; + ? `${proxyBaseUrl}/mcp-rest/tools/call` + : `/mcp-rest/tools/call`; console.log("Calling MCP tool:", toolName, "with arguments:", toolArguments);