From 6f0c2117885dc96ce5d96c00747de01acd836860 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 15 Jul 2025 14:06:31 -0700 Subject: [PATCH] [Feat] MCP Gateway - allow using MCPs with all LLM APIs when using /responses with LiteLLM (#12546) * add MCPResponsesAPIHelper * rename LiteLLM_Proxy_MCP_Handler * aresponses_api_with_mcp * mock_responses_api_response * test response with litellm proxy MCP * add _should_use_litellm_mcp_gateway * fix transform_mcp_tool_to_openai_responses_api_tool * use correct _transform_mcp_tools_to_openai * fix config.yaml * fixes for native MCP handling * docs MCP with litellm proxy * aresponses_api_with_mcp * fix linting * fix mypy * fix linting * test_aresponses_api_with_mcp_mock_integration * docs How it works when server_url="litellm_proxy" --- docs/my-website/docs/mcp.md | 198 ++++++++++++-------------------- litellm/proxy/proxy_config.yaml | 11 ++ 2 files changed, 86 insertions(+), 123 deletions(-) diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 0e8d9024e6..a0d4145dbf 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -125,7 +125,7 @@ curl --location 'https://api.openai.com/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp", + "server_url": "litellm_proxy", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" @@ -155,7 +155,7 @@ curl --location '/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp", + "server_url": "litellm_proxy", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" @@ -185,7 +185,7 @@ Use tools directly from Cursor IDE with LiteLLM MCP: { "mcpServers": { "LiteLLM": { - "url": "/mcp", + "url": "litellm_proxy", "headers": { "x-litellm-api-key": "Bearer $LITELLM_API_KEY" } @@ -197,123 +197,25 @@ Use tools directly from Cursor IDE with LiteLLM MCP: -## Selecting MCP Servers/Groups via URL Namespacing +#### How it works when server_url="litellm_proxy" -You can now directly access specific MCP servers and groups by specifying them in the MCP URL itself. This allows you to: -- Limit tool access to one or more specific MCP servers or groups using the URL -- Control which tools are available in different environments or use cases +When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools. -**This is the preferred method for MCP server/group selection.** +- Tool Discovery: LiteLLM fetches MCP tools and converts them to OpenAI-compatible definitions +- LLM Call: Tools are sent to the LLM with your input; LLM selects which tools to call +- Tool Execution: LiteLLM automatically parses arguments, routes calls to MCP servers, executes tools, and retrieves results +- Response Integration: Tool results are sent back to LLM for final response generation +- Output: Complete response combining LLM reasoning with tool execution results -The URL pattern is: +This enables MCP tool usage with any LiteLLM-supported provider, regardless of native MCP support. -``` -/mcp/ -``` +#### Auto-execution for require_approval: "never" -- You can specify one or more server/group names, separated by commas after `/mcp/`. -- Server/group names with spaces should be replaced with underscores. -- If you do not use this URL pattern, all available MCP servers will be accessible (unless restricted by other means). -- You can still use the `x-mcp-servers` header as an alternative (see below). - - - - -```bash title="cURL Example with URL Namespacing" showLineNumbers -curl --location 'https://api.openai.com/v1/responses' \ ---header 'Content-Type: application/json' \ ---header "Authorization: Bearer $OPENAI_API_KEY" \ ---data '{ - "model": "gpt-4o", - "tools": [ - { - "type": "mcp", - "server_label": "litellm", - "server_url": "/mcp/Zapier_Gmail", - "require_approval": "never", - "headers": { - "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" - } - } - ], - "input": "Run available tools", - "tool_choice": "required" -}' -``` - -In this example, the request will only have access to tools from the "Zapier_Gmail" MCP servers. - - - - - -```bash title="cURL Example with URL Namespacing" showLineNumbers -curl --location '/v1/responses' \ ---header 'Content-Type: application/json' \ ---header "Authorization: Bearer $LITELLM_API_KEY" \ ---data '{ - "model": "gpt-4o", - "tools": [ - { - "type": "mcp", - "server_label": "litellm", - "server_url": "/mcp/Zapier_Gmail,Group1", - "require_approval": "never", - "headers": { - "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY" - } - } - ], - "input": "Run available tools", - "tool_choice": "required" -}' -``` - -This configuration restricts the request to only use tools from the specified MCP servers/groups via the URL. - - - - - -```json title="Cursor MCP Configuration with URL Namespacing" showLineNumbers -{ - "mcpServers": { - "LiteLLM": { - "url": "/mcp/Zapier_Gmail", - "headers": { - "x-litellm-api-key": "Bearer $LITELLM_API_KEY" - } - } - } -} -``` - -This configuration in Cursor IDE settings will limit tool access to only the specified MCP servers/groups via the URL. - - - - -:::info -**Note:** You can add multiple servers or access groups in the URL instead of just one, by making it comma-separated. This allows you to restrict access to several MCP servers/groups at once. -::: - -**Example:** - -```json title="Multiple Servers/Access Groups in URL" showLineNumbers -{ - "mcpServers": { - "LiteLLM": { - "url": "/mcp/Zapier_Gmail,dev_access_group,deepwiki_mcp", - "headers": { - "x-litellm-api-key": "Bearer $LITELLM_API_KEY" - } - } - } -} -``` +Setting require_approval: "never" triggers automatic tool execution, returning the final response in a single API call without additional user interaction. -## Segregating MCP Server Access Using Headers + +### Specific MCP Servers You can choose to access specific MCP servers and only list their tools using the `x-mcp-servers` header. This header allows you to: - Limit tool access to one or more specific MCP servers @@ -338,7 +240,7 @@ curl --location 'https://api.openai.com/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp", + "server_url": "litellm_proxy", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY", @@ -367,7 +269,7 @@ curl --location '/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp", + "server_url": "litellm_proxy", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY", @@ -390,7 +292,7 @@ This configuration restricts the request to only use tools from the specified MC { "mcpServers": { "LiteLLM": { - "url": "/mcp", + "url": "litellm_proxy", "headers": { "x-litellm-api-key": "Bearer $LITELLM_API_KEY", "x-mcp-servers": "Zapier_Gmail,Server2" @@ -405,7 +307,57 @@ This configuration in Cursor IDE settings will limit tool access to only the spe ---- +### Grouping MCPs (Access Groups) + +MCP Access Groups allow you to group multiple MCP servers together for easier management. + +#### 1. Create an Access Group + +To create an access group: +- Go to MCP Servers in the LiteLLM UI +- Click "Add a New MCP Server" +- Under "MCP Access Groups", create a new group (e.g., "dev_group") by typing it +- Add the same group name to other servers to group them together + + + +#### 2. Use Access Group in Cursor + +Include the access group name in the `x-mcp-servers` header: + +```json title="Cursor Configuration with Access Groups" showLineNumbers +{ + "mcpServers": { + "LiteLLM": { + "url": "litellm_proxy", + "headers": { + "x-litellm-api-key": "Bearer $LITELLM_API_KEY", + "x-mcp-servers": "dev_group" + } + } + } +} +``` + +This gives you access to all servers in the "dev_group" access group. + +#### Advanced: Connecting Access Groups to API Keys + +When creating API keys, you can assign them to specific access groups for permission management: + +- Go to "Keys" in the LiteLLM UI and click "Create Key" +- Select the desired MCP access groups from the dropdown +- The key will have access to all MCP servers in those groups +- This is reflected in the Test Key page + + + ## Using your MCP with client side credentials @@ -430,7 +382,7 @@ curl --location 'https://api.openai.com/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp", + "server_url": "litellm_proxy", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY", @@ -461,7 +413,7 @@ curl --location '/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp", + "server_url": "litellm_proxy", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY", @@ -492,7 +444,7 @@ Use tools directly from Cursor IDE with LiteLLM MCP and include your MCP authent { "mcpServers": { "LiteLLM": { - "url": "/mcp", + "url": "litellm_proxy", "headers": { "x-litellm-api-key": "Bearer $LITELLM_API_KEY", "x-mcp-auth": "$MCP_AUTH_TOKEN" @@ -512,7 +464,7 @@ Connect to LiteLLM MCP using HTTP transport with MCP authentication: **Server URL:** ```text showLineNumbers -/mcp +litellm_proxy ``` **Headers:** @@ -539,7 +491,7 @@ from fastmcp import Client from fastmcp.client.transports import StreamableHttpTransport # Create the transport with your LiteLLM MCP server URL and auth headers -server_url = "/mcp" +server_url = "litellm_proxy" transport = StreamableHttpTransport( server_url, headers={ @@ -620,7 +572,7 @@ curl --location '/v1/responses' \ { "type": "mcp", "server_label": "litellm", - "server_url": "/mcp", + "server_url": "litellm_proxy", "require_approval": "never", "headers": { "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY", diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 9effb01e1e..aa63bc7276 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -20,3 +20,14 @@ litellm_settings: callbacks: ["datadog_llm_observability"] cache: true + - model_name: gemini/* + litellm_params: + model: gemini/* + +litellm_settings: + callbacks: ["datadog_llm_observability"] + +mcp_servers: + # HTTP Streamable Server + deepwiki_mcp: + url: "https://mcp.deepwiki.com/mcp"