mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-08 20:22:16 +00:00
fix: short-circuit websearch for non-Anthropic providers (github_copilot)
For providers like github_copilot that don't natively support web search, Claude Code's search sub-conversations were falling through to the adapter path which strips the web_search tool and has no stream reconversion. Instead of routing search requests through the full LLM pipeline, detect web-search-only requests early (all tools are web_search, simple prompt) and execute the search directly via Tavily/Perplexity, returning a synthetic Anthropic response. No adapter, no backend LLM call needed. Fixes #21733
This commit is contained in:
Jonathan Barazany
@@ -67,6 +67,120 @@ class WebSearchInterceptionLogger(CustomLogger):
|
||||
self.search_tool_name = search_tool_name
|
||||
self._request_has_websearch = False # Track if current request has web search
|
||||
|
||||
async def try_short_circuit_search(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[Dict],
|
||||
tools: Optional[List[Dict]],
|
||||
custom_llm_provider: Optional[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Short-circuit web-search-only requests by executing the search directly.
|
||||
|
||||
Claude Code sends web search as a separate, standalone /v1/messages
|
||||
request with a simple prompt and only web_search tool(s). For providers
|
||||
that don't natively support web search (e.g. github_copilot), there is
|
||||
no need to route this through the backend LLM — we can detect the
|
||||
pattern, execute the search via Tavily/Perplexity, and return a
|
||||
synthetic Anthropic response immediately.
|
||||
|
||||
Args:
|
||||
model: Model name from the request
|
||||
messages: Messages list from the request
|
||||
tools: Tools list from the request
|
||||
custom_llm_provider: Provider name
|
||||
|
||||
Returns:
|
||||
An AnthropicMessagesResponse dict if short-circuited, or None to
|
||||
continue normal processing.
|
||||
"""
|
||||
if not tools:
|
||||
return None
|
||||
|
||||
# Check if provider is in enabled list
|
||||
provider_str = custom_llm_provider or ""
|
||||
if (
|
||||
self.enabled_providers is not None
|
||||
and provider_str not in self.enabled_providers
|
||||
):
|
||||
return None
|
||||
|
||||
# All tools must be web search tools
|
||||
if not all(is_web_search_tool(t) for t in tools):
|
||||
return None
|
||||
|
||||
# Extract search query from the last user message
|
||||
query = self._extract_search_query(messages)
|
||||
if not query:
|
||||
return None
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search detected "
|
||||
f"(provider={provider_str}, query='{query}')"
|
||||
)
|
||||
|
||||
# Execute search
|
||||
try:
|
||||
search_result_text = await self._execute_search(query)
|
||||
except Exception as e:
|
||||
verbose_logger.error(
|
||||
f"WebSearchInterception: Short-circuit search failed: {e}"
|
||||
)
|
||||
search_result_text = f"Search failed: {e}"
|
||||
|
||||
# Build synthetic Anthropic response
|
||||
from uuid import uuid4
|
||||
|
||||
response: Dict[str, Any] = {
|
||||
"id": f"msg_{uuid4().hex[:24]}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": search_result_text}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 0, "output_tokens": 0},
|
||||
}
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search completed, "
|
||||
f"returning synthetic response ({len(search_result_text)} chars)"
|
||||
)
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _extract_search_query(messages: List[Dict]) -> Optional[str]:
|
||||
"""
|
||||
Extract the search query from messages.
|
||||
|
||||
Looks at the last user message content for the search query text.
|
||||
"""
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
# Find the last user message
|
||||
for msg in reversed(messages):
|
||||
if msg.get("role") != "user":
|
||||
continue
|
||||
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return content.strip() or None
|
||||
|
||||
# Handle list-of-blocks content
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
text = block.get("text", "").strip()
|
||||
if text:
|
||||
return text
|
||||
elif isinstance(block, str):
|
||||
text = block.strip()
|
||||
if text:
|
||||
return text
|
||||
|
||||
return None
|
||||
|
||||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: Dict[str, Any], call_type: Optional[Any]
|
||||
) -> Optional[dict]:
|
||||
|
||||
@@ -114,6 +114,54 @@ async def _execute_pre_request_hooks(
|
||||
return request_kwargs
|
||||
|
||||
|
||||
async def _try_websearch_short_circuit(
|
||||
model: str,
|
||||
messages: List[Dict],
|
||||
tools: Optional[List[Dict]],
|
||||
custom_llm_provider: Optional[str],
|
||||
stream: Optional[bool],
|
||||
) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]:
|
||||
"""
|
||||
Attempt to short-circuit a web-search-only request.
|
||||
|
||||
Claude Code sends web search as a separate, standalone /v1/messages
|
||||
request. For providers that don't natively support web search (e.g.
|
||||
github_copilot), we detect this pattern, execute the search via
|
||||
Tavily/Perplexity, and return a synthetic Anthropic response — bypassing
|
||||
the backend LLM entirely.
|
||||
|
||||
Returns the synthetic response if short-circuited, or None to continue
|
||||
normal processing.
|
||||
"""
|
||||
if not litellm.callbacks:
|
||||
return None
|
||||
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
for callback in litellm.callbacks:
|
||||
if not isinstance(callback, WebSearchInterceptionLogger):
|
||||
continue
|
||||
|
||||
response = await callback.try_short_circuit_search(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
if response is not None:
|
||||
if stream:
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
||||
return FakeAnthropicMessagesStreamIterator(response)
|
||||
return response
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@client
|
||||
async def anthropic_messages(
|
||||
max_tokens: int,
|
||||
@@ -156,6 +204,19 @@ async def anthropic_messages(
|
||||
# Merge back any other modifications
|
||||
kwargs.update(request_kwargs)
|
||||
|
||||
# Short-circuit web-search-only requests: detect the pattern, execute
|
||||
# search directly via Tavily/Perplexity, and return a synthetic response
|
||||
# without ever touching the backend LLM or the adapter path.
|
||||
short_circuit_response = await _try_websearch_short_circuit(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream=stream,
|
||||
)
|
||||
if short_circuit_response is not None:
|
||||
return short_circuit_response
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["is_async"] = True
|
||||
|
||||
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
"""
|
||||
Unit tests for WebSearch Short-Circuit
|
||||
|
||||
Tests the short-circuit path that detects web-search-only /v1/messages requests
|
||||
and executes the search directly without routing through the backend LLM.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Detection tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTryShortCircuitSearch:
|
||||
"""Tests for WebSearchInterceptionLogger.try_short_circuit_search"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_short_circuits_single_web_search_tool(self):
|
||||
"""Single web_search_20250305 tool → short-circuit fires"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
|
||||
with patch.object(
|
||||
logger, "_execute_search", new_callable=AsyncMock
|
||||
) as mock_search:
|
||||
mock_search.return_value = "Title: Result\nURL: https://example.com\nSnippet: test"
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "Search for Claude Code releases"}],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}],
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["type"] == "message"
|
||||
assert result["role"] == "assistant"
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
assert len(result["content"]) == 1
|
||||
assert result["content"][0]["type"] == "text"
|
||||
assert "Result" in result["content"][0]["text"]
|
||||
mock_search.assert_called_once_with("Search for Claude Code releases")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_mixed_tools(self):
|
||||
"""Mix of web_search and other tools → NOT short-circuited"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "Do something"}],
|
||||
tools=[
|
||||
{"type": "web_search_20250305", "name": "web_search", "max_uses": 8},
|
||||
{"name": "Read", "description": "Read a file", "input_schema": {}},
|
||||
],
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_no_tools(self):
|
||||
"""No tools → NOT short-circuited"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
tools=None,
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_empty_tools(self):
|
||||
"""Empty tools list → NOT short-circuited"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
tools=[],
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_wrong_provider(self):
|
||||
"""Provider not in enabled_providers → NOT short-circuited"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"])
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "Search for something"}],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}],
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_no_messages(self):
|
||||
"""Empty messages → NOT short-circuited"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}],
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_failure_returns_error_text(self):
|
||||
"""Search failure → response with error message, not exception"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
|
||||
with patch.object(
|
||||
logger, "_execute_search", new_callable=AsyncMock
|
||||
) as mock_search:
|
||||
mock_search.side_effect = RuntimeError("Tavily API error")
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "Search for something"}],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}],
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert "Search failed" in result["content"][0]["text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_has_valid_structure(self):
|
||||
"""Synthetic response has all required AnthropicMessagesResponse fields"""
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
|
||||
with patch.object(
|
||||
logger, "_execute_search", new_callable=AsyncMock
|
||||
) as mock_search:
|
||||
mock_search.return_value = "search results here"
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "Search query"}],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search"}],
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
# Required fields
|
||||
assert "id" in result
|
||||
assert result["id"].startswith("msg_")
|
||||
assert result["type"] == "message"
|
||||
assert result["role"] == "assistant"
|
||||
assert result["model"] == "github_copilot/claude-sonnet-4"
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
assert result["stop_sequence"] is None
|
||||
assert "usage" in result
|
||||
assert "content" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query extraction tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractSearchQuery:
|
||||
"""Tests for WebSearchInterceptionLogger._extract_search_query"""
|
||||
|
||||
def test_string_content(self):
|
||||
messages = [{"role": "user", "content": "Search for Python 3.14 features"}]
|
||||
assert (
|
||||
WebSearchInterceptionLogger._extract_search_query(messages)
|
||||
== "Search for Python 3.14 features"
|
||||
)
|
||||
|
||||
def test_list_content_with_text_block(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Perform a web search for latest news"},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert (
|
||||
WebSearchInterceptionLogger._extract_search_query(messages)
|
||||
== "Perform a web search for latest news"
|
||||
)
|
||||
|
||||
def test_takes_last_user_message(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "First message"},
|
||||
{"role": "assistant", "content": "Response"},
|
||||
{"role": "user", "content": "Second message"},
|
||||
]
|
||||
assert (
|
||||
WebSearchInterceptionLogger._extract_search_query(messages) == "Second message"
|
||||
)
|
||||
|
||||
def test_empty_messages(self):
|
||||
assert WebSearchInterceptionLogger._extract_search_query([]) is None
|
||||
|
||||
def test_no_user_messages(self):
|
||||
messages = [{"role": "assistant", "content": "Hello"}]
|
||||
assert WebSearchInterceptionLogger._extract_search_query(messages) is None
|
||||
|
||||
def test_empty_content(self):
|
||||
messages = [{"role": "user", "content": ""}]
|
||||
assert WebSearchInterceptionLogger._extract_search_query(messages) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration with entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestShortCircuitEntryPoint:
|
||||
"""Tests for _try_websearch_short_circuit in the /v1/messages handler"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_none_when_no_callbacks(self):
|
||||
"""No callbacks configured → returns None"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
_try_websearch_short_circuit,
|
||||
)
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
result = await _try_websearch_short_circuit(
|
||||
model="test",
|
||||
messages=[],
|
||||
tools=[],
|
||||
custom_llm_provider="github_copilot",
|
||||
stream=False,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_dict_when_not_streaming(self):
|
||||
"""Non-streaming short-circuit → returns dict"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
_try_websearch_short_circuit,
|
||||
)
|
||||
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
with patch.object(
|
||||
logger, "_execute_search", new_callable=AsyncMock
|
||||
) as mock_search:
|
||||
mock_search.return_value = "results"
|
||||
with patch("litellm.callbacks", [logger]):
|
||||
result = await _try_websearch_short_circuit(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "search query"}],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search"}],
|
||||
custom_llm_provider="github_copilot",
|
||||
stream=False,
|
||||
)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert result["content"][0]["text"] == "results"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_stream_iterator_when_streaming(self):
|
||||
"""Streaming short-circuit → returns FakeAnthropicMessagesStreamIterator"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
_try_websearch_short_circuit,
|
||||
)
|
||||
|
||||
logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"])
|
||||
with patch.object(
|
||||
logger, "_execute_search", new_callable=AsyncMock
|
||||
) as mock_search:
|
||||
mock_search.return_value = "streaming results"
|
||||
with patch("litellm.callbacks", [logger]):
|
||||
result = await _try_websearch_short_circuit(
|
||||
model="github_copilot/claude-sonnet-4",
|
||||
messages=[{"role": "user", "content": "search query"}],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search"}],
|
||||
custom_llm_provider="github_copilot",
|
||||
stream=True,
|
||||
)
|
||||
|
||||
assert isinstance(result, FakeAnthropicMessagesStreamIterator)
|
||||
|
||||
# Verify stream produces valid SSE events
|
||||
chunks = []
|
||||
async for chunk in result:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) > 0
|
||||
# First chunk should be message_start
|
||||
assert b"event: message_start" in chunks[0]
|
||||
# Last chunk should be message_stop
|
||||
assert b"event: message_stop" in chunks[-1]
|
||||
# Should contain the search results text
|
||||
all_data = b"".join(chunks)
|
||||
assert b"streaming results" in all_data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_non_websearch_callbacks(self):
|
||||
"""Non-WebSearchInterceptionLogger callbacks are ignored"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
_try_websearch_short_circuit,
|
||||
)
|
||||
|
||||
other_callback = MagicMock()
|
||||
with patch("litellm.callbacks", [other_callback]):
|
||||
result = await _try_websearch_short_circuit(
|
||||
model="test",
|
||||
messages=[{"role": "user", "content": "search"}],
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search"}],
|
||||
custom_llm_provider="github_copilot",
|
||||
stream=False,
|
||||
)
|
||||
assert result is None
|
||||
Reference in New Issue
Block a user