feat(tools): add Exa + Tavily web search providers with ranked ordering

Port Exa and Tavily provider clients from PR #825 into the tenant-aware
overlay architecture (builtin_tool_tenant_configs.settings).

- Add ExaConfig + TavilyConfig to WebToolsConfig with provider_order
- Add Exa HTTP client (api.exa.ai/search, x-api-key auth)
- Add Tavily HTTP client (api.tavily.com/search, Bearer auth)
- Extend Brave + DDG constructors with per-provider maxResults
- Add config_secrets plumbing for exa/tavily API keys (apply/collect/mask/strip)
- Refactor gateway_setup.go to use WebSearchConfigFromConfig
- Add NormalizeWebSearchProviderOrder (DDG always last, dedup, unknown skip)
- Extract web_search_config.go (builder, normalizer, shared helpers)
- 11 new unit tests for provider order, builder, clamp, normalize

Credit: @kaitranntt (PR #825) for the original Exa + Tavily implementation.
This commit is contained in:
viettranx
2026-04-12 11:09:32 +07:00
parent 7ae9721c2b
commit 30a40bbe59
10 changed files with 527 additions and 44 deletions
+1 -5
View File
@@ -116,11 +116,7 @@ func setupToolRegistry(
}
// Web tools (web_search + web_fetch)
webSearchTool := tools.NewWebSearchTool(tools.WebSearchConfig{
BraveEnabled: cfg.Tools.Web.Brave.Enabled,
BraveAPIKey: cfg.Tools.Web.Brave.APIKey,
DDGEnabled: cfg.Tools.Web.DuckDuckGo.Enabled,
})
webSearchTool := tools.NewWebSearchTool(tools.WebSearchConfigFromConfig(cfg))
if webSearchTool != nil {
toolsReg.Register(webSearchTool)
slog.Info("web_search tool enabled")
+17 -2
View File
@@ -432,8 +432,23 @@ type ToolPolicySpec struct {
}
type WebToolsConfig struct {
Brave BraveConfig `json:"brave"`
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
ProviderOrder []string `json:"provider_order,omitempty"`
Exa ExaConfig `json:"exa"`
Tavily TavilyConfig `json:"tavily"`
Brave BraveConfig `json:"brave"`
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
}
type ExaConfig struct {
Enabled bool `json:"enabled"`
APIKey string `json:"api_key"`
MaxResults int `json:"max_results"`
}
type TavilyConfig struct {
Enabled bool `json:"enabled"`
APIKey string `json:"api_key"`
MaxResults int `json:"max_results"`
}
type BraveConfig struct {
+10
View File
@@ -59,6 +59,8 @@ func (c *Config) MaskedCopy() *Config {
maskNonEmpty(&cp.Tts.MiniMax.APIKey)
// Mask web tool keys
maskNonEmpty(&cp.Tools.Web.Exa.APIKey)
maskNonEmpty(&cp.Tools.Web.Tavily.APIKey)
maskNonEmpty(&cp.Tools.Web.Brave.APIKey)
// Mask Tailscale auth key
@@ -109,6 +111,8 @@ func (c *Config) StripSecrets() {
c.Tts.MiniMax.APIKey = ""
// Web tool keys
c.Tools.Web.Exa.APIKey = ""
c.Tools.Web.Tavily.APIKey = ""
c.Tools.Web.Brave.APIKey = ""
// Tailscale auth key
@@ -164,6 +168,8 @@ func (c *Config) StripMaskedSecrets() {
stripIfMasked(&c.Tts.MiniMax.APIKey)
// Web tool keys
stripIfMasked(&c.Tools.Web.Exa.APIKey)
stripIfMasked(&c.Tools.Web.Tavily.APIKey)
stripIfMasked(&c.Tools.Web.Brave.APIKey)
// Tailscale auth key
@@ -185,6 +191,8 @@ func (c *Config) ApplyDBSecrets(secrets map[string]string) {
apply("tts.elevenlabs.api_key", &c.Tts.ElevenLabs.APIKey)
apply("tts.minimax.api_key", &c.Tts.MiniMax.APIKey)
apply("tts.minimax.group_id", &c.Tts.MiniMax.GroupID)
apply("tools.web.exa.api_key", &c.Tools.Web.Exa.APIKey)
apply("tools.web.tavily.api_key", &c.Tools.Web.Tavily.APIKey)
apply("tools.web.brave.api_key", &c.Tools.Web.Brave.APIKey)
apply("tailscale.auth_key", &c.Tailscale.AuthKey)
}
@@ -205,6 +213,8 @@ func (c *Config) ExtractDBSecrets() map[string]string {
collect("tts.elevenlabs.api_key", c.Tts.ElevenLabs.APIKey)
collect("tts.minimax.api_key", c.Tts.MiniMax.APIKey)
collect("tts.minimax.group_id", c.Tts.MiniMax.GroupID)
collect("tools.web.exa.api_key", c.Tools.Web.Exa.APIKey)
collect("tools.web.tavily.api_key", c.Tools.Web.Tavily.APIKey)
collect("tools.web.brave.api_key", c.Tools.Web.Brave.APIKey)
collect("tailscale.auth_key", c.Tailscale.AuthKey)
+17 -25
View File
@@ -15,9 +15,25 @@ const (
maxSearchCount = 10
searchTimeoutSeconds = 30
braveSearchEndpoint = "https://api.search.brave.com/res/v1/web/search"
exaSearchEndpoint = "https://api.exa.ai/search"
tavilySearchEndpoint = "https://api.tavily.com/search"
webSearchUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
const (
searchProviderExa = "exa"
searchProviderTavily = "tavily"
searchProviderBrave = "brave"
searchProviderDuckDuckGo = "duckduckgo"
)
var defaultSearchProviderOrder = []string{
searchProviderExa,
searchProviderTavily,
searchProviderBrave,
searchProviderDuckDuckGo,
}
// SearchProvider abstracts a web search backend.
type SearchProvider interface {
Search(ctx context.Context, params searchParams) ([]searchResult, error)
@@ -72,26 +88,8 @@ type WebSearchTool struct {
cache *webCache
}
// WebSearchConfig holds configuration for the web search tool.
type WebSearchConfig struct {
BraveAPIKey string
BraveEnabled bool
BraveMaxResults int
DDGEnabled bool
DDGMaxResults int
CacheTTL time.Duration
}
func NewWebSearchTool(cfg WebSearchConfig) *WebSearchTool {
var providers []SearchProvider
// Priority: Brave > DuckDuckGo (matching TS)
if cfg.BraveEnabled && cfg.BraveAPIKey != "" {
providers = append(providers, newBraveSearchProvider(cfg.BraveAPIKey))
}
if cfg.DDGEnabled {
providers = append(providers, newDuckDuckGoSearchProvider())
}
providers := buildSearchProviders(cfg)
if len(providers) == 0 {
return nil
@@ -246,9 +244,3 @@ func formatSearchResults(query string, results []searchResult, provider string)
return sb.String()
}
func truncateStr(s string, max int) string {
if len(s) <= max {
return s
}
return s[:max] + "..."
}
+11 -7
View File
@@ -13,23 +13,27 @@ import (
// --- Brave Search Provider ---
type braveSearchProvider struct {
apiKey string
client *http.Client
apiKey string
maxResults int
client *http.Client
}
func newBraveSearchProvider(apiKey string) *braveSearchProvider {
func newBraveSearchProvider(apiKey string, maxResults int) *braveSearchProvider {
return &braveSearchProvider{
apiKey: apiKey,
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
apiKey: apiKey,
maxResults: normalizeProviderMaxResults(maxResults),
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
}
}
func (p *braveSearchProvider) Name() string { return "brave" }
func (p *braveSearchProvider) Name() string { return searchProviderBrave }
func (p *braveSearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
count := clampProviderResultCount(params.Count, p.maxResults)
q := url.Values{}
q.Set("q", params.Query)
q.Set("count", fmt.Sprintf("%d", params.Count))
q.Set("count", fmt.Sprintf("%d", count))
if params.Country != "" {
q.Set("country", params.Country)
+145
View File
@@ -0,0 +1,145 @@
package tools
import (
"strings"
"time"
"github.com/nextlevelbuilder/goclaw/internal/config"
)
// WebSearchConfig holds configuration for the web search tool.
type WebSearchConfig struct {
ProviderOrder []string
ExaAPIKey string
ExaEnabled bool
ExaMaxResults int
TavilyAPIKey string
TavilyEnabled bool
TavilyMaxResults int
BraveAPIKey string
BraveEnabled bool
BraveMaxResults int
DDGEnabled bool
DDGMaxResults int
CacheTTL time.Duration
}
// WebSearchConfigFromConfig creates a WebSearchConfig from the global config.
func WebSearchConfigFromConfig(cfg *config.Config) WebSearchConfig {
return WebSearchConfig{
ProviderOrder: cfg.Tools.Web.ProviderOrder,
ExaEnabled: cfg.Tools.Web.Exa.Enabled,
ExaAPIKey: cfg.Tools.Web.Exa.APIKey,
ExaMaxResults: cfg.Tools.Web.Exa.MaxResults,
TavilyEnabled: cfg.Tools.Web.Tavily.Enabled,
TavilyAPIKey: cfg.Tools.Web.Tavily.APIKey,
TavilyMaxResults: cfg.Tools.Web.Tavily.MaxResults,
BraveEnabled: cfg.Tools.Web.Brave.Enabled,
BraveAPIKey: cfg.Tools.Web.Brave.APIKey,
BraveMaxResults: cfg.Tools.Web.Brave.MaxResults,
DDGEnabled: true,
DDGMaxResults: cfg.Tools.Web.DuckDuckGo.MaxResults,
}
}
func buildSearchProviders(cfg WebSearchConfig) []SearchProvider {
var providers []SearchProvider
for _, providerID := range NormalizeWebSearchProviderOrder(cfg.ProviderOrder) {
switch providerID {
case searchProviderExa:
if cfg.ExaEnabled && cfg.ExaAPIKey != "" {
providers = append(providers, newExaSearchProvider(cfg.ExaAPIKey, cfg.ExaMaxResults))
}
case searchProviderTavily:
if cfg.TavilyEnabled && cfg.TavilyAPIKey != "" {
providers = append(providers, newTavilySearchProvider(cfg.TavilyAPIKey, cfg.TavilyMaxResults))
}
case searchProviderBrave:
if cfg.BraveEnabled && cfg.BraveAPIKey != "" {
providers = append(providers, newBraveSearchProvider(cfg.BraveAPIKey, cfg.BraveMaxResults))
}
case searchProviderDuckDuckGo:
if cfg.DDGEnabled {
providers = append(providers, newDuckDuckGoSearchProvider(cfg.DDGMaxResults))
}
}
}
return providers
}
// NormalizeWebSearchProviderOrder normalizes user-specified provider order.
// Explicit providers appear first in their specified order, remaining known
// providers are appended (DuckDuckGo always last as free fallback).
func NormalizeWebSearchProviderOrder(order []string) []string {
result := make([]string, 0, len(defaultSearchProviderOrder))
seen := make(map[string]bool, len(defaultSearchProviderOrder))
for _, raw := range order {
id := strings.ToLower(strings.TrimSpace(raw))
if id == searchProviderDuckDuckGo || id == "" {
continue // DDG always last
}
if !isKnownSearchProvider(id) || seen[id] {
continue
}
result = append(result, id)
seen[id] = true
}
// Append remaining known providers not yet listed (except DDG).
for _, id := range defaultSearchProviderOrder {
if id == searchProviderDuckDuckGo {
continue
}
if !seen[id] {
result = append(result, id)
}
}
return append(result, searchProviderDuckDuckGo)
}
func isKnownSearchProvider(id string) bool {
for _, p := range defaultSearchProviderOrder {
if p == id {
return true
}
}
return false
}
// --- Shared provider helpers ---
func truncateStr(s string, max int) string {
if len(s) <= max {
return s
}
return s[:max] + "..."
}
func coalesceSearchText(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func clampProviderResultCount(requested, providerMax int) int {
if requested <= 0 {
requested = defaultSearchCount
}
if providerMax > 0 && requested > providerMax {
return providerMax
}
return requested
}
func normalizeProviderMaxResults(value int) int {
if value <= 0 {
return defaultSearchCount
}
if value > maxSearchCount {
return maxSearchCount
}
return value
}
+150
View File
@@ -0,0 +1,150 @@
package tools
import (
"reflect"
"testing"
)
func TestNormalizeWebSearchProviderOrder_Empty(t *testing.T) {
got := NormalizeWebSearchProviderOrder(nil)
want := []string{"exa", "tavily", "brave", "duckduckgo"}
if !reflect.DeepEqual(got, want) {
t.Errorf("NormalizeWebSearchProviderOrder(nil) = %v, want %v", got, want)
}
}
func TestNormalizeWebSearchProviderOrder_UserSpecified(t *testing.T) {
got := NormalizeWebSearchProviderOrder([]string{"brave", "exa"})
// brave first, exa second (user order), tavily appended, ddg last
want := []string{"brave", "exa", "tavily", "duckduckgo"}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %v, want %v", got, want)
}
}
func TestNormalizeWebSearchProviderOrder_DDGIgnored(t *testing.T) {
// DDG in user order is ignored (always last)
got := NormalizeWebSearchProviderOrder([]string{"duckduckgo", "tavily"})
want := []string{"tavily", "exa", "brave", "duckduckgo"}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %v, want %v", got, want)
}
}
func TestNormalizeWebSearchProviderOrder_Dedup(t *testing.T) {
got := NormalizeWebSearchProviderOrder([]string{"exa", "exa", "brave"})
want := []string{"exa", "brave", "tavily", "duckduckgo"}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %v, want %v", got, want)
}
}
func TestNormalizeWebSearchProviderOrder_UnknownSkipped(t *testing.T) {
got := NormalizeWebSearchProviderOrder([]string{"bing", "tavily"})
want := []string{"tavily", "exa", "brave", "duckduckgo"}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %v, want %v", got, want)
}
}
func TestBuildSearchProviders_AllEnabled(t *testing.T) {
cfg := WebSearchConfig{
ExaEnabled: true,
ExaAPIKey: "exa-key",
TavilyEnabled: true,
TavilyAPIKey: "tavily-key",
BraveEnabled: true,
BraveAPIKey: "brave-key",
DDGEnabled: true,
}
providers := buildSearchProviders(cfg)
names := make([]string, len(providers))
for i, p := range providers {
names[i] = p.Name()
}
want := []string{"exa", "tavily", "brave", "duckduckgo"}
if !reflect.DeepEqual(names, want) {
t.Errorf("got %v, want %v", names, want)
}
}
func TestBuildSearchProviders_OnlyBraveAndDDG(t *testing.T) {
cfg := WebSearchConfig{
BraveEnabled: true,
BraveAPIKey: "brave-key",
DDGEnabled: true,
}
providers := buildSearchProviders(cfg)
names := make([]string, len(providers))
for i, p := range providers {
names[i] = p.Name()
}
// Exa + Tavily not enabled, so only brave + ddg
want := []string{"brave", "duckduckgo"}
if !reflect.DeepEqual(names, want) {
t.Errorf("got %v, want %v", names, want)
}
}
func TestBuildSearchProviders_CustomOrder(t *testing.T) {
cfg := WebSearchConfig{
ProviderOrder: []string{"tavily", "brave"},
TavilyEnabled: true,
TavilyAPIKey: "key",
BraveEnabled: true,
BraveAPIKey: "key",
DDGEnabled: true,
}
providers := buildSearchProviders(cfg)
names := make([]string, len(providers))
for i, p := range providers {
names[i] = p.Name()
}
// tavily first (user), brave second, exa appended (no key so skipped), ddg last
want := []string{"tavily", "brave", "duckduckgo"}
if !reflect.DeepEqual(names, want) {
t.Errorf("got %v, want %v", names, want)
}
}
func TestBuildSearchProviders_NoProviders(t *testing.T) {
cfg := WebSearchConfig{}
providers := buildSearchProviders(cfg)
if len(providers) != 0 {
t.Errorf("expected empty providers, got %d", len(providers))
}
}
func TestClampProviderResultCount(t *testing.T) {
tests := []struct {
requested, max, want int
}{
{5, 10, 5},
{15, 10, 10},
{0, 10, defaultSearchCount},
{5, 0, 5},
}
for _, tt := range tests {
got := clampProviderResultCount(tt.requested, tt.max)
if got != tt.want {
t.Errorf("clampProviderResultCount(%d, %d) = %d, want %d", tt.requested, tt.max, got, tt.want)
}
}
}
func TestNormalizeProviderMaxResults(t *testing.T) {
tests := []struct {
input, want int
}{
{0, defaultSearchCount},
{-1, defaultSearchCount},
{7, 7},
{15, maxSearchCount},
}
for _, tt := range tests {
got := normalizeProviderMaxResults(tt.input)
if got != tt.want {
t.Errorf("normalizeProviderMaxResults(%d) = %d, want %d", tt.input, got, tt.want)
}
}
}
+8 -5
View File
@@ -14,18 +14,21 @@ import (
// --- DuckDuckGo Search Provider ---
type duckDuckGoSearchProvider struct {
client *http.Client
maxResults int
client *http.Client
}
func newDuckDuckGoSearchProvider() *duckDuckGoSearchProvider {
func newDuckDuckGoSearchProvider(maxResults int) *duckDuckGoSearchProvider {
return &duckDuckGoSearchProvider{
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
maxResults: normalizeProviderMaxResults(maxResults),
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
}
}
func (p *duckDuckGoSearchProvider) Name() string { return "duckduckgo" }
func (p *duckDuckGoSearchProvider) Name() string { return searchProviderDuckDuckGo }
func (p *duckDuckGoSearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
count := clampProviderResultCount(params.Count, p.maxResults)
searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(params.Query))
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
@@ -45,7 +48,7 @@ func (p *duckDuckGoSearchProvider) Search(ctx context.Context, params searchPara
return nil, fmt.Errorf("read response: %w", err)
}
return extractDDGResults(string(body), params.Count)
return extractDDGResults(string(body), count)
}
var (
+84
View File
@@ -0,0 +1,84 @@
package tools
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
type exaSearchProvider struct {
apiKey string
maxResults int
client *http.Client
}
func newExaSearchProvider(apiKey string, maxResults int) *exaSearchProvider {
return &exaSearchProvider{
apiKey: apiKey,
maxResults: normalizeProviderMaxResults(maxResults),
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
}
}
func (p *exaSearchProvider) Name() string { return searchProviderExa }
func (p *exaSearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
requestBody, err := json.Marshal(map[string]any{
"query": params.Query,
"type": "auto",
"numResults": clampProviderResultCount(params.Count, p.maxResults),
"text": true,
})
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, exaSearchEndpoint, bytes.NewReader(requestBody))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", webSearchUserAgent)
req.Header.Set("x-api-key", p.apiKey)
resp, err := p.client.Do(req)
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("exa API returned %d: %s", resp.StatusCode, truncateStr(string(body), 200))
}
var exaResp struct {
Results []struct {
Title string `json:"title"`
URL string `json:"url"`
Text string `json:"text"`
Summary string `json:"summary"`
} `json:"results"`
}
if err := json.Unmarshal(body, &exaResp); err != nil {
return nil, fmt.Errorf("parse response: %w", err)
}
results := make([]searchResult, 0, len(exaResp.Results))
for _, r := range exaResp.Results {
results = append(results, searchResult{
Title: coalesceSearchText(r.Title, r.URL, "Untitled"),
URL: r.URL,
Description: truncateStr(coalesceSearchText(r.Text, r.Summary, ""), 240),
})
}
return results, nil
}
+84
View File
@@ -0,0 +1,84 @@
package tools
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
type tavilySearchProvider struct {
apiKey string
maxResults int
client *http.Client
}
func newTavilySearchProvider(apiKey string, maxResults int) *tavilySearchProvider {
return &tavilySearchProvider{
apiKey: apiKey,
maxResults: normalizeProviderMaxResults(maxResults),
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
}
}
func (p *tavilySearchProvider) Name() string { return searchProviderTavily }
func (p *tavilySearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
requestBody, err := json.Marshal(map[string]any{
"query": params.Query,
"search_depth": "basic",
"max_results": clampProviderResultCount(params.Count, p.maxResults),
"include_answer": false,
"include_raw_content": false,
})
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, tavilySearchEndpoint, bytes.NewReader(requestBody))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Authorization", "Bearer "+p.apiKey)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", webSearchUserAgent)
resp, err := p.client.Do(req)
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("tavily API returned %d: %s", resp.StatusCode, truncateStr(string(body), 200))
}
var tavilyResp struct {
Results []struct {
Title string `json:"title"`
URL string `json:"url"`
Content string `json:"content"`
} `json:"results"`
}
if err := json.Unmarshal(body, &tavilyResp); err != nil {
return nil, fmt.Errorf("parse response: %w", err)
}
results := make([]searchResult, 0, len(tavilyResp.Results))
for _, r := range tavilyResp.Results {
results = append(results, searchResult{
Title: coalesceSearchText(r.Title, r.URL, "Untitled"),
URL: r.URL,
Description: truncateStr(r.Content, 240),
})
}
return results, nil
}