mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-12 13:04:57 +00:00
feat(tools): add Exa + Tavily web search providers with ranked ordering
Port Exa and Tavily provider clients from PR #825 into the tenant-aware overlay architecture (builtin_tool_tenant_configs.settings). - Add ExaConfig + TavilyConfig to WebToolsConfig with provider_order - Add Exa HTTP client (api.exa.ai/search, x-api-key auth) - Add Tavily HTTP client (api.tavily.com/search, Bearer auth) - Extend Brave + DDG constructors with per-provider maxResults - Add config_secrets plumbing for exa/tavily API keys (apply/collect/mask/strip) - Refactor gateway_setup.go to use WebSearchConfigFromConfig - Add NormalizeWebSearchProviderOrder (DDG always last, dedup, unknown skip) - Extract web_search_config.go (builder, normalizer, shared helpers) - 11 new unit tests for provider order, builder, clamp, normalize Credit: @kaitranntt (PR #825) for the original Exa + Tavily implementation.
This commit is contained in:
@@ -116,11 +116,7 @@ func setupToolRegistry(
|
||||
}
|
||||
|
||||
// Web tools (web_search + web_fetch)
|
||||
webSearchTool := tools.NewWebSearchTool(tools.WebSearchConfig{
|
||||
BraveEnabled: cfg.Tools.Web.Brave.Enabled,
|
||||
BraveAPIKey: cfg.Tools.Web.Brave.APIKey,
|
||||
DDGEnabled: cfg.Tools.Web.DuckDuckGo.Enabled,
|
||||
})
|
||||
webSearchTool := tools.NewWebSearchTool(tools.WebSearchConfigFromConfig(cfg))
|
||||
if webSearchTool != nil {
|
||||
toolsReg.Register(webSearchTool)
|
||||
slog.Info("web_search tool enabled")
|
||||
|
||||
@@ -432,8 +432,23 @@ type ToolPolicySpec struct {
|
||||
}
|
||||
|
||||
type WebToolsConfig struct {
|
||||
Brave BraveConfig `json:"brave"`
|
||||
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
||||
ProviderOrder []string `json:"provider_order,omitempty"`
|
||||
Exa ExaConfig `json:"exa"`
|
||||
Tavily TavilyConfig `json:"tavily"`
|
||||
Brave BraveConfig `json:"brave"`
|
||||
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
||||
}
|
||||
|
||||
type ExaConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
APIKey string `json:"api_key"`
|
||||
MaxResults int `json:"max_results"`
|
||||
}
|
||||
|
||||
type TavilyConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
APIKey string `json:"api_key"`
|
||||
MaxResults int `json:"max_results"`
|
||||
}
|
||||
|
||||
type BraveConfig struct {
|
||||
|
||||
@@ -59,6 +59,8 @@ func (c *Config) MaskedCopy() *Config {
|
||||
maskNonEmpty(&cp.Tts.MiniMax.APIKey)
|
||||
|
||||
// Mask web tool keys
|
||||
maskNonEmpty(&cp.Tools.Web.Exa.APIKey)
|
||||
maskNonEmpty(&cp.Tools.Web.Tavily.APIKey)
|
||||
maskNonEmpty(&cp.Tools.Web.Brave.APIKey)
|
||||
|
||||
// Mask Tailscale auth key
|
||||
@@ -109,6 +111,8 @@ func (c *Config) StripSecrets() {
|
||||
c.Tts.MiniMax.APIKey = ""
|
||||
|
||||
// Web tool keys
|
||||
c.Tools.Web.Exa.APIKey = ""
|
||||
c.Tools.Web.Tavily.APIKey = ""
|
||||
c.Tools.Web.Brave.APIKey = ""
|
||||
|
||||
// Tailscale auth key
|
||||
@@ -164,6 +168,8 @@ func (c *Config) StripMaskedSecrets() {
|
||||
stripIfMasked(&c.Tts.MiniMax.APIKey)
|
||||
|
||||
// Web tool keys
|
||||
stripIfMasked(&c.Tools.Web.Exa.APIKey)
|
||||
stripIfMasked(&c.Tools.Web.Tavily.APIKey)
|
||||
stripIfMasked(&c.Tools.Web.Brave.APIKey)
|
||||
|
||||
// Tailscale auth key
|
||||
@@ -185,6 +191,8 @@ func (c *Config) ApplyDBSecrets(secrets map[string]string) {
|
||||
apply("tts.elevenlabs.api_key", &c.Tts.ElevenLabs.APIKey)
|
||||
apply("tts.minimax.api_key", &c.Tts.MiniMax.APIKey)
|
||||
apply("tts.minimax.group_id", &c.Tts.MiniMax.GroupID)
|
||||
apply("tools.web.exa.api_key", &c.Tools.Web.Exa.APIKey)
|
||||
apply("tools.web.tavily.api_key", &c.Tools.Web.Tavily.APIKey)
|
||||
apply("tools.web.brave.api_key", &c.Tools.Web.Brave.APIKey)
|
||||
apply("tailscale.auth_key", &c.Tailscale.AuthKey)
|
||||
}
|
||||
@@ -205,6 +213,8 @@ func (c *Config) ExtractDBSecrets() map[string]string {
|
||||
collect("tts.elevenlabs.api_key", c.Tts.ElevenLabs.APIKey)
|
||||
collect("tts.minimax.api_key", c.Tts.MiniMax.APIKey)
|
||||
collect("tts.minimax.group_id", c.Tts.MiniMax.GroupID)
|
||||
collect("tools.web.exa.api_key", c.Tools.Web.Exa.APIKey)
|
||||
collect("tools.web.tavily.api_key", c.Tools.Web.Tavily.APIKey)
|
||||
collect("tools.web.brave.api_key", c.Tools.Web.Brave.APIKey)
|
||||
collect("tailscale.auth_key", c.Tailscale.AuthKey)
|
||||
|
||||
|
||||
@@ -15,9 +15,25 @@ const (
|
||||
maxSearchCount = 10
|
||||
searchTimeoutSeconds = 30
|
||||
braveSearchEndpoint = "https://api.search.brave.com/res/v1/web/search"
|
||||
exaSearchEndpoint = "https://api.exa.ai/search"
|
||||
tavilySearchEndpoint = "https://api.tavily.com/search"
|
||||
webSearchUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
const (
|
||||
searchProviderExa = "exa"
|
||||
searchProviderTavily = "tavily"
|
||||
searchProviderBrave = "brave"
|
||||
searchProviderDuckDuckGo = "duckduckgo"
|
||||
)
|
||||
|
||||
var defaultSearchProviderOrder = []string{
|
||||
searchProviderExa,
|
||||
searchProviderTavily,
|
||||
searchProviderBrave,
|
||||
searchProviderDuckDuckGo,
|
||||
}
|
||||
|
||||
// SearchProvider abstracts a web search backend.
|
||||
type SearchProvider interface {
|
||||
Search(ctx context.Context, params searchParams) ([]searchResult, error)
|
||||
@@ -72,26 +88,8 @@ type WebSearchTool struct {
|
||||
cache *webCache
|
||||
}
|
||||
|
||||
// WebSearchConfig holds configuration for the web search tool.
|
||||
type WebSearchConfig struct {
|
||||
BraveAPIKey string
|
||||
BraveEnabled bool
|
||||
BraveMaxResults int
|
||||
DDGEnabled bool
|
||||
DDGMaxResults int
|
||||
CacheTTL time.Duration
|
||||
}
|
||||
|
||||
func NewWebSearchTool(cfg WebSearchConfig) *WebSearchTool {
|
||||
var providers []SearchProvider
|
||||
|
||||
// Priority: Brave > DuckDuckGo (matching TS)
|
||||
if cfg.BraveEnabled && cfg.BraveAPIKey != "" {
|
||||
providers = append(providers, newBraveSearchProvider(cfg.BraveAPIKey))
|
||||
}
|
||||
if cfg.DDGEnabled {
|
||||
providers = append(providers, newDuckDuckGoSearchProvider())
|
||||
}
|
||||
providers := buildSearchProviders(cfg)
|
||||
|
||||
if len(providers) == 0 {
|
||||
return nil
|
||||
@@ -246,9 +244,3 @@ func formatSearchResults(query string, results []searchResult, provider string)
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func truncateStr(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "..."
|
||||
}
|
||||
|
||||
@@ -13,23 +13,27 @@ import (
|
||||
// --- Brave Search Provider ---
|
||||
|
||||
type braveSearchProvider struct {
|
||||
apiKey string
|
||||
client *http.Client
|
||||
apiKey string
|
||||
maxResults int
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newBraveSearchProvider(apiKey string) *braveSearchProvider {
|
||||
func newBraveSearchProvider(apiKey string, maxResults int) *braveSearchProvider {
|
||||
return &braveSearchProvider{
|
||||
apiKey: apiKey,
|
||||
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
|
||||
apiKey: apiKey,
|
||||
maxResults: normalizeProviderMaxResults(maxResults),
|
||||
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *braveSearchProvider) Name() string { return "brave" }
|
||||
func (p *braveSearchProvider) Name() string { return searchProviderBrave }
|
||||
|
||||
func (p *braveSearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
|
||||
count := clampProviderResultCount(params.Count, p.maxResults)
|
||||
|
||||
q := url.Values{}
|
||||
q.Set("q", params.Query)
|
||||
q.Set("count", fmt.Sprintf("%d", params.Count))
|
||||
q.Set("count", fmt.Sprintf("%d", count))
|
||||
|
||||
if params.Country != "" {
|
||||
q.Set("country", params.Country)
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
)
|
||||
|
||||
// WebSearchConfig holds configuration for the web search tool.
|
||||
type WebSearchConfig struct {
|
||||
ProviderOrder []string
|
||||
ExaAPIKey string
|
||||
ExaEnabled bool
|
||||
ExaMaxResults int
|
||||
TavilyAPIKey string
|
||||
TavilyEnabled bool
|
||||
TavilyMaxResults int
|
||||
BraveAPIKey string
|
||||
BraveEnabled bool
|
||||
BraveMaxResults int
|
||||
DDGEnabled bool
|
||||
DDGMaxResults int
|
||||
CacheTTL time.Duration
|
||||
}
|
||||
|
||||
// WebSearchConfigFromConfig creates a WebSearchConfig from the global config.
|
||||
func WebSearchConfigFromConfig(cfg *config.Config) WebSearchConfig {
|
||||
return WebSearchConfig{
|
||||
ProviderOrder: cfg.Tools.Web.ProviderOrder,
|
||||
ExaEnabled: cfg.Tools.Web.Exa.Enabled,
|
||||
ExaAPIKey: cfg.Tools.Web.Exa.APIKey,
|
||||
ExaMaxResults: cfg.Tools.Web.Exa.MaxResults,
|
||||
TavilyEnabled: cfg.Tools.Web.Tavily.Enabled,
|
||||
TavilyAPIKey: cfg.Tools.Web.Tavily.APIKey,
|
||||
TavilyMaxResults: cfg.Tools.Web.Tavily.MaxResults,
|
||||
BraveEnabled: cfg.Tools.Web.Brave.Enabled,
|
||||
BraveAPIKey: cfg.Tools.Web.Brave.APIKey,
|
||||
BraveMaxResults: cfg.Tools.Web.Brave.MaxResults,
|
||||
DDGEnabled: true,
|
||||
DDGMaxResults: cfg.Tools.Web.DuckDuckGo.MaxResults,
|
||||
}
|
||||
}
|
||||
|
||||
func buildSearchProviders(cfg WebSearchConfig) []SearchProvider {
|
||||
var providers []SearchProvider
|
||||
for _, providerID := range NormalizeWebSearchProviderOrder(cfg.ProviderOrder) {
|
||||
switch providerID {
|
||||
case searchProviderExa:
|
||||
if cfg.ExaEnabled && cfg.ExaAPIKey != "" {
|
||||
providers = append(providers, newExaSearchProvider(cfg.ExaAPIKey, cfg.ExaMaxResults))
|
||||
}
|
||||
case searchProviderTavily:
|
||||
if cfg.TavilyEnabled && cfg.TavilyAPIKey != "" {
|
||||
providers = append(providers, newTavilySearchProvider(cfg.TavilyAPIKey, cfg.TavilyMaxResults))
|
||||
}
|
||||
case searchProviderBrave:
|
||||
if cfg.BraveEnabled && cfg.BraveAPIKey != "" {
|
||||
providers = append(providers, newBraveSearchProvider(cfg.BraveAPIKey, cfg.BraveMaxResults))
|
||||
}
|
||||
case searchProviderDuckDuckGo:
|
||||
if cfg.DDGEnabled {
|
||||
providers = append(providers, newDuckDuckGoSearchProvider(cfg.DDGMaxResults))
|
||||
}
|
||||
}
|
||||
}
|
||||
return providers
|
||||
}
|
||||
|
||||
// NormalizeWebSearchProviderOrder normalizes user-specified provider order.
|
||||
// Explicit providers appear first in their specified order, remaining known
|
||||
// providers are appended (DuckDuckGo always last as free fallback).
|
||||
func NormalizeWebSearchProviderOrder(order []string) []string {
|
||||
result := make([]string, 0, len(defaultSearchProviderOrder))
|
||||
seen := make(map[string]bool, len(defaultSearchProviderOrder))
|
||||
|
||||
for _, raw := range order {
|
||||
id := strings.ToLower(strings.TrimSpace(raw))
|
||||
if id == searchProviderDuckDuckGo || id == "" {
|
||||
continue // DDG always last
|
||||
}
|
||||
if !isKnownSearchProvider(id) || seen[id] {
|
||||
continue
|
||||
}
|
||||
result = append(result, id)
|
||||
seen[id] = true
|
||||
}
|
||||
// Append remaining known providers not yet listed (except DDG).
|
||||
for _, id := range defaultSearchProviderOrder {
|
||||
if id == searchProviderDuckDuckGo {
|
||||
continue
|
||||
}
|
||||
if !seen[id] {
|
||||
result = append(result, id)
|
||||
}
|
||||
}
|
||||
return append(result, searchProviderDuckDuckGo)
|
||||
}
|
||||
|
||||
func isKnownSearchProvider(id string) bool {
|
||||
for _, p := range defaultSearchProviderOrder {
|
||||
if p == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- Shared provider helpers ---
|
||||
|
||||
func truncateStr(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "..."
|
||||
}
|
||||
|
||||
func coalesceSearchText(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func clampProviderResultCount(requested, providerMax int) int {
|
||||
if requested <= 0 {
|
||||
requested = defaultSearchCount
|
||||
}
|
||||
if providerMax > 0 && requested > providerMax {
|
||||
return providerMax
|
||||
}
|
||||
return requested
|
||||
}
|
||||
|
||||
func normalizeProviderMaxResults(value int) int {
|
||||
if value <= 0 {
|
||||
return defaultSearchCount
|
||||
}
|
||||
if value > maxSearchCount {
|
||||
return maxSearchCount
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeWebSearchProviderOrder_Empty(t *testing.T) {
|
||||
got := NormalizeWebSearchProviderOrder(nil)
|
||||
want := []string{"exa", "tavily", "brave", "duckduckgo"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("NormalizeWebSearchProviderOrder(nil) = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWebSearchProviderOrder_UserSpecified(t *testing.T) {
|
||||
got := NormalizeWebSearchProviderOrder([]string{"brave", "exa"})
|
||||
// brave first, exa second (user order), tavily appended, ddg last
|
||||
want := []string{"brave", "exa", "tavily", "duckduckgo"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWebSearchProviderOrder_DDGIgnored(t *testing.T) {
|
||||
// DDG in user order is ignored (always last)
|
||||
got := NormalizeWebSearchProviderOrder([]string{"duckduckgo", "tavily"})
|
||||
want := []string{"tavily", "exa", "brave", "duckduckgo"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWebSearchProviderOrder_Dedup(t *testing.T) {
|
||||
got := NormalizeWebSearchProviderOrder([]string{"exa", "exa", "brave"})
|
||||
want := []string{"exa", "brave", "tavily", "duckduckgo"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWebSearchProviderOrder_UnknownSkipped(t *testing.T) {
|
||||
got := NormalizeWebSearchProviderOrder([]string{"bing", "tavily"})
|
||||
want := []string{"tavily", "exa", "brave", "duckduckgo"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchProviders_AllEnabled(t *testing.T) {
|
||||
cfg := WebSearchConfig{
|
||||
ExaEnabled: true,
|
||||
ExaAPIKey: "exa-key",
|
||||
TavilyEnabled: true,
|
||||
TavilyAPIKey: "tavily-key",
|
||||
BraveEnabled: true,
|
||||
BraveAPIKey: "brave-key",
|
||||
DDGEnabled: true,
|
||||
}
|
||||
providers := buildSearchProviders(cfg)
|
||||
names := make([]string, len(providers))
|
||||
for i, p := range providers {
|
||||
names[i] = p.Name()
|
||||
}
|
||||
want := []string{"exa", "tavily", "brave", "duckduckgo"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("got %v, want %v", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchProviders_OnlyBraveAndDDG(t *testing.T) {
|
||||
cfg := WebSearchConfig{
|
||||
BraveEnabled: true,
|
||||
BraveAPIKey: "brave-key",
|
||||
DDGEnabled: true,
|
||||
}
|
||||
providers := buildSearchProviders(cfg)
|
||||
names := make([]string, len(providers))
|
||||
for i, p := range providers {
|
||||
names[i] = p.Name()
|
||||
}
|
||||
// Exa + Tavily not enabled, so only brave + ddg
|
||||
want := []string{"brave", "duckduckgo"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("got %v, want %v", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchProviders_CustomOrder(t *testing.T) {
|
||||
cfg := WebSearchConfig{
|
||||
ProviderOrder: []string{"tavily", "brave"},
|
||||
TavilyEnabled: true,
|
||||
TavilyAPIKey: "key",
|
||||
BraveEnabled: true,
|
||||
BraveAPIKey: "key",
|
||||
DDGEnabled: true,
|
||||
}
|
||||
providers := buildSearchProviders(cfg)
|
||||
names := make([]string, len(providers))
|
||||
for i, p := range providers {
|
||||
names[i] = p.Name()
|
||||
}
|
||||
// tavily first (user), brave second, exa appended (no key so skipped), ddg last
|
||||
want := []string{"tavily", "brave", "duckduckgo"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("got %v, want %v", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchProviders_NoProviders(t *testing.T) {
|
||||
cfg := WebSearchConfig{}
|
||||
providers := buildSearchProviders(cfg)
|
||||
if len(providers) != 0 {
|
||||
t.Errorf("expected empty providers, got %d", len(providers))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClampProviderResultCount(t *testing.T) {
|
||||
tests := []struct {
|
||||
requested, max, want int
|
||||
}{
|
||||
{5, 10, 5},
|
||||
{15, 10, 10},
|
||||
{0, 10, defaultSearchCount},
|
||||
{5, 0, 5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := clampProviderResultCount(tt.requested, tt.max)
|
||||
if got != tt.want {
|
||||
t.Errorf("clampProviderResultCount(%d, %d) = %d, want %d", tt.requested, tt.max, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeProviderMaxResults(t *testing.T) {
|
||||
tests := []struct {
|
||||
input, want int
|
||||
}{
|
||||
{0, defaultSearchCount},
|
||||
{-1, defaultSearchCount},
|
||||
{7, 7},
|
||||
{15, maxSearchCount},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := normalizeProviderMaxResults(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("normalizeProviderMaxResults(%d) = %d, want %d", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,18 +14,21 @@ import (
|
||||
// --- DuckDuckGo Search Provider ---
|
||||
|
||||
type duckDuckGoSearchProvider struct {
|
||||
client *http.Client
|
||||
maxResults int
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newDuckDuckGoSearchProvider() *duckDuckGoSearchProvider {
|
||||
func newDuckDuckGoSearchProvider(maxResults int) *duckDuckGoSearchProvider {
|
||||
return &duckDuckGoSearchProvider{
|
||||
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
|
||||
maxResults: normalizeProviderMaxResults(maxResults),
|
||||
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *duckDuckGoSearchProvider) Name() string { return "duckduckgo" }
|
||||
func (p *duckDuckGoSearchProvider) Name() string { return searchProviderDuckDuckGo }
|
||||
|
||||
func (p *duckDuckGoSearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
|
||||
count := clampProviderResultCount(params.Count, p.maxResults)
|
||||
searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(params.Query))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
|
||||
@@ -45,7 +48,7 @@ func (p *duckDuckGoSearchProvider) Search(ctx context.Context, params searchPara
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
return extractDDGResults(string(body), params.Count)
|
||||
return extractDDGResults(string(body), count)
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type exaSearchProvider struct {
|
||||
apiKey string
|
||||
maxResults int
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newExaSearchProvider(apiKey string, maxResults int) *exaSearchProvider {
|
||||
return &exaSearchProvider{
|
||||
apiKey: apiKey,
|
||||
maxResults: normalizeProviderMaxResults(maxResults),
|
||||
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *exaSearchProvider) Name() string { return searchProviderExa }
|
||||
|
||||
func (p *exaSearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
|
||||
requestBody, err := json.Marshal(map[string]any{
|
||||
"query": params.Query,
|
||||
"type": "auto",
|
||||
"numResults": clampProviderResultCount(params.Count, p.maxResults),
|
||||
"text": true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, exaSearchEndpoint, bytes.NewReader(requestBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", webSearchUserAgent)
|
||||
req.Header.Set("x-api-key", p.apiKey)
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("exa API returned %d: %s", resp.StatusCode, truncateStr(string(body), 200))
|
||||
}
|
||||
|
||||
var exaResp struct {
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Text string `json:"text"`
|
||||
Summary string `json:"summary"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &exaResp); err != nil {
|
||||
return nil, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
|
||||
results := make([]searchResult, 0, len(exaResp.Results))
|
||||
for _, r := range exaResp.Results {
|
||||
results = append(results, searchResult{
|
||||
Title: coalesceSearchText(r.Title, r.URL, "Untitled"),
|
||||
URL: r.URL,
|
||||
Description: truncateStr(coalesceSearchText(r.Text, r.Summary, ""), 240),
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type tavilySearchProvider struct {
|
||||
apiKey string
|
||||
maxResults int
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newTavilySearchProvider(apiKey string, maxResults int) *tavilySearchProvider {
|
||||
return &tavilySearchProvider{
|
||||
apiKey: apiKey,
|
||||
maxResults: normalizeProviderMaxResults(maxResults),
|
||||
client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *tavilySearchProvider) Name() string { return searchProviderTavily }
|
||||
|
||||
func (p *tavilySearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
|
||||
requestBody, err := json.Marshal(map[string]any{
|
||||
"query": params.Query,
|
||||
"search_depth": "basic",
|
||||
"max_results": clampProviderResultCount(params.Count, p.maxResults),
|
||||
"include_answer": false,
|
||||
"include_raw_content": false,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, tavilySearchEndpoint, bytes.NewReader(requestBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", webSearchUserAgent)
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("tavily API returned %d: %s", resp.StatusCode, truncateStr(string(body), 200))
|
||||
}
|
||||
|
||||
var tavilyResp struct {
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Content string `json:"content"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &tavilyResp); err != nil {
|
||||
return nil, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
|
||||
results := make([]searchResult, 0, len(tavilyResp.Results))
|
||||
for _, r := range tavilyResp.Results {
|
||||
results = append(results, searchResult{
|
||||
Title: coalesceSearchText(r.Title, r.URL, "Untitled"),
|
||||
URL: r.URL,
|
||||
Description: truncateStr(r.Content, 240),
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
Reference in New Issue
Block a user