diff --git a/docs/websearch.md b/docs/websearch.md index 054c75d3..ded2e1b9 100644 --- a/docs/websearch.md +++ b/docs/websearch.md @@ -1,6 +1,6 @@ # WebSearch Configuration Guide -Last Updated: 2026-04-04 +Last Updated: 2026-04-11 CCS provides automatic web search for third-party profiles that cannot access Anthropic's native WebSearch API. @@ -17,29 +17,30 @@ Third-party profiles cannot execute Anthropic's server-side WebSearch because th ## Architecture ``` -┌──────────────────────────────────────────────────────────────┐ -│ Claude Code CLI │ -│ │ -│ Search Request │ -│ │ │ -│ ├── Native Claude Account? → Anthropic WebSearch API │ -│ │ │ -│ └── Third-party Profile? → native WebSearch disabled │ -│ │ │ -│ ├── CCS MCP tool when ready│ -│ │ ccs-websearch.WebSearch│ -│ │ │ │ -│ │ ├── 1. Exa │ -│ │ ├── 2. Tavily│ -│ │ ├── 3. Brave │ -│ │ ├── 4. DuckDuckGo│ -│ │ └── 5. Legacy CLI│ -│ │ fallback │ -│ │ (Gemini/ │ -│ │ OpenCode/ │ -│ │ Grok) │ -│ └── Bash/network fallback │ -└──────────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────────┐ +│ Claude Code CLI │ +│ │ +│ Search Request │ +│ │ │ +│ ├── Native Claude Account? → Anthropic WebSearch API │ +│ │ │ +│ └── Third-party Profile? → native WebSearch disabled │ +│ │ │ +│ ├── CCS MCP tool when ready │ +│ │ ccs-websearch.WebSearch │ +│ │ │ │ +│ │ ├── 1. Exa │ +│ │ ├── 2. Tavily │ +│ │ ├── 3. Brave │ +│ │ ├── 4. SearXNG │ +│ │ ├── 5. DuckDuckGo│ +│ │ └── 6. Legacy CLI│ +│ │ fallback │ +│ │ (Gemini/ │ +│ │ OpenCode/│ +│ │ Grok) │ +│ └── Bash/network fallback │ +└──────────────────────────────────────────────────────────────────┘ ``` ## Why This Changed @@ -66,8 +67,9 @@ That shared launch helper applies to normal third-party settings profiles, CLIPr |----------|------|-------|---------|-------| | Exa | HTTP API | `EXA_API_KEY` | No | High-quality API search with extracted content | | Tavily | HTTP API | `TAVILY_API_KEY` | No | Agent-oriented search API | -| DuckDuckGo | HTML fetch | None | Yes | Built-in zero-setup fallback | | Brave Search | HTTP API | `BRAVE_API_KEY` | No | Cleaner snippets and metadata | +| SearXNG | JSON API | `providers.searxng.url` | No | Self-hosted/public SearXNG backend via `/search?format=json` | +| DuckDuckGo | HTML fetch | None | Yes | Built-in zero-setup fallback | | Gemini CLI | Legacy CLI | `npm i -g @google/gemini-cli` | No | Optional compatibility fallback | | OpenCode | Legacy CLI | `curl -fsSL https://opencode.ai/install \| bash` | No | Optional compatibility fallback | | Grok CLI | Legacy CLI | `npm i -g @vibe-kit/grok-cli` + `GROK_API_KEY` | No | Optional compatibility fallback | @@ -78,7 +80,8 @@ That shared launch helper applies to normal third-party settings profiles, CLIPr Open `ccs config` → `Settings` → `WebSearch`. -- Enable Exa, Tavily, Brave, or DuckDuckGo in the backend chain +- Enable Exa, Tavily, Brave, SearXNG, or DuckDuckGo in the backend chain +- Configure SearXNG base URL (for example `https://search.example.com`) when SearXNG is enabled - Set or rotate Exa, Tavily, and Brave API keys directly inside each provider card - Saved keys are persisted in `global_env` and injected at runtime, so readiness updates from the same screen - Review whether any legacy fallback CLIs are still enabled in config @@ -97,12 +100,16 @@ websearch: tavily: enabled: false max_results: 5 - duckduckgo: - enabled: true - max_results: 5 brave: enabled: false max_results: 5 + searxng: + enabled: false + url: "" + max_results: 5 + duckduckgo: + enabled: true + max_results: 5 gemini: enabled: false model: gemini-2.5-flash @@ -125,6 +132,8 @@ Note: `enabled: false` stops provisioning the managed local `ccs-websearch.WebSe | `EXA_API_KEY` | Enables Exa when `providers.exa.enabled: true` | | `TAVILY_API_KEY` | Enables Tavily when `providers.tavily.enabled: true` | | `BRAVE_API_KEY` | Enables Brave Search when `providers.brave.enabled: true` | +| `CCS_WEBSEARCH_SEARXNG_URL` | Runtime URL used when `providers.searxng.enabled: true` | +| `CCS_WEBSEARCH_SEARXNG_MAX_RESULTS` | Optional runtime override for SearXNG result count (clamped 1..10) | | `GROK_API_KEY` | Required only for legacy Grok CLI fallback | | `CCS_WEBSEARCH_SKIP` | Disable the CCS local WebSearch runtime for the current process; third-party launches still keep native Anthropic `WebSearch` disabled | | `CCS_DEBUG` | Verbose WebSearch runtime logging | @@ -156,6 +165,12 @@ ccs config If the dashboard says the key is stored but still not ready, check whether `Settings -> Global Env` is disabled. WebSearch reuses that injection path for dashboard-managed keys. +### SearXNG is enabled but not ready + +1. Confirm the configured base URL is valid (for example `https://search.example.com`) +2. Confirm the instance exposes `GET /search?q=&format=json` +3. If the hook reports `SearXNG returned 403: format=json is disabled on this instance`, enable JSON format on that SearXNG deployment or switch to another backend + ### I still want Gemini/OpenCode/Grok fallback Those providers remain supported, but they are no longer the primary path. Enable them explicitly in `config.yaml` if you want them as last-resort fallback. diff --git a/lib/hooks/websearch-transformer.cjs b/lib/hooks/websearch-transformer.cjs index 79c5a433..d1fae51e 100644 --- a/lib/hooks/websearch-transformer.cjs +++ b/lib/hooks/websearch-transformer.cjs @@ -6,6 +6,7 @@ * - Exa Search API * - Tavily Search API * - Brave Search API + * - SearXNG JSON API * - DuckDuckGo HTML search * * Legacy compatibility fallback: @@ -369,6 +370,20 @@ function getResultCount(provider) { return Number.isFinite(parsed) && parsed > 0 ? Math.min(parsed, 10) : DEFAULT_RESULT_COUNT; } +function getSearxngBaseUrl() { + const raw = (process.env.CCS_WEBSEARCH_SEARXNG_URL || '').trim(); + if (!raw) { + return ''; + } + + try { + const parsed = new URL(raw); + return parsed.toString().replace(/\/+$/, ''); + } catch { + return ''; + } +} + function buildPrompt(providerId, query) { const config = PROVIDER_CONFIG[providerId]; const parts = [ @@ -569,6 +584,104 @@ async function tryBraveSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { } } +async function trySearxngSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { + const baseUrl = getSearxngBaseUrl(); + if (!baseUrl) { + return { success: false, error: 'SearXNG URL is not configured' }; + } + + const params = new URLSearchParams({ + q: query, + format: 'json', + }); + + try { + const response = await fetchWithTimeout( + `${baseUrl}/search?${params.toString()}`, + { + headers: { + Accept: 'application/json', + 'User-Agent': USER_AGENT, + }, + }, + timeoutSec * 1000 + ); + + if (!response.ok) { + const body = await response.text(); + if ( + response.status === 403 && + /format(?:=|\s*)json|json[^\n]{0,40}disabled|disabled[^\n]{0,40}json/i.test(body) + ) { + return { + success: false, + error: 'SearXNG returned 403: format=json is disabled on this instance', + statusCode: response.status, + retryAfterSec: parseRetryAfterSeconds(readHeaderValue(response.headers, 'retry-after')), + }; + } + + return { + success: false, + error: `SearXNG returned ${response.status}: ${body.slice(0, 160)}`, + statusCode: response.status, + retryAfterSec: parseRetryAfterSeconds(readHeaderValue(response.headers, 'retry-after')), + }; + } + + let body; + try { + body = await response.json(); + } catch { + return { + success: false, + error: 'SearXNG returned malformed JSON payload', + }; + } + + if (!body || typeof body !== 'object' || !Array.isArray(body.results)) { + return { + success: false, + error: 'SearXNG JSON response is missing results[]', + }; + } + + const count = getResultCount('searxng'); + const results = body.results.slice(0, count).map((entry) => { + const result = entry && typeof entry === 'object' ? entry : {}; + const url = typeof result.url === 'string' ? result.url : ''; + const titleSource = + typeof result.title === 'string' && result.title.trim().length > 0 + ? result.title + : url || 'Untitled'; + const descriptionSource = + typeof result.content === 'string' + ? result.content + : typeof result.description === 'string' + ? result.description + : typeof result.snippet === 'string' + ? result.snippet + : ''; + + return { + title: compactText(titleSource, 120), + url, + description: compactText(descriptionSource, 240), + }; + }); + + return { + success: true, + content: formatStructuredSearchResults(query, 'SearXNG', results), + }; + } catch (error) { + return { + success: false, + error: error.name === 'AbortError' ? 'SearXNG timed out' : error.message, + }; + } +} + async function tryExaSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { const apiKey = getProviderApiKey('exa'); if (!apiKey) { @@ -892,6 +1005,12 @@ function getConfiguredProviders() { available: () => isProviderEnabled('brave') && Boolean(getProviderApiKey('brave')), fn: tryBraveSearch, }, + { + name: 'SearXNG', + id: 'searxng', + available: () => isProviderEnabled('searxng') && Boolean(getSearxngBaseUrl()), + fn: trySearxngSearch, + }, { name: 'DuckDuckGo', id: 'duckduckgo', @@ -1280,4 +1399,5 @@ module.exports = { tryTavilySearch, tryDuckDuckGoSearch, tryBraveSearch, + trySearxngSearch, }; diff --git a/src/config/unified-config-loader.ts b/src/config/unified-config-loader.ts index b0c3ecfd..e7a6a0bc 100644 --- a/src/config/unified-config-loader.ts +++ b/src/config/unified-config-loader.ts @@ -407,14 +407,19 @@ function mergeWithDefaults(partial: Partial): UnifiedConfig { enabled: partial.websearch?.providers?.tavily?.enabled ?? false, max_results: partial.websearch?.providers?.tavily?.max_results ?? 5, }, - duckduckgo: { - enabled: partial.websearch?.providers?.duckduckgo?.enabled ?? true, - max_results: partial.websearch?.providers?.duckduckgo?.max_results ?? 5, - }, brave: { enabled: partial.websearch?.providers?.brave?.enabled ?? false, max_results: partial.websearch?.providers?.brave?.max_results ?? 5, }, + searxng: { + enabled: partial.websearch?.providers?.searxng?.enabled ?? false, + url: partial.websearch?.providers?.searxng?.url ?? '', + max_results: partial.websearch?.providers?.searxng?.max_results ?? 5, + }, + duckduckgo: { + enabled: partial.websearch?.providers?.duckduckgo?.enabled ?? true, + max_results: partial.websearch?.providers?.duckduckgo?.max_results ?? 5, + }, gemini: { enabled: partial.websearch?.providers?.gemini?.enabled ?? @@ -1093,15 +1098,16 @@ export interface GeminiWebSearchInfo { /** * Get websearch configuration. * Returns defaults if not configured. - * Supports Gemini CLI, OpenCode, and Grok CLI providers. + * Supports deterministic providers and optional Gemini/OpenCode/Grok CLI fallbacks. */ export function getWebSearchConfig(): { enabled: boolean; providers?: { exa?: { enabled?: boolean; max_results?: number }; tavily?: { enabled?: boolean; max_results?: number }; - duckduckgo?: { enabled?: boolean; max_results?: number }; brave?: { enabled?: boolean; max_results?: number }; + searxng?: { enabled?: boolean; url?: string; max_results?: number }; + duckduckgo?: { enabled?: boolean; max_results?: number }; gemini?: GeminiWebSearchInfo; opencode?: { enabled?: boolean; model?: string; timeout?: number }; grok?: { enabled?: boolean; timeout?: number }; @@ -1132,6 +1138,12 @@ export function getWebSearchConfig(): { max_results: config.websearch?.providers?.brave?.max_results ?? 5, }; + const searxngConfig = { + enabled: config.websearch?.providers?.searxng?.enabled ?? false, + url: config.websearch?.providers?.searxng?.url ?? '', + max_results: config.websearch?.providers?.searxng?.max_results ?? 5, + }; + const geminiConfig: GeminiWebSearchInfo = { enabled: config.websearch?.providers?.gemini?.enabled ?? config.websearch?.gemini?.enabled ?? false, @@ -1155,8 +1167,9 @@ export function getWebSearchConfig(): { const anyProviderEnabled = exaConfig.enabled || tavilyConfig.enabled || - duckDuckGoConfig.enabled || braveConfig.enabled || + searxngConfig.enabled || + duckDuckGoConfig.enabled || geminiConfig.enabled || opencodeConfig.enabled || grokConfig.enabled; @@ -1167,8 +1180,9 @@ export function getWebSearchConfig(): { providers: { exa: exaConfig, tavily: tavilyConfig, - duckduckgo: duckDuckGoConfig, brave: braveConfig, + searxng: searxngConfig, + duckduckgo: duckDuckGoConfig, gemini: geminiConfig, opencode: opencodeConfig, grok: grokConfig, diff --git a/src/config/unified-config-types.ts b/src/config/unified-config-types.ts index 5ce9c304..3ab0f545 100644 --- a/src/config/unified-config-types.ts +++ b/src/config/unified-config-types.ts @@ -316,6 +316,18 @@ export interface TavilyWebSearchConfig { max_results?: number; } +/** + * SearXNG WebSearch configuration. + */ +export interface SearxngWebSearchConfig { + /** Enable SearXNG JSON search backend (default: false) */ + enabled?: boolean; + /** Base SearXNG URL, e.g. https://search.example.com (default: '') */ + url?: string; + /** Number of results to fetch (default: 5) */ + max_results?: number; +} + /** * Gemini CLI WebSearch configuration. */ @@ -359,10 +371,12 @@ export interface WebSearchProvidersConfig { exa?: ExaWebSearchConfig; /** Tavily Search API - API-backed search optimized for agent/tool usage */ tavily?: TavilyWebSearchConfig; - /** DuckDuckGo HTML search - zero setup default backend */ - duckduckgo?: DuckDuckGoWebSearchConfig; /** Brave Search API - higher quality results when BRAVE_API_KEY is set */ brave?: BraveWebSearchConfig; + /** SearXNG JSON search - self-hosted or public instance backend */ + searxng?: SearxngWebSearchConfig; + /** DuckDuckGo HTML search - zero setup default backend */ + duckduckgo?: DuckDuckGoWebSearchConfig; /** Gemini CLI - optional legacy LLM fallback */ gemini?: GeminiWebSearchConfig; /** Grok CLI - optional legacy LLM fallback */ @@ -961,14 +975,19 @@ export function createEmptyUnifiedConfig(): UnifiedConfig { enabled: false, max_results: 5, }, - duckduckgo: { - enabled: true, - max_results: 5, - }, brave: { enabled: false, max_results: 5, }, + searxng: { + enabled: false, + url: '', + max_results: 5, + }, + duckduckgo: { + enabled: true, + max_results: 5, + }, gemini: { enabled: false, model: 'gemini-2.5-flash', diff --git a/src/utils/websearch/hook-env.ts b/src/utils/websearch/hook-env.ts index d5e69018..d2b5f85c 100644 --- a/src/utils/websearch/hook-env.ts +++ b/src/utils/websearch/hook-env.ts @@ -61,6 +61,12 @@ export function getWebSearchHookEnv(): Record { env.CCS_WEBSEARCH_BRAVE_MAX_RESULTS = String(wsConfig.providers.brave.max_results || 5); } + if (wsConfig.providers?.searxng?.enabled && wsConfig.providers.searxng.url?.trim()) { + env.CCS_WEBSEARCH_SEARXNG = '1'; + env.CCS_WEBSEARCH_SEARXNG_URL = wsConfig.providers.searxng.url.trim(); + env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = String(wsConfig.providers.searxng.max_results || 5); + } + if (wsConfig.providers?.gemini?.enabled) { env.CCS_WEBSEARCH_GEMINI = '1'; if (wsConfig.providers.gemini.model) { diff --git a/src/utils/websearch/status.ts b/src/utils/websearch/status.ts index ecb5ce94..e0b9c36d 100644 --- a/src/utils/websearch/status.ts +++ b/src/utils/websearch/status.ts @@ -28,6 +28,20 @@ function hasEnvValue(name: string): boolean { return (process.env[name] || '').trim().length > 0; } +function hasValidSearxngUrl(url: string | undefined): boolean { + const normalized = String(url || '').trim(); + if (!normalized) { + return false; + } + + try { + const parsed = new URL(normalized); + return Boolean(parsed.protocol && parsed.host); + } catch { + return false; + } +} + function getProviderStatePath(): string { return join(getCcsDir(), 'cache', PROVIDER_STATE_FILE); } @@ -209,18 +223,6 @@ export function getWebSearchCliProviders(): WebSearchCliInfo[] { ? 'Stored in dashboard, but Global Env is disabled' : 'Set TAVILY_API_KEY', }, - { - id: 'duckduckgo', - kind: 'backend', - name: 'DuckDuckGo', - enabled: wsConfig.providers?.duckduckgo?.enabled ?? true, - available: wsConfig.providers?.duckduckgo?.enabled ?? true, - version: null, - docsUrl: 'https://duckduckgo.com', - requiresApiKey: false, - description: 'Default built-in HTML search backend. Zero setup.', - detail: `Built-in (${wsConfig.providers?.duckduckgo?.max_results ?? 5} results)`, - }, { id: 'brave', kind: 'backend', @@ -238,6 +240,34 @@ export function getWebSearchCliProviders(): WebSearchCliInfo[] { ? 'Stored in dashboard, but Global Env is disabled' : 'Set BRAVE_API_KEY', }, + { + id: 'searxng', + kind: 'backend', + name: 'SearXNG', + enabled: wsConfig.providers?.searxng?.enabled ?? false, + available: + (wsConfig.providers?.searxng?.enabled ?? false) && + hasValidSearxngUrl(wsConfig.providers?.searxng?.url), + version: null, + docsUrl: 'https://docs.searxng.org/dev/search_api.html', + requiresApiKey: false, + description: 'Configurable SearXNG JSON backend for self-hosted or public instances.', + detail: hasValidSearxngUrl(wsConfig.providers?.searxng?.url) + ? `Configured (${wsConfig.providers?.searxng?.max_results ?? 5} results)` + : 'Set a valid SearXNG base URL', + }, + { + id: 'duckduckgo', + kind: 'backend', + name: 'DuckDuckGo', + enabled: wsConfig.providers?.duckduckgo?.enabled ?? true, + available: wsConfig.providers?.duckduckgo?.enabled ?? true, + version: null, + docsUrl: 'https://duckduckgo.com', + requiresApiKey: false, + description: 'Default built-in HTML search backend. Zero setup.', + detail: `Built-in (${wsConfig.providers?.duckduckgo?.max_results ?? 5} results)`, + }, ]; return [...providers, ...getLegacyProviderStatuses()].map((provider) => @@ -263,6 +293,7 @@ export function getCliInstallHints(): string[] { return [ 'WebSearch: no ready providers', ' Enable DuckDuckGo in Settings > WebSearch for zero-setup search', + ' Or enable SearXNG and set a valid base URL (must support /search?format=json)', ' Or export EXA_API_KEY, TAVILY_API_KEY, or BRAVE_API_KEY for API-backed search', ' Optional legacy fallback: npm i -g @google/gemini-cli', ]; diff --git a/src/utils/websearch/types.ts b/src/utils/websearch/types.ts index 8d737617..a6ca29df 100644 --- a/src/utils/websearch/types.ts +++ b/src/utils/websearch/types.ts @@ -37,8 +37,9 @@ export type WebSearchReadiness = 'ready' | 'needs_setup' | 'unavailable'; export type WebSearchProviderId = | 'exa' | 'tavily' - | 'duckduckgo' | 'brave' + | 'searxng' + | 'duckduckgo' | 'gemini' | 'grok' | 'opencode'; @@ -107,8 +108,9 @@ export interface WebSearchConfig { providers?: { exa?: WebSearchProviderConfig; tavily?: WebSearchProviderConfig; - duckduckgo?: WebSearchProviderConfig; brave?: WebSearchProviderConfig; + searxng?: WebSearchProviderConfig; + duckduckgo?: WebSearchProviderConfig; gemini?: WebSearchProviderConfig; opencode?: WebSearchProviderConfig; grok?: WebSearchProviderConfig; diff --git a/src/web-server/health/websearch-checks.ts b/src/web-server/health/websearch-checks.ts index 3a4faa7d..b75abb1d 100644 --- a/src/web-server/health/websearch-checks.ts +++ b/src/web-server/health/websearch-checks.ts @@ -42,7 +42,7 @@ export function checkWebSearchClis(): HealthCheck[] { name: 'WebSearch Status', status: 'warning', message: 'No ready provider', - fix: 'Enable DuckDuckGo or set EXA_API_KEY, TAVILY_API_KEY, or BRAVE_API_KEY', + fix: 'Enable DuckDuckGo, configure SearXNG URL, or set EXA_API_KEY/TAVILY_API_KEY/BRAVE_API_KEY', details: 'Third-party profiles need a local WebSearch backend.', }); } diff --git a/src/web-server/routes/websearch-routes.ts b/src/web-server/routes/websearch-routes.ts index fc550868..d5bc6432 100644 --- a/src/web-server/routes/websearch-routes.ts +++ b/src/web-server/routes/websearch-routes.ts @@ -17,6 +17,8 @@ import { requireLocalAccessWhenAuthDisabled } from '../middleware/auth-middlewar const router = Router(); const WEBSEARCH_LOCAL_ACCESS_ERROR = 'WebSearch endpoints require localhost access when dashboard auth is disabled.'; +const DEFAULT_WEBSEARCH_MAX_RESULTS = 5; +const MAX_WEBSEARCH_MAX_RESULTS = 10; type WebSearchApiKeyUpdates = Partial>; @@ -28,6 +30,12 @@ function isWebSearchApiKeyProviderId(value: string): value is WebSearchApiKeyPro return Object.prototype.hasOwnProperty.call(WEBSEARCH_API_KEY_PROVIDERS, value); } +function clampWebSearchMaxResults(value: number | undefined, fallback: number): number { + const candidate = Number.isFinite(value) ? (value as number) : fallback; + const normalized = Number.isFinite(candidate) ? candidate : DEFAULT_WEBSEARCH_MAX_RESULTS; + return Math.max(1, Math.min(MAX_WEBSEARCH_MAX_RESULTS, Math.floor(normalized))); +} + router.use((req: Request, res: Response, next) => { if (requireLocalAccessWhenAuthDisabled(req, res, WEBSEARCH_LOCAL_ACCESS_ERROR)) { next(); @@ -82,6 +90,22 @@ router.put('/', (req: Request, res: Response): void => { return; } + if (providers?.searxng?.url !== undefined && typeof providers.searxng.url !== 'string') { + res.status(400).json({ error: 'Invalid value for providers.searxng.url. Must be a string.' }); + return; + } + + if ( + providers?.searxng?.max_results !== undefined && + (typeof providers.searxng.max_results !== 'number' || + !Number.isFinite(providers.searxng.max_results)) + ) { + res.status(400).json({ + error: 'Invalid value for providers.searxng.max_results. Must be a number.', + }); + return; + } + if ( apiKeys !== undefined && (apiKeys === null || Array.isArray(apiKeys) || typeof apiKeys !== 'object') @@ -144,6 +168,17 @@ router.put('/', (req: Request, res: Response): void => { config.websearch?.providers?.brave?.max_results ?? 5, }, + searxng: { + enabled: + providers.searxng?.enabled ?? + config.websearch?.providers?.searxng?.enabled ?? + false, + url: providers.searxng?.url ?? config.websearch?.providers?.searxng?.url ?? '', + max_results: clampWebSearchMaxResults( + providers.searxng?.max_results, + config.websearch?.providers?.searxng?.max_results ?? DEFAULT_WEBSEARCH_MAX_RESULTS + ), + }, gemini: { enabled: providers.gemini?.enabled ?? diff --git a/tests/unit/hooks/websearch-transformer.test.ts b/tests/unit/hooks/websearch-transformer.test.ts index 80bb74d9..2f005e0c 100644 --- a/tests/unit/hooks/websearch-transformer.test.ts +++ b/tests/unit/hooks/websearch-transformer.test.ts @@ -56,6 +56,12 @@ const hook = require('../../../lib/hooks/websearch-transformer.cjs') as { results: Array<{ title: string; url: string; description: string }> ) => string; parseRetryAfterSeconds: (rawValue: string) => number | null; + trySearxngSearch: (query: string, timeoutSec?: number) => Promise<{ + content?: string; + error?: string; + statusCode?: number; + success: boolean; + }>; }; function runHookWithMockedFetch(mode: 'success' | 'empty' | 'non-result' | 'failure') { @@ -154,6 +160,83 @@ describe('websearch-transformer hook helpers', () => { }); }); + it('queries SearXNG JSON endpoint and formats structured results', async () => { + const originalFetch = global.fetch; + const originalEnv = { + CCS_WEBSEARCH_SEARXNG_MAX_RESULTS: process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS, + CCS_WEBSEARCH_SEARXNG_URL: process.env.CCS_WEBSEARCH_SEARXNG_URL, + }; + + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com/'; + process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = '3'; + global.fetch = async () => + ({ + ok: true, + json: async () => ({ + results: [ + { + title: 'SearXNG Result', + url: 'https://example.com/searxng', + content: 'Result snippet', + }, + ], + }), + }) as Response; + + try { + const result = await hook.trySearxngSearch('btc price', 1); + expect(result.success).toBe(true); + expect(result.content).toContain('Provider: SearXNG'); + expect(result.content).toContain('URL: https://example.com/searxng'); + } finally { + global.fetch = originalFetch; + if (originalEnv.CCS_WEBSEARCH_SEARXNG_URL === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_URL; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_URL = originalEnv.CCS_WEBSEARCH_SEARXNG_URL; + } + + if (originalEnv.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = + originalEnv.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS; + } + } + }); + + it('marks SearXNG format-disabled 403 responses as non-retryable failures', async () => { + const originalFetch = global.fetch; + const originalUrl = process.env.CCS_WEBSEARCH_SEARXNG_URL; + + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com'; + global.fetch = async () => + ({ + ok: false, + status: 403, + headers: { get: () => null }, + text: async () => 'format=json disabled by this instance', + }) as Response; + + try { + const result = await hook.trySearxngSearch('btc price', 1); + expect(result.success).toBe(false); + expect(result.statusCode).toBe(403); + expect(result.error).toContain('format=json is disabled'); + expect(hook.classifyProviderFailure(result)).toMatchObject({ + kind: 'fail', + reason: 'non_retryable', + }); + } finally { + global.fetch = originalFetch; + if (originalUrl === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_URL; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_URL = originalUrl; + } + } + }); + it('extracts DuckDuckGo results and unwraps uddg redirect URLs', () => { const html = ` Example title @@ -281,6 +364,85 @@ describe('websearch-transformer hook helpers', () => { expect(output).not.toHaveProperty('additionalContext'); }); + it('falls back from SearXNG parse errors to DuckDuckGo in provider order', () => { + const tempDir = mkdtempSync(join(tmpdir(), 'websearch-hook-searxng-fallback-')); + const preloadPath = join(tempDir, 'mock-fetch.cjs'); + const requestLogPath = join(tempDir, 'requests.json'); + const html = ` + Duck title + Duck snippet + `.trim(); + + writeFileSync( + preloadPath, + ` +const fs = require('fs'); +const requestLogPath = ${JSON.stringify(requestLogPath)}; +const html = ${JSON.stringify(html)}; +function record(url) { + const requests = fs.existsSync(requestLogPath) + ? JSON.parse(fs.readFileSync(requestLogPath, 'utf8')) + : []; + requests.push(String(url)); + fs.writeFileSync(requestLogPath, JSON.stringify(requests), 'utf8'); +} +global.fetch = async (url) => { + const resolved = String(url); + record(resolved); + if (resolved.includes('/search?')) { + return { + ok: true, + json: async () => { + throw new Error('invalid json'); + }, + }; + } + return { + ok: true, + status: 200, + headers: { get: () => null }, + text: async () => html, + }; +}; + `.trimStart(), + 'utf8' + ); + + try { + const result = spawnSync('node', ['-r', preloadPath, hookPath], { + encoding: 'utf8', + input: JSON.stringify({ + tool_name: 'WebSearch', + tool_input: { query: 'btc price' }, + }), + env: { + ...process.env, + CCS_WEBSEARCH_ENABLED: '1', + CCS_WEBSEARCH_SKIP: '0', + CCS_WEBSEARCH_BRAVE: '0', + CCS_WEBSEARCH_DUCKDUCKGO: '1', + CCS_WEBSEARCH_EXA: '0', + CCS_WEBSEARCH_GEMINI: '0', + CCS_WEBSEARCH_GROK: '0', + CCS_WEBSEARCH_OPENCODE: '0', + CCS_WEBSEARCH_SEARXNG: '1', + CCS_WEBSEARCH_SEARXNG_URL: 'https://search.example.com', + CCS_WEBSEARCH_TAVILY: '0', + }, + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout.trim()) as HookOutput; + expect(output.hookSpecificOutput.additionalContext).toContain('Provider: DuckDuckGo'); + + const requests = JSON.parse(readFileSync(requestLogPath, 'utf8')) as string[]; + expect(requests[0]).toContain('search.example.com/search?'); + expect(requests[1]).toContain('duckduckgo.com'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('preserves genuine DuckDuckGo zero-result pages as successful empty searches', () => { const result = runHookWithMockedFetch('empty'); diff --git a/tests/unit/utils/websearch/status.test.ts b/tests/unit/utils/websearch/status.test.ts index b696278d..a23ea719 100644 --- a/tests/unit/utils/websearch/status.test.ts +++ b/tests/unit/utils/websearch/status.test.ts @@ -93,6 +93,28 @@ describe('websearch readiness', () => { expect(readiness.message).toContain('Exa'); }); + it('treats SearXNG as ready when enabled with a valid URL', () => { + const readiness = buildWebSearchReadiness(true, [ + provider({ + id: 'searxng', + name: 'SearXNG', + enabled: true, + available: true, + detail: 'Configured (5 results)', + }), + provider({ + id: 'duckduckgo', + name: 'DuckDuckGo', + enabled: false, + available: false, + detail: 'Built-in (5 results)', + }), + ]); + + expect(readiness.readiness).toBe('ready'); + expect(readiness.message).toContain('SearXNG'); + }); + it('treats cooled-down providers as temporarily unavailable in readiness status', () => { const tempHome = mkdtempSync(join(tmpdir(), 'websearch-status-cooldown-')); const statePath = join(tempHome, '.ccs', 'cache', 'websearch-provider-state.json'); @@ -122,8 +144,9 @@ describe('websearch readiness', () => { providers: { exa: { enabled: true, max_results: 5 }, tavily: { enabled: false, max_results: 5 }, - duckduckgo: { enabled: false, max_results: 5 }, brave: { enabled: false, max_results: 5 }, + searxng: { enabled: false, url: '', max_results: 5 }, + duckduckgo: { enabled: false, max_results: 5 }, gemini: { enabled: false }, grok: { enabled: false }, opencode: { enabled: false }, diff --git a/tests/unit/web-server/websearch-routes.test.ts b/tests/unit/web-server/websearch-routes.test.ts index f900abd0..18908aff 100644 --- a/tests/unit/web-server/websearch-routes.test.ts +++ b/tests/unit/web-server/websearch-routes.test.ts @@ -288,6 +288,33 @@ describe('websearch routes', () => { expect(config.global_env?.env.EXA_API_KEY).toBe('exa-secret-12345678'); }); + it('persists searxng provider settings and clamps max_results', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + enabled: true, + url: 'https://search.example.com', + max_results: 99, + }, + }, + }); + + expect(response.status).toBe(200); + const payload = await response.json(); + expect(payload.websearch.providers.searxng).toEqual({ + enabled: true, + url: 'https://search.example.com', + max_results: 10, + }); + + const config = loadOrCreateUnifiedConfig(); + expect(config.websearch?.providers?.searxng).toEqual({ + enabled: true, + url: 'https://search.example.com', + max_results: 10, + }); + }); + it('rejects non-object request bodies', async () => { const response = await putWebsearch('[]'); @@ -357,4 +384,34 @@ describe('websearch routes', () => { expect(await response.json()).toEqual({ error: invalidPayload.error }); } }); + + it('rejects non-string searxng url values', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + url: 123, + }, + }, + }); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: 'Invalid value for providers.searxng.url. Must be a string.', + }); + }); + + it('rejects non-number searxng max_results values', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + max_results: '5', + }, + }, + }); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: 'Invalid value for providers.searxng.max_results. Must be a number.', + }); + }); }); diff --git a/ui/src/pages/settings/sections/websearch/index.tsx b/ui/src/pages/settings/sections/websearch/index.tsx index b662d2e0..3f408749 100644 --- a/ui/src/pages/settings/sections/websearch/index.tsx +++ b/ui/src/pages/settings/sections/websearch/index.tsx @@ -28,8 +28,16 @@ import type { } from '../../types'; import { ProviderCard, type ProviderFieldConfig } from './provider-card'; -type ProviderId = 'exa' | 'tavily' | 'brave' | 'duckduckgo' | 'gemini' | 'opencode' | 'grok'; -type ProviderFieldKey = 'model' | 'timeout' | 'max_results'; +type ProviderId = + | 'exa' + | 'tavily' + | 'brave' + | 'searxng' + | 'duckduckgo' + | 'gemini' + | 'opencode' + | 'grok'; +type ProviderFieldKey = 'model' | 'timeout' | 'max_results' | 'url'; interface ProviderFieldDefinition { key: ProviderFieldKey; @@ -58,6 +66,7 @@ const CHAIN_STEPS = [ { id: 'exa', title: 'Exa', defaultEnabled: false }, { id: 'tavily', title: 'Tavily', defaultEnabled: false }, { id: 'brave', title: 'Brave', defaultEnabled: false }, + { id: 'searxng', title: 'SearXNG', defaultEnabled: false }, { id: 'duckduckgo', title: 'DuckDuckGo', defaultEnabled: true }, { id: 'legacy', title: 'Legacy CLI', defaultEnabled: false }, ] as const; @@ -130,6 +139,36 @@ const BACKEND_PROVIDERS: ProviderDefinition[] = [ }, ], }, + { + id: 'searxng', + title: 'SearXNG', + description: 'Configurable JSON backend for self-hosted or public SearXNG instances.', + badge: 'SELF-HOSTED', + badgeTone: 'cyan', + defaultEnabled: false, + fallbackDetail: 'Set a valid base URL', + footerNote: 'Runs after Brave and before DuckDuckGo when enabled and ready.', + fields: [ + { + key: 'url', + label: 'Base URL', + type: 'text', + placeholder: 'https://search.example.com', + helpText: 'Must expose /search with format=json enabled.', + defaultValue: '', + }, + { + key: 'max_results', + label: 'Max results', + type: 'number', + placeholder: '5', + helpText: 'Clamp between 1 and 10 results.', + defaultValue: 5, + min: 1, + max: 10, + }, + ], + }, { id: 'duckduckgo', title: 'DuckDuckGo', diff --git a/ui/src/pages/settings/types.ts b/ui/src/pages/settings/types.ts index 70af1d16..1cff3d33 100644 --- a/ui/src/pages/settings/types.ts +++ b/ui/src/pages/settings/types.ts @@ -9,6 +9,7 @@ import type { CliproxyServerConfig, RemoteProxyStatus } from '@/lib/api-client'; export interface ProviderConfig { enabled?: boolean; + url?: string; model?: string; timeout?: number; max_results?: number; @@ -28,8 +29,9 @@ export interface WebSearchApiKeyState { export interface WebSearchProvidersConfig { exa?: ProviderConfig; tavily?: ProviderConfig; - duckduckgo?: ProviderConfig; brave?: ProviderConfig; + searxng?: ProviderConfig; + duckduckgo?: ProviderConfig; gemini?: ProviderConfig; grok?: ProviderConfig; opencode?: ProviderConfig; @@ -48,7 +50,7 @@ export interface WebSearchSavePayload { } export interface CliStatus { - id: 'exa' | 'tavily' | 'duckduckgo' | 'brave' | 'gemini' | 'grok' | 'opencode'; + id: 'exa' | 'tavily' | 'brave' | 'searxng' | 'duckduckgo' | 'gemini' | 'grok' | 'opencode'; kind: 'backend' | 'legacy-cli'; name?: string; enabled: boolean;