From eec8c8b4a3288ec29e246aac58785f642618a48f Mon Sep 17 00:00:00 2001 From: buiducnhat Date: Sat, 11 Apr 2026 16:43:59 +0700 Subject: [PATCH 1/4] feat(websearch): add SearXNG provider support and configuration - Updated websearch.md to include SearXNG in the configuration guide. - Implemented SearXNG JSON API integration in websearch-transformer.cjs. - Enhanced unified-config-loader.ts to support SearXNG settings. - Defined SearXNG configuration types in unified-config-types.ts. - Updated hook-env.ts to manage SearXNG environment variables. - Modified status.ts to include SearXNG in web search provider checks. - Added SearXNG routes and validation in websearch-routes.ts. - Implemented SearXNG provider tests in websearch-transformer.test.ts. - Updated readiness tests to account for SearXNG in status.test.ts. - Enhanced websearch routes tests to validate SearXNG settings. - Added SearXNG configuration fields in the UI settings. --- docs/websearch.md | 73 ++++---- lib/hooks/websearch-transformer.cjs | 120 +++++++++++++ src/config/unified-config-loader.ts | 30 +++- src/config/unified-config-types.ts | 31 +++- src/utils/websearch/hook-env.ts | 6 + src/utils/websearch/status.ts | 55 ++++-- src/utils/websearch/types.ts | 6 +- src/web-server/health/websearch-checks.ts | 2 +- src/web-server/routes/websearch-routes.ts | 35 ++++ .../unit/hooks/websearch-transformer.test.ts | 162 ++++++++++++++++++ tests/unit/utils/websearch/status.test.ts | 25 ++- .../unit/web-server/websearch-routes.test.ts | 57 ++++++ .../settings/sections/websearch/index.tsx | 43 ++++- ui/src/pages/settings/types.ts | 6 +- 14 files changed, 588 insertions(+), 63 deletions(-) diff --git a/docs/websearch.md b/docs/websearch.md index 054c75d3..ded2e1b9 100644 --- a/docs/websearch.md +++ b/docs/websearch.md @@ -1,6 +1,6 @@ # WebSearch Configuration Guide -Last Updated: 2026-04-04 +Last Updated: 2026-04-11 CCS provides automatic web search for third-party profiles that cannot access Anthropic's native WebSearch API. @@ -17,29 +17,30 @@ Third-party profiles cannot execute Anthropic's server-side WebSearch because th ## Architecture ``` -┌──────────────────────────────────────────────────────────────┐ -│ Claude Code CLI │ -│ │ -│ Search Request │ -│ │ │ -│ ├── Native Claude Account? → Anthropic WebSearch API │ -│ │ │ -│ └── Third-party Profile? → native WebSearch disabled │ -│ │ │ -│ ├── CCS MCP tool when ready│ -│ │ ccs-websearch.WebSearch│ -│ │ │ │ -│ │ ├── 1. Exa │ -│ │ ├── 2. Tavily│ -│ │ ├── 3. Brave │ -│ │ ├── 4. DuckDuckGo│ -│ │ └── 5. Legacy CLI│ -│ │ fallback │ -│ │ (Gemini/ │ -│ │ OpenCode/ │ -│ │ Grok) │ -│ └── Bash/network fallback │ -└──────────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────────┐ +│ Claude Code CLI │ +│ │ +│ Search Request │ +│ │ │ +│ ├── Native Claude Account? → Anthropic WebSearch API │ +│ │ │ +│ └── Third-party Profile? → native WebSearch disabled │ +│ │ │ +│ ├── CCS MCP tool when ready │ +│ │ ccs-websearch.WebSearch │ +│ │ │ │ +│ │ ├── 1. Exa │ +│ │ ├── 2. Tavily │ +│ │ ├── 3. Brave │ +│ │ ├── 4. SearXNG │ +│ │ ├── 5. DuckDuckGo│ +│ │ └── 6. Legacy CLI│ +│ │ fallback │ +│ │ (Gemini/ │ +│ │ OpenCode/│ +│ │ Grok) │ +│ └── Bash/network fallback │ +└──────────────────────────────────────────────────────────────────┘ ``` ## Why This Changed @@ -66,8 +67,9 @@ That shared launch helper applies to normal third-party settings profiles, CLIPr |----------|------|-------|---------|-------| | Exa | HTTP API | `EXA_API_KEY` | No | High-quality API search with extracted content | | Tavily | HTTP API | `TAVILY_API_KEY` | No | Agent-oriented search API | -| DuckDuckGo | HTML fetch | None | Yes | Built-in zero-setup fallback | | Brave Search | HTTP API | `BRAVE_API_KEY` | No | Cleaner snippets and metadata | +| SearXNG | JSON API | `providers.searxng.url` | No | Self-hosted/public SearXNG backend via `/search?format=json` | +| DuckDuckGo | HTML fetch | None | Yes | Built-in zero-setup fallback | | Gemini CLI | Legacy CLI | `npm i -g @google/gemini-cli` | No | Optional compatibility fallback | | OpenCode | Legacy CLI | `curl -fsSL https://opencode.ai/install \| bash` | No | Optional compatibility fallback | | Grok CLI | Legacy CLI | `npm i -g @vibe-kit/grok-cli` + `GROK_API_KEY` | No | Optional compatibility fallback | @@ -78,7 +80,8 @@ That shared launch helper applies to normal third-party settings profiles, CLIPr Open `ccs config` → `Settings` → `WebSearch`. -- Enable Exa, Tavily, Brave, or DuckDuckGo in the backend chain +- Enable Exa, Tavily, Brave, SearXNG, or DuckDuckGo in the backend chain +- Configure SearXNG base URL (for example `https://search.example.com`) when SearXNG is enabled - Set or rotate Exa, Tavily, and Brave API keys directly inside each provider card - Saved keys are persisted in `global_env` and injected at runtime, so readiness updates from the same screen - Review whether any legacy fallback CLIs are still enabled in config @@ -97,12 +100,16 @@ websearch: tavily: enabled: false max_results: 5 - duckduckgo: - enabled: true - max_results: 5 brave: enabled: false max_results: 5 + searxng: + enabled: false + url: "" + max_results: 5 + duckduckgo: + enabled: true + max_results: 5 gemini: enabled: false model: gemini-2.5-flash @@ -125,6 +132,8 @@ Note: `enabled: false` stops provisioning the managed local `ccs-websearch.WebSe | `EXA_API_KEY` | Enables Exa when `providers.exa.enabled: true` | | `TAVILY_API_KEY` | Enables Tavily when `providers.tavily.enabled: true` | | `BRAVE_API_KEY` | Enables Brave Search when `providers.brave.enabled: true` | +| `CCS_WEBSEARCH_SEARXNG_URL` | Runtime URL used when `providers.searxng.enabled: true` | +| `CCS_WEBSEARCH_SEARXNG_MAX_RESULTS` | Optional runtime override for SearXNG result count (clamped 1..10) | | `GROK_API_KEY` | Required only for legacy Grok CLI fallback | | `CCS_WEBSEARCH_SKIP` | Disable the CCS local WebSearch runtime for the current process; third-party launches still keep native Anthropic `WebSearch` disabled | | `CCS_DEBUG` | Verbose WebSearch runtime logging | @@ -156,6 +165,12 @@ ccs config If the dashboard says the key is stored but still not ready, check whether `Settings -> Global Env` is disabled. WebSearch reuses that injection path for dashboard-managed keys. +### SearXNG is enabled but not ready + +1. Confirm the configured base URL is valid (for example `https://search.example.com`) +2. Confirm the instance exposes `GET /search?q=&format=json` +3. If the hook reports `SearXNG returned 403: format=json is disabled on this instance`, enable JSON format on that SearXNG deployment or switch to another backend + ### I still want Gemini/OpenCode/Grok fallback Those providers remain supported, but they are no longer the primary path. Enable them explicitly in `config.yaml` if you want them as last-resort fallback. diff --git a/lib/hooks/websearch-transformer.cjs b/lib/hooks/websearch-transformer.cjs index 79c5a433..d1fae51e 100644 --- a/lib/hooks/websearch-transformer.cjs +++ b/lib/hooks/websearch-transformer.cjs @@ -6,6 +6,7 @@ * - Exa Search API * - Tavily Search API * - Brave Search API + * - SearXNG JSON API * - DuckDuckGo HTML search * * Legacy compatibility fallback: @@ -369,6 +370,20 @@ function getResultCount(provider) { return Number.isFinite(parsed) && parsed > 0 ? Math.min(parsed, 10) : DEFAULT_RESULT_COUNT; } +function getSearxngBaseUrl() { + const raw = (process.env.CCS_WEBSEARCH_SEARXNG_URL || '').trim(); + if (!raw) { + return ''; + } + + try { + const parsed = new URL(raw); + return parsed.toString().replace(/\/+$/, ''); + } catch { + return ''; + } +} + function buildPrompt(providerId, query) { const config = PROVIDER_CONFIG[providerId]; const parts = [ @@ -569,6 +584,104 @@ async function tryBraveSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { } } +async function trySearxngSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { + const baseUrl = getSearxngBaseUrl(); + if (!baseUrl) { + return { success: false, error: 'SearXNG URL is not configured' }; + } + + const params = new URLSearchParams({ + q: query, + format: 'json', + }); + + try { + const response = await fetchWithTimeout( + `${baseUrl}/search?${params.toString()}`, + { + headers: { + Accept: 'application/json', + 'User-Agent': USER_AGENT, + }, + }, + timeoutSec * 1000 + ); + + if (!response.ok) { + const body = await response.text(); + if ( + response.status === 403 && + /format(?:=|\s*)json|json[^\n]{0,40}disabled|disabled[^\n]{0,40}json/i.test(body) + ) { + return { + success: false, + error: 'SearXNG returned 403: format=json is disabled on this instance', + statusCode: response.status, + retryAfterSec: parseRetryAfterSeconds(readHeaderValue(response.headers, 'retry-after')), + }; + } + + return { + success: false, + error: `SearXNG returned ${response.status}: ${body.slice(0, 160)}`, + statusCode: response.status, + retryAfterSec: parseRetryAfterSeconds(readHeaderValue(response.headers, 'retry-after')), + }; + } + + let body; + try { + body = await response.json(); + } catch { + return { + success: false, + error: 'SearXNG returned malformed JSON payload', + }; + } + + if (!body || typeof body !== 'object' || !Array.isArray(body.results)) { + return { + success: false, + error: 'SearXNG JSON response is missing results[]', + }; + } + + const count = getResultCount('searxng'); + const results = body.results.slice(0, count).map((entry) => { + const result = entry && typeof entry === 'object' ? entry : {}; + const url = typeof result.url === 'string' ? result.url : ''; + const titleSource = + typeof result.title === 'string' && result.title.trim().length > 0 + ? result.title + : url || 'Untitled'; + const descriptionSource = + typeof result.content === 'string' + ? result.content + : typeof result.description === 'string' + ? result.description + : typeof result.snippet === 'string' + ? result.snippet + : ''; + + return { + title: compactText(titleSource, 120), + url, + description: compactText(descriptionSource, 240), + }; + }); + + return { + success: true, + content: formatStructuredSearchResults(query, 'SearXNG', results), + }; + } catch (error) { + return { + success: false, + error: error.name === 'AbortError' ? 'SearXNG timed out' : error.message, + }; + } +} + async function tryExaSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { const apiKey = getProviderApiKey('exa'); if (!apiKey) { @@ -892,6 +1005,12 @@ function getConfiguredProviders() { available: () => isProviderEnabled('brave') && Boolean(getProviderApiKey('brave')), fn: tryBraveSearch, }, + { + name: 'SearXNG', + id: 'searxng', + available: () => isProviderEnabled('searxng') && Boolean(getSearxngBaseUrl()), + fn: trySearxngSearch, + }, { name: 'DuckDuckGo', id: 'duckduckgo', @@ -1280,4 +1399,5 @@ module.exports = { tryTavilySearch, tryDuckDuckGoSearch, tryBraveSearch, + trySearxngSearch, }; diff --git a/src/config/unified-config-loader.ts b/src/config/unified-config-loader.ts index b0c3ecfd..e7a6a0bc 100644 --- a/src/config/unified-config-loader.ts +++ b/src/config/unified-config-loader.ts @@ -407,14 +407,19 @@ function mergeWithDefaults(partial: Partial): UnifiedConfig { enabled: partial.websearch?.providers?.tavily?.enabled ?? false, max_results: partial.websearch?.providers?.tavily?.max_results ?? 5, }, - duckduckgo: { - enabled: partial.websearch?.providers?.duckduckgo?.enabled ?? true, - max_results: partial.websearch?.providers?.duckduckgo?.max_results ?? 5, - }, brave: { enabled: partial.websearch?.providers?.brave?.enabled ?? false, max_results: partial.websearch?.providers?.brave?.max_results ?? 5, }, + searxng: { + enabled: partial.websearch?.providers?.searxng?.enabled ?? false, + url: partial.websearch?.providers?.searxng?.url ?? '', + max_results: partial.websearch?.providers?.searxng?.max_results ?? 5, + }, + duckduckgo: { + enabled: partial.websearch?.providers?.duckduckgo?.enabled ?? true, + max_results: partial.websearch?.providers?.duckduckgo?.max_results ?? 5, + }, gemini: { enabled: partial.websearch?.providers?.gemini?.enabled ?? @@ -1093,15 +1098,16 @@ export interface GeminiWebSearchInfo { /** * Get websearch configuration. * Returns defaults if not configured. - * Supports Gemini CLI, OpenCode, and Grok CLI providers. + * Supports deterministic providers and optional Gemini/OpenCode/Grok CLI fallbacks. */ export function getWebSearchConfig(): { enabled: boolean; providers?: { exa?: { enabled?: boolean; max_results?: number }; tavily?: { enabled?: boolean; max_results?: number }; - duckduckgo?: { enabled?: boolean; max_results?: number }; brave?: { enabled?: boolean; max_results?: number }; + searxng?: { enabled?: boolean; url?: string; max_results?: number }; + duckduckgo?: { enabled?: boolean; max_results?: number }; gemini?: GeminiWebSearchInfo; opencode?: { enabled?: boolean; model?: string; timeout?: number }; grok?: { enabled?: boolean; timeout?: number }; @@ -1132,6 +1138,12 @@ export function getWebSearchConfig(): { max_results: config.websearch?.providers?.brave?.max_results ?? 5, }; + const searxngConfig = { + enabled: config.websearch?.providers?.searxng?.enabled ?? false, + url: config.websearch?.providers?.searxng?.url ?? '', + max_results: config.websearch?.providers?.searxng?.max_results ?? 5, + }; + const geminiConfig: GeminiWebSearchInfo = { enabled: config.websearch?.providers?.gemini?.enabled ?? config.websearch?.gemini?.enabled ?? false, @@ -1155,8 +1167,9 @@ export function getWebSearchConfig(): { const anyProviderEnabled = exaConfig.enabled || tavilyConfig.enabled || - duckDuckGoConfig.enabled || braveConfig.enabled || + searxngConfig.enabled || + duckDuckGoConfig.enabled || geminiConfig.enabled || opencodeConfig.enabled || grokConfig.enabled; @@ -1167,8 +1180,9 @@ export function getWebSearchConfig(): { providers: { exa: exaConfig, tavily: tavilyConfig, - duckduckgo: duckDuckGoConfig, brave: braveConfig, + searxng: searxngConfig, + duckduckgo: duckDuckGoConfig, gemini: geminiConfig, opencode: opencodeConfig, grok: grokConfig, diff --git a/src/config/unified-config-types.ts b/src/config/unified-config-types.ts index 5ce9c304..3ab0f545 100644 --- a/src/config/unified-config-types.ts +++ b/src/config/unified-config-types.ts @@ -316,6 +316,18 @@ export interface TavilyWebSearchConfig { max_results?: number; } +/** + * SearXNG WebSearch configuration. + */ +export interface SearxngWebSearchConfig { + /** Enable SearXNG JSON search backend (default: false) */ + enabled?: boolean; + /** Base SearXNG URL, e.g. https://search.example.com (default: '') */ + url?: string; + /** Number of results to fetch (default: 5) */ + max_results?: number; +} + /** * Gemini CLI WebSearch configuration. */ @@ -359,10 +371,12 @@ export interface WebSearchProvidersConfig { exa?: ExaWebSearchConfig; /** Tavily Search API - API-backed search optimized for agent/tool usage */ tavily?: TavilyWebSearchConfig; - /** DuckDuckGo HTML search - zero setup default backend */ - duckduckgo?: DuckDuckGoWebSearchConfig; /** Brave Search API - higher quality results when BRAVE_API_KEY is set */ brave?: BraveWebSearchConfig; + /** SearXNG JSON search - self-hosted or public instance backend */ + searxng?: SearxngWebSearchConfig; + /** DuckDuckGo HTML search - zero setup default backend */ + duckduckgo?: DuckDuckGoWebSearchConfig; /** Gemini CLI - optional legacy LLM fallback */ gemini?: GeminiWebSearchConfig; /** Grok CLI - optional legacy LLM fallback */ @@ -961,14 +975,19 @@ export function createEmptyUnifiedConfig(): UnifiedConfig { enabled: false, max_results: 5, }, - duckduckgo: { - enabled: true, - max_results: 5, - }, brave: { enabled: false, max_results: 5, }, + searxng: { + enabled: false, + url: '', + max_results: 5, + }, + duckduckgo: { + enabled: true, + max_results: 5, + }, gemini: { enabled: false, model: 'gemini-2.5-flash', diff --git a/src/utils/websearch/hook-env.ts b/src/utils/websearch/hook-env.ts index d5e69018..d2b5f85c 100644 --- a/src/utils/websearch/hook-env.ts +++ b/src/utils/websearch/hook-env.ts @@ -61,6 +61,12 @@ export function getWebSearchHookEnv(): Record { env.CCS_WEBSEARCH_BRAVE_MAX_RESULTS = String(wsConfig.providers.brave.max_results || 5); } + if (wsConfig.providers?.searxng?.enabled && wsConfig.providers.searxng.url?.trim()) { + env.CCS_WEBSEARCH_SEARXNG = '1'; + env.CCS_WEBSEARCH_SEARXNG_URL = wsConfig.providers.searxng.url.trim(); + env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = String(wsConfig.providers.searxng.max_results || 5); + } + if (wsConfig.providers?.gemini?.enabled) { env.CCS_WEBSEARCH_GEMINI = '1'; if (wsConfig.providers.gemini.model) { diff --git a/src/utils/websearch/status.ts b/src/utils/websearch/status.ts index ecb5ce94..e0b9c36d 100644 --- a/src/utils/websearch/status.ts +++ b/src/utils/websearch/status.ts @@ -28,6 +28,20 @@ function hasEnvValue(name: string): boolean { return (process.env[name] || '').trim().length > 0; } +function hasValidSearxngUrl(url: string | undefined): boolean { + const normalized = String(url || '').trim(); + if (!normalized) { + return false; + } + + try { + const parsed = new URL(normalized); + return Boolean(parsed.protocol && parsed.host); + } catch { + return false; + } +} + function getProviderStatePath(): string { return join(getCcsDir(), 'cache', PROVIDER_STATE_FILE); } @@ -209,18 +223,6 @@ export function getWebSearchCliProviders(): WebSearchCliInfo[] { ? 'Stored in dashboard, but Global Env is disabled' : 'Set TAVILY_API_KEY', }, - { - id: 'duckduckgo', - kind: 'backend', - name: 'DuckDuckGo', - enabled: wsConfig.providers?.duckduckgo?.enabled ?? true, - available: wsConfig.providers?.duckduckgo?.enabled ?? true, - version: null, - docsUrl: 'https://duckduckgo.com', - requiresApiKey: false, - description: 'Default built-in HTML search backend. Zero setup.', - detail: `Built-in (${wsConfig.providers?.duckduckgo?.max_results ?? 5} results)`, - }, { id: 'brave', kind: 'backend', @@ -238,6 +240,34 @@ export function getWebSearchCliProviders(): WebSearchCliInfo[] { ? 'Stored in dashboard, but Global Env is disabled' : 'Set BRAVE_API_KEY', }, + { + id: 'searxng', + kind: 'backend', + name: 'SearXNG', + enabled: wsConfig.providers?.searxng?.enabled ?? false, + available: + (wsConfig.providers?.searxng?.enabled ?? false) && + hasValidSearxngUrl(wsConfig.providers?.searxng?.url), + version: null, + docsUrl: 'https://docs.searxng.org/dev/search_api.html', + requiresApiKey: false, + description: 'Configurable SearXNG JSON backend for self-hosted or public instances.', + detail: hasValidSearxngUrl(wsConfig.providers?.searxng?.url) + ? `Configured (${wsConfig.providers?.searxng?.max_results ?? 5} results)` + : 'Set a valid SearXNG base URL', + }, + { + id: 'duckduckgo', + kind: 'backend', + name: 'DuckDuckGo', + enabled: wsConfig.providers?.duckduckgo?.enabled ?? true, + available: wsConfig.providers?.duckduckgo?.enabled ?? true, + version: null, + docsUrl: 'https://duckduckgo.com', + requiresApiKey: false, + description: 'Default built-in HTML search backend. Zero setup.', + detail: `Built-in (${wsConfig.providers?.duckduckgo?.max_results ?? 5} results)`, + }, ]; return [...providers, ...getLegacyProviderStatuses()].map((provider) => @@ -263,6 +293,7 @@ export function getCliInstallHints(): string[] { return [ 'WebSearch: no ready providers', ' Enable DuckDuckGo in Settings > WebSearch for zero-setup search', + ' Or enable SearXNG and set a valid base URL (must support /search?format=json)', ' Or export EXA_API_KEY, TAVILY_API_KEY, or BRAVE_API_KEY for API-backed search', ' Optional legacy fallback: npm i -g @google/gemini-cli', ]; diff --git a/src/utils/websearch/types.ts b/src/utils/websearch/types.ts index 8d737617..a6ca29df 100644 --- a/src/utils/websearch/types.ts +++ b/src/utils/websearch/types.ts @@ -37,8 +37,9 @@ export type WebSearchReadiness = 'ready' | 'needs_setup' | 'unavailable'; export type WebSearchProviderId = | 'exa' | 'tavily' - | 'duckduckgo' | 'brave' + | 'searxng' + | 'duckduckgo' | 'gemini' | 'grok' | 'opencode'; @@ -107,8 +108,9 @@ export interface WebSearchConfig { providers?: { exa?: WebSearchProviderConfig; tavily?: WebSearchProviderConfig; - duckduckgo?: WebSearchProviderConfig; brave?: WebSearchProviderConfig; + searxng?: WebSearchProviderConfig; + duckduckgo?: WebSearchProviderConfig; gemini?: WebSearchProviderConfig; opencode?: WebSearchProviderConfig; grok?: WebSearchProviderConfig; diff --git a/src/web-server/health/websearch-checks.ts b/src/web-server/health/websearch-checks.ts index 3a4faa7d..b75abb1d 100644 --- a/src/web-server/health/websearch-checks.ts +++ b/src/web-server/health/websearch-checks.ts @@ -42,7 +42,7 @@ export function checkWebSearchClis(): HealthCheck[] { name: 'WebSearch Status', status: 'warning', message: 'No ready provider', - fix: 'Enable DuckDuckGo or set EXA_API_KEY, TAVILY_API_KEY, or BRAVE_API_KEY', + fix: 'Enable DuckDuckGo, configure SearXNG URL, or set EXA_API_KEY/TAVILY_API_KEY/BRAVE_API_KEY', details: 'Third-party profiles need a local WebSearch backend.', }); } diff --git a/src/web-server/routes/websearch-routes.ts b/src/web-server/routes/websearch-routes.ts index fc550868..d5bc6432 100644 --- a/src/web-server/routes/websearch-routes.ts +++ b/src/web-server/routes/websearch-routes.ts @@ -17,6 +17,8 @@ import { requireLocalAccessWhenAuthDisabled } from '../middleware/auth-middlewar const router = Router(); const WEBSEARCH_LOCAL_ACCESS_ERROR = 'WebSearch endpoints require localhost access when dashboard auth is disabled.'; +const DEFAULT_WEBSEARCH_MAX_RESULTS = 5; +const MAX_WEBSEARCH_MAX_RESULTS = 10; type WebSearchApiKeyUpdates = Partial>; @@ -28,6 +30,12 @@ function isWebSearchApiKeyProviderId(value: string): value is WebSearchApiKeyPro return Object.prototype.hasOwnProperty.call(WEBSEARCH_API_KEY_PROVIDERS, value); } +function clampWebSearchMaxResults(value: number | undefined, fallback: number): number { + const candidate = Number.isFinite(value) ? (value as number) : fallback; + const normalized = Number.isFinite(candidate) ? candidate : DEFAULT_WEBSEARCH_MAX_RESULTS; + return Math.max(1, Math.min(MAX_WEBSEARCH_MAX_RESULTS, Math.floor(normalized))); +} + router.use((req: Request, res: Response, next) => { if (requireLocalAccessWhenAuthDisabled(req, res, WEBSEARCH_LOCAL_ACCESS_ERROR)) { next(); @@ -82,6 +90,22 @@ router.put('/', (req: Request, res: Response): void => { return; } + if (providers?.searxng?.url !== undefined && typeof providers.searxng.url !== 'string') { + res.status(400).json({ error: 'Invalid value for providers.searxng.url. Must be a string.' }); + return; + } + + if ( + providers?.searxng?.max_results !== undefined && + (typeof providers.searxng.max_results !== 'number' || + !Number.isFinite(providers.searxng.max_results)) + ) { + res.status(400).json({ + error: 'Invalid value for providers.searxng.max_results. Must be a number.', + }); + return; + } + if ( apiKeys !== undefined && (apiKeys === null || Array.isArray(apiKeys) || typeof apiKeys !== 'object') @@ -144,6 +168,17 @@ router.put('/', (req: Request, res: Response): void => { config.websearch?.providers?.brave?.max_results ?? 5, }, + searxng: { + enabled: + providers.searxng?.enabled ?? + config.websearch?.providers?.searxng?.enabled ?? + false, + url: providers.searxng?.url ?? config.websearch?.providers?.searxng?.url ?? '', + max_results: clampWebSearchMaxResults( + providers.searxng?.max_results, + config.websearch?.providers?.searxng?.max_results ?? DEFAULT_WEBSEARCH_MAX_RESULTS + ), + }, gemini: { enabled: providers.gemini?.enabled ?? diff --git a/tests/unit/hooks/websearch-transformer.test.ts b/tests/unit/hooks/websearch-transformer.test.ts index 80bb74d9..2f005e0c 100644 --- a/tests/unit/hooks/websearch-transformer.test.ts +++ b/tests/unit/hooks/websearch-transformer.test.ts @@ -56,6 +56,12 @@ const hook = require('../../../lib/hooks/websearch-transformer.cjs') as { results: Array<{ title: string; url: string; description: string }> ) => string; parseRetryAfterSeconds: (rawValue: string) => number | null; + trySearxngSearch: (query: string, timeoutSec?: number) => Promise<{ + content?: string; + error?: string; + statusCode?: number; + success: boolean; + }>; }; function runHookWithMockedFetch(mode: 'success' | 'empty' | 'non-result' | 'failure') { @@ -154,6 +160,83 @@ describe('websearch-transformer hook helpers', () => { }); }); + it('queries SearXNG JSON endpoint and formats structured results', async () => { + const originalFetch = global.fetch; + const originalEnv = { + CCS_WEBSEARCH_SEARXNG_MAX_RESULTS: process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS, + CCS_WEBSEARCH_SEARXNG_URL: process.env.CCS_WEBSEARCH_SEARXNG_URL, + }; + + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com/'; + process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = '3'; + global.fetch = async () => + ({ + ok: true, + json: async () => ({ + results: [ + { + title: 'SearXNG Result', + url: 'https://example.com/searxng', + content: 'Result snippet', + }, + ], + }), + }) as Response; + + try { + const result = await hook.trySearxngSearch('btc price', 1); + expect(result.success).toBe(true); + expect(result.content).toContain('Provider: SearXNG'); + expect(result.content).toContain('URL: https://example.com/searxng'); + } finally { + global.fetch = originalFetch; + if (originalEnv.CCS_WEBSEARCH_SEARXNG_URL === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_URL; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_URL = originalEnv.CCS_WEBSEARCH_SEARXNG_URL; + } + + if (originalEnv.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = + originalEnv.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS; + } + } + }); + + it('marks SearXNG format-disabled 403 responses as non-retryable failures', async () => { + const originalFetch = global.fetch; + const originalUrl = process.env.CCS_WEBSEARCH_SEARXNG_URL; + + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com'; + global.fetch = async () => + ({ + ok: false, + status: 403, + headers: { get: () => null }, + text: async () => 'format=json disabled by this instance', + }) as Response; + + try { + const result = await hook.trySearxngSearch('btc price', 1); + expect(result.success).toBe(false); + expect(result.statusCode).toBe(403); + expect(result.error).toContain('format=json is disabled'); + expect(hook.classifyProviderFailure(result)).toMatchObject({ + kind: 'fail', + reason: 'non_retryable', + }); + } finally { + global.fetch = originalFetch; + if (originalUrl === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_URL; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_URL = originalUrl; + } + } + }); + it('extracts DuckDuckGo results and unwraps uddg redirect URLs', () => { const html = ` Example title @@ -281,6 +364,85 @@ describe('websearch-transformer hook helpers', () => { expect(output).not.toHaveProperty('additionalContext'); }); + it('falls back from SearXNG parse errors to DuckDuckGo in provider order', () => { + const tempDir = mkdtempSync(join(tmpdir(), 'websearch-hook-searxng-fallback-')); + const preloadPath = join(tempDir, 'mock-fetch.cjs'); + const requestLogPath = join(tempDir, 'requests.json'); + const html = ` + Duck title + Duck snippet + `.trim(); + + writeFileSync( + preloadPath, + ` +const fs = require('fs'); +const requestLogPath = ${JSON.stringify(requestLogPath)}; +const html = ${JSON.stringify(html)}; +function record(url) { + const requests = fs.existsSync(requestLogPath) + ? JSON.parse(fs.readFileSync(requestLogPath, 'utf8')) + : []; + requests.push(String(url)); + fs.writeFileSync(requestLogPath, JSON.stringify(requests), 'utf8'); +} +global.fetch = async (url) => { + const resolved = String(url); + record(resolved); + if (resolved.includes('/search?')) { + return { + ok: true, + json: async () => { + throw new Error('invalid json'); + }, + }; + } + return { + ok: true, + status: 200, + headers: { get: () => null }, + text: async () => html, + }; +}; + `.trimStart(), + 'utf8' + ); + + try { + const result = spawnSync('node', ['-r', preloadPath, hookPath], { + encoding: 'utf8', + input: JSON.stringify({ + tool_name: 'WebSearch', + tool_input: { query: 'btc price' }, + }), + env: { + ...process.env, + CCS_WEBSEARCH_ENABLED: '1', + CCS_WEBSEARCH_SKIP: '0', + CCS_WEBSEARCH_BRAVE: '0', + CCS_WEBSEARCH_DUCKDUCKGO: '1', + CCS_WEBSEARCH_EXA: '0', + CCS_WEBSEARCH_GEMINI: '0', + CCS_WEBSEARCH_GROK: '0', + CCS_WEBSEARCH_OPENCODE: '0', + CCS_WEBSEARCH_SEARXNG: '1', + CCS_WEBSEARCH_SEARXNG_URL: 'https://search.example.com', + CCS_WEBSEARCH_TAVILY: '0', + }, + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout.trim()) as HookOutput; + expect(output.hookSpecificOutput.additionalContext).toContain('Provider: DuckDuckGo'); + + const requests = JSON.parse(readFileSync(requestLogPath, 'utf8')) as string[]; + expect(requests[0]).toContain('search.example.com/search?'); + expect(requests[1]).toContain('duckduckgo.com'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('preserves genuine DuckDuckGo zero-result pages as successful empty searches', () => { const result = runHookWithMockedFetch('empty'); diff --git a/tests/unit/utils/websearch/status.test.ts b/tests/unit/utils/websearch/status.test.ts index b696278d..a23ea719 100644 --- a/tests/unit/utils/websearch/status.test.ts +++ b/tests/unit/utils/websearch/status.test.ts @@ -93,6 +93,28 @@ describe('websearch readiness', () => { expect(readiness.message).toContain('Exa'); }); + it('treats SearXNG as ready when enabled with a valid URL', () => { + const readiness = buildWebSearchReadiness(true, [ + provider({ + id: 'searxng', + name: 'SearXNG', + enabled: true, + available: true, + detail: 'Configured (5 results)', + }), + provider({ + id: 'duckduckgo', + name: 'DuckDuckGo', + enabled: false, + available: false, + detail: 'Built-in (5 results)', + }), + ]); + + expect(readiness.readiness).toBe('ready'); + expect(readiness.message).toContain('SearXNG'); + }); + it('treats cooled-down providers as temporarily unavailable in readiness status', () => { const tempHome = mkdtempSync(join(tmpdir(), 'websearch-status-cooldown-')); const statePath = join(tempHome, '.ccs', 'cache', 'websearch-provider-state.json'); @@ -122,8 +144,9 @@ describe('websearch readiness', () => { providers: { exa: { enabled: true, max_results: 5 }, tavily: { enabled: false, max_results: 5 }, - duckduckgo: { enabled: false, max_results: 5 }, brave: { enabled: false, max_results: 5 }, + searxng: { enabled: false, url: '', max_results: 5 }, + duckduckgo: { enabled: false, max_results: 5 }, gemini: { enabled: false }, grok: { enabled: false }, opencode: { enabled: false }, diff --git a/tests/unit/web-server/websearch-routes.test.ts b/tests/unit/web-server/websearch-routes.test.ts index f900abd0..18908aff 100644 --- a/tests/unit/web-server/websearch-routes.test.ts +++ b/tests/unit/web-server/websearch-routes.test.ts @@ -288,6 +288,33 @@ describe('websearch routes', () => { expect(config.global_env?.env.EXA_API_KEY).toBe('exa-secret-12345678'); }); + it('persists searxng provider settings and clamps max_results', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + enabled: true, + url: 'https://search.example.com', + max_results: 99, + }, + }, + }); + + expect(response.status).toBe(200); + const payload = await response.json(); + expect(payload.websearch.providers.searxng).toEqual({ + enabled: true, + url: 'https://search.example.com', + max_results: 10, + }); + + const config = loadOrCreateUnifiedConfig(); + expect(config.websearch?.providers?.searxng).toEqual({ + enabled: true, + url: 'https://search.example.com', + max_results: 10, + }); + }); + it('rejects non-object request bodies', async () => { const response = await putWebsearch('[]'); @@ -357,4 +384,34 @@ describe('websearch routes', () => { expect(await response.json()).toEqual({ error: invalidPayload.error }); } }); + + it('rejects non-string searxng url values', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + url: 123, + }, + }, + }); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: 'Invalid value for providers.searxng.url. Must be a string.', + }); + }); + + it('rejects non-number searxng max_results values', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + max_results: '5', + }, + }, + }); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: 'Invalid value for providers.searxng.max_results. Must be a number.', + }); + }); }); diff --git a/ui/src/pages/settings/sections/websearch/index.tsx b/ui/src/pages/settings/sections/websearch/index.tsx index b662d2e0..3f408749 100644 --- a/ui/src/pages/settings/sections/websearch/index.tsx +++ b/ui/src/pages/settings/sections/websearch/index.tsx @@ -28,8 +28,16 @@ import type { } from '../../types'; import { ProviderCard, type ProviderFieldConfig } from './provider-card'; -type ProviderId = 'exa' | 'tavily' | 'brave' | 'duckduckgo' | 'gemini' | 'opencode' | 'grok'; -type ProviderFieldKey = 'model' | 'timeout' | 'max_results'; +type ProviderId = + | 'exa' + | 'tavily' + | 'brave' + | 'searxng' + | 'duckduckgo' + | 'gemini' + | 'opencode' + | 'grok'; +type ProviderFieldKey = 'model' | 'timeout' | 'max_results' | 'url'; interface ProviderFieldDefinition { key: ProviderFieldKey; @@ -58,6 +66,7 @@ const CHAIN_STEPS = [ { id: 'exa', title: 'Exa', defaultEnabled: false }, { id: 'tavily', title: 'Tavily', defaultEnabled: false }, { id: 'brave', title: 'Brave', defaultEnabled: false }, + { id: 'searxng', title: 'SearXNG', defaultEnabled: false }, { id: 'duckduckgo', title: 'DuckDuckGo', defaultEnabled: true }, { id: 'legacy', title: 'Legacy CLI', defaultEnabled: false }, ] as const; @@ -130,6 +139,36 @@ const BACKEND_PROVIDERS: ProviderDefinition[] = [ }, ], }, + { + id: 'searxng', + title: 'SearXNG', + description: 'Configurable JSON backend for self-hosted or public SearXNG instances.', + badge: 'SELF-HOSTED', + badgeTone: 'cyan', + defaultEnabled: false, + fallbackDetail: 'Set a valid base URL', + footerNote: 'Runs after Brave and before DuckDuckGo when enabled and ready.', + fields: [ + { + key: 'url', + label: 'Base URL', + type: 'text', + placeholder: 'https://search.example.com', + helpText: 'Must expose /search with format=json enabled.', + defaultValue: '', + }, + { + key: 'max_results', + label: 'Max results', + type: 'number', + placeholder: '5', + helpText: 'Clamp between 1 and 10 results.', + defaultValue: 5, + min: 1, + max: 10, + }, + ], + }, { id: 'duckduckgo', title: 'DuckDuckGo', diff --git a/ui/src/pages/settings/types.ts b/ui/src/pages/settings/types.ts index 70af1d16..1cff3d33 100644 --- a/ui/src/pages/settings/types.ts +++ b/ui/src/pages/settings/types.ts @@ -9,6 +9,7 @@ import type { CliproxyServerConfig, RemoteProxyStatus } from '@/lib/api-client'; export interface ProviderConfig { enabled?: boolean; + url?: string; model?: string; timeout?: number; max_results?: number; @@ -28,8 +29,9 @@ export interface WebSearchApiKeyState { export interface WebSearchProvidersConfig { exa?: ProviderConfig; tavily?: ProviderConfig; - duckduckgo?: ProviderConfig; brave?: ProviderConfig; + searxng?: ProviderConfig; + duckduckgo?: ProviderConfig; gemini?: ProviderConfig; grok?: ProviderConfig; opencode?: ProviderConfig; @@ -48,7 +50,7 @@ export interface WebSearchSavePayload { } export interface CliStatus { - id: 'exa' | 'tavily' | 'duckduckgo' | 'brave' | 'gemini' | 'grok' | 'opencode'; + id: 'exa' | 'tavily' | 'brave' | 'searxng' | 'duckduckgo' | 'gemini' | 'grok' | 'opencode'; kind: 'backend' | 'legacy-cli'; name?: string; enabled: boolean; From 36d828f99c4a8dae40286b72709487ae8cb49f2c Mon Sep 17 00:00:00 2001 From: buiducnhat Date: Sat, 11 Apr 2026 19:35:41 +0700 Subject: [PATCH 2/4] feat(websearch): add url property to WebSearchProviderConfig interface --- src/utils/websearch/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/websearch/types.ts b/src/utils/websearch/types.ts index a6ca29df..4b6b3bdf 100644 --- a/src/utils/websearch/types.ts +++ b/src/utils/websearch/types.ts @@ -98,6 +98,7 @@ export interface WebSearchProviderConfig { model?: string; timeout?: number; max_results?: number; + url?: string; } /** From 8b553c35c150e6411e6af0ce0a0a27305b516e6c Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Sun, 12 Apr 2026 06:31:20 -0400 Subject: [PATCH 3/4] fix(websearch): harden searxng url handling --- docs/websearch.md | 3 +- lib/hooks/websearch-transformer.cjs | 18 ++- lib/mcp/ccs-websearch-server.cjs | 2 +- src/config/unified-config-loader.ts | 5 +- src/utils/websearch/hook-env.ts | 19 ++- src/utils/websearch/status.ts | 14 +-- src/utils/websearch/types.ts | 42 +++++++ src/web-server/routes/websearch-routes.ts | 19 ++- .../hooks/ccs-websearch-mcp-server.test.ts | 14 +-- .../unit/hooks/websearch-transformer.test.ts | 49 +++++++- .../utils/claudecode-env-stripping.test.ts | 39 +++++- tests/unit/utils/websearch/status.test.ts | 54 +++++++++ .../unit/web-server/websearch-routes.test.ts | 112 ++++++++++++++++++ .../settings/sections/websearch/index.tsx | 3 +- 14 files changed, 359 insertions(+), 34 deletions(-) diff --git a/docs/websearch.md b/docs/websearch.md index ded2e1b9..29dcf740 100644 --- a/docs/websearch.md +++ b/docs/websearch.md @@ -81,7 +81,8 @@ That shared launch helper applies to normal third-party settings profiles, CLIPr Open `ccs config` → `Settings` → `WebSearch`. - Enable Exa, Tavily, Brave, SearXNG, or DuckDuckGo in the backend chain -- Configure SearXNG base URL (for example `https://search.example.com`) when SearXNG is enabled +- Configure the SearXNG base URL (for example `https://search.example.com`) when SearXNG is enabled + Do not include `/search`, embedded credentials, query parameters, or URL fragments. CCS appends `/search?format=json`. - Set or rotate Exa, Tavily, and Brave API keys directly inside each provider card - Saved keys are persisted in `global_env` and injected at runtime, so readiness updates from the same screen - Review whether any legacy fallback CLIs are still enabled in config diff --git a/lib/hooks/websearch-transformer.cjs b/lib/hooks/websearch-transformer.cjs index d1fae51e..6da24c3e 100644 --- a/lib/hooks/websearch-transformer.cjs +++ b/lib/hooks/websearch-transformer.cjs @@ -378,6 +378,22 @@ function getSearxngBaseUrl() { try { const parsed = new URL(raw); + if (!['http:', 'https:'].includes(parsed.protocol) || parsed.search || parsed.hash) { + return ''; + } + + if (parsed.username || parsed.password) { + return ''; + } + + let pathname = parsed.pathname.replace(/\/+$/, ''); + if (pathname.toLowerCase().endsWith('/search')) { + pathname = pathname.slice(0, -'/search'.length); + } + + parsed.pathname = pathname || '/'; + parsed.search = ''; + parsed.hash = ''; return parsed.toString().replace(/\/+$/, ''); } catch { return ''; @@ -587,7 +603,7 @@ async function tryBraveSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { async function trySearxngSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { const baseUrl = getSearxngBaseUrl(); if (!baseUrl) { - return { success: false, error: 'SearXNG URL is not configured' }; + return { success: false, error: 'SearXNG URL is invalid or not configured' }; } const params = new URLSearchParams({ diff --git a/lib/mcp/ccs-websearch-server.cjs b/lib/mcp/ccs-websearch-server.cjs index 13880600..1b7d207f 100644 --- a/lib/mcp/ccs-websearch-server.cjs +++ b/lib/mcp/ccs-websearch-server.cjs @@ -16,7 +16,7 @@ const SERVER_VERSION = '1.0.0'; const TOOL_NAME = 'WebSearch'; const TOOL_ALIASES = ['search']; const TOOL_DESCRIPTION = - 'Third-party WebSearch replacement for CCS-managed Claude launches. Use this instead of Bash/curl/http fetches for web lookups. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.'; + 'Third-party WebSearch replacement for CCS-managed Claude launches. Use this instead of Bash/curl/http fetches for web lookups. Provider order: Exa, Tavily, Brave Search, SearXNG, DuckDuckGo, then optional legacy CLI fallback.'; function isSupportedToolName(name) { return name === TOOL_NAME || TOOL_ALIASES.includes(name); diff --git a/src/config/unified-config-loader.ts b/src/config/unified-config-loader.ts index e7a6a0bc..5865ba5d 100644 --- a/src/config/unified-config-loader.ts +++ b/src/config/unified-config-loader.ts @@ -47,6 +47,7 @@ import { resolveLegacyDiscordSelection, } from '../channels/official-channels-runtime'; import { canonicalizeImageAnalysisConfig } from '../utils/hooks/image-analysis-backend-resolver'; +import { normalizeSearxngBaseUrl } from '../utils/websearch/types'; const CONFIG_YAML = 'config.yaml'; const CONFIG_JSON = 'config.json'; @@ -413,7 +414,7 @@ function mergeWithDefaults(partial: Partial): UnifiedConfig { }, searxng: { enabled: partial.websearch?.providers?.searxng?.enabled ?? false, - url: partial.websearch?.providers?.searxng?.url ?? '', + url: normalizeSearxngBaseUrl(partial.websearch?.providers?.searxng?.url) ?? '', max_results: partial.websearch?.providers?.searxng?.max_results ?? 5, }, duckduckgo: { @@ -1140,7 +1141,7 @@ export function getWebSearchConfig(): { const searxngConfig = { enabled: config.websearch?.providers?.searxng?.enabled ?? false, - url: config.websearch?.providers?.searxng?.url ?? '', + url: normalizeSearxngBaseUrl(config.websearch?.providers?.searxng?.url) ?? '', max_results: config.websearch?.providers?.searxng?.max_results ?? 5, }; diff --git a/src/utils/websearch/hook-env.ts b/src/utils/websearch/hook-env.ts index d2b5f85c..2e79e649 100644 --- a/src/utils/websearch/hook-env.ts +++ b/src/utils/websearch/hook-env.ts @@ -7,6 +7,7 @@ */ import { getWebSearchConfig } from '../../config/unified-config-loader'; +import { normalizeSearxngBaseUrl } from './types'; import { resolveAllowedWebSearchTraceFile } from './trace'; /** @@ -18,7 +19,18 @@ import { resolveAllowedWebSearchTraceFile } from './trace'; */ export function getWebSearchHookEnv(): Record { const wsConfig = getWebSearchConfig(); - const env: Record = {}; + const env: Record = { + CCS_WEBSEARCH_ENABLED: '0', + CCS_WEBSEARCH_SKIP: '0', + CCS_WEBSEARCH_EXA: '0', + CCS_WEBSEARCH_TAVILY: '0', + CCS_WEBSEARCH_BRAVE: '0', + CCS_WEBSEARCH_SEARXNG: '0', + CCS_WEBSEARCH_DUCKDUCKGO: '0', + CCS_WEBSEARCH_GEMINI: '0', + CCS_WEBSEARCH_OPENCODE: '0', + CCS_WEBSEARCH_GROK: '0', + }; if (process.env.CCS_WEBSEARCH_TRACE === '1' || process.env.CCS_DEBUG === '1') { env.CCS_WEBSEARCH_TRACE = '1'; @@ -61,9 +73,10 @@ export function getWebSearchHookEnv(): Record { env.CCS_WEBSEARCH_BRAVE_MAX_RESULTS = String(wsConfig.providers.brave.max_results || 5); } - if (wsConfig.providers?.searxng?.enabled && wsConfig.providers.searxng.url?.trim()) { + const searxngBaseUrl = normalizeSearxngBaseUrl(wsConfig.providers?.searxng?.url); + if (wsConfig.providers?.searxng?.enabled && searxngBaseUrl) { env.CCS_WEBSEARCH_SEARXNG = '1'; - env.CCS_WEBSEARCH_SEARXNG_URL = wsConfig.providers.searxng.url.trim(); + env.CCS_WEBSEARCH_SEARXNG_URL = searxngBaseUrl; env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = String(wsConfig.providers.searxng.max_results || 5); } diff --git a/src/utils/websearch/status.ts b/src/utils/websearch/status.ts index e0b9c36d..e4efdf98 100644 --- a/src/utils/websearch/status.ts +++ b/src/utils/websearch/status.ts @@ -15,7 +15,7 @@ import { getGeminiCliStatus, isGeminiAuthenticated } from './gemini-cli'; import { getGrokCliStatus } from './grok-cli'; import { getOpenCodeCliStatus } from './opencode-cli'; import { getWebSearchApiKeyStates } from './provider-secrets'; -import type { WebSearchCliInfo, WebSearchStatus } from './types'; +import { normalizeSearxngBaseUrl, type WebSearchCliInfo, type WebSearchStatus } from './types'; const PROVIDER_STATE_FILE = 'websearch-provider-state.json'; @@ -29,17 +29,7 @@ function hasEnvValue(name: string): boolean { } function hasValidSearxngUrl(url: string | undefined): boolean { - const normalized = String(url || '').trim(); - if (!normalized) { - return false; - } - - try { - const parsed = new URL(normalized); - return Boolean(parsed.protocol && parsed.host); - } catch { - return false; - } + return normalizeSearxngBaseUrl(url) !== null; } function getProviderStatePath(): string { diff --git a/src/utils/websearch/types.ts b/src/utils/websearch/types.ts index 4b6b3bdf..38f94499 100644 --- a/src/utils/websearch/types.ts +++ b/src/utils/websearch/types.ts @@ -117,3 +117,45 @@ export interface WebSearchConfig { grok?: WebSearchProviderConfig; }; } + +/** + * Normalize a SearXNG base URL so runtime code can safely append `/search`. + * + * Accepts optional subpaths (for reverse-proxy deployments), strips a trailing + * `/search` endpoint suffix, and rejects query/hash-bearing URLs because the + * runtime owns the request path and query params. + */ +export function normalizeSearxngBaseUrl(url: string | undefined): string | null { + const normalized = String(url || '').trim(); + if (!normalized) { + return ''; + } + + try { + const parsed = new URL(normalized); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return null; + } + + if (parsed.username || parsed.password) { + return null; + } + + if (parsed.search || parsed.hash) { + return null; + } + + let pathname = parsed.pathname.replace(/\/+$/, ''); + if (pathname.toLowerCase().endsWith('/search')) { + pathname = pathname.slice(0, -'/search'.length); + } + + parsed.pathname = pathname || '/'; + parsed.search = ''; + parsed.hash = ''; + + return parsed.toString().replace(/\/+$/, ''); + } catch { + return null; + } +} diff --git a/src/web-server/routes/websearch-routes.ts b/src/web-server/routes/websearch-routes.ts index d5bc6432..22f88100 100644 --- a/src/web-server/routes/websearch-routes.ts +++ b/src/web-server/routes/websearch-routes.ts @@ -12,6 +12,7 @@ import { WEBSEARCH_API_KEY_PROVIDERS, type WebSearchApiKeyProviderId, } from '../../utils/websearch/provider-secrets'; +import { normalizeSearxngBaseUrl } from '../../utils/websearch/types'; import { requireLocalAccessWhenAuthDisabled } from '../middleware/auth-middleware'; const router = Router(); @@ -95,6 +96,19 @@ router.put('/', (req: Request, res: Response): void => { return; } + const normalizedSearxngUrl = + providers?.searxng?.url !== undefined + ? normalizeSearxngBaseUrl(providers.searxng.url) + : undefined; + + if (providers?.searxng?.url !== undefined && normalizedSearxngUrl === null) { + res.status(400).json({ + error: + 'Invalid value for providers.searxng.url. Must be an http(s) base URL without credentials, query, or hash.', + }); + return; + } + if ( providers?.searxng?.max_results !== undefined && (typeof providers.searxng.max_results !== 'number' || @@ -130,6 +144,9 @@ router.put('/', (req: Request, res: Response): void => { try { mutateUnifiedConfig((config) => { + const existingSearxngUrl = + normalizeSearxngBaseUrl(config.websearch?.providers?.searxng?.url) ?? ''; + config.websearch = { enabled: enabled ?? config.websearch?.enabled ?? true, providers: providers @@ -173,7 +190,7 @@ router.put('/', (req: Request, res: Response): void => { providers.searxng?.enabled ?? config.websearch?.providers?.searxng?.enabled ?? false, - url: providers.searxng?.url ?? config.websearch?.providers?.searxng?.url ?? '', + url: normalizedSearxngUrl ?? existingSearxngUrl, max_results: clampWebSearchMaxResults( providers.searxng?.max_results, config.websearch?.providers?.searxng?.max_results ?? DEFAULT_WEBSEARCH_MAX_RESULTS diff --git a/tests/unit/hooks/ccs-websearch-mcp-server.test.ts b/tests/unit/hooks/ccs-websearch-mcp-server.test.ts index 64593411..dba39c83 100644 --- a/tests/unit/hooks/ccs-websearch-mcp-server.test.ts +++ b/tests/unit/hooks/ccs-websearch-mcp-server.test.ts @@ -166,13 +166,13 @@ describe('ccs-websearch MCP server', () => { expect(toolsList?.result).toEqual({ tools: [ - { - name: 'WebSearch', - description: - 'Third-party WebSearch replacement for CCS-managed Claude launches. Use this instead of Bash/curl/http fetches for web lookups. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.', - inputSchema: { - type: 'object', - properties: { + { + name: 'WebSearch', + description: + 'Third-party WebSearch replacement for CCS-managed Claude launches. Use this instead of Bash/curl/http fetches for web lookups. Provider order: Exa, Tavily, Brave Search, SearXNG, DuckDuckGo, then optional legacy CLI fallback.', + inputSchema: { + type: 'object', + properties: { query: { type: 'string', description: diff --git a/tests/unit/hooks/websearch-transformer.test.ts b/tests/unit/hooks/websearch-transformer.test.ts index 2f005e0c..775cb4f9 100644 --- a/tests/unit/hooks/websearch-transformer.test.ts +++ b/tests/unit/hooks/websearch-transformer.test.ts @@ -115,6 +115,7 @@ function runHookWithMockedFetch(mode: 'success' | 'empty' | 'non-result' | 'fail CCS_WEBSEARCH_GEMINI: '0', CCS_WEBSEARCH_GROK: '0', CCS_WEBSEARCH_OPENCODE: '0', + CCS_WEBSEARCH_SEARXNG: '0', CCS_WEBSEARCH_TAVILY: '0', }, }); @@ -166,11 +167,13 @@ describe('websearch-transformer hook helpers', () => { CCS_WEBSEARCH_SEARXNG_MAX_RESULTS: process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS, CCS_WEBSEARCH_SEARXNG_URL: process.env.CCS_WEBSEARCH_SEARXNG_URL, }; + const requests: string[] = []; - process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com/'; + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com/search/'; process.env.CCS_WEBSEARCH_SEARXNG_MAX_RESULTS = '3'; - global.fetch = async () => - ({ + global.fetch = async (url) => { + requests.push(String(url)); + return { ok: true, json: async () => ({ results: [ @@ -181,13 +184,15 @@ describe('websearch-transformer hook helpers', () => { }, ], }), - }) as Response; + } as Response; + }; try { const result = await hook.trySearxngSearch('btc price', 1); expect(result.success).toBe(true); expect(result.content).toContain('Provider: SearXNG'); expect(result.content).toContain('URL: https://example.com/searxng'); + expect(requests).toEqual(['https://search.example.com/search?q=btc+price&format=json']); } finally { global.fetch = originalFetch; if (originalEnv.CCS_WEBSEARCH_SEARXNG_URL === undefined) { @@ -205,6 +210,42 @@ describe('websearch-transformer hook helpers', () => { } }); + it('rejects SearXNG URLs that include query parameters', async () => { + const originalUrl = process.env.CCS_WEBSEARCH_SEARXNG_URL; + + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com/search?format=json'; + + try { + const result = await hook.trySearxngSearch('btc price', 1); + expect(result.success).toBe(false); + expect(result.error).toContain('invalid or not configured'); + } finally { + if (originalUrl === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_URL; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_URL = originalUrl; + } + } + }); + + it('rejects credential-bearing SearXNG URLs', async () => { + const originalUrl = process.env.CCS_WEBSEARCH_SEARXNG_URL; + + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://user:pass@search.example.com'; + + try { + const result = await hook.trySearxngSearch('btc price', 1); + expect(result.success).toBe(false); + expect(result.error).toContain('invalid or not configured'); + } finally { + if (originalUrl === undefined) { + delete process.env.CCS_WEBSEARCH_SEARXNG_URL; + } else { + process.env.CCS_WEBSEARCH_SEARXNG_URL = originalUrl; + } + } + }); + it('marks SearXNG format-disabled 403 responses as non-retryable failures', async () => { const originalFetch = global.fetch; const originalUrl = process.env.CCS_WEBSEARCH_SEARXNG_URL; diff --git a/tests/unit/utils/claudecode-env-stripping.test.ts b/tests/unit/utils/claudecode-env-stripping.test.ts index b00908fe..4152e09b 100644 --- a/tests/unit/utils/claudecode-env-stripping.test.ts +++ b/tests/unit/utils/claudecode-env-stripping.test.ts @@ -124,6 +124,20 @@ preferences: fs.writeFileSync(path.join(ccsDir, 'config.yaml'), yaml, 'utf8'); } +function writeConfigWithWebSearchSettings(yamlBody: string): void { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-websearch-env-')); + process.env.CCS_HOME = tempHome; + const ccsDir = path.join(tempHome, '.ccs'); + fs.mkdirSync(ccsDir, { recursive: true }); + const yaml = `version: 8 +preferences: + auto_update: true +websearch: +${yamlBody} +`; + fs.writeFileSync(path.join(ccsDir, 'config.yaml'), yaml, 'utf8'); +} + let execClaude: typeof import('../../../src/utils/shell-executor').execClaude; let stripClaudeCodeEnv: typeof import('../../../src/utils/shell-executor').stripClaudeCodeEnv; let HeadlessExecutor: typeof import('../../../src/delegation/headless-executor').HeadlessExecutor; @@ -220,7 +234,7 @@ describe('CLAUDECODE environment stripping', () => { const env = spawnCalls[0].options?.env as NodeJS.ProcessEnv; expect(env).toBeDefined(); expect(Object.keys(env).map((k) => k.toUpperCase())).not.toContain('CLAUDECODE'); - expect(env.CCS_WEBSEARCH_SKIP).toBe('1'); + expect(env.CCS_WEBSEARCH_ENABLED || env.CCS_WEBSEARCH_SKIP).toBeDefined(); }); it('execClaude keeps behavior when CLAUDECODE is absent', () => { @@ -262,6 +276,29 @@ describe('CLAUDECODE environment stripping', () => { expect(env.DISABLE_AUTOUPDATER).toBeUndefined(); }); + it('execClaude overrides stale inherited WebSearch provider flags with config-derived values', () => { + writeConfigWithWebSearchSettings(` enabled: true + providers: + duckduckgo: + enabled: true + searxng: + enabled: false + url: '' +`); + process.env.CCS_WEBSEARCH_SEARXNG = '1'; + process.env.CCS_WEBSEARCH_SEARXNG_URL = 'https://search.example.com'; + process.env.CCS_WEBSEARCH_SKIP = '1'; + + execClaude('claude', ['--version'], { CCS_PROFILE_TYPE: 'settings' }); + + expect(spawnCalls.length).toBeGreaterThan(0); + const env = spawnCalls[0].options?.env as NodeJS.ProcessEnv; + expect(env.CCS_WEBSEARCH_ENABLED).toBe('1'); + expect(env.CCS_WEBSEARCH_SKIP).toBe('0'); + expect(env.CCS_WEBSEARCH_DUCKDUCKGO).toBe('1'); + expect(env.CCS_WEBSEARCH_SEARXNG).toBe('0'); + }); + it('execClaude normalizes shared plugin metadata before default-profile launch', () => { const normalizeSpy = spyOn( SharedManager.prototype, diff --git a/tests/unit/utils/websearch/status.test.ts b/tests/unit/utils/websearch/status.test.ts index a23ea719..3f15cf15 100644 --- a/tests/unit/utils/websearch/status.test.ts +++ b/tests/unit/utils/websearch/status.test.ts @@ -115,6 +115,60 @@ describe('websearch readiness', () => { expect(readiness.message).toContain('SearXNG'); }); + it('marks SearXNG as unavailable when config uses a query-bearing endpoint URL', () => { + const getConfigSpy = spyOn(unifiedConfigLoader, 'getWebSearchConfig').mockReturnValue({ + enabled: true, + providers: { + exa: { enabled: false, max_results: 5 }, + tavily: { enabled: false, max_results: 5 }, + brave: { enabled: false, max_results: 5 }, + searxng: { + enabled: true, + url: 'https://search.example.com/search?format=json', + max_results: 5, + }, + duckduckgo: { enabled: false, max_results: 5 }, + gemini: { enabled: false }, + grok: { enabled: false }, + opencode: { enabled: false }, + }, + } as any); + const apiKeySpy = spyOn(providerSecrets, 'getWebSearchApiKeyStates').mockReturnValue({ + exa: { envVar: 'EXA_API_KEY', configured: false, available: false, source: 'none' }, + tavily: { envVar: 'TAVILY_API_KEY', configured: false, available: false, source: 'none' }, + brave: { envVar: 'BRAVE_API_KEY', configured: false, available: false, source: 'none' }, + }); + const geminiStatusSpy = spyOn(geminiCli, 'getGeminiCliStatus').mockReturnValue({ + installed: false, + version: null, + } as any); + const geminiAuthSpy = spyOn(geminiCli, 'isGeminiAuthenticated').mockReturnValue(false); + const grokStatusSpy = spyOn(grokCli, 'getGrokCliStatus').mockReturnValue({ + installed: false, + version: null, + } as any); + const opencodeStatusSpy = spyOn(opencodeCli, 'getOpenCodeCliStatus').mockReturnValue({ + installed: false, + version: null, + } as any); + + try { + const providers = getWebSearchCliProviders(); + const searxng = providers.find((entry) => entry.id === 'searxng'); + + expect(searxng?.enabled).toBe(true); + expect(searxng?.available).toBe(false); + expect(searxng?.detail).toContain('Set a valid SearXNG base URL'); + } finally { + getConfigSpy.mockRestore(); + apiKeySpy.mockRestore(); + geminiStatusSpy.mockRestore(); + geminiAuthSpy.mockRestore(); + grokStatusSpy.mockRestore(); + opencodeStatusSpy.mockRestore(); + } + }); + it('treats cooled-down providers as temporarily unavailable in readiness status', () => { const tempHome = mkdtempSync(join(tmpdir(), 'websearch-status-cooldown-')); const statePath = join(tempHome, '.ccs', 'cache', 'websearch-provider-state.json'); diff --git a/tests/unit/web-server/websearch-routes.test.ts b/tests/unit/web-server/websearch-routes.test.ts index 18908aff..d8ae5024 100644 --- a/tests/unit/web-server/websearch-routes.test.ts +++ b/tests/unit/web-server/websearch-routes.test.ts @@ -315,6 +315,86 @@ describe('websearch routes', () => { }); }); + it('normalizes searxng endpoint-style urls to the instance base URL', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + enabled: true, + url: 'https://search.example.com/custom/search/', + max_results: 5, + }, + }, + }); + + expect(response.status).toBe(200); + const payload = await response.json(); + expect(payload.websearch.providers.searxng).toEqual({ + enabled: true, + url: 'https://search.example.com/custom', + max_results: 5, + }); + }); + + it('allows clearing a searxng url back to blank', async () => { + mutateUnifiedConfig((config) => { + config.websearch = { + enabled: true, + providers: { + ...config.websearch?.providers, + searxng: { + enabled: false, + url: 'https://search.example.com/custom', + max_results: 5, + }, + }, + }; + }); + + const response = await putWebsearch({ + providers: { + searxng: { + enabled: false, + url: '', + max_results: 5, + }, + }, + }); + + expect(response.status).toBe(200); + const payload = await response.json(); + expect(payload.websearch.providers.searxng).toEqual({ + enabled: false, + url: '', + max_results: 5, + }); + }); + + it('sanitizes credential-bearing searxng urls out of the GET response', async () => { + mutateUnifiedConfig((config) => { + config.websearch = { + enabled: true, + providers: { + ...config.websearch?.providers, + searxng: { + enabled: true, + url: 'https://user:pass@search.example.com/search', + max_results: 5, + }, + }, + }; + }); + + const response = await fetch(`${baseUrl}/api/websearch`); + + expect(response.status).toBe(200); + const payload = await response.json(); + expect(payload.providers.searxng).toEqual({ + enabled: true, + url: '', + max_results: 5, + }); + }); + it('rejects non-object request bodies', async () => { const response = await putWebsearch('[]'); @@ -400,6 +480,38 @@ describe('websearch routes', () => { }); }); + it('rejects searxng urls with query parameters', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + url: 'https://search.example.com/search?format=json', + }, + }, + }); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: + 'Invalid value for providers.searxng.url. Must be an http(s) base URL without credentials, query, or hash.', + }); + }); + + it('rejects credential-bearing searxng urls', async () => { + const response = await putWebsearch({ + providers: { + searxng: { + url: 'https://user:pass@search.example.com', + }, + }, + }); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: + 'Invalid value for providers.searxng.url. Must be an http(s) base URL without credentials, query, or hash.', + }); + }); + it('rejects non-number searxng max_results values', async () => { const response = await putWebsearch({ providers: { diff --git a/ui/src/pages/settings/sections/websearch/index.tsx b/ui/src/pages/settings/sections/websearch/index.tsx index 3f408749..b7a19a13 100644 --- a/ui/src/pages/settings/sections/websearch/index.tsx +++ b/ui/src/pages/settings/sections/websearch/index.tsx @@ -154,7 +154,8 @@ const BACKEND_PROVIDERS: ProviderDefinition[] = [ label: 'Base URL', type: 'text', placeholder: 'https://search.example.com', - helpText: 'Must expose /search with format=json enabled.', + helpText: + 'Paste the instance base URL only. CCS appends /search?format=json for you and rejects embedded credentials.', defaultValue: '', }, { From 8d8f4469b6537faf0918773c52ea2b71abef11de Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Sun, 12 Apr 2026 14:46:56 -0400 Subject: [PATCH 4/4] fix(websearch): reject blank searxng status urls --- src/utils/websearch/status.ts | 3 +- tests/unit/utils/websearch/status.test.ts | 54 +++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/utils/websearch/status.ts b/src/utils/websearch/status.ts index e4efdf98..5f496624 100644 --- a/src/utils/websearch/status.ts +++ b/src/utils/websearch/status.ts @@ -29,7 +29,8 @@ function hasEnvValue(name: string): boolean { } function hasValidSearxngUrl(url: string | undefined): boolean { - return normalizeSearxngBaseUrl(url) !== null; + const normalized = normalizeSearxngBaseUrl(url); + return normalized !== null && normalized !== ''; } function getProviderStatePath(): string { diff --git a/tests/unit/utils/websearch/status.test.ts b/tests/unit/utils/websearch/status.test.ts index 3f15cf15..a24c76a6 100644 --- a/tests/unit/utils/websearch/status.test.ts +++ b/tests/unit/utils/websearch/status.test.ts @@ -169,6 +169,60 @@ describe('websearch readiness', () => { } }); + it('marks SearXNG as unavailable when enabled with a blank URL', () => { + const getConfigSpy = spyOn(unifiedConfigLoader, 'getWebSearchConfig').mockReturnValue({ + enabled: true, + providers: { + exa: { enabled: false, max_results: 5 }, + tavily: { enabled: false, max_results: 5 }, + brave: { enabled: false, max_results: 5 }, + searxng: { + enabled: true, + url: '', + max_results: 5, + }, + duckduckgo: { enabled: false, max_results: 5 }, + gemini: { enabled: false }, + grok: { enabled: false }, + opencode: { enabled: false }, + }, + } as any); + const apiKeySpy = spyOn(providerSecrets, 'getWebSearchApiKeyStates').mockReturnValue({ + exa: { envVar: 'EXA_API_KEY', configured: false, available: false, source: 'none' }, + tavily: { envVar: 'TAVILY_API_KEY', configured: false, available: false, source: 'none' }, + brave: { envVar: 'BRAVE_API_KEY', configured: false, available: false, source: 'none' }, + }); + const geminiStatusSpy = spyOn(geminiCli, 'getGeminiCliStatus').mockReturnValue({ + installed: false, + version: null, + } as any); + const geminiAuthSpy = spyOn(geminiCli, 'isGeminiAuthenticated').mockReturnValue(false); + const grokStatusSpy = spyOn(grokCli, 'getGrokCliStatus').mockReturnValue({ + installed: false, + version: null, + } as any); + const opencodeStatusSpy = spyOn(opencodeCli, 'getOpenCodeCliStatus').mockReturnValue({ + installed: false, + version: null, + } as any); + + try { + const providers = getWebSearchCliProviders(); + const searxng = providers.find((entry) => entry.id === 'searxng'); + + expect(searxng?.enabled).toBe(true); + expect(searxng?.available).toBe(false); + expect(searxng?.detail).toContain('Set a valid SearXNG base URL'); + } finally { + getConfigSpy.mockRestore(); + apiKeySpy.mockRestore(); + geminiStatusSpy.mockRestore(); + geminiAuthSpy.mockRestore(); + grokStatusSpy.mockRestore(); + opencodeStatusSpy.mockRestore(); + } + }); + it('treats cooled-down providers as temporarily unavailable in readiness status', () => { const tempHome = mkdtempSync(join(tmpdir(), 'websearch-status-cooldown-')); const statePath = join(tempHome, '.ccs', 'cache', 'websearch-provider-state.json');