From de7171d81076ecfd4e5e6474ce062c44a4c8be9a Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Mon, 30 Mar 2026 22:41:51 -0400 Subject: [PATCH] feat(websearch): finish managed third-party rollout - steer third-party launches toward the managed WebSearch MCP tool - add opt-in trace diagnostics across launch, MCP, provider, and headless paths - extend docs and regression coverage for the first-class runtime Refs #862 --- README.md | 10 +- docs/websearch.md | 28 ++- lib/hooks/websearch-transformer.cjs | 169 ++++++++++++- lib/mcp/ccs-websearch-server.cjs | 130 +++++++++- src/ccs.ts | 19 +- src/cliproxy/executor/index.ts | 35 ++- src/copilot/copilot-executor.ts | 14 +- src/delegation/executor/result-aggregator.ts | 27 ++- src/delegation/executor/types.ts | 7 + src/delegation/headless-executor.ts | 73 +++++- src/utils/websearch-manager.ts | 8 + src/utils/websearch/claude-tool-args.ts | 117 +++++++-- src/utils/websearch/hook-env.ts | 7 + src/utils/websearch/index.ts | 8 + src/utils/websearch/mcp-installer.ts | 30 ++- src/utils/websearch/trace.ts | 223 ++++++++++++++++++ .../hooks/ccs-websearch-mcp-server.test.ts | 186 ++++++++++++++- .../unit/hooks/websearch-transformer.test.ts | 76 +++++- .../settings-profile-websearch-launch.test.ts | 48 +++- .../utils/claudecode-env-stripping.test.ts | 49 ++++ .../utils/websearch/claude-tool-args.test.ts | 109 ++++++++- 21 files changed, 1281 insertions(+), 92 deletions(-) create mode 100644 src/utils/websearch/trace.ts diff --git a/README.md b/README.md index 34c10a75..7de245fd 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,9 @@ The dashboard provides visual management for all account types: **Ollama Integration**: Run local open-source models (qwen3-coder, gpt-oss:20b) with full privacy. Use `ccs api create --preset ollama` - requires [Ollama v0.14.0+](https://ollama.com) installed. For cloud models, use `ccs api create --preset ollama-cloud`. +> **Third-party WebSearch steering:** Claude-backed third-party launches keep Anthropic's native `WebSearch` disabled, expose `ccs-websearch.WebSearch`, and append a short system hint so Claude prefers that managed tool over ad hoc Bash or `curl` lookups whenever current web information is needed. +> Setting `websearch.enabled: false` disables the managed local runtime, but CCS still suppresses Anthropic's native `WebSearch` on third-party backends because those providers cannot execute it correctly. + > **Copilot config behavior:** Opening the dashboard or other read-only Copilot endpoints does not rewrite `~/.ccs/copilot.settings.json`. If CCS detects deprecated Copilot model IDs such as `raptor-mini`, it shows warnings immediately and only persists replacements when you explicitly save the Copilot configuration. **llama.cpp Integration**: Run a local llama.cpp OpenAI-compatible server and create a profile with `ccs api create --preset llamacpp`. CCS defaults to `http://127.0.0.1:8080`, matching the standard llama.cpp server port. @@ -637,11 +640,11 @@ Third-party profiles (Gemini, Codex, GLM, etc.) cannot use Anthropic's native We | Profile Type | WebSearch Method | |--------------|------------------| | Claude (native) | Anthropic WebSearch API | -| Third-party profiles | CCS local MCP search tool | +| Third-party profiles | CCS local MCP `WebSearch` tool | ### Local Search Backend Chain -For third-party profiles, Claude uses the managed `ccs-websearch` MCP tool. CCS then routes that request through deterministic search providers in this order: +For third-party profiles, Claude uses the managed `ccs-websearch.WebSearch` MCP tool. The tool is intentionally named to match the native `WebSearch` concept, which helps Claude prefer it over ad hoc Bash or `curl` fetches. CCS then routes that request through deterministic search providers in this order: | Priority | Provider | Setup | Notes | |----------|----------|-------|-------| @@ -675,6 +678,9 @@ websearch: > **DuckDuckGo** still works out of the box. Add **Exa**, **Tavily**, or **Brave Search** if you want API-backed results, then keep Gemini/OpenCode/Grok only if you explicitly want legacy fallback behavior. > CCS manages the user-scope MCP entry in `~/.claude.json` and syncs it into isolated account configs when needed. +> [!NOTE] +> Set `CCS_WEBSEARCH_TRACE=1` to write correlated launch, MCP, provider, and headless summary records to `~/.ccs/logs/websearch-trace.jsonl`. That trace is designed to answer whether CCS exposed the managed tool, whether Claude called it, which provider won, and when a headless run likely bypassed it via `Bash` or `WebFetch`. + See [docs/websearch.md](./docs/websearch.md) for detailed configuration and troubleshooting.
diff --git a/docs/websearch.md b/docs/websearch.md index 7c83632b..a3976e38 100644 --- a/docs/websearch.md +++ b/docs/websearch.md @@ -12,7 +12,7 @@ Native Claude subscription accounts still use Anthropic's server-side WebSearch ### Third-Party Profiles -Third-party profiles cannot execute Anthropic's server-side WebSearch because the tool never reaches their backend. CCS now solves that by provisioning a first-class local MCP tool, suppressing native `WebSearch` for those launches, and running real local search providers directly. +Third-party profiles cannot execute Anthropic's server-side WebSearch because the tool never reaches their backend. CCS now solves that by provisioning a first-class local MCP tool, suppressing native `WebSearch` for those launches, appending a short launch-time steering hint, and running real local search providers directly. ## Architecture @@ -27,7 +27,7 @@ Third-party profiles cannot execute Anthropic's server-side WebSearch because th │ └── Third-party Profile? → native WebSearch disabled │ │ │ │ │ └── CCS MCP tool │ -│ ccs-websearch.search │ +│ ccs-websearch.WebSearch │ │ │ │ │ ├── 1. Exa │ │ ├── 2. Tavily │ @@ -52,6 +52,13 @@ The previous design asked another model CLI to perform web search and summarize The new flow matches the `goclaw` model more closely: web search is treated as a first-class deterministic capability, not an LLM-to-LLM workaround or a denied native tool call. +The managed MCP tool is exposed as `ccs-websearch.WebSearch`, not a generic `search` helper. That naming is deliberate: it gives Claude a tool that matches the native `WebSearch` concept more directly, which should reduce cases where the model reaches for ad hoc Bash or `curl` fetches instead. + +CCS also appends a third-party-only `--append-system-prompt` hint telling Claude to prefer that managed `WebSearch` tool for web lookups and current-information requests. This is soft steering only: if the user explicitly asks for shell commands, or the tool is unavailable, Claude can still fall back to Bash/network tools. +That shared launch helper applies to normal third-party settings profiles, CLIProxy/Copilot-backed Claude launches, and CCS headless/delegation runs that execute through a settings profile. + +`websearch.enabled: false` disables the managed local runtime, but CCS still suppresses Anthropic's native `WebSearch` on third-party profiles. That native tool cannot be satisfied by Exa, Tavily, Brave, DuckDuckGo, or other non-Anthropic backends, so CCS avoids sending a broken native-tool request and lets Claude fall back to normal shell/network tools instead. + ## Providers | Provider | Type | Setup | Default | Notes | @@ -108,6 +115,8 @@ websearch: timeout: 55 ``` +Note: `enabled: false` stops provisioning the managed local `ccs-websearch.WebSearch` runtime. It does not re-enable Anthropic's native `WebSearch` for third-party backends. + ## Environment Variables | Variable | Description | @@ -118,6 +127,8 @@ websearch: | `GROK_API_KEY` | Required only for legacy Grok CLI fallback | | `CCS_WEBSEARCH_SKIP` | Disable the CCS local WebSearch runtime for the current process | | `CCS_DEBUG` | Verbose WebSearch runtime logging | +| `CCS_WEBSEARCH_TRACE` | Write opt-in JSONL trace records under `~/.ccs/logs/websearch-trace.jsonl` | +| `CCS_WEBSEARCH_TRACE_FILE` | Override the trace file path (must stay inside `~/.ccs/`, `/tmp`, or `/var/log`) | ## Managed Runtime Files @@ -148,12 +159,23 @@ If the dashboard says the key is stored but still not ready, check whether `Sett Those providers remain supported, but they are no longer the primary path. Enable them explicitly in `config.yaml` if you want them as last-resort fallback. +### I need to see whether CCS exposed WebSearch or the model bypassed it + +Run the launch with `CCS_WEBSEARCH_TRACE=1` (or `CCS_DEBUG=1`). CCS writes a JSONL trace to `~/.ccs/logs/websearch-trace.jsonl` with: + +1. source-side launch records from CCS (`ccs_websearch_launch`) +2. MCP exposure and call records (`mcp_initialize`, `mcp_tools_list`, `mcp_tool_call_*`) +3. provider attempt and winner records (`websearch_provider_attempt`, `websearch_provider_success`) +4. session summaries (`mcp_session_summary`, and headless `headless_websearch_summary` when applicable) + +Queries are fingerprinted (`queryHash`, `queryLength`) instead of logged raw by default. For headless/delegation runs, `headless_websearch_summary.likelyBypassed=true` means the MCP tool was exposed, no WebSearch call occurred, and Claude fell back to `Bash` or `WebFetch`. + ### WebSearch returns no results 1. Check `websearch.enabled: true` 2. Keep DuckDuckGo enabled unless you have a strong reason to disable it 3. If using Exa, Tavily, or Brave, verify the matching API key -4. Run with `CCS_DEBUG=1` for runtime logs +4. Run with `CCS_DEBUG=1` for runtime logs, or `CCS_WEBSEARCH_TRACE=1` for correlated launch/MCP/provider traces ## Security Considerations diff --git a/lib/hooks/websearch-transformer.cjs b/lib/hooks/websearch-transformer.cjs index 9b7faf42..6b8e023d 100644 --- a/lib/hooks/websearch-transformer.cjs +++ b/lib/hooks/websearch-transformer.cjs @@ -15,6 +15,9 @@ */ const { spawnSync } = require('child_process'); +const { createHash } = require('crypto'); +const fs = require('fs'); +const path = require('path'); const isWindows = process.platform === 'win32'; const DEFAULT_TIMEOUT_SEC = 55; @@ -64,12 +67,96 @@ function debug(message) { } } -function shouldSkipHook() { - if (process.env.CCS_WEBSEARCH_SKIP === '1') return true; +function getCcsDirPath() { + if ((process.env.CCS_DIR || '').trim()) { + return path.resolve(process.env.CCS_DIR.trim()); + } + + if ((process.env.CCS_HOME || '').trim()) { + return path.join(path.resolve(process.env.CCS_HOME.trim()), '.ccs'); + } + + const home = (process.env.HOME || process.env.USERPROFILE || '').trim(); + if (home) { + return path.join(home, '.ccs'); + } + + return path.join(process.cwd(), '.ccs'); +} + +function isTraceEnabled() { + return process.env.CCS_WEBSEARCH_TRACE === '1' || process.env.CCS_DEBUG === '1'; +} + +function getTraceFilePath() { + const fallback = path.join(getCcsDirPath(), 'logs', 'websearch-trace.jsonl'); + const configured = (process.env.CCS_WEBSEARCH_TRACE_FILE || '').trim(); + if (!configured) { + return fallback; + } + + const resolved = path.resolve(configured); + const safePrefixes = [ + `${path.resolve(getCcsDirPath())}${path.sep}`, + `${path.resolve(process.env.HOME || process.env.USERPROFILE || process.cwd())}${path.sep}`, + '/tmp/', + '/var/log/', + ]; + + if (safePrefixes.some((prefix) => resolved.startsWith(prefix))) { + return resolved; + } + + return fallback; +} + +function traceWebSearchEvent(event, payload = {}) { + if (!isTraceEnabled()) { + return; + } + + try { + const traceFilePath = getTraceFilePath(); + fs.mkdirSync(path.dirname(traceFilePath), { recursive: true }); + fs.appendFileSync( + traceFilePath, + JSON.stringify({ + at: new Date().toISOString(), + event, + launchId: process.env.CCS_WEBSEARCH_TRACE_LAUNCH_ID || null, + launcher: process.env.CCS_WEBSEARCH_TRACE_LAUNCHER || null, + profileType: process.env.CCS_PROFILE_TYPE || null, + pid: process.pid, + ...payload, + }) + '\n', + 'utf8' + ); + } catch { + // Best-effort only. + } +} + +function getQueryFingerprint(query) { + const normalizedQuery = typeof query === 'string' ? query.trim() : ''; + return { + queryHash: normalizedQuery + ? createHash('sha256').update(normalizedQuery).digest('hex').slice(0, 16) + : null, + queryLength: normalizedQuery.length, + }; +} + +function getSkipReason() { + if (process.env.CCS_WEBSEARCH_SKIP === '1') return 'skip_flag'; const profileType = process.env.CCS_PROFILE_TYPE; - if (profileType === 'account' || profileType === 'default') return true; - if (process.env.CCS_WEBSEARCH_ENABLED === '0') return true; - return false; + if (profileType === 'account') return 'native_account_profile'; + if (profileType === 'default') return 'native_default_profile'; + if (process.env.CCS_WEBSEARCH_ENABLED === '0') return 'disabled'; + return null; +} + +function shouldSkipHook() { + return getSkipReason() !== null; } function isCliAvailable(cmd) { @@ -627,26 +714,53 @@ function getActiveProviders() { return getConfiguredProviders().filter((provider) => provider.available()); } +function getActiveProviderIds() { + return getActiveProviders().map((provider) => provider.id); +} + function hasAnyActiveProviders() { return getActiveProviders().length > 0; } async function runLocalWebSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { const activeProviders = getActiveProviders(); + const fingerprint = getQueryFingerprint(query); debug( `Enabled providers: ${activeProviders.map((provider) => provider.name).join(', ') || 'none'}` ); + traceWebSearchEvent('websearch_provider_run_started', { + source: 'provider', + activeProviderIds: activeProviders.map((provider) => provider.id), + ...fingerprint, + }); if (activeProviders.length === 0) { + traceWebSearchEvent('websearch_provider_run_unavailable', { + source: 'provider', + activeProviderIds: [], + ...fingerprint, + }); return { success: false, noActiveProviders: true, errors: [] }; } const errors = []; for (const provider of activeProviders) { debug(`Trying ${provider.name}`); + traceWebSearchEvent('websearch_provider_attempt', { + source: 'provider', + providerId: provider.id, + providerName: provider.name, + ...fingerprint, + }); const result = await provider.fn(query, timeoutSec); if (result.success) { + traceWebSearchEvent('websearch_provider_success', { + source: 'provider', + providerId: provider.id, + providerName: provider.name, + ...fingerprint, + }); return { success: true, providerId: provider.id, @@ -654,15 +768,32 @@ async function runLocalWebSearch(query, timeoutSec = DEFAULT_TIMEOUT_SEC) { content: result.content, }; } + traceWebSearchEvent('websearch_provider_failure', { + source: 'provider', + providerId: provider.id, + providerName: provider.name, + error: result.error, + ...fingerprint, + }); errors.push({ provider: provider.name, error: result.error }); } + traceWebSearchEvent('websearch_provider_run_failed', { + source: 'provider', + errorCount: errors.length, + activeProviderIds: activeProviders.map((provider) => provider.id), + ...fingerprint, + }); return { success: false, noActiveProviders: false, errors }; } async function processHook(input) { try { if (shouldSkipHook()) { + traceWebSearchEvent('websearch_hook_skipped', { + source: 'hook', + reason: getSkipReason(), + }); process.exit(0); } @@ -676,23 +807,47 @@ async function processHook(input) { process.exit(0); } + traceWebSearchEvent('websearch_hook_invoked', { + source: 'hook', + ...getQueryFingerprint(query), + }); + const timeout = Number.parseInt( process.env.CCS_WEBSEARCH_TIMEOUT || `${DEFAULT_TIMEOUT_SEC}`, 10 ); const result = await runLocalWebSearch(query, timeout); if (result.noActiveProviders) { + traceWebSearchEvent('websearch_hook_no_active_providers', { + source: 'hook', + ...getQueryFingerprint(query), + }); process.exit(0); } if (result.success) { + traceWebSearchEvent('websearch_hook_success', { + source: 'hook', + providerId: result.providerId, + providerName: result.providerName, + ...getQueryFingerprint(query), + }); outputSuccess(query, result.content, result.providerName); return; } + traceWebSearchEvent('websearch_hook_failure', { + source: 'hook', + errorCount: result.errors.length, + ...getQueryFingerprint(query), + }); outputAllFailedMessage(query, result.errors); } catch (error) { debug(`Hook error: ${error.message}`); + traceWebSearchEvent('websearch_hook_error', { + source: 'hook', + error: error.message, + }); process.exit(0); } } @@ -724,6 +879,10 @@ module.exports = { hasAnyActiveProviders, runLocalWebSearch, shouldSkipHook, + getActiveProviderIds, + getQueryFingerprint, + getSkipReason, + traceWebSearchEvent, tryExaSearch, tryTavilySearch, tryDuckDuckGoSearch, diff --git a/lib/mcp/ccs-websearch-server.cjs b/lib/mcp/ccs-websearch-server.cjs index 4d891899..c3f7c137 100644 --- a/lib/mcp/ccs-websearch-server.cjs +++ b/lib/mcp/ccs-websearch-server.cjs @@ -1,19 +1,35 @@ #!/usr/bin/env node const { + getActiveProviderIds, + getQueryFingerprint, + getSkipReason, hasAnyActiveProviders, runLocalWebSearch, shouldSkipHook, + traceWebSearchEvent, } = require('../hooks/websearch-transformer.cjs'); const PROTOCOL_VERSION = '2024-11-05'; const SERVER_NAME = 'ccs-websearch'; const SERVER_VERSION = '1.0.0'; -const TOOL_NAME = 'search'; +const TOOL_NAME = 'WebSearch'; +const TOOL_ALIASES = ['search']; const TOOL_DESCRIPTION = - 'Search the web through CCS-managed providers. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.'; + 'Third-party WebSearch replacement for CCS-managed Claude launches. Use this instead of Bash/curl/http fetches for web lookups. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.'; + +function isSupportedToolName(name) { + return name === TOOL_NAME || TOOL_ALIASES.includes(name); +} let inputBuffer = Buffer.alloc(0); +const sessionState = { + initializeCount: 0, + toolsListCount: 0, + exposed: false, + toolCalls: 0, +}; +let sessionSummaryWritten = false; function shouldExposeTools() { return !shouldSkipHook() && hasAnyActiveProviders(); @@ -33,7 +49,8 @@ function getTools() { properties: { query: { type: 'string', - description: 'The search query to run against CCS WebSearch providers.', + description: + 'Web query to resolve through CCS providers. Prefer this tool over ad hoc Bash/curl lookups when you need current web information.', }, }, required: ['query'], @@ -72,13 +89,36 @@ async function handleToolCall(message) { const id = message.id; const params = message.params || {}; const toolArgs = params.arguments || {}; + const toolName = params.name || ''; + const query = typeof toolArgs.query === 'string' ? toolArgs.query.trim() : ''; + const fingerprint = getQueryFingerprint(query); - if (params.name !== TOOL_NAME) { - writeError(id, -32602, `Unknown tool: ${params.name || ''}`); + if (!isSupportedToolName(toolName)) { + traceWebSearchEvent('mcp_tool_call_rejected', { + source: 'mcp', + reason: 'unknown_tool', + toolName, + }); + writeError(id, -32602, `Unknown tool: ${toolName}`); return; } + sessionState.toolCalls += 1; + traceWebSearchEvent('mcp_tool_call_received', { + source: 'mcp', + toolName, + ...fingerprint, + }); + if (!shouldExposeTools()) { + traceWebSearchEvent('mcp_tool_call_unavailable', { + source: 'mcp', + toolName, + exposed: false, + skipReason: getSkipReason(), + activeProviderIds: getActiveProviderIds(), + ...fingerprint, + }); writeResponse(id, { content: [ { @@ -91,20 +131,41 @@ async function handleToolCall(message) { return; } - const query = typeof toolArgs.query === 'string' ? toolArgs.query.trim() : ''; if (!query) { - writeError(id, -32602, 'Tool "search" requires a non-empty string query.'); + traceWebSearchEvent('mcp_tool_call_rejected', { + source: 'mcp', + reason: 'empty_query', + toolName, + }); + writeError(id, -32602, `Tool "${TOOL_NAME}" requires a non-empty string query.`); return; } const result = await runLocalWebSearch(query); if (result.success) { + traceWebSearchEvent('mcp_tool_call_result', { + source: 'mcp', + toolName, + success: true, + providerId: result.providerId, + providerName: result.providerName, + ...fingerprint, + }); writeResponse(id, { content: [{ type: 'text', text: result.content }], }); return; } + traceWebSearchEvent('mcp_tool_call_result', { + source: 'mcp', + toolName, + success: false, + noActiveProviders: Boolean(result.noActiveProviders), + errorCount: result.errors.length, + ...fingerprint, + }); + const errorDetail = result.noActiveProviders || result.errors.length === 0 ? 'No active WebSearch providers are ready.' @@ -128,6 +189,14 @@ async function handleMessage(message) { switch (message.method) { case 'initialize': + sessionState.initializeCount += 1; + sessionState.exposed = sessionState.exposed || shouldExposeTools(); + traceWebSearchEvent('mcp_initialize', { + source: 'mcp', + exposed: shouldExposeTools(), + skipReason: getSkipReason(), + activeProviderIds: getActiveProviderIds(), + }); writeResponse(message.id, { protocolVersion: PROTOCOL_VERSION, capabilities: { @@ -145,7 +214,20 @@ async function handleMessage(message) { writeResponse(message.id, {}); return; case 'tools/list': - writeResponse(message.id, { tools: getTools() }); + sessionState.toolsListCount += 1; + { + const tools = getTools(); + const exposed = tools.length > 0; + sessionState.exposed = sessionState.exposed || exposed; + traceWebSearchEvent('mcp_tools_list', { + source: 'mcp', + exposed, + toolNames: tools.map((tool) => tool.name), + activeProviderIds: getActiveProviderIds(), + skipReason: getSkipReason(), + }); + writeResponse(message.id, { tools }); + } return; case 'tools/call': await handleToolCall(message); @@ -157,6 +239,27 @@ async function handleMessage(message) { } } +function writeSessionSummary(exitCodeOrSignal) { + if (sessionSummaryWritten) { + return; + } + + sessionSummaryWritten = true; + traceWebSearchEvent('mcp_session_summary', { + source: 'mcp', + initializeCount: sessionState.initializeCount, + toolsListCount: sessionState.toolsListCount, + exposed: sessionState.exposed, + toolCalls: sessionState.toolCalls, + calledWebSearch: sessionState.toolCalls > 0, + likelyBypassed: sessionState.exposed && sessionState.toolCalls === 0 ? 'unknown' : false, + activeProviderIds: getActiveProviderIds(), + skipReason: getSkipReason(), + exitCode: typeof exitCodeOrSignal === 'number' ? exitCodeOrSignal : null, + exitSignal: typeof exitCodeOrSignal === 'string' ? exitCodeOrSignal : null, + }); +} + function parseMessages() { while (true) { const headerEnd = inputBuffer.indexOf('\r\n\r\n'); @@ -204,4 +307,15 @@ process.stdin.on('error', () => { process.exit(0); }); +process.on('exit', (code) => { + writeSessionSummary(code); +}); + +['SIGINT', 'SIGTERM', 'SIGHUP'].forEach((signal) => { + process.on(signal, () => { + writeSessionSummary(signal); + process.exit(0); + }); +}); + process.stdin.resume(); diff --git a/src/ccs.ts b/src/ccs.ts index 32d94649..d427d234 100644 --- a/src/ccs.ts +++ b/src/ccs.ts @@ -29,6 +29,7 @@ import { getWebSearchHookEnv, syncWebSearchMcpToConfigDir, appendThirdPartyWebSearchToolArgs, + createWebSearchTraceContext, } from './utils/websearch-manager'; import { getGlobalEnvConfig, getOfficialChannelsConfig } from './config/unified-config-loader'; import { ensureProfileHooks as ensureImageAnalyzerHooks } from './utils/hooks/image-analyzer-profile-hook-injector'; @@ -1053,11 +1054,19 @@ async function main(): Promise { return; } - execClaude( - claudeCli, - ['--settings', expandedSettingsPath, ...appendThirdPartyWebSearchToolArgs(remainingArgs)], - envVars - ); + const launchArgs = [ + '--settings', + expandedSettingsPath, + ...appendThirdPartyWebSearchToolArgs(remainingArgs), + ]; + const traceEnv = createWebSearchTraceContext({ + launcher: 'ccs.settings-profile', + args: launchArgs, + profile: profileInfo.name, + profileType: profileInfo.type, + settingsPath: expandedSettingsPath, + }); + execClaude(claudeCli, launchArgs, { ...envVars, ...traceEnv }); } else if (profileInfo.type === 'account') { // NEW FLOW: Account-based profile (work, personal) // All platforms: Use instance isolation with CLAUDE_CONFIG_DIR diff --git a/src/cliproxy/executor/index.ts b/src/cliproxy/executor/index.ts index e8e07862..74efd266 100644 --- a/src/cliproxy/executor/index.ts +++ b/src/cliproxy/executor/index.ts @@ -56,6 +56,7 @@ import { ensureWebSearchMcpOrThrow, displayWebSearchStatus, appendThirdPartyWebSearchToolArgs, + createWebSearchTraceContext, } from '../../utils/websearch-manager'; import { loadOrCreateUnifiedConfig, getThinkingConfig } from '../../config/unified-config-loader'; import { installImageAnalyzerHook } from '../../utils/hooks'; @@ -1034,31 +1035,29 @@ export async function execClaudeWithCLIProxy( : getProviderSettingsPath(provider); let claude: ChildProcess; + const launchArgs = ['--settings', settingsPath, ...appendThirdPartyWebSearchToolArgs(claudeArgs)]; + const traceEnv = createWebSearchTraceContext({ + launcher: 'cliproxy.executor', + args: launchArgs, + profile: provider, + profileType: 'settings', + settingsPath, + }); + const tracedEnv = { ...env, ...traceEnv }; if (needsShell) { - const cmdString = [ - claudeCli, - '--settings', - settingsPath, - ...appendThirdPartyWebSearchToolArgs(claudeArgs), - ] - .map(escapeShellArg) - .join(' '); + const cmdString = [claudeCli, ...launchArgs].map(escapeShellArg).join(' '); claude = spawn(cmdString, { stdio: 'inherit', windowsHide: true, shell: true, - env, + env: tracedEnv, }); } else { - claude = spawn( - claudeCli, - ['--settings', settingsPath, ...appendThirdPartyWebSearchToolArgs(claudeArgs)], - { - stdio: 'inherit', - windowsHide: true, - env, - } - ); + claude = spawn(claudeCli, launchArgs, { + stdio: 'inherit', + windowsHide: true, + env: tracedEnv, + }); } // 12b. Start runtime quota monitor (adaptive polling during session) diff --git a/src/copilot/copilot-executor.ts b/src/copilot/copilot-executor.ts index 47a2df5e..d1487cbf 100644 --- a/src/copilot/copilot-executor.ts +++ b/src/copilot/copilot-executor.ts @@ -17,6 +17,7 @@ import { fail, info, ok, warn } from '../utils/ui'; import { getWebSearchHookEnv, appendThirdPartyWebSearchToolArgs, + createWebSearchTraceContext, syncWebSearchMcpToConfigDir, } from '../utils/websearch-manager'; import { getImageAnalysisHookEnv } from '../utils/hooks'; @@ -181,9 +182,18 @@ export async function executeCopilotProfile( // Spawn Claude CLI return new Promise((resolve) => { - const proc = spawn(claudeCliPath, appendThirdPartyWebSearchToolArgs(claudeArgs), { + const launchArgs = appendThirdPartyWebSearchToolArgs(claudeArgs); + const traceEnv = createWebSearchTraceContext({ + launcher: 'copilot.executor', + args: launchArgs, + profile: 'copilot', + profileType: 'copilot', + claudeConfigDir, + }); + + const proc = spawn(claudeCliPath, launchArgs, { stdio: 'inherit', - env, + env: { ...env, ...traceEnv }, shell: process.platform === 'win32', }); diff --git a/src/delegation/executor/result-aggregator.ts b/src/delegation/executor/result-aggregator.ts index c7415ba6..821313cd 100644 --- a/src/delegation/executor/result-aggregator.ts +++ b/src/delegation/executor/result-aggregator.ts @@ -2,9 +2,33 @@ * Result aggregation utilities for headless executor */ -import type { ExecutionResult, StreamMessage } from './types'; +import type { ExecutionResult, StreamMessage, ToolUsageSummary } from './types'; import { warn } from '../../utils/ui'; +const WEBSEARCH_FALLBACK_TOOLS = new Set(['Bash', 'WebFetch']); + +export function summarizeToolUsage(messages: StreamMessage[]): ToolUsageSummary { + const toolNames = new Set(); + + for (const message of messages) { + const content = message.message?.content || []; + for (const entry of content) { + if (entry.type === 'tool_use' && entry.name) { + toolNames.add(entry.name); + } + } + } + + const orderedToolNames = [...toolNames]; + return { + toolNames: orderedToolNames, + calledWebSearch: toolNames.has('WebSearch') || toolNames.has('search'), + fallbackToolsUsed: orderedToolNames.filter((toolName) => + WEBSEARCH_FALLBACK_TOOLS.has(toolName) + ), + }; +} + /** * Build execution result from stream messages * @param params - Parameters for building result @@ -32,6 +56,7 @@ export function buildExecutionResult(params: { timedOut, success: exitCode === 0 && !timedOut, messages, + toolUsageSummary: summarizeToolUsage(messages), }; // Extract metadata from final 'result' message in stream-json diff --git a/src/delegation/executor/types.ts b/src/delegation/executor/types.ts index 0bb0b922..00162015 100644 --- a/src/delegation/executor/types.ts +++ b/src/delegation/executor/types.ts @@ -44,6 +44,12 @@ export interface ExecutionError { [key: string]: unknown; } +export interface ToolUsageSummary { + toolNames: string[]; + calledWebSearch: boolean; + fallbackToolsUsed: string[]; +} + /** * Options for headless execution */ @@ -86,6 +92,7 @@ export interface ExecutionResult { permissionDenials?: PermissionDenial[]; errors?: ExecutionError[]; content?: string; + toolUsageSummary?: ToolUsageSummary; } /** diff --git a/src/delegation/headless-executor.ts b/src/delegation/headless-executor.ts index 85265534..2386b6de 100644 --- a/src/delegation/headless-executor.ts +++ b/src/delegation/headless-executor.ts @@ -18,6 +18,13 @@ import { buildExecutionResult } from './executor/result-aggregator'; import { getCcsDir, getModelDisplayName } from '../utils/config-manager'; import { getProfileLookupCandidates } from '../utils/profile-compat'; import { getClaudeLaunchEnvOverrides, stripClaudeCodeEnv } from '../utils/shell-executor'; +import { + appendThirdPartyWebSearchToolArgs, + appendWebSearchTrace, + createWebSearchTraceContext, + getWebSearchHookEnv, + readWebSearchTraceRecords, +} from '../utils/websearch-manager'; // Re-export types for consumers export type { ExecutionOptions, ExecutionResult, StreamMessage } from './executor/types'; @@ -163,21 +170,32 @@ export class HeadlessExecutor { } } + const launchArgs = appendThirdPartyWebSearchToolArgs(args); + const traceEnv = createWebSearchTraceContext({ + launcher: 'delegation.headless-executor', + args: launchArgs, + cwd, + profile, + profileType: 'settings', + settingsPath, + }); + if (process.env.CCS_DEBUG) { - console.error(info(`Claude CLI args: ${args.join(' ')}`)); + console.error(info(`Claude CLI args: ${launchArgs.join(' ')}`)); } // Initialize UI before spawning await ui.init(); // Execute with spawn - return this._spawnAndExecute(claudeCli, args, { + return this._spawnAndExecute(claudeCli, launchArgs, { cwd, profile, timeout, resumeSession, sessionId, sessionMgr, + traceEnv, }); } @@ -194,9 +212,10 @@ export class HeadlessExecutor { resumeSession: boolean; sessionId: string | null; sessionMgr: SessionManager; + traceEnv?: Record; } ): Promise { - const { cwd, profile, timeout, resumeSession, sessionId, sessionMgr } = ctx; + const { cwd, profile, timeout, resumeSession, sessionId, sessionMgr, traceEnv = {} } = ctx; return new Promise((resolve, reject) => { const startTime = Date.now(); @@ -213,6 +232,9 @@ export class HeadlessExecutor { const cleanEnv = stripClaudeCodeEnv({ ...process.env, ...getClaudeLaunchEnvOverrides(), + ...getWebSearchHookEnv(), + ...traceEnv, + CCS_PROFILE_TYPE: 'settings', }); const proc = spawn(claudeCli, args, { @@ -313,6 +335,51 @@ export class HeadlessExecutor { messages, }); + const launchId = traceEnv.CCS_WEBSEARCH_TRACE_LAUNCH_ID; + if (launchId) { + const launchTraceRecords = readWebSearchTraceRecords(launchId, { + ...process.env, + ...traceEnv, + CCS_PROFILE_TYPE: 'settings', + }); + const mcpSessionSummary = [...launchTraceRecords] + .reverse() + .find((record) => record.event === 'mcp_session_summary'); + const providerSuccess = [...launchTraceRecords] + .reverse() + .find((record) => record.event === 'websearch_provider_success'); + + const exposed = mcpSessionSummary?.exposed === true; + const calledWebSearch = result.toolUsageSummary?.calledWebSearch === true; + const fallbackToolsUsed = result.toolUsageSummary?.fallbackToolsUsed || []; + + appendWebSearchTrace( + 'headless_websearch_summary', + { + profile, + sessionId: result.sessionId || null, + calledWebSearch, + fallbackToolsUsed, + providerUsed: + typeof providerSuccess?.providerName === 'string' + ? providerSuccess.providerName + : null, + exposed, + likelyBypassed: + exposed && !calledWebSearch + ? fallbackToolsUsed.length > 0 + ? true + : 'unknown' + : false, + }, + { + ...process.env, + ...traceEnv, + CCS_PROFILE_TYPE: 'settings', + } + ); + } + // Store session if (result.sessionId) { if (resumeSession || sessionId) { diff --git a/src/utils/websearch-manager.ts b/src/utils/websearch-manager.ts index 1503695a..e78bf4f8 100644 --- a/src/utils/websearch-manager.ts +++ b/src/utils/websearch-manager.ts @@ -70,6 +70,14 @@ export { // Re-export Claude launch arg helpers export { appendThirdPartyWebSearchToolArgs } from './websearch/claude-tool-args'; +// Re-export trace helpers +export { + appendWebSearchTrace, + createWebSearchTraceContext, + isWebSearchTraceEnabled, + readWebSearchTraceRecords, +} from './websearch/trace'; + // Re-export status and readiness functions export { getWebSearchCliProviders, diff --git a/src/utils/websearch/claude-tool-args.ts b/src/utils/websearch/claude-tool-args.ts index 62de05ce..15028178 100644 --- a/src/utils/websearch/claude-tool-args.ts +++ b/src/utils/websearch/claude-tool-args.ts @@ -4,6 +4,9 @@ const NATIVE_WEBSEARCH_TOOL = 'WebSearch'; const DISALLOWED_TOOLS_FLAG = '--disallowedTools'; +const APPEND_SYSTEM_PROMPT_FLAG = '--append-system-prompt'; +const THIRD_PARTY_WEBSEARCH_STEERING_PROMPT = + 'For web lookup or current-information requests, prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches. If the user explicitly wants shell commands, or WebSearch is unavailable or fails, you may fall back to Bash/network tools.'; function parseToolValue(rawValue: string): string[] { return rawValue @@ -20,19 +23,34 @@ function mergeToolValues(rawValues: string[], toolName: string): string { return merged.join(','); } +function splitArgsAtTerminator(args: string[]): { optionArgs: string[]; trailingArgs: string[] } { + const terminatorIndex = args.indexOf('--'); + if (terminatorIndex === -1) { + return { optionArgs: args, trailingArgs: [] }; + } + + return { + optionArgs: args.slice(0, terminatorIndex), + trailingArgs: args.slice(terminatorIndex), + }; +} + +function getImmediateFlagValue(args: string[], index: number): string | null { + const value = args[index + 1]; + if (value === undefined || value === '--' || value.startsWith('--')) { + return null; + } + return value; +} + function hasToolInFlag(args: string[], flag: string, toolName: string): boolean { for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === flag) { - for (let cursor = index + 1; cursor < args.length; cursor += 1) { - const value = args[cursor]; - if (value.startsWith('--')) { - break; - } - if (parseToolValue(value).includes(toolName)) { - return true; - } + const value = getImmediateFlagValue(args, index); + if (value && parseToolValue(value).includes(toolName)) { + return true; } continue; } @@ -50,39 +68,86 @@ function hasToolInFlag(args: string[], flag: string, toolName: string): boolean return false; } -export function appendThirdPartyWebSearchToolArgs(args: string[]): string[] { - if (hasToolInFlag(args, DISALLOWED_TOOLS_FLAG, NATIVE_WEBSEARCH_TOOL)) { - return args; - } - +function hasExactFlagValue(args: string[], flag: string, expectedValue: string): boolean { for (let index = 0; index < args.length; index += 1) { const arg = args[index]; - if (arg === DISALLOWED_TOOLS_FLAG) { - let cursor = index + 1; - const rawValues: string[] = []; - - while (cursor < args.length && !args[cursor].startsWith('--')) { - rawValues.push(args[cursor]); - cursor += 1; + if (arg === flag) { + const value = getImmediateFlagValue(args, index); + if (value === expectedValue) { + return true; } + continue; + } + + if (arg === `${flag}=${expectedValue}`) { + return true; + } + + if (arg.startsWith(`${flag}=`) && arg.slice(flag.length + 1) === expectedValue) { + return true; + } + } + + return false; +} + +function ensureDisallowedNativeWebSearchTool(args: string[]): string[] { + const { optionArgs, trailingArgs } = splitArgsAtTerminator(args); + + if (hasToolInFlag(optionArgs, DISALLOWED_TOOLS_FLAG, NATIVE_WEBSEARCH_TOOL)) { + return args; + } + + for (let index = 0; index < optionArgs.length; index += 1) { + const arg = optionArgs[index]; + + if (arg === DISALLOWED_TOOLS_FLAG) { + const currentValue = getImmediateFlagValue(optionArgs, index); + const mergedValue = mergeToolValues( + currentValue ? [currentValue] : [], + NATIVE_WEBSEARCH_TOOL + ); return [ - ...args.slice(0, index + 1), - mergeToolValues(rawValues, NATIVE_WEBSEARCH_TOOL), - ...args.slice(cursor), + ...optionArgs.slice(0, index + 1), + mergedValue, + ...optionArgs.slice(currentValue === null ? index + 1 : index + 2), + ...trailingArgs, ]; } if (arg.startsWith(`${DISALLOWED_TOOLS_FLAG}=`)) { const rawValue = arg.slice(DISALLOWED_TOOLS_FLAG.length + 1); return [ - ...args.slice(0, index), + ...optionArgs.slice(0, index), `${DISALLOWED_TOOLS_FLAG}=${mergeToolValues([rawValue], NATIVE_WEBSEARCH_TOOL)}`, - ...args.slice(index + 1), + ...optionArgs.slice(index + 1), + ...trailingArgs, ]; } } - return [...args, DISALLOWED_TOOLS_FLAG, NATIVE_WEBSEARCH_TOOL]; + return [...optionArgs, DISALLOWED_TOOLS_FLAG, NATIVE_WEBSEARCH_TOOL, ...trailingArgs]; +} + +function ensureWebSearchSteeringPrompt(args: string[]): string[] { + const { optionArgs, trailingArgs } = splitArgsAtTerminator(args); + + if ( + hasExactFlagValue(optionArgs, APPEND_SYSTEM_PROMPT_FLAG, THIRD_PARTY_WEBSEARCH_STEERING_PROMPT) + ) { + return args; + } + + return [ + ...optionArgs, + APPEND_SYSTEM_PROMPT_FLAG, + THIRD_PARTY_WEBSEARCH_STEERING_PROMPT, + ...trailingArgs, + ]; +} + +export function appendThirdPartyWebSearchToolArgs(args: string[]): string[] { + return ensureWebSearchSteeringPrompt(ensureDisallowedNativeWebSearchTool(args)); } diff --git a/src/utils/websearch/hook-env.ts b/src/utils/websearch/hook-env.ts index ac0e9e1b..78fff9ca 100644 --- a/src/utils/websearch/hook-env.ts +++ b/src/utils/websearch/hook-env.ts @@ -19,6 +19,13 @@ export function getWebSearchHookEnv(): Record { const wsConfig = getWebSearchConfig(); const env: Record = {}; + if (process.env.CCS_WEBSEARCH_TRACE === '1' || process.env.CCS_DEBUG === '1') { + env.CCS_WEBSEARCH_TRACE = '1'; + } + if (process.env.CCS_WEBSEARCH_TRACE_FILE) { + env.CCS_WEBSEARCH_TRACE_FILE = process.env.CCS_WEBSEARCH_TRACE_FILE; + } + // Skip hook entirely if disabled if (!wsConfig.enabled) { env.CCS_WEBSEARCH_SKIP = '1'; diff --git a/src/utils/websearch/index.ts b/src/utils/websearch/index.ts index 5cd4cd27..0d375dfc 100644 --- a/src/utils/websearch/index.ts +++ b/src/utils/websearch/index.ts @@ -67,6 +67,14 @@ export { // Claude launch args export { appendThirdPartyWebSearchToolArgs } from './claude-tool-args'; +// Trace helpers +export { + appendWebSearchTrace, + createWebSearchTraceContext, + isWebSearchTraceEnabled, + readWebSearchTraceRecords, +} from './trace'; + // Status and Readiness export { getWebSearchCliProviders, diff --git a/src/utils/websearch/mcp-installer.ts b/src/utils/websearch/mcp-installer.ts index ff45b31f..a86e0900 100644 --- a/src/utils/websearch/mcp-installer.ts +++ b/src/utils/websearch/mcp-installer.ts @@ -10,6 +10,7 @@ import { getClaudeUserConfigPath } from '../claude-config-path'; import { info, warn } from '../ui'; import { InstanceManager } from '../../management/instance-manager'; import { installWebSearchHook } from './hook-installer'; +import { appendWebSearchTrace } from './trace'; const WEBSEARCH_MCP_SERVER = 'ccs-websearch-server.cjs'; const WEBSEARCH_MCP_SERVER_NAME = 'ccs-websearch'; @@ -154,10 +155,12 @@ function removeManagedServerConfig(configPath: string): boolean { export function installWebSearchMcpServer(): boolean { const wsConfig = getWebSearchConfig(); if (!wsConfig.enabled) { + appendWebSearchTrace('websearch_mcp_install_skipped', { reason: 'disabled' }); return false; } if (!installWebSearchHook()) { + appendWebSearchTrace('websearch_mcp_install_failed', { reason: 'hook_unavailable' }); if (process.env.CCS_DEBUG) { console.error( warn('WebSearch MCP server install skipped because hook runtime is unavailable') @@ -168,6 +171,7 @@ export function installWebSearchMcpServer(): boolean { const sourcePath = resolveBundledServerSourcePath(); if (!sourcePath) { + appendWebSearchTrace('websearch_mcp_install_failed', { reason: 'source_missing' }); if (process.env.CCS_DEBUG) { console.error(warn(`WebSearch MCP server source not found: ${WEBSEARCH_MCP_SERVER}`)); } @@ -181,6 +185,7 @@ export function installWebSearchMcpServer(): boolean { const serverPath = getWebSearchMcpServerPath(); if (hasMatchingContents(sourcePath, serverPath)) { + appendWebSearchTrace('websearch_mcp_install_ready', { serverPath }); return true; } @@ -190,8 +195,13 @@ export function installWebSearchMcpServer(): boolean { fs.copyFileSync(sourcePath, tempPath); fs.chmodSync(tempPath, 0o755); fs.renameSync(tempPath, serverPath); + appendWebSearchTrace('websearch_mcp_install_ready', { serverPath }); return true; } catch (error) { + appendWebSearchTrace('websearch_mcp_install_failed', { + reason: 'copy_failed', + error: (error as Error).message, + }); if (process.env.CCS_DEBUG) { console.error(warn(`Failed to install WebSearch MCP server: ${(error as Error).message}`)); } @@ -206,6 +216,7 @@ export function installWebSearchMcpServer(): boolean { export function ensureWebSearchMcpConfig(): boolean { const wsConfig = getWebSearchConfig(); if (!wsConfig.enabled) { + appendWebSearchTrace('websearch_mcp_config_skipped', { reason: 'disabled' }); return false; } @@ -214,6 +225,7 @@ export function ensureWebSearchMcpConfig(): boolean { const config = readClaudeUserConfig(claudeUserConfigPath); if (config === null) { + appendWebSearchTrace('websearch_mcp_config_failed', { reason: 'malformed_user_config' }); if (process.env.CCS_DEBUG) { console.error(warn('Malformed ~/.claude.json prevents WebSearch MCP provisioning')); } @@ -239,6 +251,7 @@ export function ensureWebSearchMcpConfig(): boolean { currentConfig !== null && JSON.stringify(currentConfig) === JSON.stringify(desiredServerConfig) ) { + appendWebSearchTrace('websearch_mcp_config_ready', { configPath: claudeUserConfigPath }); return true; } @@ -252,11 +265,17 @@ export function ensureWebSearchMcpConfig(): boolean { try { writeClaudeUserConfig(claudeUserConfigPath, nextConfig); + appendWebSearchTrace('websearch_mcp_config_ready', { configPath: claudeUserConfigPath }); if (process.env.CCS_DEBUG) { console.error(info(`Ensured WebSearch MCP config in ${claudeUserConfigPath}`)); } return true; } catch (error) { + appendWebSearchTrace('websearch_mcp_config_failed', { + reason: 'write_failed', + configPath: claudeUserConfigPath, + error: (error as Error).message, + }); if (process.env.CCS_DEBUG) { console.error(warn(`Failed to update ~/.claude.json: ${(error as Error).message}`)); } @@ -267,18 +286,25 @@ export function ensureWebSearchMcpConfig(): boolean { export function ensureWebSearchMcp(): boolean { const wsConfig = getWebSearchConfig(); if (!wsConfig.enabled) { + appendWebSearchTrace('websearch_mcp_ensure_skipped', { reason: 'disabled' }); return false; } - return installWebSearchMcpServer() && ensureWebSearchMcpConfig(); + const installed = installWebSearchMcpServer(); + const configured = installed && ensureWebSearchMcpConfig(); + appendWebSearchTrace('websearch_mcp_ensure_result', { installed, configured }); + return installed && configured; } export function syncWebSearchMcpToConfigDir(claudeConfigDir: string | undefined): boolean { if (!claudeConfigDir) { + appendWebSearchTrace('websearch_mcp_sync_skipped', { reason: 'missing_config_dir' }); return false; } - return new InstanceManager().syncMcpServers(claudeConfigDir); + const synced = new InstanceManager().syncMcpServers(claudeConfigDir); + appendWebSearchTrace('websearch_mcp_sync_result', { claudeConfigDir, synced }); + return synced; } export function uninstallWebSearchMcpServer(): boolean { diff --git a/src/utils/websearch/trace.ts b/src/utils/websearch/trace.ts new file mode 100644 index 00000000..86b4ce2c --- /dev/null +++ b/src/utils/websearch/trace.ts @@ -0,0 +1,223 @@ +/** + * Best-effort WebSearch trace helpers. + * + * Writes opt-in JSONL trace records to ~/.ccs/logs/websearch-trace.jsonl so + * CCS can explain launch intent, MCP exposure, provider selection, and likely + * bypass scenarios without polluting Claude/MCP stdout. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { getCcsDir } from '../config-manager'; + +const TRACE_FILE_NAME = 'websearch-trace.jsonl'; +const SAFE_SYSTEM_TRACE_PREFIXES = ['/tmp/', '/var/log/']; +const NATIVE_WEBSEARCH_TOOL = 'WebSearch'; +const DISALLOWED_TOOLS_FLAG = '--disallowedTools'; +const APPEND_SYSTEM_PROMPT_FLAG = '--append-system-prompt'; +const THIRD_PARTY_WEBSEARCH_STEERING_PROMPT = + 'For web lookup or current-information requests, prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches. If the user explicitly wants shell commands, or WebSearch is unavailable or fails, you may fall back to Bash/network tools.'; + +function parseToolValue(rawValue: string): string[] { + return rawValue + .split(',') + .map((value) => value.trim()) + .filter((value) => value.length > 0); +} + +function getImmediateFlagValue(args: string[], index: number): string | null { + const value = args[index + 1]; + if (value === undefined || value === '--' || value.startsWith('--')) { + return null; + } + return value; +} + +function hasToolInFlag(args: string[], flag: string, toolName: string): boolean { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + + if (arg === flag) { + const value = getImmediateFlagValue(args, index); + if (value && parseToolValue(value).includes(toolName)) { + return true; + } + continue; + } + + if (arg.startsWith(`${flag}=`)) { + const rawValue = arg.slice(flag.length + 1); + if (parseToolValue(rawValue).includes(toolName)) { + return true; + } + } + } + + return false; +} + +function hasExactFlagValue(args: string[], flag: string, expectedValue: string): boolean { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + + if (arg === flag) { + if (getImmediateFlagValue(args, index) === expectedValue) { + return true; + } + continue; + } + + if (arg === `${flag}=${expectedValue}`) { + return true; + } + } + + return false; +} + +function getTraceFilePath(env: NodeJS.ProcessEnv): string { + const configured = env.CCS_WEBSEARCH_TRACE_FILE?.trim(); + if (!configured) { + return path.join(getCcsDir(), 'logs', TRACE_FILE_NAME); + } + + const resolved = path.resolve(configured); + const home = env.HOME || env.USERPROFILE || ''; + const normalizedHome = home ? `${path.resolve(home)}${path.sep}` : ''; + const normalizedCcsDir = `${path.resolve(getCcsDir())}${path.sep}`; + + if ( + resolved.startsWith(normalizedCcsDir) || + (normalizedHome && resolved.startsWith(normalizedHome)) || + SAFE_SYSTEM_TRACE_PREFIXES.some((prefix) => resolved.startsWith(prefix)) + ) { + return resolved; + } + + return path.join(getCcsDir(), 'logs', TRACE_FILE_NAME); +} + +export function isWebSearchTraceEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return env.CCS_WEBSEARCH_TRACE === '1' || env.CCS_DEBUG === '1'; +} + +export function appendWebSearchTrace( + event: string, + payload: Record = {}, + env: NodeJS.ProcessEnv = process.env +): void { + if (!isWebSearchTraceEnabled(env)) { + return; + } + + try { + const traceFilePath = getTraceFilePath(env); + fs.mkdirSync(path.dirname(traceFilePath), { recursive: true }); + fs.appendFileSync( + traceFilePath, + JSON.stringify({ + at: new Date().toISOString(), + event, + launchId: env.CCS_WEBSEARCH_TRACE_LAUNCH_ID || null, + launcher: env.CCS_WEBSEARCH_TRACE_LAUNCHER || null, + profileType: env.CCS_PROFILE_TYPE || null, + pid: process.pid, + ...payload, + }) + '\n', + 'utf8' + ); + } catch { + // Tracing must never affect launch behavior. + } +} + +export function readWebSearchTraceRecords( + launchId: string, + env: NodeJS.ProcessEnv = process.env +): Array> { + if (!launchId) { + return []; + } + + try { + const traceFilePath = getTraceFilePath(env); + if (!fs.existsSync(traceFilePath)) { + return []; + } + + return fs + .readFileSync(traceFilePath, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record) + .filter((record) => record.launchId === launchId); + } catch { + return []; + } +} + +function buildLaunchId(): string { + const random = Math.random().toString(36).slice(2, 10); + return `websearch-${Date.now()}-${process.pid}-${random}`; +} + +function summarizeLaunchArgs(args: string[]): Record { + return { + argCount: args.length, + hasSettingsFlag: args.includes('--settings'), + nativeWebSearchDisallowed: hasToolInFlag(args, DISALLOWED_TOOLS_FLAG, NATIVE_WEBSEARCH_TOOL), + steeringPromptApplied: hasExactFlagValue( + args, + APPEND_SYSTEM_PROMPT_FLAG, + THIRD_PARTY_WEBSEARCH_STEERING_PROMPT + ), + }; +} + +export function createWebSearchTraceContext(params: { + launcher: string; + args: string[]; + cwd?: string; + profile?: string; + profileType?: string; + settingsPath?: string; + claudeConfigDir?: string; + env?: NodeJS.ProcessEnv; +}): Record { + const env = params.env ?? process.env; + if (!isWebSearchTraceEnabled(env)) { + return {}; + } + + const launchId = buildLaunchId(); + const traceEnv: Record = { + CCS_WEBSEARCH_TRACE: '1', + CCS_WEBSEARCH_TRACE_LAUNCH_ID: launchId, + CCS_WEBSEARCH_TRACE_LAUNCHER: params.launcher, + }; + + if (env.CCS_WEBSEARCH_TRACE_FILE) { + traceEnv.CCS_WEBSEARCH_TRACE_FILE = env.CCS_WEBSEARCH_TRACE_FILE; + } + + appendWebSearchTrace( + 'ccs_websearch_launch', + { + launcher: params.launcher, + profile: params.profile || null, + profileType: params.profileType || null, + cwd: params.cwd || null, + settingsPath: params.settingsPath || null, + claudeConfigDir: params.claudeConfigDir || null, + ...summarizeLaunchArgs(params.args), + }, + { + ...env, + ...traceEnv, + CCS_PROFILE_TYPE: params.profileType || env.CCS_PROFILE_TYPE || '', + } + ); + + return traceEnv; +} diff --git a/tests/unit/hooks/ccs-websearch-mcp-server.test.ts b/tests/unit/hooks/ccs-websearch-mcp-server.test.ts index 9b0e35df..79f4fdaf 100644 --- a/tests/unit/hooks/ccs-websearch-mcp-server.test.ts +++ b/tests/unit/hooks/ccs-websearch-mcp-server.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'bun:test'; import { spawn } from 'child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -73,8 +73,15 @@ function collectResponses( }); } +function waitForClose(child: ReturnType): Promise { + return new Promise((resolve, reject) => { + child.once('close', (code) => resolve(code)); + child.once('error', reject); + }); +} + describe('ccs-websearch MCP server', () => { - it('lists the CCS search tool and returns provider-backed results', async () => { + it('lists the CCS WebSearch tool and returns provider-backed results', async () => { const tempDir = mkdtempSync(join(tmpdir(), 'ccs-websearch-mcp-server-')); const preloadPath = join(tempDir, 'mock-fetch.cjs'); const html = ` @@ -124,7 +131,7 @@ describe('ccs-websearch MCP server', () => { jsonrpc: '2.0', id: 3, method: 'tools/call', - params: { name: 'search', arguments: { query: 'btc price' } }, + params: { name: 'WebSearch', arguments: { query: 'btc price' } }, }) ); @@ -135,15 +142,16 @@ describe('ccs-websearch MCP server', () => { expect(toolsList?.result).toEqual({ tools: [ { - name: 'search', + name: 'WebSearch', description: - 'Search the web through CCS-managed providers. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.', + 'Third-party WebSearch replacement for CCS-managed Claude launches. Use this instead of Bash/curl/http fetches for web lookups. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.', inputSchema: { type: 'object', properties: { query: { type: 'string', - description: 'The search query to run against CCS WebSearch providers.', + description: + 'Web query to resolve through CCS providers. Prefer this tool over ad hoc Bash/curl lookups when you need current web information.', }, }, required: ['query'], @@ -165,6 +173,75 @@ describe('ccs-websearch MCP server', () => { } }); + it('accepts the legacy search alias for direct calls', async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'ccs-websearch-mcp-server-')); + const preloadPath = join(tempDir, 'mock-fetch.cjs'); + const html = ` + Example title + Example snippet + `.trim(); + writeFileSync( + preloadPath, + `global.fetch = async () => ({ ok: true, text: async () => ${JSON.stringify(html)} });\n`, + 'utf8' + ); + + const child = spawn('node', ['-r', preloadPath, serverPath], { + env: { + ...process.env, + CCS_PROFILE_TYPE: 'settings', + CCS_WEBSEARCH_ENABLED: '1', + CCS_WEBSEARCH_SKIP: '0', + CCS_WEBSEARCH_BRAVE: '0', + CCS_WEBSEARCH_DUCKDUCKGO: '1', + CCS_WEBSEARCH_EXA: '0', + CCS_WEBSEARCH_GEMINI: '0', + CCS_WEBSEARCH_GROK: '0', + CCS_WEBSEARCH_OPENCODE: '0', + CCS_WEBSEARCH_TAVILY: '0', + }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + + try { + const responsesPromise = collectResponses(child, 2); + child.stdin.write( + encodeMessage({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'bun-test', version: '1.0.0' }, + }, + }) + ); + child.stdin.write( + encodeMessage({ + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'search', arguments: { query: 'btc price' } }, + }) + ); + + const responses = await responsesPromise; + const toolCall = responses.find((message) => message.id === 2); + + expect(toolCall?.result).toBeDefined(); + expect( + ((toolCall?.result as { content: Array<{ text: string }> }).content[0] || {}).text + ).toContain('CCS local WebSearch evidence'); + expect( + ((toolCall?.result as { content: Array<{ text: string }> }).content[0] || {}).text + ).toContain('Provider: DuckDuckGo'); + } finally { + child.kill(); + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('hides the tool for native account profiles', async () => { const child = spawn('node', [serverPath], { env: { @@ -200,4 +277,101 @@ describe('ccs-websearch MCP server', () => { child.kill(); } }); + + it('writes trace records for exposure, tool calls, provider success, and session summary', async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'ccs-websearch-mcp-trace-')); + const preloadPath = join(tempDir, 'mock-fetch.cjs'); + const ccsHome = join(tempDir, 'home'); + const tracePath = join(ccsHome, '.ccs', 'logs', 'websearch-trace.jsonl'); + const html = ` + Example title + Example snippet + `.trim(); + writeFileSync( + preloadPath, + `global.fetch = async () => ({ ok: true, text: async () => ${JSON.stringify(html)} });\n`, + 'utf8' + ); + + const child = spawn('node', ['-r', preloadPath, serverPath], { + env: { + ...process.env, + CCS_HOME: ccsHome, + CCS_PROFILE_TYPE: 'settings', + CCS_WEBSEARCH_TRACE: '1', + CCS_WEBSEARCH_TRACE_LAUNCH_ID: 'mcp-trace-test', + CCS_WEBSEARCH_TRACE_LAUNCHER: 'unit-test', + CCS_WEBSEARCH_ENABLED: '1', + CCS_WEBSEARCH_SKIP: '0', + CCS_WEBSEARCH_BRAVE: '0', + CCS_WEBSEARCH_DUCKDUCKGO: '1', + CCS_WEBSEARCH_EXA: '0', + CCS_WEBSEARCH_GEMINI: '0', + CCS_WEBSEARCH_GROK: '0', + CCS_WEBSEARCH_OPENCODE: '0', + CCS_WEBSEARCH_TAVILY: '0', + }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + + try { + const responsesPromise = collectResponses(child, 3); + child.stdin.write( + encodeMessage({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'bun-test', version: '1.0.0' }, + }, + }) + ); + child.stdin.write(encodeMessage({ jsonrpc: '2.0', id: 2, method: 'tools/list' })); + child.stdin.write( + encodeMessage({ + jsonrpc: '2.0', + id: 3, + method: 'tools/call', + params: { name: 'WebSearch', arguments: { query: 'btc price' } }, + }) + ); + + await responsesPromise; + } finally { + child.kill(); + await waitForClose(child); + } + + const traceEvents = readFileSync(tracePath, 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line) as Record); + + expect( + traceEvents.some((event) => event.event === 'mcp_tools_list' && event.exposed === true) + ).toBe(true); + expect( + traceEvents.some( + (event) => event.event === 'mcp_tool_call_received' && event.toolName === 'WebSearch' + ) + ).toBe(true); + expect( + traceEvents.some( + (event) => + event.event === 'websearch_provider_success' && event.providerName === 'DuckDuckGo' + ) + ).toBe(true); + expect( + traceEvents.some( + (event) => + event.event === 'mcp_session_summary' && + event.calledWebSearch === true && + event.toolCalls === 1 + ) + ).toBe(true); + + rmSync(tempDir, { recursive: true, force: true }); + }); }); diff --git a/tests/unit/hooks/websearch-transformer.test.ts b/tests/unit/hooks/websearch-transformer.test.ts index e731e197..2c25302e 100644 --- a/tests/unit/hooks/websearch-transformer.test.ts +++ b/tests/unit/hooks/websearch-transformer.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { spawnSync } from 'node:child_process'; @@ -193,4 +193,78 @@ describe('websearch-transformer hook helpers', () => { ); expect(output).not.toHaveProperty('additionalContext'); }); + + it('writes opt-in trace records with redacted query fingerprints', () => { + const tempDir = mkdtempSync(join(tmpdir(), 'websearch-hook-trace-')); + const preloadPath = join(tempDir, 'mock-fetch.cjs'); + const ccsHome = join(tempDir, 'home'); + const tracePath = join(ccsHome, '.ccs', 'logs', 'websearch-trace.jsonl'); + const html = ` + Example title + Example snippet + `.trim(); + + writeFileSync( + preloadPath, + `global.fetch = async () => ({ ok: true, text: async () => ${JSON.stringify(html)} });\n`, + 'utf8' + ); + + try { + const result = spawnSync('node', ['-r', preloadPath, hookPath], { + encoding: 'utf8', + input: JSON.stringify({ + tool_name: 'WebSearch', + tool_input: { query: 'btc price' }, + }), + env: { + ...process.env, + CCS_HOME: ccsHome, + CCS_WEBSEARCH_TRACE: '1', + CCS_WEBSEARCH_TRACE_LAUNCH_ID: 'hook-trace-test', + CCS_WEBSEARCH_TRACE_LAUNCHER: 'unit-test', + CCS_WEBSEARCH_ENABLED: '1', + CCS_WEBSEARCH_SKIP: '0', + CCS_WEBSEARCH_BRAVE: '0', + CCS_WEBSEARCH_DUCKDUCKGO: '1', + CCS_WEBSEARCH_EXA: '0', + CCS_WEBSEARCH_GEMINI: '0', + CCS_WEBSEARCH_GROK: '0', + CCS_WEBSEARCH_OPENCODE: '0', + CCS_WEBSEARCH_TAVILY: '0', + }, + }); + + expect(result.status).toBe(0); + + const traceContents = readFileSync(tracePath, 'utf8'); + expect(traceContents).not.toContain('btc price'); + + const traceEvents = traceContents + .trim() + .split('\n') + .map((line) => JSON.parse(line) as Record); + + expect(traceEvents.some((event) => event.event === 'websearch_hook_invoked')).toBe(true); + expect( + traceEvents.some( + (event) => + event.event === 'websearch_provider_attempt' && event.providerName === 'DuckDuckGo' + ) + ).toBe(true); + expect( + traceEvents.some( + (event) => + event.event === 'websearch_provider_success' && event.providerName === 'DuckDuckGo' + ) + ).toBe(true); + const fingerprintEvent = traceEvents.find( + (event) => event.event === 'websearch_hook_invoked' + ) as { queryHash?: string; queryLength?: number } | undefined; + expect(fingerprintEvent?.queryHash).toBeString(); + expect(fingerprintEvent?.queryLength).toBe(9); + } finally { + rmSync(tempDir, { force: true, recursive: true }); + } + }); }); diff --git a/tests/unit/targets/settings-profile-websearch-launch.test.ts b/tests/unit/targets/settings-profile-websearch-launch.test.ts index e0ab84f4..045757ef 100644 --- a/tests/unit/targets/settings-profile-websearch-launch.test.ts +++ b/tests/unit/targets/settings-profile-websearch-launch.test.ts @@ -4,6 +4,8 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; +const STEERING_PROMPT_SNIPPET = 'prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches'; + interface RunResult { status: number | null; stdout: string; @@ -25,6 +27,15 @@ function runCcs(args: string[], env: NodeJS.ProcessEnv): RunResult { }; } +function readTraceEvents(tracePath: string): Array> { + return fs + .readFileSync(tracePath, 'utf8') + .trim() + .split('\n') + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record); +} + describe('settings profile WebSearch launch', () => { let tmpHome = ''; let ccsDir = ''; @@ -119,7 +130,40 @@ exit 0 expect(result.status).toBe(0); expect(result.stderr).not.toContain('could not prepare the local WebSearch tool'); expect(fs.existsSync(claudeArgsLogPath)).toBe(true); - expect(fs.readFileSync(claudeArgsLogPath, 'utf8')).toContain('--disallowedTools'); - expect(fs.readFileSync(claudeArgsLogPath, 'utf8')).toContain('WebSearch'); + const launchedArgs = fs.readFileSync(claudeArgsLogPath, 'utf8'); + expect(launchedArgs).toContain('--disallowedTools'); + expect(launchedArgs).toContain('WebSearch'); + expect(launchedArgs).toContain('--append-system-prompt'); + expect(launchedArgs).toContain(STEERING_PROMPT_SNIPPET); + }); + + it('writes a source-side launch trace for settings profiles when tracing is enabled', () => { + if (process.platform === 'win32') return; + + const tracePath = path.join(ccsDir, 'logs', 'websearch-trace.jsonl'); + const result = runCcs(['glm', 'smoke'], { + ...baseEnv, + CCS_WEBSEARCH_TRACE: '1', + }); + + expect(result.status).toBe(0); + expect(fs.existsSync(tracePath)).toBe(true); + + const traceEvents = readTraceEvents(tracePath); + const launchEvent = traceEvents.find( + (event) => event.event === 'ccs_websearch_launch' + ) as + | { + launcher?: string; + nativeWebSearchDisallowed?: boolean; + steeringPromptApplied?: boolean; + settingsPath?: string; + } + | undefined; + + expect(launchEvent?.launcher).toBe('ccs.settings-profile'); + expect(launchEvent?.nativeWebSearchDisallowed).toBe(true); + expect(launchEvent?.steeringPromptApplied).toBe(true); + expect(launchEvent?.settingsPath).toBe(settingsPath); }); }); diff --git a/tests/unit/utils/claudecode-env-stripping.test.ts b/tests/unit/utils/claudecode-env-stripping.test.ts index 0d10ebe0..38109ecb 100644 --- a/tests/unit/utils/claudecode-env-stripping.test.ts +++ b/tests/unit/utils/claudecode-env-stripping.test.ts @@ -21,12 +21,14 @@ type SpawnCall = { options: Record | undefined; }; +const STEERING_PROMPT_SNIPPET = 'prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches'; const spawnCalls: SpawnCall[] = []; const originalPlatform = process.platform; let baselineSigintListeners: Array<(...args: unknown[]) => void> = []; let baselineSigtermListeners: Array<(...args: unknown[]) => void> = []; let baselineSighupListeners: Array<(...args: unknown[]) => void> = []; let originalCcsHome: string | undefined; +let originalCcsClaudePath: string | undefined; let originalDisableAutoUpdater: string | undefined; const realSpawn = childProcess.spawn.bind(childProcess); const realSpawnSync = childProcess.spawnSync.bind(childProcess); @@ -150,6 +152,7 @@ describe('CLAUDECODE environment stripping', () => { spawnCalls.length = 0; process.env.CCS_QUIET = '1'; originalCcsHome = process.env.CCS_HOME; + originalCcsClaudePath = process.env.CCS_CLAUDE_PATH; originalDisableAutoUpdater = process.env.DISABLE_AUTOUPDATER; delete process.env.DISABLE_AUTOUPDATER; baselineSigintListeners = process.listeners('SIGINT'); @@ -162,8 +165,11 @@ describe('CLAUDECODE environment stripping', () => { delete process.env.CLAUDECODE; delete process.env.claudecode; delete process.env.CCS_QUIET; + delete process.env.CCS_WEBSEARCH_TRACE; if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome; else delete process.env.CCS_HOME; + if (originalCcsClaudePath !== undefined) process.env.CCS_CLAUDE_PATH = originalCcsClaudePath; + else delete process.env.CCS_CLAUDE_PATH; if (originalDisableAutoUpdater !== undefined) { process.env.DISABLE_AUTOUPDATER = originalDisableAutoUpdater; } else { @@ -325,4 +331,47 @@ describe('CLAUDECODE environment stripping', () => { expect(Object.keys(env).map((k) => k.toUpperCase())).not.toContain('CLAUDECODE'); expect(env.DISABLE_AUTOUPDATER).toBe('1'); }); + + it('headless executor adds third-party WebSearch steering args and env', async () => { + writeConfigWithAutoUpdatePreference(false); + const ccsDir = path.join(process.env.CCS_HOME as string, '.ccs'); + fs.writeFileSync(path.join(ccsDir, 'glm.settings.json'), '{}\n', 'utf8'); + process.env.CCS_CLAUDE_PATH = 'claude'; + + const result = await HeadlessExecutor.execute('glm', 'latest AI chip news', { + permissionMode: 'default', + timeout: 1000, + }); + + expect(result.success).toBe(true); + expect(spawnCalls.length).toBeGreaterThan(0); + const launch = spawnCalls[0]; + expect(launch.args).toContain('--disallowedTools'); + expect(launch.args).toContain('WebSearch'); + expect(launch.args).toContain('--append-system-prompt'); + expect(launch.args.join(' ')).toContain(STEERING_PROMPT_SNIPPET); + const env = launch.options?.env as NodeJS.ProcessEnv; + expect(env.CCS_PROFILE_TYPE).toBe('settings'); + expect(env.CCS_WEBSEARCH_ENABLED || env.CCS_WEBSEARCH_SKIP).toBeDefined(); + }); + + it('headless executor propagates a WebSearch trace launch id when tracing is enabled', async () => { + writeConfigWithAutoUpdatePreference(false); + const ccsDir = path.join(process.env.CCS_HOME as string, '.ccs'); + fs.writeFileSync(path.join(ccsDir, 'glm.settings.json'), '{}\n', 'utf8'); + process.env.CCS_CLAUDE_PATH = 'claude'; + process.env.CCS_WEBSEARCH_TRACE = '1'; + + const result = await HeadlessExecutor.execute('glm', 'latest AI chip news', { + permissionMode: 'default', + timeout: 1000, + }); + + expect(result.success).toBe(true); + expect(spawnCalls.length).toBeGreaterThan(0); + const env = spawnCalls[0].options?.env as NodeJS.ProcessEnv; + expect(env.CCS_WEBSEARCH_TRACE).toBe('1'); + expect(env.CCS_WEBSEARCH_TRACE_LAUNCH_ID).toBeString(); + expect(env.CCS_WEBSEARCH_TRACE_LAUNCHER).toBe('delegation.headless-executor'); + }); }); diff --git a/tests/unit/utils/websearch/claude-tool-args.test.ts b/tests/unit/utils/websearch/claude-tool-args.test.ts index 6eb2367c..9f902b80 100644 --- a/tests/unit/utils/websearch/claude-tool-args.test.ts +++ b/tests/unit/utils/websearch/claude-tool-args.test.ts @@ -1,25 +1,47 @@ import { describe, expect, it } from 'bun:test'; import { appendThirdPartyWebSearchToolArgs } from '../../../../src/utils/websearch/claude-tool-args'; +const STEERING_PROMPT = + 'For web lookup or current-information requests, prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches. If the user explicitly wants shell commands, or WebSearch is unavailable or fails, you may fall back to Bash/network tools.'; + describe('appendThirdPartyWebSearchToolArgs', () => { - it('appends native WebSearch suppression when no tool flags are present', () => { + it('appends native WebSearch suppression and steering prompt when no tool flags are present', () => { expect(appendThirdPartyWebSearchToolArgs(['smoke'])).toEqual([ 'smoke', '--disallowedTools', 'WebSearch', + '--append-system-prompt', + STEERING_PROMPT, ]); }); - it('does not append duplicate suppression when WebSearch is already disallowed', () => { - expect(appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools', 'WebSearch'])).toEqual( - ['smoke', '--disallowedTools', 'WebSearch'] - ); + it('does not append duplicate suppression or steering prompt when both are already present', () => { + expect( + appendThirdPartyWebSearchToolArgs([ + 'smoke', + '--disallowedTools', + 'WebSearch', + '--append-system-prompt', + STEERING_PROMPT, + ]) + ).toEqual([ + 'smoke', + '--disallowedTools', + 'WebSearch', + '--append-system-prompt', + STEERING_PROMPT, + ]); }); it('detects comma-separated disallowed tool values', () => { - expect(appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools=Read,WebSearch'])).toEqual( - ['smoke', '--disallowedTools=Read,WebSearch'] - ); + expect( + appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools=Read,WebSearch']) + ).toEqual([ + 'smoke', + '--disallowedTools=Read,WebSearch', + '--append-system-prompt', + STEERING_PROMPT, + ]); }); it('merges WebSearch into an existing space-separated disallowed tool flag', () => { @@ -27,6 +49,8 @@ describe('appendThirdPartyWebSearchToolArgs', () => { 'smoke', '--disallowedTools', 'Read,WebSearch', + '--append-system-prompt', + STEERING_PROMPT, ]); }); @@ -34,6 +58,75 @@ describe('appendThirdPartyWebSearchToolArgs', () => { expect(appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools=Read'])).toEqual([ 'smoke', '--disallowedTools=Read,WebSearch', + '--append-system-prompt', + STEERING_PROMPT, + ]); + }); + + it('preserves user-supplied append-system-prompt values and adds the CCS steering hint once', () => { + expect( + appendThirdPartyWebSearchToolArgs([ + 'smoke', + '--append-system-prompt', + 'User-provided instruction', + ]) + ).toEqual([ + 'smoke', + '--append-system-prompt', + 'User-provided instruction', + '--disallowedTools', + 'WebSearch', + '--append-system-prompt', + STEERING_PROMPT, + ]); + }); + + it('does not duplicate the steering prompt when it already exists in equals form', () => { + expect( + appendThirdPartyWebSearchToolArgs([ + 'smoke', + '--disallowedTools', + 'WebSearch', + `--append-system-prompt=${STEERING_PROMPT}`, + ]) + ).toEqual([ + 'smoke', + '--disallowedTools', + 'WebSearch', + `--append-system-prompt=${STEERING_PROMPT}`, + ]); + }); + + it('does not consume positional args after a disallowed-tools flag value', () => { + expect( + appendThirdPartyWebSearchToolArgs(['--disallowedTools', 'Read', 'latest AI news']) + ).toEqual([ + '--disallowedTools', + 'Read,WebSearch', + 'latest AI news', + '--append-system-prompt', + STEERING_PROMPT, + ]); + }); + + it('injects synthetic flags before an end-of-options marker', () => { + expect(appendThirdPartyWebSearchToolArgs(['--', 'latest AI news'])).toEqual([ + '--disallowedTools', + 'WebSearch', + '--append-system-prompt', + STEERING_PROMPT, + '--', + 'latest AI news', + ]); + }); + + it('inserts the WebSearch disallow value when the flag is present without one', () => { + expect(appendThirdPartyWebSearchToolArgs(['--disallowedTools', '--verbose'])).toEqual([ + '--disallowedTools', + 'WebSearch', + '--verbose', + '--append-system-prompt', + STEERING_PROMPT, ]); }); });