feat(websearch): finish managed third-party rollout

- steer third-party launches toward the managed WebSearch MCP tool

- add opt-in trace diagnostics across launch, MCP, provider, and headless paths

- extend docs and regression coverage for the first-class runtime

Refs #862
This commit is contained in:
Tam Nhu Tran
2026-03-30 22:41:51 -04:00
parent b087c02738
commit de7171d810
21 changed files with 1281 additions and 92 deletions
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'bun:test';
import { spawn } from 'child_process';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
@@ -73,8 +73,15 @@ function collectResponses(
});
}
function waitForClose(child: ReturnType<typeof spawn>): Promise<number | null> {
return new Promise((resolve, reject) => {
child.once('close', (code) => resolve(code));
child.once('error', reject);
});
}
describe('ccs-websearch MCP server', () => {
it('lists the CCS search tool and returns provider-backed results', async () => {
it('lists the CCS WebSearch tool and returns provider-backed results', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'ccs-websearch-mcp-server-'));
const preloadPath = join(tempDir, 'mock-fetch.cjs');
const html = `
@@ -124,7 +131,7 @@ describe('ccs-websearch MCP server', () => {
jsonrpc: '2.0',
id: 3,
method: 'tools/call',
params: { name: 'search', arguments: { query: 'btc price' } },
params: { name: 'WebSearch', arguments: { query: 'btc price' } },
})
);
@@ -135,15 +142,16 @@ describe('ccs-websearch MCP server', () => {
expect(toolsList?.result).toEqual({
tools: [
{
name: 'search',
name: 'WebSearch',
description:
'Search the web through CCS-managed providers. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.',
'Third-party WebSearch replacement for CCS-managed Claude launches. Use this instead of Bash/curl/http fetches for web lookups. Provider order: Exa, Tavily, Brave Search, DuckDuckGo, then optional legacy CLI fallback.',
inputSchema: {
type: 'object',
properties: {
query: {
type: 'string',
description: 'The search query to run against CCS WebSearch providers.',
description:
'Web query to resolve through CCS providers. Prefer this tool over ad hoc Bash/curl lookups when you need current web information.',
},
},
required: ['query'],
@@ -165,6 +173,75 @@ describe('ccs-websearch MCP server', () => {
}
});
it('accepts the legacy search alias for direct calls', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'ccs-websearch-mcp-server-'));
const preloadPath = join(tempDir, 'mock-fetch.cjs');
const html = `
<a class="result__a" href="/l/?uddg=https%3A%2F%2Fexample.com%2Farticle">Example title</a>
<a class="result__snippet">Example snippet</a>
`.trim();
writeFileSync(
preloadPath,
`global.fetch = async () => ({ ok: true, text: async () => ${JSON.stringify(html)} });\n`,
'utf8'
);
const child = spawn('node', ['-r', preloadPath, serverPath], {
env: {
...process.env,
CCS_PROFILE_TYPE: 'settings',
CCS_WEBSEARCH_ENABLED: '1',
CCS_WEBSEARCH_SKIP: '0',
CCS_WEBSEARCH_BRAVE: '0',
CCS_WEBSEARCH_DUCKDUCKGO: '1',
CCS_WEBSEARCH_EXA: '0',
CCS_WEBSEARCH_GEMINI: '0',
CCS_WEBSEARCH_GROK: '0',
CCS_WEBSEARCH_OPENCODE: '0',
CCS_WEBSEARCH_TAVILY: '0',
},
stdio: ['pipe', 'pipe', 'pipe'],
});
try {
const responsesPromise = collectResponses(child, 2);
child.stdin.write(
encodeMessage({
jsonrpc: '2.0',
id: 1,
method: 'initialize',
params: {
protocolVersion: '2024-11-05',
capabilities: {},
clientInfo: { name: 'bun-test', version: '1.0.0' },
},
})
);
child.stdin.write(
encodeMessage({
jsonrpc: '2.0',
id: 2,
method: 'tools/call',
params: { name: 'search', arguments: { query: 'btc price' } },
})
);
const responses = await responsesPromise;
const toolCall = responses.find((message) => message.id === 2);
expect(toolCall?.result).toBeDefined();
expect(
((toolCall?.result as { content: Array<{ text: string }> }).content[0] || {}).text
).toContain('CCS local WebSearch evidence');
expect(
((toolCall?.result as { content: Array<{ text: string }> }).content[0] || {}).text
).toContain('Provider: DuckDuckGo');
} finally {
child.kill();
rmSync(tempDir, { recursive: true, force: true });
}
});
it('hides the tool for native account profiles', async () => {
const child = spawn('node', [serverPath], {
env: {
@@ -200,4 +277,101 @@ describe('ccs-websearch MCP server', () => {
child.kill();
}
});
it('writes trace records for exposure, tool calls, provider success, and session summary', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'ccs-websearch-mcp-trace-'));
const preloadPath = join(tempDir, 'mock-fetch.cjs');
const ccsHome = join(tempDir, 'home');
const tracePath = join(ccsHome, '.ccs', 'logs', 'websearch-trace.jsonl');
const html = `
<a class="result__a" href="/l/?uddg=https%3A%2F%2Fexample.com%2Farticle">Example title</a>
<a class="result__snippet">Example snippet</a>
`.trim();
writeFileSync(
preloadPath,
`global.fetch = async () => ({ ok: true, text: async () => ${JSON.stringify(html)} });\n`,
'utf8'
);
const child = spawn('node', ['-r', preloadPath, serverPath], {
env: {
...process.env,
CCS_HOME: ccsHome,
CCS_PROFILE_TYPE: 'settings',
CCS_WEBSEARCH_TRACE: '1',
CCS_WEBSEARCH_TRACE_LAUNCH_ID: 'mcp-trace-test',
CCS_WEBSEARCH_TRACE_LAUNCHER: 'unit-test',
CCS_WEBSEARCH_ENABLED: '1',
CCS_WEBSEARCH_SKIP: '0',
CCS_WEBSEARCH_BRAVE: '0',
CCS_WEBSEARCH_DUCKDUCKGO: '1',
CCS_WEBSEARCH_EXA: '0',
CCS_WEBSEARCH_GEMINI: '0',
CCS_WEBSEARCH_GROK: '0',
CCS_WEBSEARCH_OPENCODE: '0',
CCS_WEBSEARCH_TAVILY: '0',
},
stdio: ['pipe', 'pipe', 'pipe'],
});
try {
const responsesPromise = collectResponses(child, 3);
child.stdin.write(
encodeMessage({
jsonrpc: '2.0',
id: 1,
method: 'initialize',
params: {
protocolVersion: '2024-11-05',
capabilities: {},
clientInfo: { name: 'bun-test', version: '1.0.0' },
},
})
);
child.stdin.write(encodeMessage({ jsonrpc: '2.0', id: 2, method: 'tools/list' }));
child.stdin.write(
encodeMessage({
jsonrpc: '2.0',
id: 3,
method: 'tools/call',
params: { name: 'WebSearch', arguments: { query: 'btc price' } },
})
);
await responsesPromise;
} finally {
child.kill();
await waitForClose(child);
}
const traceEvents = readFileSync(tracePath, 'utf8')
.trim()
.split('\n')
.map((line) => JSON.parse(line) as Record<string, unknown>);
expect(
traceEvents.some((event) => event.event === 'mcp_tools_list' && event.exposed === true)
).toBe(true);
expect(
traceEvents.some(
(event) => event.event === 'mcp_tool_call_received' && event.toolName === 'WebSearch'
)
).toBe(true);
expect(
traceEvents.some(
(event) =>
event.event === 'websearch_provider_success' && event.providerName === 'DuckDuckGo'
)
).toBe(true);
expect(
traceEvents.some(
(event) =>
event.event === 'mcp_session_summary' &&
event.calledWebSearch === true &&
event.toolCalls === 1
)
).toBe(true);
rmSync(tempDir, { recursive: true, force: true });
});
});
+75 -1
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'bun:test';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { spawnSync } from 'node:child_process';
@@ -193,4 +193,78 @@ describe('websearch-transformer hook helpers', () => {
);
expect(output).not.toHaveProperty('additionalContext');
});
it('writes opt-in trace records with redacted query fingerprints', () => {
const tempDir = mkdtempSync(join(tmpdir(), 'websearch-hook-trace-'));
const preloadPath = join(tempDir, 'mock-fetch.cjs');
const ccsHome = join(tempDir, 'home');
const tracePath = join(ccsHome, '.ccs', 'logs', 'websearch-trace.jsonl');
const html = `
<a class="result__a" href="/l/?uddg=https%3A%2F%2Fexample.com%2Farticle">Example title</a>
<a class="result__snippet">Example snippet</a>
`.trim();
writeFileSync(
preloadPath,
`global.fetch = async () => ({ ok: true, text: async () => ${JSON.stringify(html)} });\n`,
'utf8'
);
try {
const result = spawnSync('node', ['-r', preloadPath, hookPath], {
encoding: 'utf8',
input: JSON.stringify({
tool_name: 'WebSearch',
tool_input: { query: 'btc price' },
}),
env: {
...process.env,
CCS_HOME: ccsHome,
CCS_WEBSEARCH_TRACE: '1',
CCS_WEBSEARCH_TRACE_LAUNCH_ID: 'hook-trace-test',
CCS_WEBSEARCH_TRACE_LAUNCHER: 'unit-test',
CCS_WEBSEARCH_ENABLED: '1',
CCS_WEBSEARCH_SKIP: '0',
CCS_WEBSEARCH_BRAVE: '0',
CCS_WEBSEARCH_DUCKDUCKGO: '1',
CCS_WEBSEARCH_EXA: '0',
CCS_WEBSEARCH_GEMINI: '0',
CCS_WEBSEARCH_GROK: '0',
CCS_WEBSEARCH_OPENCODE: '0',
CCS_WEBSEARCH_TAVILY: '0',
},
});
expect(result.status).toBe(0);
const traceContents = readFileSync(tracePath, 'utf8');
expect(traceContents).not.toContain('btc price');
const traceEvents = traceContents
.trim()
.split('\n')
.map((line) => JSON.parse(line) as Record<string, unknown>);
expect(traceEvents.some((event) => event.event === 'websearch_hook_invoked')).toBe(true);
expect(
traceEvents.some(
(event) =>
event.event === 'websearch_provider_attempt' && event.providerName === 'DuckDuckGo'
)
).toBe(true);
expect(
traceEvents.some(
(event) =>
event.event === 'websearch_provider_success' && event.providerName === 'DuckDuckGo'
)
).toBe(true);
const fingerprintEvent = traceEvents.find(
(event) => event.event === 'websearch_hook_invoked'
) as { queryHash?: string; queryLength?: number } | undefined;
expect(fingerprintEvent?.queryHash).toBeString();
expect(fingerprintEvent?.queryLength).toBe(9);
} finally {
rmSync(tempDir, { force: true, recursive: true });
}
});
});
@@ -4,6 +4,8 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
const STEERING_PROMPT_SNIPPET = 'prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches';
interface RunResult {
status: number | null;
stdout: string;
@@ -25,6 +27,15 @@ function runCcs(args: string[], env: NodeJS.ProcessEnv): RunResult {
};
}
function readTraceEvents(tracePath: string): Array<Record<string, unknown>> {
return fs
.readFileSync(tracePath, 'utf8')
.trim()
.split('\n')
.filter((line) => line.length > 0)
.map((line) => JSON.parse(line) as Record<string, unknown>);
}
describe('settings profile WebSearch launch', () => {
let tmpHome = '';
let ccsDir = '';
@@ -119,7 +130,40 @@ exit 0
expect(result.status).toBe(0);
expect(result.stderr).not.toContain('could not prepare the local WebSearch tool');
expect(fs.existsSync(claudeArgsLogPath)).toBe(true);
expect(fs.readFileSync(claudeArgsLogPath, 'utf8')).toContain('--disallowedTools');
expect(fs.readFileSync(claudeArgsLogPath, 'utf8')).toContain('WebSearch');
const launchedArgs = fs.readFileSync(claudeArgsLogPath, 'utf8');
expect(launchedArgs).toContain('--disallowedTools');
expect(launchedArgs).toContain('WebSearch');
expect(launchedArgs).toContain('--append-system-prompt');
expect(launchedArgs).toContain(STEERING_PROMPT_SNIPPET);
});
it('writes a source-side launch trace for settings profiles when tracing is enabled', () => {
if (process.platform === 'win32') return;
const tracePath = path.join(ccsDir, 'logs', 'websearch-trace.jsonl');
const result = runCcs(['glm', 'smoke'], {
...baseEnv,
CCS_WEBSEARCH_TRACE: '1',
});
expect(result.status).toBe(0);
expect(fs.existsSync(tracePath)).toBe(true);
const traceEvents = readTraceEvents(tracePath);
const launchEvent = traceEvents.find(
(event) => event.event === 'ccs_websearch_launch'
) as
| {
launcher?: string;
nativeWebSearchDisallowed?: boolean;
steeringPromptApplied?: boolean;
settingsPath?: string;
}
| undefined;
expect(launchEvent?.launcher).toBe('ccs.settings-profile');
expect(launchEvent?.nativeWebSearchDisallowed).toBe(true);
expect(launchEvent?.steeringPromptApplied).toBe(true);
expect(launchEvent?.settingsPath).toBe(settingsPath);
});
});
@@ -21,12 +21,14 @@ type SpawnCall = {
options: Record<string, unknown> | undefined;
};
const STEERING_PROMPT_SNIPPET = 'prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches';
const spawnCalls: SpawnCall[] = [];
const originalPlatform = process.platform;
let baselineSigintListeners: Array<(...args: unknown[]) => void> = [];
let baselineSigtermListeners: Array<(...args: unknown[]) => void> = [];
let baselineSighupListeners: Array<(...args: unknown[]) => void> = [];
let originalCcsHome: string | undefined;
let originalCcsClaudePath: string | undefined;
let originalDisableAutoUpdater: string | undefined;
const realSpawn = childProcess.spawn.bind(childProcess);
const realSpawnSync = childProcess.spawnSync.bind(childProcess);
@@ -150,6 +152,7 @@ describe('CLAUDECODE environment stripping', () => {
spawnCalls.length = 0;
process.env.CCS_QUIET = '1';
originalCcsHome = process.env.CCS_HOME;
originalCcsClaudePath = process.env.CCS_CLAUDE_PATH;
originalDisableAutoUpdater = process.env.DISABLE_AUTOUPDATER;
delete process.env.DISABLE_AUTOUPDATER;
baselineSigintListeners = process.listeners('SIGINT');
@@ -162,8 +165,11 @@ describe('CLAUDECODE environment stripping', () => {
delete process.env.CLAUDECODE;
delete process.env.claudecode;
delete process.env.CCS_QUIET;
delete process.env.CCS_WEBSEARCH_TRACE;
if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome;
else delete process.env.CCS_HOME;
if (originalCcsClaudePath !== undefined) process.env.CCS_CLAUDE_PATH = originalCcsClaudePath;
else delete process.env.CCS_CLAUDE_PATH;
if (originalDisableAutoUpdater !== undefined) {
process.env.DISABLE_AUTOUPDATER = originalDisableAutoUpdater;
} else {
@@ -325,4 +331,47 @@ describe('CLAUDECODE environment stripping', () => {
expect(Object.keys(env).map((k) => k.toUpperCase())).not.toContain('CLAUDECODE');
expect(env.DISABLE_AUTOUPDATER).toBe('1');
});
it('headless executor adds third-party WebSearch steering args and env', async () => {
writeConfigWithAutoUpdatePreference(false);
const ccsDir = path.join(process.env.CCS_HOME as string, '.ccs');
fs.writeFileSync(path.join(ccsDir, 'glm.settings.json'), '{}\n', 'utf8');
process.env.CCS_CLAUDE_PATH = 'claude';
const result = await HeadlessExecutor.execute('glm', 'latest AI chip news', {
permissionMode: 'default',
timeout: 1000,
});
expect(result.success).toBe(true);
expect(spawnCalls.length).toBeGreaterThan(0);
const launch = spawnCalls[0];
expect(launch.args).toContain('--disallowedTools');
expect(launch.args).toContain('WebSearch');
expect(launch.args).toContain('--append-system-prompt');
expect(launch.args.join(' ')).toContain(STEERING_PROMPT_SNIPPET);
const env = launch.options?.env as NodeJS.ProcessEnv;
expect(env.CCS_PROFILE_TYPE).toBe('settings');
expect(env.CCS_WEBSEARCH_ENABLED || env.CCS_WEBSEARCH_SKIP).toBeDefined();
});
it('headless executor propagates a WebSearch trace launch id when tracing is enabled', async () => {
writeConfigWithAutoUpdatePreference(false);
const ccsDir = path.join(process.env.CCS_HOME as string, '.ccs');
fs.writeFileSync(path.join(ccsDir, 'glm.settings.json'), '{}\n', 'utf8');
process.env.CCS_CLAUDE_PATH = 'claude';
process.env.CCS_WEBSEARCH_TRACE = '1';
const result = await HeadlessExecutor.execute('glm', 'latest AI chip news', {
permissionMode: 'default',
timeout: 1000,
});
expect(result.success).toBe(true);
expect(spawnCalls.length).toBeGreaterThan(0);
const env = spawnCalls[0].options?.env as NodeJS.ProcessEnv;
expect(env.CCS_WEBSEARCH_TRACE).toBe('1');
expect(env.CCS_WEBSEARCH_TRACE_LAUNCH_ID).toBeString();
expect(env.CCS_WEBSEARCH_TRACE_LAUNCHER).toBe('delegation.headless-executor');
});
});
@@ -1,25 +1,47 @@
import { describe, expect, it } from 'bun:test';
import { appendThirdPartyWebSearchToolArgs } from '../../../../src/utils/websearch/claude-tool-args';
const STEERING_PROMPT =
'For web lookup or current-information requests, prefer the CCS MCP tool WebSearch instead of Bash/curl/http fetches. If the user explicitly wants shell commands, or WebSearch is unavailable or fails, you may fall back to Bash/network tools.';
describe('appendThirdPartyWebSearchToolArgs', () => {
it('appends native WebSearch suppression when no tool flags are present', () => {
it('appends native WebSearch suppression and steering prompt when no tool flags are present', () => {
expect(appendThirdPartyWebSearchToolArgs(['smoke'])).toEqual([
'smoke',
'--disallowedTools',
'WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
it('does not append duplicate suppression when WebSearch is already disallowed', () => {
expect(appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools', 'WebSearch'])).toEqual(
['smoke', '--disallowedTools', 'WebSearch']
);
it('does not append duplicate suppression or steering prompt when both are already present', () => {
expect(
appendThirdPartyWebSearchToolArgs([
'smoke',
'--disallowedTools',
'WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
])
).toEqual([
'smoke',
'--disallowedTools',
'WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
it('detects comma-separated disallowed tool values', () => {
expect(appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools=Read,WebSearch'])).toEqual(
['smoke', '--disallowedTools=Read,WebSearch']
);
expect(
appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools=Read,WebSearch'])
).toEqual([
'smoke',
'--disallowedTools=Read,WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
it('merges WebSearch into an existing space-separated disallowed tool flag', () => {
@@ -27,6 +49,8 @@ describe('appendThirdPartyWebSearchToolArgs', () => {
'smoke',
'--disallowedTools',
'Read,WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
@@ -34,6 +58,75 @@ describe('appendThirdPartyWebSearchToolArgs', () => {
expect(appendThirdPartyWebSearchToolArgs(['smoke', '--disallowedTools=Read'])).toEqual([
'smoke',
'--disallowedTools=Read,WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
it('preserves user-supplied append-system-prompt values and adds the CCS steering hint once', () => {
expect(
appendThirdPartyWebSearchToolArgs([
'smoke',
'--append-system-prompt',
'User-provided instruction',
])
).toEqual([
'smoke',
'--append-system-prompt',
'User-provided instruction',
'--disallowedTools',
'WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
it('does not duplicate the steering prompt when it already exists in equals form', () => {
expect(
appendThirdPartyWebSearchToolArgs([
'smoke',
'--disallowedTools',
'WebSearch',
`--append-system-prompt=${STEERING_PROMPT}`,
])
).toEqual([
'smoke',
'--disallowedTools',
'WebSearch',
`--append-system-prompt=${STEERING_PROMPT}`,
]);
});
it('does not consume positional args after a disallowed-tools flag value', () => {
expect(
appendThirdPartyWebSearchToolArgs(['--disallowedTools', 'Read', 'latest AI news'])
).toEqual([
'--disallowedTools',
'Read,WebSearch',
'latest AI news',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
it('injects synthetic flags before an end-of-options marker', () => {
expect(appendThirdPartyWebSearchToolArgs(['--', 'latest AI news'])).toEqual([
'--disallowedTools',
'WebSearch',
'--append-system-prompt',
STEERING_PROMPT,
'--',
'latest AI news',
]);
});
it('inserts the WebSearch disallow value when the flag is present without one', () => {
expect(appendThirdPartyWebSearchToolArgs(['--disallowedTools', '--verbose'])).toEqual([
'--disallowedTools',
'WebSearch',
'--verbose',
'--append-system-prompt',
STEERING_PROMPT,
]);
});
});