feat(browser): 完成 browser MCP 第四阶段输入能力

补齐 browser_click 的精细点击参数,并新增 browser_press_key 与 browser_scroll,继续扩展 browser MCP 的真实交互能力。
同步更新单测与浏览器自动化文档,确保 Phase 4 能力完成并通过验证。

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
walker1211
2026-04-18 13:59:15 +08:00
co-authored by Claude Opus 4.7
parent cf2375e169
commit e92a2c11bf
3 changed files with 868 additions and 25 deletions
+8 -2
View File
@@ -1,6 +1,6 @@
# Browser Automation
Last Updated: 2026-04-17
Last Updated: 2026-04-18
CCS provides browser automation through two separate runtime paths:
@@ -23,7 +23,7 @@ enabled. A recent Chrome update alone is not sufficient.
The managed `ccs-browser` runtime currently exposes five tool groups:
- **Session inspection**: `browser_get_session_info`, `browser_get_url_and_title`, `browser_get_visible_text`, `browser_get_dom_snapshot`
- **Navigation and interaction**: `browser_navigate`, `browser_click`, `browser_type`, `browser_take_screenshot`
- **Navigation and interaction**: `browser_navigate`, `browser_click`, `browser_type`, `browser_press_key`, `browser_scroll`, `browser_take_screenshot`
- **Hover diagnostics**: `browser_hover`, `browser_query`, `browser_take_element_screenshot`
- **Readiness and page evaluation**: `browser_wait_for`, `browser_eval`
- **Event observation**: `browser_wait_for_event`
@@ -41,6 +41,12 @@ Phase 3 capability details:
- the same scoped-selector tools accept optional `pierceShadow: true` to search open shadow roots beneath the selected root
- `browser_wait_for_event` observes typed page or browser events for dialogs, navigation, requests, and downloads
Phase 4 capability details:
- `browser_click` also accepts optional `offsetX`, `offsetY`, `button`, and `clickCount` for more precise element-relative click control
- `browser_press_key` sends real browser-level key events, supports modifier combinations plus repeat counts, and covers a focused set of common special keys such as `Enter`, `Tab`, `Escape`, and arrow keys
- `browser_scroll` supports page-level `by-offset` scrolling and element-scoped `by-offset` or `into-view` scrolling, including same-origin iframe scoping
A common hover-debug workflow is:
1. call `browser_hover` to move the browser pointer onto the card or trigger
+414 -20
View File
@@ -43,6 +43,8 @@ const TOOL_DOM_SNAPSHOT = 'browser_get_dom_snapshot';
const TOOL_NAVIGATE = 'browser_navigate';
const TOOL_CLICK = 'browser_click';
const TOOL_TYPE = 'browser_type';
const TOOL_PRESS_KEY = 'browser_press_key';
const TOOL_SCROLL = 'browser_scroll';
const TOOL_TAKE_SCREENSHOT = 'browser_take_screenshot';
const TOOL_WAIT_FOR = 'browser_wait_for';
const TOOL_EVAL = 'browser_eval';
@@ -58,6 +60,8 @@ const TOOL_NAMES = [
TOOL_NAVIGATE,
TOOL_CLICK,
TOOL_TYPE,
TOOL_PRESS_KEY,
TOOL_SCROLL,
TOOL_TAKE_SCREENSHOT,
TOOL_WAIT_FOR,
TOOL_EVAL,
@@ -278,6 +282,24 @@ function getTools() {
type: 'boolean',
description: 'When true, search open shadow roots beneath the selected root.',
},
offsetX: {
type: 'number',
description: "Optional horizontal offset in CSS pixels from the target element's left edge.",
},
offsetY: {
type: 'number',
description: "Optional vertical offset in CSS pixels from the target element's top edge.",
},
button: {
type: 'string',
enum: ['left', 'middle', 'right'],
description: 'Optional mouse button. Defaults to left.',
},
clickCount: {
type: 'integer',
minimum: 1,
description: 'Optional click count. Defaults to 1.',
},
},
required: ['selector'],
additionalProperties: false,
@@ -312,6 +334,82 @@ function getTools() {
additionalProperties: false,
},
},
{
name: TOOL_PRESS_KEY,
description:
'Press a key or key combination in the selected page using real keyboard-style events. Optionally choose a page by index.',
inputSchema: {
type: 'object',
properties: {
pageIndex: {
type: 'integer',
minimum: 0,
description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.',
},
key: {
type: 'string',
description: 'Required primary key to press.',
},
modifiers: {
type: 'array',
items: {
type: 'string',
enum: ['Alt', 'Control', 'Meta', 'Shift'],
},
description: 'Optional modifier keys such as Alt, Control, Meta, or Shift.',
},
repeat: {
type: 'integer',
minimum: 1,
description: 'Optional repeat count. Defaults to 1.',
},
},
required: ['key'],
additionalProperties: false,
},
},
{
name: TOOL_SCROLL,
description:
'Scroll the selected page or a matched element. Supports explicit deltas or scrolling an element into view.',
inputSchema: {
type: 'object',
properties: {
pageIndex: {
type: 'integer',
minimum: 0,
description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.',
},
selector: {
type: 'string',
description: 'Optional CSS selector for an element-scoped scroll target.',
},
frameSelector: {
type: 'string',
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
},
pierceShadow: {
type: 'boolean',
description: 'When true, search open shadow roots beneath the selected root.',
},
behavior: {
type: 'string',
enum: ['into-view', 'by-offset'],
description: 'Required scroll behavior.',
},
deltaX: {
type: 'number',
description: 'Optional horizontal scroll delta for by-offset behavior.',
},
deltaY: {
type: 'number',
description: 'Optional vertical scroll delta for by-offset behavior.',
},
},
required: ['behavior'],
additionalProperties: false,
},
},
{
name: TOOL_TAKE_SCREENSHOT,
description:
@@ -795,6 +893,49 @@ function requirePositiveIntegerOrDefault(value, label, fallback) {
return value;
}
function requirePositiveInteger(value, label) {
if (!Number.isInteger(value) || value <= 0) {
throw new Error(`${label} must be a positive integer`);
}
return value;
}
function requireFiniteNumber(value, label) {
if (typeof value !== 'number' || !Number.isFinite(value)) {
throw new Error(`${label} must be a finite number`);
}
return value;
}
function requireEnumString(value, label, allowedValues) {
if (value === undefined) {
return undefined;
}
const normalized = requireNonEmptyString(value, label);
if (!allowedValues.includes(normalized)) {
throw new Error(`${label} must be one of: ${allowedValues.join(', ')}`);
}
return normalized;
}
function requireOptionalStringArray(value, label, allowedValues) {
if (value === undefined) {
return [];
}
if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || item.trim() === '')) {
throw new Error(`${label} must be an array of non-empty strings`);
}
const normalized = value.map((item) => item.trim());
if (Array.isArray(allowedValues) && allowedValues.length > 0) {
for (const item of normalized) {
if (!allowedValues.includes(item)) {
throw new Error(`${label} must only contain: ${allowedValues.join(', ')}`);
}
}
}
return normalized;
}
function getBrowserEvalMode() {
const raw = String(process.env.CCS_BROWSER_EVAL_MODE || 'readonly').trim();
if (raw === 'disabled' || raw === 'readonly' || raw === 'readwrite') {
@@ -1058,12 +1199,20 @@ async function handleClick(toolArgs) {
const targetIndex = nth ?? 0;
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const offsetX = toolArgs.offsetX === undefined ? undefined : requireFiniteNumber(toolArgs.offsetX, 'offsetX');
const offsetY = toolArgs.offsetY === undefined ? undefined : requireFiniteNumber(toolArgs.offsetY, 'offsetY');
const button = requireEnumString(toolArgs.button, 'button', ['left', 'middle', 'right']) || 'left';
const clickCount = toolArgs.clickCount === undefined ? 1 : requirePositiveInteger(toolArgs.clickCount, 'clickCount');
const expression = `(() => {
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
const nth = ${nth === undefined ? 'undefined' : String(nth)};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
const offsetX = ${offsetX === undefined ? 'undefined' : String(offsetX)};
const offsetY = ${offsetY === undefined ? 'undefined' : String(offsetY)};
const button = JSON.parse(${JSON.stringify(JSON.stringify(button))});
const clickCount = ${clickCount};
const visitRoots = (root) => {
const roots = [root];
@@ -1113,52 +1262,104 @@ async function handleClick(toolArgs) {
throw new Error('element is disabled for selector: ' + selector);
}
const style = window.getComputedStyle(element);
const rect = element.getBoundingClientRect();
const initialRect = element.getBoundingClientRect();
if (
style.display === 'none' ||
style.visibility === 'hidden' ||
rect.width <= 0 ||
rect.height <= 0
initialRect.width <= 0 ||
initialRect.height <= 0
) {
throw new Error('element is hidden or not interactable for selector: ' + selector);
}
element.scrollIntoView({ block: 'center', inline: 'center' });
const rect = element.getBoundingClientRect();
const dispatchMouseEvent = (type, init) => {
const resolvedOffsetX = offsetX === undefined ? rect.width / 2 : offsetX;
const resolvedOffsetY = offsetY === undefined ? rect.height / 2 : offsetY;
const clientX = rect.left + resolvedOffsetX;
const clientY = rect.top + resolvedOffsetY;
const buttonCode = button === 'middle' ? 1 : button === 'right' ? 2 : 0;
const buttonsMask = buttonCode === 1 ? 4 : buttonCode === 2 ? 2 : 1;
const dispatchMouseEvent = (type, detail, init) => {
const event = new MouseEvent(type, {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: 1,
detail,
clientX,
clientY,
button: buttonCode,
buttons: type === 'mousedown' ? buttonsMask : 0,
...init,
});
return element.dispatchEvent(event);
};
try {
const dispatchResult = {
shouldActivate:
dispatchMouseEvent('mousedown', { button: 0, buttons: 1 }) &&
dispatchMouseEvent('mouseup', { button: 0, buttons: 0 }),
};
if (!dispatchResult.shouldActivate) {
return 'ok';
for (let index = 1; index <= clickCount; index += 1) {
const dispatchResult = {
shouldActivate:
dispatchMouseEvent('mousedown', index, {}) &&
dispatchMouseEvent('mouseup', index, {}),
};
if (!dispatchResult.shouldActivate) {
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
}
if (!element.isConnected) {
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
}
if (button === 'left') {
element.click();
} else if (button === 'right') {
element.dispatchEvent(new MouseEvent('contextmenu', {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: index,
clientX,
clientY,
button: 2,
buttons: 0,
}));
} else {
element.dispatchEvent(new MouseEvent('auxclick', {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: index,
clientX,
clientY,
button: 1,
buttons: 0,
}));
}
}
if (!element.isConnected) {
return 'ok';
if (button === 'left' && clickCount === 2) {
element.dispatchEvent(new MouseEvent('dblclick', {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: 2,
clientX,
clientY,
button: 0,
buttons: 0,
}));
}
} catch (mouseError) {
// Fall through to the native activation path below.
}
element.click();
return 'ok';
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
})()`;
await evaluateExpression(page, expression);
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`;
const raw = await evaluateExpression(page, expression);
const parsed = JSON.parse(raw);
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}\noffsetX: ${parsed.resolvedOffsetX}\noffsetY: ${parsed.resolvedOffsetY}\nbutton: ${parsed.button}\nclickCount: ${parsed.clickCount}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`;
}
async function handleType(toolArgs) {
@@ -1240,6 +1441,183 @@ async function handleType(toolArgs) {
return `pageIndex: ${pageIndex}\nselector: ${selector}\ntypedLength: ${typedLength}\nstatus: typed`;
}
async function handlePressKey(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const key = requireNonEmptyString(toolArgs.key, 'key');
const modifiers = requireOptionalStringArray(toolArgs.modifiers, 'modifiers', [
'Alt',
'Control',
'Meta',
'Shift',
]);
const repeat = toolArgs.repeat === undefined ? 1 : requirePositiveInteger(toolArgs.repeat, 'repeat');
const modifierMask =
(modifiers.includes('Alt') ? 1 : 0) |
(modifiers.includes('Control') ? 2 : 0) |
(modifiers.includes('Meta') ? 4 : 0) |
(modifiers.includes('Shift') ? 8 : 0);
const specialKeyMap = {
Enter: { code: 'Enter', keyCode: 13, text: '\r' },
Tab: { code: 'Tab', keyCode: 9, text: '' },
Escape: { code: 'Escape', keyCode: 27, text: '' },
ArrowUp: { code: 'ArrowUp', keyCode: 38, text: '' },
ArrowDown: { code: 'ArrowDown', keyCode: 40, text: '' },
ArrowLeft: { code: 'ArrowLeft', keyCode: 37, text: '' },
ArrowRight: { code: 'ArrowRight', keyCode: 39, text: '' },
Backspace: { code: 'Backspace', keyCode: 8, text: '' },
Delete: { code: 'Delete', keyCode: 46, text: '' },
Space: { code: 'Space', keyCode: 32, text: ' ' },
};
const keyDescriptor =
key.length === 1
? {
code: `Key${key.toUpperCase()}`,
keyCode: key.toUpperCase().charCodeAt(0),
text: key,
}
: specialKeyMap[key];
if (!keyDescriptor) {
throw new Error(`unsupported key: ${key}`);
}
const normalizedKey = key;
const normalizedText = keyDescriptor.text;
const code = keyDescriptor.code;
const keyCode = keyDescriptor.keyCode;
for (let index = 0; index < repeat; index += 1) {
await sendCdpCommand(page, 'Input.dispatchKeyEvent', {
type: 'keyDown',
key: normalizedKey,
code,
text: normalizedText,
unmodifiedText: normalizedText,
windowsVirtualKeyCode: keyCode,
nativeVirtualKeyCode: keyCode,
modifiers: modifierMask,
autoRepeat: index > 0,
});
await sendCdpCommand(page, 'Input.dispatchKeyEvent', {
type: 'keyUp',
key: normalizedKey,
code,
windowsVirtualKeyCode: keyCode,
nativeVirtualKeyCode: keyCode,
modifiers: modifierMask,
});
}
const modifierText = modifiers.length > 0 ? modifiers.join(',') : 'none';
return `pageIndex: ${pageIndex}\nkey: ${key}\nmodifiers: ${modifierText}\nrepeat: ${repeat}\nstatus: key-pressed`;
}
async function handleScroll(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const selector = parseOptionalNonEmptyString(toolArgs.selector);
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const behavior = requireEnumString(toolArgs.behavior, 'behavior', ['into-view', 'by-offset']);
const deltaX = toolArgs.deltaX === undefined ? 0 : requireFiniteNumber(toolArgs.deltaX, 'deltaX');
const deltaY = toolArgs.deltaY === undefined ? 0 : requireFiniteNumber(toolArgs.deltaY, 'deltaY');
const expression = `(() => {
const selector = ${selector ? `JSON.parse(${JSON.stringify(JSON.stringify(selector))})` : 'undefined'};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
const behavior = JSON.parse(${JSON.stringify(JSON.stringify(behavior))});
const deltaX = ${deltaX};
const deltaY = ${deltaY};
const visitRoots = (root) => {
const roots = [root];
if (!pierceShadow) {
return roots;
}
const queue = [root];
while (queue.length > 0) {
const current = queue.shift();
const elements = Array.from(current.querySelectorAll('*'));
for (const element of elements) {
if (element.shadowRoot) {
roots.push(element.shadowRoot);
queue.push(element.shadowRoot);
}
}
}
return roots;
};
let root = document;
let scrollWindow = window;
if (frameSelector) {
const frame = document.querySelector(frameSelector);
if (!frame) {
throw new Error('frame not found for selector: ' + frameSelector);
}
const frameDocument = frame.contentDocument;
if (!frameDocument) {
throw new Error('frame document is unavailable for selector: ' + frameSelector);
}
if (!frame.contentWindow) {
throw new Error('frame window is unavailable for selector: ' + frameSelector);
}
root = frameDocument;
scrollWindow = frame.contentWindow;
}
if (!selector) {
if (behavior !== 'by-offset') {
throw new Error('selector is required for behavior: ' + behavior);
}
scrollWindow.scrollBy(deltaX, deltaY);
return JSON.stringify({ scope: 'page', behavior, deltaX, deltaY });
}
const roots = visitRoots(root);
const matches = [];
for (const currentRoot of roots) {
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
}
const element = matches[0];
if (!element) {
throw new Error('element not found for selector: ' + selector);
}
if (behavior === 'into-view') {
element.scrollIntoView({ block: 'center', inline: 'center' });
return JSON.stringify({ scope: 'element', selector, behavior });
}
if (typeof element.scrollBy === 'function') {
element.scrollBy(deltaX, deltaY);
return JSON.stringify({ scope: 'element', selector, behavior, deltaX, deltaY });
}
throw new Error('element does not support scrollBy for selector: ' + selector);
})()`;
const raw = await evaluateExpression(page, expression);
const parsed = JSON.parse(raw);
const lines = [`pageIndex: ${pageIndex}`];
if (parsed.selector) {
lines.push(`selector: ${parsed.selector}`);
}
lines.push(`behavior: ${parsed.behavior}`);
if (typeof parsed.deltaX === 'number') {
lines.push(`deltaX: ${parsed.deltaX}`);
}
if (typeof parsed.deltaY === 'number') {
lines.push(`deltaY: ${parsed.deltaY}`);
}
if (parsed.scope === 'element') {
const scopedSuffix = formatScopedSelectorSuffix(frameSelector, pierceShadow);
if (scopedSuffix) {
lines.push(scopedSuffix.slice(1));
}
}
lines.push('status: scrolled');
return lines.join('\n');
}
async function handleScreenshot(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const fullPage = toolArgs.fullPage === true;
@@ -1865,6 +2243,22 @@ async function handleToolCall(message) {
return;
}
if (toolName === TOOL_PRESS_KEY) {
const text = await handlePressKey(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_SCROLL) {
const text = await handleScroll(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_TAKE_SCREENSHOT) {
const text = await handleScreenshot(toolArgs);
writeResponse(id, {
+446 -3
View File
@@ -48,6 +48,10 @@ type MockClickState = {
detachAfterMouseDown?: boolean;
mouseSequenceError?: string;
label?: string;
expectedOffset?: { x: number; y: number };
expectedButton?: 'left' | 'middle' | 'right';
expectedClickCount?: number;
requireDoubleClickEvent?: boolean;
};
type MockClickPlan = MockClickState | MockClickState[];
@@ -144,6 +148,20 @@ type MockPageState = {
focused?: boolean;
}
>;
keyboard?: {
expectedKey?: string;
expectedModifiers?: string[];
expectedRepeat?: number;
_seenKeyDownCount?: number;
};
scroll?: Record<
string,
{
expectedBehavior?: 'into-view' | 'by-offset';
expectedDeltaX?: number;
expectedDeltaY?: number;
}
>;
};
const bundledServerPath = join(process.cwd(), 'lib', 'mcp', 'ccs-browser-server.cjs');
@@ -521,6 +539,45 @@ function createMockBrowser(pagesInput: MockPageState[]) {
return;
}
if (message.method === 'Input.dispatchKeyEvent') {
const type = typeof message.params?.type === 'string' ? message.params.type : '';
const key = typeof message.params?.key === 'string' ? message.params.key : '';
const modifiersMask = Number(message.params?.modifiers || 0);
const modifiers = [
...(modifiersMask & 1 ? ['Alt'] : []),
...(modifiersMask & 2 ? ['Control'] : []),
...(modifiersMask & 4 ? ['Meta'] : []),
...(modifiersMask & 8 ? ['Shift'] : []),
];
const keyboardPlan = page.keyboard;
if (type === 'keyDown') {
if (keyboardPlan?.expectedKey && key !== keyboardPlan.expectedKey) {
replyError(`unexpected key: ${key}`);
return;
}
if (
keyboardPlan?.expectedModifiers &&
JSON.stringify(modifiers) !== JSON.stringify(keyboardPlan.expectedModifiers)
) {
replyError(`unexpected modifiers for key: ${key}`);
return;
}
if (keyboardPlan) {
keyboardPlan._seenKeyDownCount = (keyboardPlan._seenKeyDownCount || 0) + 1;
}
}
if (
type === 'keyUp' &&
typeof keyboardPlan?.expectedRepeat === 'number' &&
(keyboardPlan._seenKeyDownCount || 0) !== keyboardPlan.expectedRepeat
) {
replyError(`unexpected repeat for key: ${key}`);
return;
}
reply({});
return;
}
if (message.method === 'Page.enable' || message.method === 'Network.enable') {
reply({});
if (message.method === 'Page.enable') {
@@ -627,7 +684,7 @@ function createMockBrowser(pagesInput: MockPageState[]) {
return;
}
if (expression.includes('scrollIntoView') && expression.includes('.click()')) {
if (expression.includes('scrollIntoView') && expression.includes('resolvedOffsetX')) {
const selector = parseJsonArgument(expression, 'selector') || '';
const nth = parseNumberArgument(expression, 'nth') ?? 0;
const clickPlan = page.click?.[selector];
@@ -636,6 +693,7 @@ function createMockBrowser(pagesInput: MockPageState[]) {
const attemptedMouseUp = expression.includes("dispatchMouseEvent('mouseup'");
const attemptedMouseSequence = attemptedMouseDown && attemptedMouseUp;
const attemptedClickEvent = expression.includes("dispatchMouseEvent('click'");
const attemptedDoubleClickEvent = expression.includes("new MouseEvent('dblclick'");
const readsDispatchResult = expression.includes('const dispatchResult = {');
const gatesNativeClickOnDispatchResult = expression.includes(
'if (!dispatchResult.shouldActivate)'
@@ -662,6 +720,10 @@ function createMockBrowser(pagesInput: MockPageState[]) {
index < catchIndex ||
index > catchBlockEnd
);
const offsetX = parseNumberArgument(expression, 'offsetX');
const offsetY = parseNumberArgument(expression, 'offsetY');
const button = parseJsonArgument(expression, 'button') || 'left';
const clickCount = parseNumberArgument(expression, 'clickCount') ?? 1;
if (!resolvedClickPlan) {
replyError(`element index ${nth} is out of range for selector: ${selector}`);
return;
@@ -712,13 +774,49 @@ function createMockBrowser(pagesInput: MockPageState[]) {
replyError(`native click required for selector: ${selector}`);
return;
}
if (
resolvedClickPlan.expectedOffset &&
(offsetX !== resolvedClickPlan.expectedOffset.x || offsetY !== resolvedClickPlan.expectedOffset.y)
) {
replyError(`unexpected click offset for selector: ${selector}`);
return;
}
if (resolvedClickPlan.expectedButton && button !== resolvedClickPlan.expectedButton) {
replyError(`unexpected click button for selector: ${selector}`);
return;
}
if (
typeof resolvedClickPlan.expectedClickCount === 'number' &&
clickCount !== resolvedClickPlan.expectedClickCount
) {
replyError(`unexpected click count for selector: ${selector}`);
return;
}
if (
resolvedClickPlan.requireDoubleClickEvent &&
clickCount === 2 &&
!attemptedDoubleClickEvent
) {
replyError(`dblclick event required for selector: ${selector}`);
return;
}
if (resolvedClickPlan.mouseSequenceError) {
if (!attemptedMouseSequence) {
replyError(`mousedown/mouseup required for selector: ${selector}`);
return;
}
if (attemptedFallbackClick || attemptedNativeClickOutsideCatch) {
reply({ result: { type: 'string', value: 'ok' } });
reply({
result: {
type: 'string',
value: JSON.stringify({
resolvedOffsetX: offsetX ?? 50,
resolvedOffsetY: offsetY ?? 10,
button,
clickCount,
}),
},
});
return;
}
replyError(resolvedClickPlan.mouseSequenceError);
@@ -728,7 +826,17 @@ function createMockBrowser(pagesInput: MockPageState[]) {
replyError(resolvedClickPlan.error);
return;
}
reply({ result: { type: 'string', value: 'ok' } });
reply({
result: {
type: 'string',
value: JSON.stringify({
resolvedOffsetX: offsetX ?? 50,
resolvedOffsetY: offsetY ?? 10,
button,
clickCount,
}),
},
});
return;
}
@@ -950,6 +1058,80 @@ function createMockBrowser(pagesInput: MockPageState[]) {
});
return;
}
if (expression.includes('new KeyboardEvent(')) {
const key = parseJsonArgument(expression, 'key') || '';
const repeat = parseNumberArgument(expression, 'repeat') ?? 1;
const modifiersMatch = expression.match(/const modifiers = (\[[^\n;]*\]);/);
const modifiers = modifiersMatch ? (JSON.parse(modifiersMatch[1]) as string[]) : [];
const keyboardPlan = page.keyboard;
if (keyboardPlan?.expectedKey && key !== keyboardPlan.expectedKey) {
replyError(`unexpected key: ${key}`);
return;
}
if (
keyboardPlan?.expectedModifiers &&
JSON.stringify(modifiers) !== JSON.stringify(keyboardPlan.expectedModifiers)
) {
replyError(`unexpected modifiers for key: ${key}`);
return;
}
if (
typeof keyboardPlan?.expectedRepeat === 'number' &&
repeat !== keyboardPlan.expectedRepeat
) {
replyError(`unexpected repeat for key: ${key}`);
return;
}
reply({
result: {
type: 'string',
value: JSON.stringify({ key, modifiers, repeat }),
},
});
return;
}
if (expression.includes('window.scrollBy(deltaX, deltaY)') || expression.includes('element.scrollBy(deltaX, deltaY)')) {
const selector = parseJsonArgument(expression, 'selector');
const behavior = parseJsonArgument(expression, 'behavior') || '';
const deltaX = Number(expression.match(/const deltaX = (-?[0-9]+(?:\.[0-9]+)?);/)?.[1] || 0);
const deltaY = Number(expression.match(/const deltaY = (-?[0-9]+(?:\.[0-9]+)?);/)?.[1] || 0);
const scrollPlan = selector ? page.scroll?.[selector] : undefined;
if (selector && !scrollPlan) {
replyError(`element not found for selector: ${selector}`);
return;
}
if (scrollPlan?.expectedBehavior && behavior !== scrollPlan.expectedBehavior) {
replyError(`unexpected scroll behavior for selector: ${selector}`);
return;
}
if (
typeof scrollPlan?.expectedDeltaX === 'number' &&
deltaX !== scrollPlan.expectedDeltaX
) {
replyError(`unexpected deltaX for selector: ${selector}`);
return;
}
if (
typeof scrollPlan?.expectedDeltaY === 'number' &&
deltaY !== scrollPlan.expectedDeltaY
) {
replyError(`unexpected deltaY for selector: ${selector}`);
return;
}
reply({
result: {
type: 'string',
value: JSON.stringify(
selector
? { scope: 'element', selector, behavior, deltaX, deltaY }
: { scope: 'page', behavior, deltaX, deltaY }
),
},
});
return;
}
});
});
@@ -1056,6 +1238,8 @@ describe('ccs-browser MCP server', () => {
'browser_navigate',
'browser_click',
'browser_type',
'browser_press_key',
'browser_scroll',
'browser_take_screenshot',
'browser_wait_for',
'browser_eval',
@@ -1068,6 +1252,21 @@ describe('ccs-browser MCP server', () => {
const clickTool = tools.find((tool) => tool.name === 'browser_click');
expect(clickTool?.description).toContain('mouse event chain');
expect(clickTool?.description).not.toContain('synthetic element.click()');
expect(clickTool?.inputSchema?.properties?.offsetX).toMatchObject({ type: 'number' });
expect(clickTool?.inputSchema?.properties?.offsetY).toMatchObject({ type: 'number' });
expect(clickTool?.inputSchema?.properties?.button).toMatchObject({ type: 'string' });
expect(clickTool?.inputSchema?.properties?.clickCount).toMatchObject({
type: 'integer',
minimum: 1,
});
const keyTool = tools.find((tool) => tool.name === 'browser_press_key');
expect(keyTool?.inputSchema?.properties?.key).toMatchObject({ type: 'string' });
expect(keyTool?.inputSchema?.properties?.modifiers).toMatchObject({ type: 'array' });
const scrollTool = tools.find((tool) => tool.name === 'browser_scroll');
expect(scrollTool?.inputSchema?.properties?.deltaX).toMatchObject({ type: 'number' });
expect(scrollTool?.inputSchema?.properties?.deltaY).toMatchObject({ type: 'number' });
const queryTool = tools.find((tool) => tool.name === 'browser_query');
expect(queryTool?.inputSchema?.properties?.fields).toMatchObject({
@@ -1638,6 +1837,250 @@ describe('ccs-browser MCP server', () => {
);
});
it('clicks with element-relative offsets and richer click options', async () => {
const responses = await runMcpRequests(
[
{
id: 'page-1',
title: 'Click Page',
currentUrl: 'https://example.com/',
click: {
'#offset-target': {
requireMouseSequence: true,
expectedOffset: { x: 12, y: 8 },
expectedButton: 'right',
expectedClickCount: 2,
},
'#double-left': {
requireMouseSequence: true,
expectedButton: 'left',
expectedClickCount: 2,
requireDoubleClickEvent: true,
},
},
},
],
[
{
jsonrpc: '2.0',
id: 70,
method: 'tools/call',
params: {
name: 'browser_click',
arguments: {
selector: '#offset-target',
offsetX: 12,
offsetY: 8,
button: 'right',
clickCount: 2,
},
},
},
{
jsonrpc: '2.0',
id: 700,
method: 'tools/call',
params: {
name: 'browser_click',
arguments: {
selector: '#double-left',
clickCount: 2,
},
},
},
]
);
const text = getResponseText(responses.find((message) => message.id === 70));
expect(text).toContain('status: clicked');
expect(text).toContain('offsetX: 12');
expect(text).toContain('offsetY: 8');
expect(text).toContain('button: right');
expect(text).toContain('clickCount: 2');
const doubleText = getResponseText(responses.find((message) => message.id === 700));
expect(doubleText).toContain('status: clicked');
expect(doubleText).toContain('button: left');
expect(doubleText).toContain('clickCount: 2');
});
it('presses a key combination with browser_press_key', async () => {
const responses = await runMcpRequests(
[
{
id: 'page-1',
title: 'Keyboard Page',
currentUrl: 'https://example.com/',
keyboard: {
expectedKey: 'k',
expectedModifiers: ['Meta'],
expectedRepeat: 2,
},
},
],
[
{
jsonrpc: '2.0',
id: 71,
method: 'tools/call',
params: {
name: 'browser_press_key',
arguments: {
key: 'k',
modifiers: ['Meta'],
repeat: 2,
},
},
},
]
);
const text = getResponseText(responses.find((message) => message.id === 71));
expect(text).toContain('status: key-pressed');
expect(text).toContain('key: k');
expect(text).toContain('modifiers: Meta');
expect(text).toContain('repeat: 2');
});
it('supports common special keys with browser_press_key', async () => {
const responses = await runMcpRequests(
[
{
id: 'page-1',
title: 'Special Key Page',
currentUrl: 'https://example.com/',
keyboard: {
expectedKey: 'Enter',
expectedRepeat: 1,
},
},
],
[
{
jsonrpc: '2.0',
id: 711,
method: 'tools/call',
params: {
name: 'browser_press_key',
arguments: {
key: 'Enter',
},
},
},
]
);
const text = getResponseText(responses.find((message) => message.id === 711));
expect(text).toContain('status: key-pressed');
expect(text).toContain('key: Enter');
});
it('rejects unsupported modifiers for browser_press_key', async () => {
const responses = await runMcpRequests(
[{ id: 'page-1', title: 'Invalid Modifier Page', currentUrl: 'https://example.com/' }],
[
{
jsonrpc: '2.0',
id: 712,
method: 'tools/call',
params: {
name: 'browser_press_key',
arguments: {
key: 'k',
modifiers: ['Cmd'],
},
},
},
]
);
const response = responses.find((message) => message.id === 712);
expect((response?.result as { isError?: boolean }).isError).toBe(true);
expect(getResponseText(response)).toContain(
'modifiers must only contain: Alt, Control, Meta, Shift'
);
});
it('scrolls an element into view with browser_scroll', async () => {
const responses = await runMcpRequests(
[
{
id: 'page-1',
title: 'Scroll Page',
currentUrl: 'https://example.com/',
scroll: {
'#results': {
expectedBehavior: 'into-view',
},
},
},
],
[
{
jsonrpc: '2.0',
id: 72,
method: 'tools/call',
params: {
name: 'browser_scroll',
arguments: {
selector: '#results',
behavior: 'into-view',
},
},
},
]
);
const text = getResponseText(responses.find((message) => message.id === 72));
expect(text).toContain('status: scrolled');
expect(text).toContain('selector: #results');
expect(text).toContain('behavior: into-view');
});
it('scrolls an iframe page by offset with browser_scroll', async () => {
const responses = await runMcpRequests(
[
{
id: 'page-1',
title: 'Iframe Scroll Page',
currentUrl: 'https://example.com/',
frames: [
{
selector: '#preview-frame',
query: {
'#preview-frame': {
exists: true,
},
},
},
],
},
],
[
{
jsonrpc: '2.0',
id: 73,
method: 'tools/call',
params: {
name: 'browser_scroll',
arguments: {
frameSelector: '#preview-frame',
behavior: 'by-offset',
deltaX: 5,
deltaY: 40,
},
},
},
]
);
const text = getResponseText(responses.find((message) => message.id === 73));
expect(text).toContain('status: scrolled');
expect(text).toContain('behavior: by-offset');
expect(text).toContain('deltaX: 5');
expect(text).toContain('deltaY: 40');
});
it('requires real mouse movement for hover-only targets and reports hover failures', async () => {
const responses = await runMcpRequests(
[