mirror of
https://github.com/tiennm99/ccs.git
synced 2026-07-16 04:18:05 +00:00
feat(browser): 完成 browser MCP 第四阶段输入能力
补齐 browser_click 的精细点击参数,并新增 browser_press_key 与 browser_scroll,继续扩展 browser MCP 的真实交互能力。 同步更新单测与浏览器自动化文档,确保 Phase 4 能力完成并通过验证。 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
cf2375e169
commit
e92a2c11bf
@@ -1,6 +1,6 @@
|
||||
# Browser Automation
|
||||
|
||||
Last Updated: 2026-04-17
|
||||
Last Updated: 2026-04-18
|
||||
|
||||
CCS provides browser automation through two separate runtime paths:
|
||||
|
||||
@@ -23,7 +23,7 @@ enabled. A recent Chrome update alone is not sufficient.
|
||||
The managed `ccs-browser` runtime currently exposes five tool groups:
|
||||
|
||||
- **Session inspection**: `browser_get_session_info`, `browser_get_url_and_title`, `browser_get_visible_text`, `browser_get_dom_snapshot`
|
||||
- **Navigation and interaction**: `browser_navigate`, `browser_click`, `browser_type`, `browser_take_screenshot`
|
||||
- **Navigation and interaction**: `browser_navigate`, `browser_click`, `browser_type`, `browser_press_key`, `browser_scroll`, `browser_take_screenshot`
|
||||
- **Hover diagnostics**: `browser_hover`, `browser_query`, `browser_take_element_screenshot`
|
||||
- **Readiness and page evaluation**: `browser_wait_for`, `browser_eval`
|
||||
- **Event observation**: `browser_wait_for_event`
|
||||
@@ -41,6 +41,12 @@ Phase 3 capability details:
|
||||
- the same scoped-selector tools accept optional `pierceShadow: true` to search open shadow roots beneath the selected root
|
||||
- `browser_wait_for_event` observes typed page or browser events for dialogs, navigation, requests, and downloads
|
||||
|
||||
Phase 4 capability details:
|
||||
|
||||
- `browser_click` also accepts optional `offsetX`, `offsetY`, `button`, and `clickCount` for more precise element-relative click control
|
||||
- `browser_press_key` sends real browser-level key events, supports modifier combinations plus repeat counts, and covers a focused set of common special keys such as `Enter`, `Tab`, `Escape`, and arrow keys
|
||||
- `browser_scroll` supports page-level `by-offset` scrolling and element-scoped `by-offset` or `into-view` scrolling, including same-origin iframe scoping
|
||||
|
||||
A common hover-debug workflow is:
|
||||
|
||||
1. call `browser_hover` to move the browser pointer onto the card or trigger
|
||||
|
||||
+414
-20
@@ -43,6 +43,8 @@ const TOOL_DOM_SNAPSHOT = 'browser_get_dom_snapshot';
|
||||
const TOOL_NAVIGATE = 'browser_navigate';
|
||||
const TOOL_CLICK = 'browser_click';
|
||||
const TOOL_TYPE = 'browser_type';
|
||||
const TOOL_PRESS_KEY = 'browser_press_key';
|
||||
const TOOL_SCROLL = 'browser_scroll';
|
||||
const TOOL_TAKE_SCREENSHOT = 'browser_take_screenshot';
|
||||
const TOOL_WAIT_FOR = 'browser_wait_for';
|
||||
const TOOL_EVAL = 'browser_eval';
|
||||
@@ -58,6 +60,8 @@ const TOOL_NAMES = [
|
||||
TOOL_NAVIGATE,
|
||||
TOOL_CLICK,
|
||||
TOOL_TYPE,
|
||||
TOOL_PRESS_KEY,
|
||||
TOOL_SCROLL,
|
||||
TOOL_TAKE_SCREENSHOT,
|
||||
TOOL_WAIT_FOR,
|
||||
TOOL_EVAL,
|
||||
@@ -278,6 +282,24 @@ function getTools() {
|
||||
type: 'boolean',
|
||||
description: 'When true, search open shadow roots beneath the selected root.',
|
||||
},
|
||||
offsetX: {
|
||||
type: 'number',
|
||||
description: "Optional horizontal offset in CSS pixels from the target element's left edge.",
|
||||
},
|
||||
offsetY: {
|
||||
type: 'number',
|
||||
description: "Optional vertical offset in CSS pixels from the target element's top edge.",
|
||||
},
|
||||
button: {
|
||||
type: 'string',
|
||||
enum: ['left', 'middle', 'right'],
|
||||
description: 'Optional mouse button. Defaults to left.',
|
||||
},
|
||||
clickCount: {
|
||||
type: 'integer',
|
||||
minimum: 1,
|
||||
description: 'Optional click count. Defaults to 1.',
|
||||
},
|
||||
},
|
||||
required: ['selector'],
|
||||
additionalProperties: false,
|
||||
@@ -312,6 +334,82 @@ function getTools() {
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: TOOL_PRESS_KEY,
|
||||
description:
|
||||
'Press a key or key combination in the selected page using real keyboard-style events. Optionally choose a page by index.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
pageIndex: {
|
||||
type: 'integer',
|
||||
minimum: 0,
|
||||
description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.',
|
||||
},
|
||||
key: {
|
||||
type: 'string',
|
||||
description: 'Required primary key to press.',
|
||||
},
|
||||
modifiers: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'string',
|
||||
enum: ['Alt', 'Control', 'Meta', 'Shift'],
|
||||
},
|
||||
description: 'Optional modifier keys such as Alt, Control, Meta, or Shift.',
|
||||
},
|
||||
repeat: {
|
||||
type: 'integer',
|
||||
minimum: 1,
|
||||
description: 'Optional repeat count. Defaults to 1.',
|
||||
},
|
||||
},
|
||||
required: ['key'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: TOOL_SCROLL,
|
||||
description:
|
||||
'Scroll the selected page or a matched element. Supports explicit deltas or scrolling an element into view.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
pageIndex: {
|
||||
type: 'integer',
|
||||
minimum: 0,
|
||||
description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.',
|
||||
},
|
||||
selector: {
|
||||
type: 'string',
|
||||
description: 'Optional CSS selector for an element-scoped scroll target.',
|
||||
},
|
||||
frameSelector: {
|
||||
type: 'string',
|
||||
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
|
||||
},
|
||||
pierceShadow: {
|
||||
type: 'boolean',
|
||||
description: 'When true, search open shadow roots beneath the selected root.',
|
||||
},
|
||||
behavior: {
|
||||
type: 'string',
|
||||
enum: ['into-view', 'by-offset'],
|
||||
description: 'Required scroll behavior.',
|
||||
},
|
||||
deltaX: {
|
||||
type: 'number',
|
||||
description: 'Optional horizontal scroll delta for by-offset behavior.',
|
||||
},
|
||||
deltaY: {
|
||||
type: 'number',
|
||||
description: 'Optional vertical scroll delta for by-offset behavior.',
|
||||
},
|
||||
},
|
||||
required: ['behavior'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: TOOL_TAKE_SCREENSHOT,
|
||||
description:
|
||||
@@ -795,6 +893,49 @@ function requirePositiveIntegerOrDefault(value, label, fallback) {
|
||||
return value;
|
||||
}
|
||||
|
||||
function requirePositiveInteger(value, label) {
|
||||
if (!Number.isInteger(value) || value <= 0) {
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireFiniteNumber(value, label) {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
||||
throw new Error(`${label} must be a finite number`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireEnumString(value, label, allowedValues) {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = requireNonEmptyString(value, label);
|
||||
if (!allowedValues.includes(normalized)) {
|
||||
throw new Error(`${label} must be one of: ${allowedValues.join(', ')}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireOptionalStringArray(value, label, allowedValues) {
|
||||
if (value === undefined) {
|
||||
return [];
|
||||
}
|
||||
if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || item.trim() === '')) {
|
||||
throw new Error(`${label} must be an array of non-empty strings`);
|
||||
}
|
||||
const normalized = value.map((item) => item.trim());
|
||||
if (Array.isArray(allowedValues) && allowedValues.length > 0) {
|
||||
for (const item of normalized) {
|
||||
if (!allowedValues.includes(item)) {
|
||||
throw new Error(`${label} must only contain: ${allowedValues.join(', ')}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function getBrowserEvalMode() {
|
||||
const raw = String(process.env.CCS_BROWSER_EVAL_MODE || 'readonly').trim();
|
||||
if (raw === 'disabled' || raw === 'readonly' || raw === 'readwrite') {
|
||||
@@ -1058,12 +1199,20 @@ async function handleClick(toolArgs) {
|
||||
const targetIndex = nth ?? 0;
|
||||
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
|
||||
const pierceShadow = toolArgs.pierceShadow === true;
|
||||
const offsetX = toolArgs.offsetX === undefined ? undefined : requireFiniteNumber(toolArgs.offsetX, 'offsetX');
|
||||
const offsetY = toolArgs.offsetY === undefined ? undefined : requireFiniteNumber(toolArgs.offsetY, 'offsetY');
|
||||
const button = requireEnumString(toolArgs.button, 'button', ['left', 'middle', 'right']) || 'left';
|
||||
const clickCount = toolArgs.clickCount === undefined ? 1 : requirePositiveInteger(toolArgs.clickCount, 'clickCount');
|
||||
|
||||
const expression = `(() => {
|
||||
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
|
||||
const nth = ${nth === undefined ? 'undefined' : String(nth)};
|
||||
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
|
||||
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
|
||||
const offsetX = ${offsetX === undefined ? 'undefined' : String(offsetX)};
|
||||
const offsetY = ${offsetY === undefined ? 'undefined' : String(offsetY)};
|
||||
const button = JSON.parse(${JSON.stringify(JSON.stringify(button))});
|
||||
const clickCount = ${clickCount};
|
||||
|
||||
const visitRoots = (root) => {
|
||||
const roots = [root];
|
||||
@@ -1113,52 +1262,104 @@ async function handleClick(toolArgs) {
|
||||
throw new Error('element is disabled for selector: ' + selector);
|
||||
}
|
||||
const style = window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
const initialRect = element.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
initialRect.width <= 0 ||
|
||||
initialRect.height <= 0
|
||||
) {
|
||||
throw new Error('element is hidden or not interactable for selector: ' + selector);
|
||||
}
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
const rect = element.getBoundingClientRect();
|
||||
|
||||
const dispatchMouseEvent = (type, init) => {
|
||||
const resolvedOffsetX = offsetX === undefined ? rect.width / 2 : offsetX;
|
||||
const resolvedOffsetY = offsetY === undefined ? rect.height / 2 : offsetY;
|
||||
const clientX = rect.left + resolvedOffsetX;
|
||||
const clientY = rect.top + resolvedOffsetY;
|
||||
const buttonCode = button === 'middle' ? 1 : button === 'right' ? 2 : 0;
|
||||
const buttonsMask = buttonCode === 1 ? 4 : buttonCode === 2 ? 2 : 1;
|
||||
|
||||
const dispatchMouseEvent = (type, detail, init) => {
|
||||
const event = new MouseEvent(type, {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
composed: true,
|
||||
view: window,
|
||||
detail: 1,
|
||||
detail,
|
||||
clientX,
|
||||
clientY,
|
||||
button: buttonCode,
|
||||
buttons: type === 'mousedown' ? buttonsMask : 0,
|
||||
...init,
|
||||
});
|
||||
return element.dispatchEvent(event);
|
||||
};
|
||||
|
||||
try {
|
||||
const dispatchResult = {
|
||||
shouldActivate:
|
||||
dispatchMouseEvent('mousedown', { button: 0, buttons: 1 }) &&
|
||||
dispatchMouseEvent('mouseup', { button: 0, buttons: 0 }),
|
||||
};
|
||||
if (!dispatchResult.shouldActivate) {
|
||||
return 'ok';
|
||||
for (let index = 1; index <= clickCount; index += 1) {
|
||||
const dispatchResult = {
|
||||
shouldActivate:
|
||||
dispatchMouseEvent('mousedown', index, {}) &&
|
||||
dispatchMouseEvent('mouseup', index, {}),
|
||||
};
|
||||
if (!dispatchResult.shouldActivate) {
|
||||
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
|
||||
}
|
||||
if (!element.isConnected) {
|
||||
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
|
||||
}
|
||||
if (button === 'left') {
|
||||
element.click();
|
||||
} else if (button === 'right') {
|
||||
element.dispatchEvent(new MouseEvent('contextmenu', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
composed: true,
|
||||
view: window,
|
||||
detail: index,
|
||||
clientX,
|
||||
clientY,
|
||||
button: 2,
|
||||
buttons: 0,
|
||||
}));
|
||||
} else {
|
||||
element.dispatchEvent(new MouseEvent('auxclick', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
composed: true,
|
||||
view: window,
|
||||
detail: index,
|
||||
clientX,
|
||||
clientY,
|
||||
button: 1,
|
||||
buttons: 0,
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (!element.isConnected) {
|
||||
return 'ok';
|
||||
if (button === 'left' && clickCount === 2) {
|
||||
element.dispatchEvent(new MouseEvent('dblclick', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
composed: true,
|
||||
view: window,
|
||||
detail: 2,
|
||||
clientX,
|
||||
clientY,
|
||||
button: 0,
|
||||
buttons: 0,
|
||||
}));
|
||||
}
|
||||
} catch (mouseError) {
|
||||
// Fall through to the native activation path below.
|
||||
}
|
||||
|
||||
element.click();
|
||||
|
||||
return 'ok';
|
||||
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
|
||||
})()`;
|
||||
|
||||
await evaluateExpression(page, expression);
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`;
|
||||
const raw = await evaluateExpression(page, expression);
|
||||
const parsed = JSON.parse(raw);
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}\noffsetX: ${parsed.resolvedOffsetX}\noffsetY: ${parsed.resolvedOffsetY}\nbutton: ${parsed.button}\nclickCount: ${parsed.clickCount}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`;
|
||||
}
|
||||
|
||||
async function handleType(toolArgs) {
|
||||
@@ -1240,6 +1441,183 @@ async function handleType(toolArgs) {
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}\ntypedLength: ${typedLength}\nstatus: typed`;
|
||||
}
|
||||
|
||||
async function handlePressKey(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const key = requireNonEmptyString(toolArgs.key, 'key');
|
||||
const modifiers = requireOptionalStringArray(toolArgs.modifiers, 'modifiers', [
|
||||
'Alt',
|
||||
'Control',
|
||||
'Meta',
|
||||
'Shift',
|
||||
]);
|
||||
const repeat = toolArgs.repeat === undefined ? 1 : requirePositiveInteger(toolArgs.repeat, 'repeat');
|
||||
const modifierMask =
|
||||
(modifiers.includes('Alt') ? 1 : 0) |
|
||||
(modifiers.includes('Control') ? 2 : 0) |
|
||||
(modifiers.includes('Meta') ? 4 : 0) |
|
||||
(modifiers.includes('Shift') ? 8 : 0);
|
||||
const specialKeyMap = {
|
||||
Enter: { code: 'Enter', keyCode: 13, text: '\r' },
|
||||
Tab: { code: 'Tab', keyCode: 9, text: '' },
|
||||
Escape: { code: 'Escape', keyCode: 27, text: '' },
|
||||
ArrowUp: { code: 'ArrowUp', keyCode: 38, text: '' },
|
||||
ArrowDown: { code: 'ArrowDown', keyCode: 40, text: '' },
|
||||
ArrowLeft: { code: 'ArrowLeft', keyCode: 37, text: '' },
|
||||
ArrowRight: { code: 'ArrowRight', keyCode: 39, text: '' },
|
||||
Backspace: { code: 'Backspace', keyCode: 8, text: '' },
|
||||
Delete: { code: 'Delete', keyCode: 46, text: '' },
|
||||
Space: { code: 'Space', keyCode: 32, text: ' ' },
|
||||
};
|
||||
const keyDescriptor =
|
||||
key.length === 1
|
||||
? {
|
||||
code: `Key${key.toUpperCase()}`,
|
||||
keyCode: key.toUpperCase().charCodeAt(0),
|
||||
text: key,
|
||||
}
|
||||
: specialKeyMap[key];
|
||||
if (!keyDescriptor) {
|
||||
throw new Error(`unsupported key: ${key}`);
|
||||
}
|
||||
const normalizedKey = key;
|
||||
const normalizedText = keyDescriptor.text;
|
||||
const code = keyDescriptor.code;
|
||||
const keyCode = keyDescriptor.keyCode;
|
||||
|
||||
for (let index = 0; index < repeat; index += 1) {
|
||||
await sendCdpCommand(page, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyDown',
|
||||
key: normalizedKey,
|
||||
code,
|
||||
text: normalizedText,
|
||||
unmodifiedText: normalizedText,
|
||||
windowsVirtualKeyCode: keyCode,
|
||||
nativeVirtualKeyCode: keyCode,
|
||||
modifiers: modifierMask,
|
||||
autoRepeat: index > 0,
|
||||
});
|
||||
await sendCdpCommand(page, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyUp',
|
||||
key: normalizedKey,
|
||||
code,
|
||||
windowsVirtualKeyCode: keyCode,
|
||||
nativeVirtualKeyCode: keyCode,
|
||||
modifiers: modifierMask,
|
||||
});
|
||||
}
|
||||
|
||||
const modifierText = modifiers.length > 0 ? modifiers.join(',') : 'none';
|
||||
return `pageIndex: ${pageIndex}\nkey: ${key}\nmodifiers: ${modifierText}\nrepeat: ${repeat}\nstatus: key-pressed`;
|
||||
}
|
||||
|
||||
async function handleScroll(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const selector = parseOptionalNonEmptyString(toolArgs.selector);
|
||||
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
|
||||
const pierceShadow = toolArgs.pierceShadow === true;
|
||||
const behavior = requireEnumString(toolArgs.behavior, 'behavior', ['into-view', 'by-offset']);
|
||||
const deltaX = toolArgs.deltaX === undefined ? 0 : requireFiniteNumber(toolArgs.deltaX, 'deltaX');
|
||||
const deltaY = toolArgs.deltaY === undefined ? 0 : requireFiniteNumber(toolArgs.deltaY, 'deltaY');
|
||||
|
||||
const expression = `(() => {
|
||||
const selector = ${selector ? `JSON.parse(${JSON.stringify(JSON.stringify(selector))})` : 'undefined'};
|
||||
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
|
||||
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
|
||||
const behavior = JSON.parse(${JSON.stringify(JSON.stringify(behavior))});
|
||||
const deltaX = ${deltaX};
|
||||
const deltaY = ${deltaY};
|
||||
|
||||
const visitRoots = (root) => {
|
||||
const roots = [root];
|
||||
if (!pierceShadow) {
|
||||
return roots;
|
||||
}
|
||||
const queue = [root];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
const elements = Array.from(current.querySelectorAll('*'));
|
||||
for (const element of elements) {
|
||||
if (element.shadowRoot) {
|
||||
roots.push(element.shadowRoot);
|
||||
queue.push(element.shadowRoot);
|
||||
}
|
||||
}
|
||||
}
|
||||
return roots;
|
||||
};
|
||||
|
||||
let root = document;
|
||||
let scrollWindow = window;
|
||||
if (frameSelector) {
|
||||
const frame = document.querySelector(frameSelector);
|
||||
if (!frame) {
|
||||
throw new Error('frame not found for selector: ' + frameSelector);
|
||||
}
|
||||
const frameDocument = frame.contentDocument;
|
||||
if (!frameDocument) {
|
||||
throw new Error('frame document is unavailable for selector: ' + frameSelector);
|
||||
}
|
||||
if (!frame.contentWindow) {
|
||||
throw new Error('frame window is unavailable for selector: ' + frameSelector);
|
||||
}
|
||||
root = frameDocument;
|
||||
scrollWindow = frame.contentWindow;
|
||||
}
|
||||
|
||||
if (!selector) {
|
||||
if (behavior !== 'by-offset') {
|
||||
throw new Error('selector is required for behavior: ' + behavior);
|
||||
}
|
||||
scrollWindow.scrollBy(deltaX, deltaY);
|
||||
return JSON.stringify({ scope: 'page', behavior, deltaX, deltaY });
|
||||
}
|
||||
|
||||
const roots = visitRoots(root);
|
||||
const matches = [];
|
||||
for (const currentRoot of roots) {
|
||||
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
|
||||
}
|
||||
const element = matches[0];
|
||||
if (!element) {
|
||||
throw new Error('element not found for selector: ' + selector);
|
||||
}
|
||||
|
||||
if (behavior === 'into-view') {
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
return JSON.stringify({ scope: 'element', selector, behavior });
|
||||
}
|
||||
|
||||
if (typeof element.scrollBy === 'function') {
|
||||
element.scrollBy(deltaX, deltaY);
|
||||
return JSON.stringify({ scope: 'element', selector, behavior, deltaX, deltaY });
|
||||
}
|
||||
|
||||
throw new Error('element does not support scrollBy for selector: ' + selector);
|
||||
})()`;
|
||||
|
||||
const raw = await evaluateExpression(page, expression);
|
||||
const parsed = JSON.parse(raw);
|
||||
const lines = [`pageIndex: ${pageIndex}`];
|
||||
if (parsed.selector) {
|
||||
lines.push(`selector: ${parsed.selector}`);
|
||||
}
|
||||
lines.push(`behavior: ${parsed.behavior}`);
|
||||
if (typeof parsed.deltaX === 'number') {
|
||||
lines.push(`deltaX: ${parsed.deltaX}`);
|
||||
}
|
||||
if (typeof parsed.deltaY === 'number') {
|
||||
lines.push(`deltaY: ${parsed.deltaY}`);
|
||||
}
|
||||
if (parsed.scope === 'element') {
|
||||
const scopedSuffix = formatScopedSelectorSuffix(frameSelector, pierceShadow);
|
||||
if (scopedSuffix) {
|
||||
lines.push(scopedSuffix.slice(1));
|
||||
}
|
||||
}
|
||||
lines.push('status: scrolled');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function handleScreenshot(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const fullPage = toolArgs.fullPage === true;
|
||||
@@ -1865,6 +2243,22 @@ async function handleToolCall(message) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (toolName === TOOL_PRESS_KEY) {
|
||||
const text = await handlePressKey(toolArgs);
|
||||
writeResponse(id, {
|
||||
content: [{ type: 'text', text }],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (toolName === TOOL_SCROLL) {
|
||||
const text = await handleScroll(toolArgs);
|
||||
writeResponse(id, {
|
||||
content: [{ type: 'text', text }],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (toolName === TOOL_TAKE_SCREENSHOT) {
|
||||
const text = await handleScreenshot(toolArgs);
|
||||
writeResponse(id, {
|
||||
|
||||
@@ -48,6 +48,10 @@ type MockClickState = {
|
||||
detachAfterMouseDown?: boolean;
|
||||
mouseSequenceError?: string;
|
||||
label?: string;
|
||||
expectedOffset?: { x: number; y: number };
|
||||
expectedButton?: 'left' | 'middle' | 'right';
|
||||
expectedClickCount?: number;
|
||||
requireDoubleClickEvent?: boolean;
|
||||
};
|
||||
|
||||
type MockClickPlan = MockClickState | MockClickState[];
|
||||
@@ -144,6 +148,20 @@ type MockPageState = {
|
||||
focused?: boolean;
|
||||
}
|
||||
>;
|
||||
keyboard?: {
|
||||
expectedKey?: string;
|
||||
expectedModifiers?: string[];
|
||||
expectedRepeat?: number;
|
||||
_seenKeyDownCount?: number;
|
||||
};
|
||||
scroll?: Record<
|
||||
string,
|
||||
{
|
||||
expectedBehavior?: 'into-view' | 'by-offset';
|
||||
expectedDeltaX?: number;
|
||||
expectedDeltaY?: number;
|
||||
}
|
||||
>;
|
||||
};
|
||||
|
||||
const bundledServerPath = join(process.cwd(), 'lib', 'mcp', 'ccs-browser-server.cjs');
|
||||
@@ -521,6 +539,45 @@ function createMockBrowser(pagesInput: MockPageState[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.method === 'Input.dispatchKeyEvent') {
|
||||
const type = typeof message.params?.type === 'string' ? message.params.type : '';
|
||||
const key = typeof message.params?.key === 'string' ? message.params.key : '';
|
||||
const modifiersMask = Number(message.params?.modifiers || 0);
|
||||
const modifiers = [
|
||||
...(modifiersMask & 1 ? ['Alt'] : []),
|
||||
...(modifiersMask & 2 ? ['Control'] : []),
|
||||
...(modifiersMask & 4 ? ['Meta'] : []),
|
||||
...(modifiersMask & 8 ? ['Shift'] : []),
|
||||
];
|
||||
const keyboardPlan = page.keyboard;
|
||||
if (type === 'keyDown') {
|
||||
if (keyboardPlan?.expectedKey && key !== keyboardPlan.expectedKey) {
|
||||
replyError(`unexpected key: ${key}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
keyboardPlan?.expectedModifiers &&
|
||||
JSON.stringify(modifiers) !== JSON.stringify(keyboardPlan.expectedModifiers)
|
||||
) {
|
||||
replyError(`unexpected modifiers for key: ${key}`);
|
||||
return;
|
||||
}
|
||||
if (keyboardPlan) {
|
||||
keyboardPlan._seenKeyDownCount = (keyboardPlan._seenKeyDownCount || 0) + 1;
|
||||
}
|
||||
}
|
||||
if (
|
||||
type === 'keyUp' &&
|
||||
typeof keyboardPlan?.expectedRepeat === 'number' &&
|
||||
(keyboardPlan._seenKeyDownCount || 0) !== keyboardPlan.expectedRepeat
|
||||
) {
|
||||
replyError(`unexpected repeat for key: ${key}`);
|
||||
return;
|
||||
}
|
||||
reply({});
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.method === 'Page.enable' || message.method === 'Network.enable') {
|
||||
reply({});
|
||||
if (message.method === 'Page.enable') {
|
||||
@@ -627,7 +684,7 @@ function createMockBrowser(pagesInput: MockPageState[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (expression.includes('scrollIntoView') && expression.includes('.click()')) {
|
||||
if (expression.includes('scrollIntoView') && expression.includes('resolvedOffsetX')) {
|
||||
const selector = parseJsonArgument(expression, 'selector') || '';
|
||||
const nth = parseNumberArgument(expression, 'nth') ?? 0;
|
||||
const clickPlan = page.click?.[selector];
|
||||
@@ -636,6 +693,7 @@ function createMockBrowser(pagesInput: MockPageState[]) {
|
||||
const attemptedMouseUp = expression.includes("dispatchMouseEvent('mouseup'");
|
||||
const attemptedMouseSequence = attemptedMouseDown && attemptedMouseUp;
|
||||
const attemptedClickEvent = expression.includes("dispatchMouseEvent('click'");
|
||||
const attemptedDoubleClickEvent = expression.includes("new MouseEvent('dblclick'");
|
||||
const readsDispatchResult = expression.includes('const dispatchResult = {');
|
||||
const gatesNativeClickOnDispatchResult = expression.includes(
|
||||
'if (!dispatchResult.shouldActivate)'
|
||||
@@ -662,6 +720,10 @@ function createMockBrowser(pagesInput: MockPageState[]) {
|
||||
index < catchIndex ||
|
||||
index > catchBlockEnd
|
||||
);
|
||||
const offsetX = parseNumberArgument(expression, 'offsetX');
|
||||
const offsetY = parseNumberArgument(expression, 'offsetY');
|
||||
const button = parseJsonArgument(expression, 'button') || 'left';
|
||||
const clickCount = parseNumberArgument(expression, 'clickCount') ?? 1;
|
||||
if (!resolvedClickPlan) {
|
||||
replyError(`element index ${nth} is out of range for selector: ${selector}`);
|
||||
return;
|
||||
@@ -712,13 +774,49 @@ function createMockBrowser(pagesInput: MockPageState[]) {
|
||||
replyError(`native click required for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
resolvedClickPlan.expectedOffset &&
|
||||
(offsetX !== resolvedClickPlan.expectedOffset.x || offsetY !== resolvedClickPlan.expectedOffset.y)
|
||||
) {
|
||||
replyError(`unexpected click offset for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (resolvedClickPlan.expectedButton && button !== resolvedClickPlan.expectedButton) {
|
||||
replyError(`unexpected click button for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
typeof resolvedClickPlan.expectedClickCount === 'number' &&
|
||||
clickCount !== resolvedClickPlan.expectedClickCount
|
||||
) {
|
||||
replyError(`unexpected click count for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
resolvedClickPlan.requireDoubleClickEvent &&
|
||||
clickCount === 2 &&
|
||||
!attemptedDoubleClickEvent
|
||||
) {
|
||||
replyError(`dblclick event required for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (resolvedClickPlan.mouseSequenceError) {
|
||||
if (!attemptedMouseSequence) {
|
||||
replyError(`mousedown/mouseup required for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (attemptedFallbackClick || attemptedNativeClickOutsideCatch) {
|
||||
reply({ result: { type: 'string', value: 'ok' } });
|
||||
reply({
|
||||
result: {
|
||||
type: 'string',
|
||||
value: JSON.stringify({
|
||||
resolvedOffsetX: offsetX ?? 50,
|
||||
resolvedOffsetY: offsetY ?? 10,
|
||||
button,
|
||||
clickCount,
|
||||
}),
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
replyError(resolvedClickPlan.mouseSequenceError);
|
||||
@@ -728,7 +826,17 @@ function createMockBrowser(pagesInput: MockPageState[]) {
|
||||
replyError(resolvedClickPlan.error);
|
||||
return;
|
||||
}
|
||||
reply({ result: { type: 'string', value: 'ok' } });
|
||||
reply({
|
||||
result: {
|
||||
type: 'string',
|
||||
value: JSON.stringify({
|
||||
resolvedOffsetX: offsetX ?? 50,
|
||||
resolvedOffsetY: offsetY ?? 10,
|
||||
button,
|
||||
clickCount,
|
||||
}),
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -950,6 +1058,80 @@ function createMockBrowser(pagesInput: MockPageState[]) {
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (expression.includes('new KeyboardEvent(')) {
|
||||
const key = parseJsonArgument(expression, 'key') || '';
|
||||
const repeat = parseNumberArgument(expression, 'repeat') ?? 1;
|
||||
const modifiersMatch = expression.match(/const modifiers = (\[[^\n;]*\]);/);
|
||||
const modifiers = modifiersMatch ? (JSON.parse(modifiersMatch[1]) as string[]) : [];
|
||||
const keyboardPlan = page.keyboard;
|
||||
if (keyboardPlan?.expectedKey && key !== keyboardPlan.expectedKey) {
|
||||
replyError(`unexpected key: ${key}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
keyboardPlan?.expectedModifiers &&
|
||||
JSON.stringify(modifiers) !== JSON.stringify(keyboardPlan.expectedModifiers)
|
||||
) {
|
||||
replyError(`unexpected modifiers for key: ${key}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
typeof keyboardPlan?.expectedRepeat === 'number' &&
|
||||
repeat !== keyboardPlan.expectedRepeat
|
||||
) {
|
||||
replyError(`unexpected repeat for key: ${key}`);
|
||||
return;
|
||||
}
|
||||
reply({
|
||||
result: {
|
||||
type: 'string',
|
||||
value: JSON.stringify({ key, modifiers, repeat }),
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (expression.includes('window.scrollBy(deltaX, deltaY)') || expression.includes('element.scrollBy(deltaX, deltaY)')) {
|
||||
const selector = parseJsonArgument(expression, 'selector');
|
||||
const behavior = parseJsonArgument(expression, 'behavior') || '';
|
||||
const deltaX = Number(expression.match(/const deltaX = (-?[0-9]+(?:\.[0-9]+)?);/)?.[1] || 0);
|
||||
const deltaY = Number(expression.match(/const deltaY = (-?[0-9]+(?:\.[0-9]+)?);/)?.[1] || 0);
|
||||
const scrollPlan = selector ? page.scroll?.[selector] : undefined;
|
||||
if (selector && !scrollPlan) {
|
||||
replyError(`element not found for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (scrollPlan?.expectedBehavior && behavior !== scrollPlan.expectedBehavior) {
|
||||
replyError(`unexpected scroll behavior for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
typeof scrollPlan?.expectedDeltaX === 'number' &&
|
||||
deltaX !== scrollPlan.expectedDeltaX
|
||||
) {
|
||||
replyError(`unexpected deltaX for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
typeof scrollPlan?.expectedDeltaY === 'number' &&
|
||||
deltaY !== scrollPlan.expectedDeltaY
|
||||
) {
|
||||
replyError(`unexpected deltaY for selector: ${selector}`);
|
||||
return;
|
||||
}
|
||||
reply({
|
||||
result: {
|
||||
type: 'string',
|
||||
value: JSON.stringify(
|
||||
selector
|
||||
? { scope: 'element', selector, behavior, deltaX, deltaY }
|
||||
: { scope: 'page', behavior, deltaX, deltaY }
|
||||
),
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1056,6 +1238,8 @@ describe('ccs-browser MCP server', () => {
|
||||
'browser_navigate',
|
||||
'browser_click',
|
||||
'browser_type',
|
||||
'browser_press_key',
|
||||
'browser_scroll',
|
||||
'browser_take_screenshot',
|
||||
'browser_wait_for',
|
||||
'browser_eval',
|
||||
@@ -1068,6 +1252,21 @@ describe('ccs-browser MCP server', () => {
|
||||
const clickTool = tools.find((tool) => tool.name === 'browser_click');
|
||||
expect(clickTool?.description).toContain('mouse event chain');
|
||||
expect(clickTool?.description).not.toContain('synthetic element.click()');
|
||||
expect(clickTool?.inputSchema?.properties?.offsetX).toMatchObject({ type: 'number' });
|
||||
expect(clickTool?.inputSchema?.properties?.offsetY).toMatchObject({ type: 'number' });
|
||||
expect(clickTool?.inputSchema?.properties?.button).toMatchObject({ type: 'string' });
|
||||
expect(clickTool?.inputSchema?.properties?.clickCount).toMatchObject({
|
||||
type: 'integer',
|
||||
minimum: 1,
|
||||
});
|
||||
|
||||
const keyTool = tools.find((tool) => tool.name === 'browser_press_key');
|
||||
expect(keyTool?.inputSchema?.properties?.key).toMatchObject({ type: 'string' });
|
||||
expect(keyTool?.inputSchema?.properties?.modifiers).toMatchObject({ type: 'array' });
|
||||
|
||||
const scrollTool = tools.find((tool) => tool.name === 'browser_scroll');
|
||||
expect(scrollTool?.inputSchema?.properties?.deltaX).toMatchObject({ type: 'number' });
|
||||
expect(scrollTool?.inputSchema?.properties?.deltaY).toMatchObject({ type: 'number' });
|
||||
|
||||
const queryTool = tools.find((tool) => tool.name === 'browser_query');
|
||||
expect(queryTool?.inputSchema?.properties?.fields).toMatchObject({
|
||||
@@ -1638,6 +1837,250 @@ describe('ccs-browser MCP server', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('clicks with element-relative offsets and richer click options', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Click Page',
|
||||
currentUrl: 'https://example.com/',
|
||||
click: {
|
||||
'#offset-target': {
|
||||
requireMouseSequence: true,
|
||||
expectedOffset: { x: 12, y: 8 },
|
||||
expectedButton: 'right',
|
||||
expectedClickCount: 2,
|
||||
},
|
||||
'#double-left': {
|
||||
requireMouseSequence: true,
|
||||
expectedButton: 'left',
|
||||
expectedClickCount: 2,
|
||||
requireDoubleClickEvent: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 70,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_click',
|
||||
arguments: {
|
||||
selector: '#offset-target',
|
||||
offsetX: 12,
|
||||
offsetY: 8,
|
||||
button: 'right',
|
||||
clickCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 700,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_click',
|
||||
arguments: {
|
||||
selector: '#double-left',
|
||||
clickCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const text = getResponseText(responses.find((message) => message.id === 70));
|
||||
expect(text).toContain('status: clicked');
|
||||
expect(text).toContain('offsetX: 12');
|
||||
expect(text).toContain('offsetY: 8');
|
||||
expect(text).toContain('button: right');
|
||||
expect(text).toContain('clickCount: 2');
|
||||
|
||||
const doubleText = getResponseText(responses.find((message) => message.id === 700));
|
||||
expect(doubleText).toContain('status: clicked');
|
||||
expect(doubleText).toContain('button: left');
|
||||
expect(doubleText).toContain('clickCount: 2');
|
||||
});
|
||||
|
||||
it('presses a key combination with browser_press_key', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Keyboard Page',
|
||||
currentUrl: 'https://example.com/',
|
||||
keyboard: {
|
||||
expectedKey: 'k',
|
||||
expectedModifiers: ['Meta'],
|
||||
expectedRepeat: 2,
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 71,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_press_key',
|
||||
arguments: {
|
||||
key: 'k',
|
||||
modifiers: ['Meta'],
|
||||
repeat: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const text = getResponseText(responses.find((message) => message.id === 71));
|
||||
expect(text).toContain('status: key-pressed');
|
||||
expect(text).toContain('key: k');
|
||||
expect(text).toContain('modifiers: Meta');
|
||||
expect(text).toContain('repeat: 2');
|
||||
});
|
||||
|
||||
it('supports common special keys with browser_press_key', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Special Key Page',
|
||||
currentUrl: 'https://example.com/',
|
||||
keyboard: {
|
||||
expectedKey: 'Enter',
|
||||
expectedRepeat: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 711,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_press_key',
|
||||
arguments: {
|
||||
key: 'Enter',
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const text = getResponseText(responses.find((message) => message.id === 711));
|
||||
expect(text).toContain('status: key-pressed');
|
||||
expect(text).toContain('key: Enter');
|
||||
});
|
||||
|
||||
it('rejects unsupported modifiers for browser_press_key', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[{ id: 'page-1', title: 'Invalid Modifier Page', currentUrl: 'https://example.com/' }],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 712,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_press_key',
|
||||
arguments: {
|
||||
key: 'k',
|
||||
modifiers: ['Cmd'],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const response = responses.find((message) => message.id === 712);
|
||||
expect((response?.result as { isError?: boolean }).isError).toBe(true);
|
||||
expect(getResponseText(response)).toContain(
|
||||
'modifiers must only contain: Alt, Control, Meta, Shift'
|
||||
);
|
||||
});
|
||||
|
||||
it('scrolls an element into view with browser_scroll', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Scroll Page',
|
||||
currentUrl: 'https://example.com/',
|
||||
scroll: {
|
||||
'#results': {
|
||||
expectedBehavior: 'into-view',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 72,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_scroll',
|
||||
arguments: {
|
||||
selector: '#results',
|
||||
behavior: 'into-view',
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const text = getResponseText(responses.find((message) => message.id === 72));
|
||||
expect(text).toContain('status: scrolled');
|
||||
expect(text).toContain('selector: #results');
|
||||
expect(text).toContain('behavior: into-view');
|
||||
});
|
||||
|
||||
it('scrolls an iframe page by offset with browser_scroll', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Iframe Scroll Page',
|
||||
currentUrl: 'https://example.com/',
|
||||
frames: [
|
||||
{
|
||||
selector: '#preview-frame',
|
||||
query: {
|
||||
'#preview-frame': {
|
||||
exists: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 73,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_scroll',
|
||||
arguments: {
|
||||
frameSelector: '#preview-frame',
|
||||
behavior: 'by-offset',
|
||||
deltaX: 5,
|
||||
deltaY: 40,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const text = getResponseText(responses.find((message) => message.id === 73));
|
||||
expect(text).toContain('status: scrolled');
|
||||
expect(text).toContain('behavior: by-offset');
|
||||
expect(text).toContain('deltaX: 5');
|
||||
expect(text).toContain('deltaY: 40');
|
||||
});
|
||||
|
||||
it('requires real mouse movement for hover-only targets and reports hover failures', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user