From e92a2c11bfedd8c4d18cf591baa15e9236aacb68 Mon Sep 17 00:00:00 2001 From: walker1211 <13750528578@163.com> Date: Sat, 18 Apr 2026 13:59:15 +0800 Subject: [PATCH] =?UTF-8?q?feat(browser):=20=E5=AE=8C=E6=88=90=20browser?= =?UTF-8?q?=20MCP=20=E7=AC=AC=E5=9B=9B=E9=98=B6=E6=AE=B5=E8=BE=93=E5=85=A5?= =?UTF-8?q?=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补齐 browser_click 的精细点击参数，并新增 browser_press_key 与 browser_scroll，继续扩展 browser MCP 的真实交互能力。 同步更新单测与浏览器自动化文档，确保 Phase 4 能力完成并通过验证。 Co-Authored-By: Claude Opus 4.7 --- docs/browser-automation.md | 10 +- lib/mcp/ccs-browser-server.cjs | 434 ++++++++++++++++- .../unit/hooks/ccs-browser-mcp-server.test.ts | 449 +++++++++++++++++- 3 files changed, 868 insertions(+), 25 deletions(-) diff --git a/docs/browser-automation.md b/docs/browser-automation.md index f4a72d3d..4928b65a 100644 --- a/docs/browser-automation.md +++ b/docs/browser-automation.md @@ -1,6 +1,6 @@ # Browser Automation -Last Updated: 2026-04-17 +Last Updated: 2026-04-18 CCS provides browser automation through two separate runtime paths: @@ -23,7 +23,7 @@ enabled. A recent Chrome update alone is not sufficient. The managed `ccs-browser` runtime currently exposes five tool groups: - **Session inspection**: `browser_get_session_info`, `browser_get_url_and_title`, `browser_get_visible_text`, `browser_get_dom_snapshot` -- **Navigation and interaction**: `browser_navigate`, `browser_click`, `browser_type`, `browser_take_screenshot` +- **Navigation and interaction**: `browser_navigate`, `browser_click`, `browser_type`, `browser_press_key`, `browser_scroll`, `browser_take_screenshot` - **Hover diagnostics**: `browser_hover`, `browser_query`, `browser_take_element_screenshot` - **Readiness and page evaluation**: `browser_wait_for`, `browser_eval` - **Event observation**: `browser_wait_for_event` @@ -41,6 +41,12 @@ Phase 3 capability details: - the same scoped-selector tools accept optional `pierceShadow: true` to search open shadow roots beneath the selected root - `browser_wait_for_event` observes typed page or browser events for dialogs, navigation, requests, and downloads +Phase 4 capability details: + +- `browser_click` also accepts optional `offsetX`, `offsetY`, `button`, and `clickCount` for more precise element-relative click control +- `browser_press_key` sends real browser-level key events, supports modifier combinations plus repeat counts, and covers a focused set of common special keys such as `Enter`, `Tab`, `Escape`, and arrow keys +- `browser_scroll` supports page-level `by-offset` scrolling and element-scoped `by-offset` or `into-view` scrolling, including same-origin iframe scoping + A common hover-debug workflow is: 1. call `browser_hover` to move the browser pointer onto the card or trigger diff --git a/lib/mcp/ccs-browser-server.cjs b/lib/mcp/ccs-browser-server.cjs index 759cf306..b39b33f3 100755 --- a/lib/mcp/ccs-browser-server.cjs +++ b/lib/mcp/ccs-browser-server.cjs @@ -43,6 +43,8 @@ const TOOL_DOM_SNAPSHOT = 'browser_get_dom_snapshot'; const TOOL_NAVIGATE = 'browser_navigate'; const TOOL_CLICK = 'browser_click'; const TOOL_TYPE = 'browser_type'; +const TOOL_PRESS_KEY = 'browser_press_key'; +const TOOL_SCROLL = 'browser_scroll'; const TOOL_TAKE_SCREENSHOT = 'browser_take_screenshot'; const TOOL_WAIT_FOR = 'browser_wait_for'; const TOOL_EVAL = 'browser_eval'; @@ -58,6 +60,8 @@ const TOOL_NAMES = [ TOOL_NAVIGATE, TOOL_CLICK, TOOL_TYPE, + TOOL_PRESS_KEY, + TOOL_SCROLL, TOOL_TAKE_SCREENSHOT, TOOL_WAIT_FOR, TOOL_EVAL, @@ -278,6 +282,24 @@ function getTools() { type: 'boolean', description: 'When true, search open shadow roots beneath the selected root.', }, + offsetX: { + type: 'number', + description: "Optional horizontal offset in CSS pixels from the target element's left edge.", + }, + offsetY: { + type: 'number', + description: "Optional vertical offset in CSS pixels from the target element's top edge.", + }, + button: { + type: 'string', + enum: ['left', 'middle', 'right'], + description: 'Optional mouse button. Defaults to left.', + }, + clickCount: { + type: 'integer', + minimum: 1, + description: 'Optional click count. Defaults to 1.', + }, }, required: ['selector'], additionalProperties: false, @@ -312,6 +334,82 @@ function getTools() { additionalProperties: false, }, }, + { + name: TOOL_PRESS_KEY, + description: + 'Press a key or key combination in the selected page using real keyboard-style events. Optionally choose a page by index.', + inputSchema: { + type: 'object', + properties: { + pageIndex: { + type: 'integer', + minimum: 0, + description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.', + }, + key: { + type: 'string', + description: 'Required primary key to press.', + }, + modifiers: { + type: 'array', + items: { + type: 'string', + enum: ['Alt', 'Control', 'Meta', 'Shift'], + }, + description: 'Optional modifier keys such as Alt, Control, Meta, or Shift.', + }, + repeat: { + type: 'integer', + minimum: 1, + description: 'Optional repeat count. Defaults to 1.', + }, + }, + required: ['key'], + additionalProperties: false, + }, + }, + { + name: TOOL_SCROLL, + description: + 'Scroll the selected page or a matched element. Supports explicit deltas or scrolling an element into view.', + inputSchema: { + type: 'object', + properties: { + pageIndex: { + type: 'integer', + minimum: 0, + description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.', + }, + selector: { + type: 'string', + description: 'Optional CSS selector for an element-scoped scroll target.', + }, + frameSelector: { + type: 'string', + description: 'Optional CSS selector for an iframe whose document should be used as the query root.', + }, + pierceShadow: { + type: 'boolean', + description: 'When true, search open shadow roots beneath the selected root.', + }, + behavior: { + type: 'string', + enum: ['into-view', 'by-offset'], + description: 'Required scroll behavior.', + }, + deltaX: { + type: 'number', + description: 'Optional horizontal scroll delta for by-offset behavior.', + }, + deltaY: { + type: 'number', + description: 'Optional vertical scroll delta for by-offset behavior.', + }, + }, + required: ['behavior'], + additionalProperties: false, + }, + }, { name: TOOL_TAKE_SCREENSHOT, description: @@ -795,6 +893,49 @@ function requirePositiveIntegerOrDefault(value, label, fallback) { return value; } +function requirePositiveInteger(value, label) { + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +function requireFiniteNumber(value, label) { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`${label} must be a finite number`); + } + return value; +} + +function requireEnumString(value, label, allowedValues) { + if (value === undefined) { + return undefined; + } + const normalized = requireNonEmptyString(value, label); + if (!allowedValues.includes(normalized)) { + throw new Error(`${label} must be one of: ${allowedValues.join(', ')}`); + } + return normalized; +} + +function requireOptionalStringArray(value, label, allowedValues) { + if (value === undefined) { + return []; + } + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || item.trim() === '')) { + throw new Error(`${label} must be an array of non-empty strings`); + } + const normalized = value.map((item) => item.trim()); + if (Array.isArray(allowedValues) && allowedValues.length > 0) { + for (const item of normalized) { + if (!allowedValues.includes(item)) { + throw new Error(`${label} must only contain: ${allowedValues.join(', ')}`); + } + } + } + return normalized; +} + function getBrowserEvalMode() { const raw = String(process.env.CCS_BROWSER_EVAL_MODE || 'readonly').trim(); if (raw === 'disabled' || raw === 'readonly' || raw === 'readwrite') { @@ -1058,12 +1199,20 @@ async function handleClick(toolArgs) { const targetIndex = nth ?? 0; const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector); const pierceShadow = toolArgs.pierceShadow === true; + const offsetX = toolArgs.offsetX === undefined ? undefined : requireFiniteNumber(toolArgs.offsetX, 'offsetX'); + const offsetY = toolArgs.offsetY === undefined ? undefined : requireFiniteNumber(toolArgs.offsetY, 'offsetY'); + const button = requireEnumString(toolArgs.button, 'button', ['left', 'middle', 'right']) || 'left'; + const clickCount = toolArgs.clickCount === undefined ? 1 : requirePositiveInteger(toolArgs.clickCount, 'clickCount'); const expression = `(() => { const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))}); const nth = ${nth === undefined ? 'undefined' : String(nth)}; const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'}; const pierceShadow = ${pierceShadow ? 'true' : 'false'}; + const offsetX = ${offsetX === undefined ? 'undefined' : String(offsetX)}; + const offsetY = ${offsetY === undefined ? 'undefined' : String(offsetY)}; + const button = JSON.parse(${JSON.stringify(JSON.stringify(button))}); + const clickCount = ${clickCount}; const visitRoots = (root) => { const roots = [root]; @@ -1113,52 +1262,104 @@ async function handleClick(toolArgs) { throw new Error('element is disabled for selector: ' + selector); } const style = window.getComputedStyle(element); - const rect = element.getBoundingClientRect(); + const initialRect = element.getBoundingClientRect(); if ( style.display === 'none' || style.visibility === 'hidden' || - rect.width <= 0 || - rect.height <= 0 + initialRect.width <= 0 || + initialRect.height <= 0 ) { throw new Error('element is hidden or not interactable for selector: ' + selector); } element.scrollIntoView({ block: 'center', inline: 'center' }); + const rect = element.getBoundingClientRect(); - const dispatchMouseEvent = (type, init) => { + const resolvedOffsetX = offsetX === undefined ? rect.width / 2 : offsetX; + const resolvedOffsetY = offsetY === undefined ? rect.height / 2 : offsetY; + const clientX = rect.left + resolvedOffsetX; + const clientY = rect.top + resolvedOffsetY; + const buttonCode = button === 'middle' ? 1 : button === 'right' ? 2 : 0; + const buttonsMask = buttonCode === 1 ? 4 : buttonCode === 2 ? 2 : 1; + + const dispatchMouseEvent = (type, detail, init) => { const event = new MouseEvent(type, { bubbles: true, cancelable: true, composed: true, view: window, - detail: 1, + detail, + clientX, + clientY, + button: buttonCode, + buttons: type === 'mousedown' ? buttonsMask : 0, ...init, }); return element.dispatchEvent(event); }; try { - const dispatchResult = { - shouldActivate: - dispatchMouseEvent('mousedown', { button: 0, buttons: 1 }) && - dispatchMouseEvent('mouseup', { button: 0, buttons: 0 }), - }; - if (!dispatchResult.shouldActivate) { - return 'ok'; + for (let index = 1; index <= clickCount; index += 1) { + const dispatchResult = { + shouldActivate: + dispatchMouseEvent('mousedown', index, {}) && + dispatchMouseEvent('mouseup', index, {}), + }; + if (!dispatchResult.shouldActivate) { + return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount }); + } + if (!element.isConnected) { + return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount }); + } + if (button === 'left') { + element.click(); + } else if (button === 'right') { + element.dispatchEvent(new MouseEvent('contextmenu', { + bubbles: true, + cancelable: true, + composed: true, + view: window, + detail: index, + clientX, + clientY, + button: 2, + buttons: 0, + })); + } else { + element.dispatchEvent(new MouseEvent('auxclick', { + bubbles: true, + cancelable: true, + composed: true, + view: window, + detail: index, + clientX, + clientY, + button: 1, + buttons: 0, + })); + } } - if (!element.isConnected) { - return 'ok'; + if (button === 'left' && clickCount === 2) { + element.dispatchEvent(new MouseEvent('dblclick', { + bubbles: true, + cancelable: true, + composed: true, + view: window, + detail: 2, + clientX, + clientY, + button: 0, + buttons: 0, + })); } } catch (mouseError) { - // Fall through to the native activation path below. } - element.click(); - - return 'ok'; + return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount }); })()`; - await evaluateExpression(page, expression); - return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`; + const raw = await evaluateExpression(page, expression); + const parsed = JSON.parse(raw); + return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}\noffsetX: ${parsed.resolvedOffsetX}\noffsetY: ${parsed.resolvedOffsetY}\nbutton: ${parsed.button}\nclickCount: ${parsed.clickCount}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`; } async function handleType(toolArgs) { @@ -1240,6 +1441,183 @@ async function handleType(toolArgs) { return `pageIndex: ${pageIndex}\nselector: ${selector}\ntypedLength: ${typedLength}\nstatus: typed`; } +async function handlePressKey(toolArgs) { + const { page, pageIndex } = await getSelectedPage(toolArgs); + const key = requireNonEmptyString(toolArgs.key, 'key'); + const modifiers = requireOptionalStringArray(toolArgs.modifiers, 'modifiers', [ + 'Alt', + 'Control', + 'Meta', + 'Shift', + ]); + const repeat = toolArgs.repeat === undefined ? 1 : requirePositiveInteger(toolArgs.repeat, 'repeat'); + const modifierMask = + (modifiers.includes('Alt') ? 1 : 0) | + (modifiers.includes('Control') ? 2 : 0) | + (modifiers.includes('Meta') ? 4 : 0) | + (modifiers.includes('Shift') ? 8 : 0); + const specialKeyMap = { + Enter: { code: 'Enter', keyCode: 13, text: '\r' }, + Tab: { code: 'Tab', keyCode: 9, text: '' }, + Escape: { code: 'Escape', keyCode: 27, text: '' }, + ArrowUp: { code: 'ArrowUp', keyCode: 38, text: '' }, + ArrowDown: { code: 'ArrowDown', keyCode: 40, text: '' }, + ArrowLeft: { code: 'ArrowLeft', keyCode: 37, text: '' }, + ArrowRight: { code: 'ArrowRight', keyCode: 39, text: '' }, + Backspace: { code: 'Backspace', keyCode: 8, text: '' }, + Delete: { code: 'Delete', keyCode: 46, text: '' }, + Space: { code: 'Space', keyCode: 32, text: ' ' }, + }; + const keyDescriptor = + key.length === 1 + ? { + code: `Key${key.toUpperCase()}`, + keyCode: key.toUpperCase().charCodeAt(0), + text: key, + } + : specialKeyMap[key]; + if (!keyDescriptor) { + throw new Error(`unsupported key: ${key}`); + } + const normalizedKey = key; + const normalizedText = keyDescriptor.text; + const code = keyDescriptor.code; + const keyCode = keyDescriptor.keyCode; + + for (let index = 0; index < repeat; index += 1) { + await sendCdpCommand(page, 'Input.dispatchKeyEvent', { + type: 'keyDown', + key: normalizedKey, + code, + text: normalizedText, + unmodifiedText: normalizedText, + windowsVirtualKeyCode: keyCode, + nativeVirtualKeyCode: keyCode, + modifiers: modifierMask, + autoRepeat: index > 0, + }); + await sendCdpCommand(page, 'Input.dispatchKeyEvent', { + type: 'keyUp', + key: normalizedKey, + code, + windowsVirtualKeyCode: keyCode, + nativeVirtualKeyCode: keyCode, + modifiers: modifierMask, + }); + } + + const modifierText = modifiers.length > 0 ? modifiers.join(',') : 'none'; + return `pageIndex: ${pageIndex}\nkey: ${key}\nmodifiers: ${modifierText}\nrepeat: ${repeat}\nstatus: key-pressed`; +} + +async function handleScroll(toolArgs) { + const { page, pageIndex } = await getSelectedPage(toolArgs); + const selector = parseOptionalNonEmptyString(toolArgs.selector); + const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector); + const pierceShadow = toolArgs.pierceShadow === true; + const behavior = requireEnumString(toolArgs.behavior, 'behavior', ['into-view', 'by-offset']); + const deltaX = toolArgs.deltaX === undefined ? 0 : requireFiniteNumber(toolArgs.deltaX, 'deltaX'); + const deltaY = toolArgs.deltaY === undefined ? 0 : requireFiniteNumber(toolArgs.deltaY, 'deltaY'); + + const expression = `(() => { + const selector = ${selector ? `JSON.parse(${JSON.stringify(JSON.stringify(selector))})` : 'undefined'}; + const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'}; + const pierceShadow = ${pierceShadow ? 'true' : 'false'}; + const behavior = JSON.parse(${JSON.stringify(JSON.stringify(behavior))}); + const deltaX = ${deltaX}; + const deltaY = ${deltaY}; + + const visitRoots = (root) => { + const roots = [root]; + if (!pierceShadow) { + return roots; + } + const queue = [root]; + while (queue.length > 0) { + const current = queue.shift(); + const elements = Array.from(current.querySelectorAll('*')); + for (const element of elements) { + if (element.shadowRoot) { + roots.push(element.shadowRoot); + queue.push(element.shadowRoot); + } + } + } + return roots; + }; + + let root = document; + let scrollWindow = window; + if (frameSelector) { + const frame = document.querySelector(frameSelector); + if (!frame) { + throw new Error('frame not found for selector: ' + frameSelector); + } + const frameDocument = frame.contentDocument; + if (!frameDocument) { + throw new Error('frame document is unavailable for selector: ' + frameSelector); + } + if (!frame.contentWindow) { + throw new Error('frame window is unavailable for selector: ' + frameSelector); + } + root = frameDocument; + scrollWindow = frame.contentWindow; + } + + if (!selector) { + if (behavior !== 'by-offset') { + throw new Error('selector is required for behavior: ' + behavior); + } + scrollWindow.scrollBy(deltaX, deltaY); + return JSON.stringify({ scope: 'page', behavior, deltaX, deltaY }); + } + + const roots = visitRoots(root); + const matches = []; + for (const currentRoot of roots) { + matches.push(...Array.from(currentRoot.querySelectorAll(selector))); + } + const element = matches[0]; + if (!element) { + throw new Error('element not found for selector: ' + selector); + } + + if (behavior === 'into-view') { + element.scrollIntoView({ block: 'center', inline: 'center' }); + return JSON.stringify({ scope: 'element', selector, behavior }); + } + + if (typeof element.scrollBy === 'function') { + element.scrollBy(deltaX, deltaY); + return JSON.stringify({ scope: 'element', selector, behavior, deltaX, deltaY }); + } + + throw new Error('element does not support scrollBy for selector: ' + selector); + })()`; + + const raw = await evaluateExpression(page, expression); + const parsed = JSON.parse(raw); + const lines = [`pageIndex: ${pageIndex}`]; + if (parsed.selector) { + lines.push(`selector: ${parsed.selector}`); + } + lines.push(`behavior: ${parsed.behavior}`); + if (typeof parsed.deltaX === 'number') { + lines.push(`deltaX: ${parsed.deltaX}`); + } + if (typeof parsed.deltaY === 'number') { + lines.push(`deltaY: ${parsed.deltaY}`); + } + if (parsed.scope === 'element') { + const scopedSuffix = formatScopedSelectorSuffix(frameSelector, pierceShadow); + if (scopedSuffix) { + lines.push(scopedSuffix.slice(1)); + } + } + lines.push('status: scrolled'); + return lines.join('\n'); +} + async function handleScreenshot(toolArgs) { const { page, pageIndex } = await getSelectedPage(toolArgs); const fullPage = toolArgs.fullPage === true; @@ -1865,6 +2243,22 @@ async function handleToolCall(message) { return; } + if (toolName === TOOL_PRESS_KEY) { + const text = await handlePressKey(toolArgs); + writeResponse(id, { + content: [{ type: 'text', text }], + }); + return; + } + + if (toolName === TOOL_SCROLL) { + const text = await handleScroll(toolArgs); + writeResponse(id, { + content: [{ type: 'text', text }], + }); + return; + } + if (toolName === TOOL_TAKE_SCREENSHOT) { const text = await handleScreenshot(toolArgs); writeResponse(id, { diff --git a/tests/unit/hooks/ccs-browser-mcp-server.test.ts b/tests/unit/hooks/ccs-browser-mcp-server.test.ts index 9afc8d3e..a2740854 100644 --- a/tests/unit/hooks/ccs-browser-mcp-server.test.ts +++ b/tests/unit/hooks/ccs-browser-mcp-server.test.ts @@ -48,6 +48,10 @@ type MockClickState = { detachAfterMouseDown?: boolean; mouseSequenceError?: string; label?: string; + expectedOffset?: { x: number; y: number }; + expectedButton?: 'left' | 'middle' | 'right'; + expectedClickCount?: number; + requireDoubleClickEvent?: boolean; }; type MockClickPlan = MockClickState | MockClickState[]; @@ -144,6 +148,20 @@ type MockPageState = { focused?: boolean; } >; + keyboard?: { + expectedKey?: string; + expectedModifiers?: string[]; + expectedRepeat?: number; + _seenKeyDownCount?: number; + }; + scroll?: Record< + string, + { + expectedBehavior?: 'into-view' | 'by-offset'; + expectedDeltaX?: number; + expectedDeltaY?: number; + } + >; }; const bundledServerPath = join(process.cwd(), 'lib', 'mcp', 'ccs-browser-server.cjs'); @@ -521,6 +539,45 @@ function createMockBrowser(pagesInput: MockPageState[]) { return; } + if (message.method === 'Input.dispatchKeyEvent') { + const type = typeof message.params?.type === 'string' ? message.params.type : ''; + const key = typeof message.params?.key === 'string' ? message.params.key : ''; + const modifiersMask = Number(message.params?.modifiers || 0); + const modifiers = [ + ...(modifiersMask & 1 ? ['Alt'] : []), + ...(modifiersMask & 2 ? ['Control'] : []), + ...(modifiersMask & 4 ? ['Meta'] : []), + ...(modifiersMask & 8 ? ['Shift'] : []), + ]; + const keyboardPlan = page.keyboard; + if (type === 'keyDown') { + if (keyboardPlan?.expectedKey && key !== keyboardPlan.expectedKey) { + replyError(`unexpected key: ${key}`); + return; + } + if ( + keyboardPlan?.expectedModifiers && + JSON.stringify(modifiers) !== JSON.stringify(keyboardPlan.expectedModifiers) + ) { + replyError(`unexpected modifiers for key: ${key}`); + return; + } + if (keyboardPlan) { + keyboardPlan._seenKeyDownCount = (keyboardPlan._seenKeyDownCount || 0) + 1; + } + } + if ( + type === 'keyUp' && + typeof keyboardPlan?.expectedRepeat === 'number' && + (keyboardPlan._seenKeyDownCount || 0) !== keyboardPlan.expectedRepeat + ) { + replyError(`unexpected repeat for key: ${key}`); + return; + } + reply({}); + return; + } + if (message.method === 'Page.enable' || message.method === 'Network.enable') { reply({}); if (message.method === 'Page.enable') { @@ -627,7 +684,7 @@ function createMockBrowser(pagesInput: MockPageState[]) { return; } - if (expression.includes('scrollIntoView') && expression.includes('.click()')) { + if (expression.includes('scrollIntoView') && expression.includes('resolvedOffsetX')) { const selector = parseJsonArgument(expression, 'selector') || ''; const nth = parseNumberArgument(expression, 'nth') ?? 0; const clickPlan = page.click?.[selector]; @@ -636,6 +693,7 @@ function createMockBrowser(pagesInput: MockPageState[]) { const attemptedMouseUp = expression.includes("dispatchMouseEvent('mouseup'"); const attemptedMouseSequence = attemptedMouseDown && attemptedMouseUp; const attemptedClickEvent = expression.includes("dispatchMouseEvent('click'"); + const attemptedDoubleClickEvent = expression.includes("new MouseEvent('dblclick'"); const readsDispatchResult = expression.includes('const dispatchResult = {'); const gatesNativeClickOnDispatchResult = expression.includes( 'if (!dispatchResult.shouldActivate)' @@ -662,6 +720,10 @@ function createMockBrowser(pagesInput: MockPageState[]) { index < catchIndex || index > catchBlockEnd ); + const offsetX = parseNumberArgument(expression, 'offsetX'); + const offsetY = parseNumberArgument(expression, 'offsetY'); + const button = parseJsonArgument(expression, 'button') || 'left'; + const clickCount = parseNumberArgument(expression, 'clickCount') ?? 1; if (!resolvedClickPlan) { replyError(`element index ${nth} is out of range for selector: ${selector}`); return; @@ -712,13 +774,49 @@ function createMockBrowser(pagesInput: MockPageState[]) { replyError(`native click required for selector: ${selector}`); return; } + if ( + resolvedClickPlan.expectedOffset && + (offsetX !== resolvedClickPlan.expectedOffset.x || offsetY !== resolvedClickPlan.expectedOffset.y) + ) { + replyError(`unexpected click offset for selector: ${selector}`); + return; + } + if (resolvedClickPlan.expectedButton && button !== resolvedClickPlan.expectedButton) { + replyError(`unexpected click button for selector: ${selector}`); + return; + } + if ( + typeof resolvedClickPlan.expectedClickCount === 'number' && + clickCount !== resolvedClickPlan.expectedClickCount + ) { + replyError(`unexpected click count for selector: ${selector}`); + return; + } + if ( + resolvedClickPlan.requireDoubleClickEvent && + clickCount === 2 && + !attemptedDoubleClickEvent + ) { + replyError(`dblclick event required for selector: ${selector}`); + return; + } if (resolvedClickPlan.mouseSequenceError) { if (!attemptedMouseSequence) { replyError(`mousedown/mouseup required for selector: ${selector}`); return; } if (attemptedFallbackClick || attemptedNativeClickOutsideCatch) { - reply({ result: { type: 'string', value: 'ok' } }); + reply({ + result: { + type: 'string', + value: JSON.stringify({ + resolvedOffsetX: offsetX ?? 50, + resolvedOffsetY: offsetY ?? 10, + button, + clickCount, + }), + }, + }); return; } replyError(resolvedClickPlan.mouseSequenceError); @@ -728,7 +826,17 @@ function createMockBrowser(pagesInput: MockPageState[]) { replyError(resolvedClickPlan.error); return; } - reply({ result: { type: 'string', value: 'ok' } }); + reply({ + result: { + type: 'string', + value: JSON.stringify({ + resolvedOffsetX: offsetX ?? 50, + resolvedOffsetY: offsetY ?? 10, + button, + clickCount, + }), + }, + }); return; } @@ -950,6 +1058,80 @@ function createMockBrowser(pagesInput: MockPageState[]) { }); return; } + + if (expression.includes('new KeyboardEvent(')) { + const key = parseJsonArgument(expression, 'key') || ''; + const repeat = parseNumberArgument(expression, 'repeat') ?? 1; + const modifiersMatch = expression.match(/const modifiers = (\[[^\n;]*\]);/); + const modifiers = modifiersMatch ? (JSON.parse(modifiersMatch[1]) as string[]) : []; + const keyboardPlan = page.keyboard; + if (keyboardPlan?.expectedKey && key !== keyboardPlan.expectedKey) { + replyError(`unexpected key: ${key}`); + return; + } + if ( + keyboardPlan?.expectedModifiers && + JSON.stringify(modifiers) !== JSON.stringify(keyboardPlan.expectedModifiers) + ) { + replyError(`unexpected modifiers for key: ${key}`); + return; + } + if ( + typeof keyboardPlan?.expectedRepeat === 'number' && + repeat !== keyboardPlan.expectedRepeat + ) { + replyError(`unexpected repeat for key: ${key}`); + return; + } + reply({ + result: { + type: 'string', + value: JSON.stringify({ key, modifiers, repeat }), + }, + }); + return; + } + + if (expression.includes('window.scrollBy(deltaX, deltaY)') || expression.includes('element.scrollBy(deltaX, deltaY)')) { + const selector = parseJsonArgument(expression, 'selector'); + const behavior = parseJsonArgument(expression, 'behavior') || ''; + const deltaX = Number(expression.match(/const deltaX = (-?[0-9]+(?:\.[0-9]+)?);/)?.[1] || 0); + const deltaY = Number(expression.match(/const deltaY = (-?[0-9]+(?:\.[0-9]+)?);/)?.[1] || 0); + const scrollPlan = selector ? page.scroll?.[selector] : undefined; + if (selector && !scrollPlan) { + replyError(`element not found for selector: ${selector}`); + return; + } + if (scrollPlan?.expectedBehavior && behavior !== scrollPlan.expectedBehavior) { + replyError(`unexpected scroll behavior for selector: ${selector}`); + return; + } + if ( + typeof scrollPlan?.expectedDeltaX === 'number' && + deltaX !== scrollPlan.expectedDeltaX + ) { + replyError(`unexpected deltaX for selector: ${selector}`); + return; + } + if ( + typeof scrollPlan?.expectedDeltaY === 'number' && + deltaY !== scrollPlan.expectedDeltaY + ) { + replyError(`unexpected deltaY for selector: ${selector}`); + return; + } + reply({ + result: { + type: 'string', + value: JSON.stringify( + selector + ? { scope: 'element', selector, behavior, deltaX, deltaY } + : { scope: 'page', behavior, deltaX, deltaY } + ), + }, + }); + return; + } }); }); @@ -1056,6 +1238,8 @@ describe('ccs-browser MCP server', () => { 'browser_navigate', 'browser_click', 'browser_type', + 'browser_press_key', + 'browser_scroll', 'browser_take_screenshot', 'browser_wait_for', 'browser_eval', @@ -1068,6 +1252,21 @@ describe('ccs-browser MCP server', () => { const clickTool = tools.find((tool) => tool.name === 'browser_click'); expect(clickTool?.description).toContain('mouse event chain'); expect(clickTool?.description).not.toContain('synthetic element.click()'); + expect(clickTool?.inputSchema?.properties?.offsetX).toMatchObject({ type: 'number' }); + expect(clickTool?.inputSchema?.properties?.offsetY).toMatchObject({ type: 'number' }); + expect(clickTool?.inputSchema?.properties?.button).toMatchObject({ type: 'string' }); + expect(clickTool?.inputSchema?.properties?.clickCount).toMatchObject({ + type: 'integer', + minimum: 1, + }); + + const keyTool = tools.find((tool) => tool.name === 'browser_press_key'); + expect(keyTool?.inputSchema?.properties?.key).toMatchObject({ type: 'string' }); + expect(keyTool?.inputSchema?.properties?.modifiers).toMatchObject({ type: 'array' }); + + const scrollTool = tools.find((tool) => tool.name === 'browser_scroll'); + expect(scrollTool?.inputSchema?.properties?.deltaX).toMatchObject({ type: 'number' }); + expect(scrollTool?.inputSchema?.properties?.deltaY).toMatchObject({ type: 'number' }); const queryTool = tools.find((tool) => tool.name === 'browser_query'); expect(queryTool?.inputSchema?.properties?.fields).toMatchObject({ @@ -1638,6 +1837,250 @@ describe('ccs-browser MCP server', () => { ); }); + it('clicks with element-relative offsets and richer click options', async () => { + const responses = await runMcpRequests( + [ + { + id: 'page-1', + title: 'Click Page', + currentUrl: 'https://example.com/', + click: { + '#offset-target': { + requireMouseSequence: true, + expectedOffset: { x: 12, y: 8 }, + expectedButton: 'right', + expectedClickCount: 2, + }, + '#double-left': { + requireMouseSequence: true, + expectedButton: 'left', + expectedClickCount: 2, + requireDoubleClickEvent: true, + }, + }, + }, + ], + [ + { + jsonrpc: '2.0', + id: 70, + method: 'tools/call', + params: { + name: 'browser_click', + arguments: { + selector: '#offset-target', + offsetX: 12, + offsetY: 8, + button: 'right', + clickCount: 2, + }, + }, + }, + { + jsonrpc: '2.0', + id: 700, + method: 'tools/call', + params: { + name: 'browser_click', + arguments: { + selector: '#double-left', + clickCount: 2, + }, + }, + }, + ] + ); + + const text = getResponseText(responses.find((message) => message.id === 70)); + expect(text).toContain('status: clicked'); + expect(text).toContain('offsetX: 12'); + expect(text).toContain('offsetY: 8'); + expect(text).toContain('button: right'); + expect(text).toContain('clickCount: 2'); + + const doubleText = getResponseText(responses.find((message) => message.id === 700)); + expect(doubleText).toContain('status: clicked'); + expect(doubleText).toContain('button: left'); + expect(doubleText).toContain('clickCount: 2'); + }); + + it('presses a key combination with browser_press_key', async () => { + const responses = await runMcpRequests( + [ + { + id: 'page-1', + title: 'Keyboard Page', + currentUrl: 'https://example.com/', + keyboard: { + expectedKey: 'k', + expectedModifiers: ['Meta'], + expectedRepeat: 2, + }, + }, + ], + [ + { + jsonrpc: '2.0', + id: 71, + method: 'tools/call', + params: { + name: 'browser_press_key', + arguments: { + key: 'k', + modifiers: ['Meta'], + repeat: 2, + }, + }, + }, + ] + ); + + const text = getResponseText(responses.find((message) => message.id === 71)); + expect(text).toContain('status: key-pressed'); + expect(text).toContain('key: k'); + expect(text).toContain('modifiers: Meta'); + expect(text).toContain('repeat: 2'); + }); + + it('supports common special keys with browser_press_key', async () => { + const responses = await runMcpRequests( + [ + { + id: 'page-1', + title: 'Special Key Page', + currentUrl: 'https://example.com/', + keyboard: { + expectedKey: 'Enter', + expectedRepeat: 1, + }, + }, + ], + [ + { + jsonrpc: '2.0', + id: 711, + method: 'tools/call', + params: { + name: 'browser_press_key', + arguments: { + key: 'Enter', + }, + }, + }, + ] + ); + + const text = getResponseText(responses.find((message) => message.id === 711)); + expect(text).toContain('status: key-pressed'); + expect(text).toContain('key: Enter'); + }); + + it('rejects unsupported modifiers for browser_press_key', async () => { + const responses = await runMcpRequests( + [{ id: 'page-1', title: 'Invalid Modifier Page', currentUrl: 'https://example.com/' }], + [ + { + jsonrpc: '2.0', + id: 712, + method: 'tools/call', + params: { + name: 'browser_press_key', + arguments: { + key: 'k', + modifiers: ['Cmd'], + }, + }, + }, + ] + ); + + const response = responses.find((message) => message.id === 712); + expect((response?.result as { isError?: boolean }).isError).toBe(true); + expect(getResponseText(response)).toContain( + 'modifiers must only contain: Alt, Control, Meta, Shift' + ); + }); + + it('scrolls an element into view with browser_scroll', async () => { + const responses = await runMcpRequests( + [ + { + id: 'page-1', + title: 'Scroll Page', + currentUrl: 'https://example.com/', + scroll: { + '#results': { + expectedBehavior: 'into-view', + }, + }, + }, + ], + [ + { + jsonrpc: '2.0', + id: 72, + method: 'tools/call', + params: { + name: 'browser_scroll', + arguments: { + selector: '#results', + behavior: 'into-view', + }, + }, + }, + ] + ); + + const text = getResponseText(responses.find((message) => message.id === 72)); + expect(text).toContain('status: scrolled'); + expect(text).toContain('selector: #results'); + expect(text).toContain('behavior: into-view'); + }); + + it('scrolls an iframe page by offset with browser_scroll', async () => { + const responses = await runMcpRequests( + [ + { + id: 'page-1', + title: 'Iframe Scroll Page', + currentUrl: 'https://example.com/', + frames: [ + { + selector: '#preview-frame', + query: { + '#preview-frame': { + exists: true, + }, + }, + }, + ], + }, + ], + [ + { + jsonrpc: '2.0', + id: 73, + method: 'tools/call', + params: { + name: 'browser_scroll', + arguments: { + frameSelector: '#preview-frame', + behavior: 'by-offset', + deltaX: 5, + deltaY: 40, + }, + }, + }, + ] + ); + + const text = getResponseText(responses.find((message) => message.id === 73)); + expect(text).toContain('status: scrolled'); + expect(text).toContain('behavior: by-offset'); + expect(text).toContain('deltaX: 5'); + expect(text).toContain('deltaY: 40'); + }); + it('requires real mouse movement for hover-only targets and reports hover failures', async () => { const responses = await runMcpRequests( [