From 844b5f6c428e2d2db3e301c805790689c4020661 Mon Sep 17 00:00:00 2001 From: walker1211 <13750528578@163.com> Date: Sun, 19 Apr 2026 18:41:18 +0800 Subject: [PATCH] =?UTF-8?q?feat(browser):=20=E5=AE=8C=E6=88=90=20Phase=209?= =?UTF-8?q?=20=E6=8B=96=E6=8B=BD=E4=B8=8E=E6=8C=87=E9=92=88=E5=8A=A8?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补齐文件拖拽、元素拖拽与受限 pointer 原语，让 Browser MCP 能覆盖常见上传与页面内拖放交互。 并补足 pageId 路由与回归测试，保持与现有 selected-page 语义一致。 Co-Authored-By: Claude Opus 4.7 --- lib/mcp/ccs-browser-server.cjs | 502 +++++++++++- .../unit/hooks/ccs-browser-mcp-server.test.ts | 733 +++++++++++++++++- 2 files changed, 1220 insertions(+), 15 deletions(-) diff --git a/lib/mcp/ccs-browser-server.cjs b/lib/mcp/ccs-browser-server.cjs index fee8a06e..7e80b71b 100755 --- a/lib/mcp/ccs-browser-server.cjs +++ b/lib/mcp/ccs-browser-server.cjs @@ -59,6 +59,9 @@ const TOOL_SET_DOWNLOAD_BEHAVIOR = 'browser_set_download_behavior'; const TOOL_LIST_DOWNLOADS = 'browser_list_downloads'; const TOOL_CANCEL_DOWNLOAD = 'browser_cancel_download'; const TOOL_SET_FILE_INPUT = 'browser_set_file_input'; +const TOOL_DRAG_FILES = 'browser_drag_files'; +const TOOL_DRAG_ELEMENT = 'browser_drag_element'; +const TOOL_POINTER_ACTION = 'browser_pointer_action'; const TOOL_TAKE_SCREENSHOT = 'browser_take_screenshot'; const TOOL_WAIT_FOR = 'browser_wait_for'; const TOOL_EVAL = 'browser_eval'; @@ -87,6 +90,9 @@ const TOOL_NAMES = [ TOOL_LIST_DOWNLOADS, TOOL_CANCEL_DOWNLOAD, TOOL_SET_FILE_INPUT, + TOOL_DRAG_FILES, + TOOL_DRAG_ELEMENT, + TOOL_POINTER_ACTION, TOOL_TAKE_SCREENSHOT, TOOL_WAIT_FOR, TOOL_EVAL, @@ -113,6 +119,8 @@ const CDP_TIMEOUT_MS = 5000; const NAVIGATION_POLL_INTERVAL_MS = 100; const DEFAULT_WAIT_TIMEOUT_MS = 2000; const DEFAULT_WAIT_POLL_INTERVAL_MS = 100; +const DEFAULT_DRAG_STEPS = 5; +const MAX_POINTER_ACTIONS = 25; let inputBuffer = Buffer.alloc(0); let requestCounter = 0; @@ -624,6 +632,83 @@ function getTools() { additionalProperties: false, }, }, + { + name: TOOL_DRAG_FILES, + description: + 'Drag one or more local files onto a matched page drop target using page-side File and DataTransfer injection.', + inputSchema: { + type: 'object', + properties: { + selector: { type: 'string' }, + files: { + type: 'array', + items: { type: 'string' }, + }, + pageIndex: { type: 'integer', minimum: 0 }, + pageId: { type: 'string' }, + nth: { type: 'integer', minimum: 0 }, + frameSelector: { type: 'string' }, + pierceShadow: { type: 'boolean' }, + }, + required: ['selector', 'files'], + additionalProperties: false, + }, + }, + { + name: TOOL_DRAG_ELEMENT, + description: + 'Drag a matched element to another matched element or explicit coordinates using browser mouse events.', + inputSchema: { + type: 'object', + properties: { + selector: { type: 'string' }, + targetSelector: { type: 'string' }, + targetX: { type: 'number' }, + targetY: { type: 'number' }, + pageIndex: { type: 'integer', minimum: 0 }, + pageId: { type: 'string' }, + nth: { type: 'integer', minimum: 0 }, + targetNth: { type: 'integer', minimum: 0 }, + frameSelector: { type: 'string' }, + pierceShadow: { type: 'boolean' }, + steps: { type: 'integer', minimum: 1 }, + }, + required: ['selector'], + additionalProperties: false, + }, + }, + { + name: TOOL_POINTER_ACTION, + description: 'Run a limited pointer action sequence using browser mouse events.', + inputSchema: { + type: 'object', + properties: { + pageIndex: { type: 'integer', minimum: 0 }, + pageId: { type: 'string' }, + actions: { + type: 'array', + items: { + type: 'object', + properties: { + type: { type: 'string', enum: ['move', 'down', 'up', 'pause'] }, + selector: { type: 'string' }, + nth: { type: 'integer', minimum: 0 }, + frameSelector: { type: 'string' }, + pierceShadow: { type: 'boolean' }, + x: { type: 'number' }, + y: { type: 'number' }, + button: { type: 'string', enum: ['left', 'middle', 'right'] }, + durationMs: { type: 'integer', minimum: 0 }, + }, + required: ['type'], + additionalProperties: false, + }, + }, + }, + required: ['actions'], + additionalProperties: false, + }, + }, { name: TOOL_TAKE_SCREENSHOT, description: @@ -2029,6 +2114,122 @@ function formatFileInputResult({ pageIndex, selector, nth, frameSelector, pierce return lines.join('\n'); } +function inferMimeType(filePath) { + const ext = path.extname(filePath).toLowerCase(); + if (ext === '.pdf') return 'application/pdf'; + if (ext === '.png') return 'image/png'; + if (ext === '.jpg' || ext === '.jpeg') return 'image/jpeg'; + if (ext === '.txt') return 'text/plain'; + if (ext === '.json') return 'application/json'; + return 'application/octet-stream'; +} + +function readLocalFilesForDrop(files) { + return validateLocalFiles(files).map((resolvedPath) => { + const buffer = fs.readFileSync(resolvedPath); + return { + path: resolvedPath, + name: path.basename(resolvedPath), + mimeType: inferMimeType(resolvedPath), + size: buffer.length, + base64: buffer.toString('base64'), + }; + }); +} + +function buildDropFilesExpression(selector, nth, frameSelector, pierceShadow, files) { + return `(() => { + const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))}); + const nth = ${nth === undefined ? 'undefined' : String(nth)}; + const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'}; + const pierceShadow = ${pierceShadow ? 'true' : 'false'}; + const filePayloads = JSON.parse(${JSON.stringify(JSON.stringify(files))}); + + const visitRoots = (root) => { + const roots = [root]; + if (!pierceShadow) { + return roots; + } + const queue = [root]; + while (queue.length > 0) { + const current = queue.shift(); + const elements = Array.from(current.querySelectorAll('*')); + for (const element of elements) { + if (element.shadowRoot) { + roots.push(element.shadowRoot); + queue.push(element.shadowRoot); + } + } + } + return roots; + }; + + let root = document; + if (frameSelector) { + const frame = document.querySelector(frameSelector); + if (!frame) { + throw new Error('frame not found for selector: ' + frameSelector); + } + const frameDocument = frame.contentDocument; + if (!frameDocument) { + throw new Error('frame document is unavailable for selector: ' + frameSelector); + } + root = frameDocument; + } + + const roots = visitRoots(root); + const matches = []; + for (const currentRoot of roots) { + matches.push(...Array.from(currentRoot.querySelectorAll(selector))); + } + + const targetIndex = nth ?? 0; + const element = matches[targetIndex]; + if (!element) { + throw new Error('element not found for selector: ' + selector); + } + + const files = filePayloads.map((file) => { + const binary = atob(file.base64); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index); + } + return new File([bytes], file.name, { type: file.mimeType }); + }); + + const dataTransfer = new DataTransfer(); + for (const file of files) { + dataTransfer.items.add(file); + } + + element.dispatchEvent(new DragEvent('dragenter', { bubbles: true, cancelable: true, dataTransfer })); + element.dispatchEvent(new DragEvent('dragover', { bubbles: true, cancelable: true, dataTransfer })); + const accepted = element.dispatchEvent( + new DragEvent('drop', { bubbles: true, cancelable: true, dataTransfer }) + ); + + return { accepted }; + })()`; +} + +function formatDragFilesResult({ pageIndex, selector, nth, frameSelector, pierceShadow, files }) { + const lines = [ + `pageIndex: ${pageIndex}`, + `selector: ${selector}`, + `nth: ${nth}`, + `fileCount: ${files.length}`, + ]; + if (frameSelector) { + lines.push(`frameSelector: ${frameSelector}`); + } + if (pierceShadow) { + lines.push('pierceShadow: true'); + } + lines.push('status: files-dropped'); + return lines.join('\n'); +} + function parseEventCondition(value) { if (!value || typeof value !== 'object') { throw new Error('event is required'); @@ -2613,9 +2814,10 @@ async function getElementDiagnostics(page, selector, nth, frameSelector = '', pi return await getScopedDiagnostics(page, selector, nth, frameSelector, pierceShadow); } -async function getScrolledElementState(page, selector, frameSelector = '', pierceShadow = false) { +async function getScrolledElementStateAt(page, selector, nth, frameSelector = '', pierceShadow = false) { const expression = `(() => { const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))}); + const nth = ${nth === undefined ? 'undefined' : String(nth)}; const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'}; const pierceShadow = ${pierceShadow ? 'true' : 'false'}; @@ -2659,7 +2861,8 @@ async function getScrolledElementState(page, selector, frameSelector = '', pierc for (const currentRoot of roots) { matches.push(...Array.from(currentRoot.querySelectorAll(selector))); } - const element = matches[0]; + const targetIndex = nth ?? 0; + const element = matches[targetIndex]; if (!element) { return JSON.stringify({ exists: false }); } @@ -2713,6 +2916,10 @@ async function getScrolledElementState(page, selector, frameSelector = '', pierc return JSON.parse(await evaluateExpression(page, expression)); } +async function getScrolledElementState(page, selector, frameSelector = '', pierceShadow = false) { + return await getScrolledElementStateAt(page, selector, undefined, frameSelector, pierceShadow); +} + function formatQueryValue(field, value) { if (field === 'boundingClientRect') { return JSON.stringify(value); @@ -2920,6 +3127,220 @@ async function handleEval(toolArgs) { return `pageIndex: ${pageIndex}\nmode: ${mode}\nvalue: ${JSON.stringify(value)}`; } +function getButtonMask(button) { + if (button === 'right') { + return 2; + } + if (button === 'middle') { + return 4; + } + return 1; +} + +async function dispatchMousePointerEvent(page, type, x, y, button = 'left', isPressed = false) { + await sendCdpCommand(page, 'Input.dispatchMouseEvent', { + type, + x, + y, + button: type === 'mouseMoved' && !isPressed ? 'none' : button, + buttons: isPressed ? getButtonMask(button) : 0, + pointerType: 'mouse', + clickCount: 1, + }); +} + +function interpolatePoints(sourcePoint, targetPoint, steps) { + const points = []; + for (let index = 1; index <= steps; index += 1) { + const ratio = index / steps; + points.push({ + x: Math.round(sourcePoint.x + (targetPoint.x - sourcePoint.x) * ratio), + y: Math.round(sourcePoint.y + (targetPoint.y - sourcePoint.y) * ratio), + }); + } + return points; +} + +async function resolveElementCenterPoint(page, selector, nth, frameSelector, pierceShadow, missingLabel) { + const state = await getScrolledElementStateAt(page, selector, nth, frameSelector, pierceShadow); + if (!state.exists) { + throw new Error(missingLabel); + } + if (state.connected !== true) { + throw new Error(`drag coordinates unavailable for selector: ${selector}`); + } + if (state.interactable !== true || !state.centerPoint) { + throw new Error(`drag coordinates unavailable for selector: ${selector}`); + } + return state.centerPoint; +} + +async function handlePointerAction(toolArgs) { + const pages = await listPageTargets(); + if (pages.length === 0) { + throw new Error('Browser MCP did not find any page targets in the current Chrome session.'); + } + + const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId); + if (!page.webSocketDebuggerUrl) { + throw new Error(`Browser MCP page ${pageIndex} does not expose a websocket debugger URL.`); + } + + const actions = Array.isArray(toolArgs.actions) ? toolArgs.actions : null; + if (!actions || actions.length === 0) { + throw new Error('actions must be a non-empty array'); + } + if (actions.length > MAX_POINTER_ACTIONS) { + throw new Error(`actions exceeds maximum of ${MAX_POINTER_ACTIONS}`); + } + + let pointerX = null; + let pointerY = null; + let pressedButton = null; + + for (const action of actions) { + if (!action || typeof action !== 'object') { + throw new Error('each action must be an object'); + } + if (!['move', 'down', 'up', 'pause'].includes(action.type)) { + throw new Error(`unsupported pointer action: ${String(action.type || '')}`); + } + + if (action.type === 'pause') { + const durationMs = action.durationMs === undefined ? 0 : requireNonNegativeInteger(action.durationMs, 'durationMs'); + if (durationMs > 0) { + await sleep(durationMs); + } + continue; + } + + if (action.type === 'move') { + let point = null; + if (typeof action.selector === 'string' && action.selector.trim() !== '') { + point = await resolveElementCenterPoint( + page, + requireNonEmptyString(action.selector, 'actions.selector'), + requireOptionalNonNegativeInteger(action.nth, 'actions.nth'), + parseOptionalNonEmptyString(action.frameSelector), + action.pierceShadow === true, + 'drag coordinates unavailable' + ); + } else if (typeof action.x === 'number' && typeof action.y === 'number') { + point = { + x: requireFiniteNumber(action.x, 'actions.x'), + y: requireFiniteNumber(action.y, 'actions.y'), + }; + } else { + throw new Error('drag coordinates unavailable'); + } + + pointerX = point.x; + pointerY = point.y; + await dispatchMousePointerEvent(page, 'mouseMoved', pointerX, pointerY, pressedButton || 'left', Boolean(pressedButton)); + continue; + } + + if (pointerX === null || pointerY === null) { + throw new Error('pointer state error'); + } + + if (action.type === 'down') { + if (pressedButton) { + throw new Error('pointer state error'); + } + pressedButton = requireEnumString(action.button, 'button', ['left', 'middle', 'right']) || 'left'; + await dispatchMousePointerEvent(page, 'mousePressed', pointerX, pointerY, pressedButton, true); + continue; + } + + if (!pressedButton) { + throw new Error('pointer state error'); + } + const releaseButton = requireEnumString(action.button, 'button', ['left', 'middle', 'right']) || pressedButton; + await dispatchMousePointerEvent(page, 'mouseReleased', pointerX, pointerY, releaseButton, false); + pressedButton = null; + } + + if (pressedButton) { + throw new Error('pointer state error'); + } + + return `pageIndex: ${pageIndex}\nactionCount: ${actions.length}\nstatus: pointer-actions-completed`; +} + +async function handleDragElement(toolArgs) { + const pages = await listPageTargets(); + if (pages.length === 0) { + throw new Error('Browser MCP did not find any page targets in the current Chrome session.'); + } + + const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId); + if (!page.webSocketDebuggerUrl) { + throw new Error(`Browser MCP page ${pageIndex} does not expose a websocket debugger URL.`); + } + + const selector = requireNonEmptyString(toolArgs.selector, 'selector'); + const nth = requireOptionalNonNegativeInteger(toolArgs.nth, 'nth'); + const targetNth = requireOptionalNonNegativeInteger(toolArgs.targetNth, 'targetNth'); + const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector); + const pierceShadow = toolArgs.pierceShadow === true; + const targetSelector = parseOptionalNonEmptyString(toolArgs.targetSelector); + const hasTargetX = Object.prototype.hasOwnProperty.call(toolArgs, 'targetX'); + const hasTargetY = Object.prototype.hasOwnProperty.call(toolArgs, 'targetY'); + const hasTargetCoordinates = hasTargetX || hasTargetY; + if (targetSelector && hasTargetCoordinates) { + throw new Error('targetSelector and targetX/targetY cannot be used together'); + } + if (!targetSelector && !hasTargetCoordinates) { + throw new Error('targetSelector or targetX/targetY is required'); + } + if (hasTargetX !== hasTargetY) { + throw new Error('targetX and targetY must be provided together'); + } + + const sourcePoint = await resolveElementCenterPoint( + page, + selector, + nth, + frameSelector, + pierceShadow, + 'source element not found' + ); + + const targetPoint = targetSelector + ? await resolveElementCenterPoint( + page, + targetSelector, + targetNth, + frameSelector, + pierceShadow, + 'target element not found' + ) + : { + x: requireFiniteNumber(toolArgs.targetX, 'targetX'), + y: requireFiniteNumber(toolArgs.targetY, 'targetY'), + }; + + const steps = toolArgs.steps === undefined ? DEFAULT_DRAG_STEPS : requirePositiveInteger(toolArgs.steps, 'steps'); + + await dispatchMousePointerEvent(page, 'mouseMoved', sourcePoint.x, sourcePoint.y, 'left', false); + await dispatchMousePointerEvent(page, 'mousePressed', sourcePoint.x, sourcePoint.y, 'left', true); + for (const point of interpolatePoints(sourcePoint, targetPoint, steps)) { + await dispatchMousePointerEvent(page, 'mouseMoved', point.x, point.y, 'left', true); + } + await dispatchMousePointerEvent(page, 'mouseReleased', targetPoint.x, targetPoint.y, 'left', false); + + return [ + `pageIndex: ${pageIndex}`, + `selector: ${selector}`, + `nth: ${nth ?? 0}`, + targetSelector ? `targetSelector: ${targetSelector}` : `targetX: ${targetPoint.x}`, + targetSelector ? `targetNth: ${targetNth ?? 0}` : `targetY: ${targetPoint.y}`, + `steps: ${steps}`, + `${frameSelector ? `frameSelector: ${frameSelector}\n` : ''}${pierceShadow ? 'pierceShadow: true\n' : ''}status: dragged`, + ].join('\n'); +} + async function handleHover(toolArgs) { const { page, pageIndex } = await getSelectedPage(toolArgs); const selector = requireNonEmptyString(toolArgs.selector, 'selector'); @@ -2936,14 +3357,7 @@ async function handleHover(toolArgs) { throw new Error(`element is hidden or not interactable for selector: ${selector}`); } - await sendCdpCommand(page, 'Input.dispatchMouseEvent', { - type: 'mouseMoved', - x: state.centerPoint.x, - y: state.centerPoint.y, - button: 'none', - buttons: 0, - pointerType: 'mouse', - }); + await dispatchMousePointerEvent(page, 'mouseMoved', state.centerPoint.x, state.centerPoint.y, 'left', false); return `pageIndex: ${pageIndex}\nselector: ${selector}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: hovered`; } @@ -3684,6 +4098,50 @@ async function handleSetFileInput(toolArgs) { }); } +async function handleDragFiles(toolArgs) { + const pages = await listPageTargets(); + if (pages.length === 0) { + throw new Error('Browser MCP did not find any page targets in the current Chrome session.'); + } + + const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId); + const selector = requireNonEmptyString(toolArgs.selector, 'selector'); + const files = readLocalFilesForDrop(requireNonEmptyStringArray(toolArgs.files, 'files')); + const nth = toolArgs.nth === undefined ? 0 : requireNonNegativeInteger(toolArgs.nth, 'nth'); + const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector); + const pierceShadow = toolArgs.pierceShadow === true; + + const response = await sendCdpCommand(page, 'Runtime.evaluate', { + expression: buildDropFilesExpression(selector, nth, frameSelector, pierceShadow, files), + returnByValue: true, + awaitPromise: true, + }); + + if (response?.exceptionDetails) { + throw new Error(response.exceptionDetails.text || 'drop target evaluation failed'); + } + + const result = response?.result || null; + if (!result) { + throw new Error('Browser MCP received an invalid DevTools evaluation response.'); + } + if (result.subtype === 'error') { + throw new Error(result.description || 'drop target evaluation failed'); + } + if (result.value?.accepted === false) { + throw new Error('drop target rejected files'); + } + + return formatDragFilesResult({ + pageIndex, + selector, + nth, + frameSelector, + pierceShadow, + files, + }); +} + async function handleToolCall(message) { const id = message.id; const params = message.params || {}; @@ -3848,6 +4306,30 @@ async function handleToolCall(message) { return; } + if (toolName === TOOL_DRAG_FILES) { + const text = await handleDragFiles(toolArgs); + writeResponse(id, { + content: [{ type: 'text', text }], + }); + return; + } + + if (toolName === TOOL_DRAG_ELEMENT) { + const text = await handleDragElement(toolArgs); + writeResponse(id, { + content: [{ type: 'text', text }], + }); + return; + } + + if (toolName === TOOL_POINTER_ACTION) { + const text = await handlePointerAction(toolArgs); + writeResponse(id, { + content: [{ type: 'text', text }], + }); + return; + } + if (toolName === TOOL_TAKE_SCREENSHOT) { const text = await handleScreenshot(toolArgs); writeResponse(id, { diff --git a/tests/unit/hooks/ccs-browser-mcp-server.test.ts b/tests/unit/hooks/ccs-browser-mcp-server.test.ts index 71a4a490..fbae2d58 100644 --- a/tests/unit/hooks/ccs-browser-mcp-server.test.ts +++ b/tests/unit/hooks/ccs-browser-mcp-server.test.ts @@ -111,17 +111,40 @@ type MockFileInputState = { type MockFileInputPlan = MockFileInputState | MockFileInputState[]; +type MockDropzoneState = { + accepted?: boolean; + requireFiles?: boolean; + receivedEventTypes?: string[]; + receivedFiles?: Array<{ name: string; size: number; type: string }>; + error?: string; +}; + +type MockDropzonePlan = MockDropzoneState | MockDropzoneState[]; + +type MockPointerActionRecord = { + type: string; + x?: number; + y?: number; + button?: string; +}; + +type MockDragPlan = { + recordedActions?: MockPointerActionRecord[]; +}; + type MockFrameState = { selector: string; query?: Record; visibleText?: string; fileInputs?: Record; + dropzones?: Record; }; type MockShadowRootState = { hostSelector: string; query?: Record; fileInputs?: Record; + dropzones?: Record; }; type MockEvalPlan = Record< @@ -175,6 +198,8 @@ type MockPageState = { eval?: MockEvalPlan; frames?: MockFrameState[]; shadowRoots?: MockShadowRootState[]; + dropzones?: Record; + drag?: MockDragPlan; events?: MockPageEventPlan; intercept?: MockInterceptState; screenshot?: { @@ -378,6 +403,36 @@ function parseNumberArgument(expression: string, key: string): number | undefine return Number.parseInt(match[1], 10); } +function parseParsedJsonArrayArgument(expression: string, key: string): T[] { + const marker = `const ${key} = JSON.parse(`; + const start = expression.indexOf(marker); + if (start === -1) { + return []; + } + + const quoteStart = start + marker.length; + const quote = expression[quoteStart]; + if (quote !== '"' && quote !== "'") { + return []; + } + + let index = quoteStart + 1; + while (index < expression.length) { + if (expression[index] === '\\') { + index += 2; + continue; + } + if (expression[index] === quote) { + const encoded = expression.slice(quoteStart, index + 1); + const decoded = JSON.parse(encoded) as string; + return JSON.parse(decoded) as T[]; + } + index += 1; + } + + return []; +} + function pickMockMatch( plan: T | T[] | undefined, nth = 0 @@ -410,6 +465,41 @@ function getMockShadowRoot(page: MockPageState): MockShadowRootState | undefined return page.shadowRoots?.[0]; } +function getMockDropzoneState( + page: Pick, + selector: string, + options: { nth?: number; frameSelector?: string; pierceShadow?: boolean } = {} +): MockDropzoneState | null { + const nth = options.nth ?? 0; + if (options.frameSelector) { + const frame = page.frames?.find((entry) => entry.selector === options.frameSelector); + const plan = frame?.dropzones?.[selector]; + if (!plan) { + return null; + } + return Array.isArray(plan) ? (plan[nth] ?? null) : plan; + } + if (options.pierceShadow) { + for (const shadowRoot of page.shadowRoots || []) { + const plan = shadowRoot.dropzones?.[selector]; + if (plan) { + return Array.isArray(plan) ? (plan[nth] ?? null) : plan; + } + } + } + const plan = page.dropzones?.[selector]; + if (!plan) { + return null; + } + return Array.isArray(plan) ? (plan[nth] ?? null) : plan; +} + +function pushPointerAction(page: MockPageState, action: MockPointerActionRecord): void { + page.drag = page.drag || {}; + page.drag.recordedActions = page.drag.recordedActions || []; + page.drag.recordedActions.push(action); +} + function shiftPageText(page: MockPageState): string { const queue = page.wait?.pageTextSequence; if (!queue || queue.length === 0) { @@ -443,11 +533,13 @@ function createMockBrowser(pagesInput: MockPageState[]) { let nextPageCounter = pagesInput.length + 1; for (const [index, page] of pagesInput.entries()) { - pageStates.set(`/devtools/page/${index + 1}`, { - visibleText: 'Hello from visible text', - domSnapshot: 'Hello from DOM snapshot', - ...page, - }); + if (page.visibleText === undefined) { + page.visibleText = 'Hello from visible text'; + } + if (page.domSnapshot === undefined) { + page.domSnapshot = 'Hello from DOM snapshot'; + } + pageStates.set(`/devtools/page/${index + 1}`, page); } async function start(options: RunMcpRequestsOptions = {}) { @@ -688,11 +780,13 @@ function createMockBrowser(pagesInput: MockPageState[]) { const type = typeof message.params?.type === 'string' ? message.params.type : ''; const x = Number(message.params?.x); const y = Number(message.params?.y); + const button = typeof message.params?.button === 'string' ? message.params.button : undefined; for (const hoverPlan of Object.values(page.hover || {})) { if (type === 'mouseMoved') { hoverPlan.lastMouseMove = { x, y }; } } + pushPointerAction(page, { type, x, y, button }); reply({}); return; } @@ -868,6 +962,51 @@ function createMockBrowser(pagesInput: MockPageState[]) { const expression = String(message.params?.expression || ''); + if (expression.includes('new DragEvent') && expression.includes('new DataTransfer()')) { + const selector = parseJsonArgument(expression, 'selector') || ''; + const nth = parseNumberArgument(expression, 'nth') ?? 0; + const frameSelector = parseJsonArgument(expression, 'frameSelector') || ''; + const pierceShadow = expression.includes('const pierceShadow = true'); + const filePayloads = parseParsedJsonArrayArgument<{ + name: string; + size: number; + mimeType: string; + }>(expression, 'filePayloads'); + const eventTypes = Array.from( + expression.matchAll(/new DragEvent\('(dragenter|dragover|drop)'/g) + ).map((match) => match[1]); + + const target = getMockDropzoneState(page, selector, { + nth, + frameSelector, + pierceShadow, + }); + if (!target) { + replyError(`element not found for selector: ${selector}`); + return; + } + if (target.error) { + replyError(target.error); + return; + } + if (eventTypes.length === 0) { + replyError(`drag event sequence not found for selector: ${selector}`); + return; + } + if (target.requireFiles && filePayloads.length === 0) { + reply({ result: { type: 'object', value: { accepted: false } } }); + return; + } + target.receivedEventTypes = eventTypes; + target.receivedFiles = filePayloads.map((file) => ({ + name: file.name, + size: file.size, + type: file.mimeType, + })); + reply({ result: { type: 'object', value: { accepted: target.accepted !== false } } }); + return; + } + if (page.eval?.[expression]) { const evalPlan = page.eval[expression]; if (evalPlan.error) { @@ -1535,6 +1674,9 @@ describe('ccs-browser MCP server', () => { 'browser_list_downloads', 'browser_cancel_download', 'browser_set_file_input', + 'browser_drag_files', + 'browser_drag_element', + 'browser_pointer_action', 'browser_take_screenshot', 'browser_wait_for', 'browser_eval', @@ -1624,6 +1766,25 @@ describe('ccs-browser MCP server', () => { expect(uploadTool?.inputSchema?.properties?.frameSelector).toMatchObject({ type: 'string' }); expect(uploadTool?.inputSchema?.properties?.pierceShadow).toMatchObject({ type: 'boolean' }); + const dragFilesTool = tools.find((tool) => tool.name === 'browser_drag_files'); + expect(dragFilesTool?.inputSchema?.properties?.selector).toMatchObject({ type: 'string' }); + expect(dragFilesTool?.inputSchema?.properties?.files).toMatchObject({ type: 'array' }); + expect(dragFilesTool?.inputSchema?.properties?.pageIndex).toMatchObject({ type: 'integer' }); + expect(dragFilesTool?.inputSchema?.properties?.pageId).toMatchObject({ type: 'string' }); + expect(dragFilesTool?.inputSchema?.properties?.nth).toMatchObject({ type: 'integer' }); + expect(dragFilesTool?.inputSchema?.properties?.frameSelector).toMatchObject({ type: 'string' }); + expect(dragFilesTool?.inputSchema?.properties?.pierceShadow).toMatchObject({ type: 'boolean' }); + + const dragElementTool = tools.find((tool) => tool.name === 'browser_drag_element'); + expect(dragElementTool?.inputSchema?.properties?.selector).toMatchObject({ type: 'string' }); + expect(dragElementTool?.inputSchema?.properties?.targetSelector).toMatchObject({ type: 'string' }); + expect(dragElementTool?.inputSchema?.properties?.targetX).toMatchObject({ type: 'number' }); + expect(dragElementTool?.inputSchema?.properties?.targetY).toMatchObject({ type: 'number' }); + expect(dragElementTool?.inputSchema?.properties?.steps).toMatchObject({ type: 'integer' }); + + const pointerTool = tools.find((tool) => tool.name === 'browser_pointer_action'); + expect(pointerTool?.inputSchema?.properties?.actions).toMatchObject({ type: 'array' }); + const queryTool = tools.find((tool) => tool.name === 'browser_query'); expect(queryTool?.inputSchema?.properties?.fields).toMatchObject({ type: 'array', @@ -5277,6 +5438,568 @@ describe('ccs-browser MCP server', () => { ); }); + it('drags local files onto normal, frame, and shadow dropzones', async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'ccs-browser-drag-files-')); + const invoicePath = join(tempDir, 'invoice.pdf'); + const receiptPath = join(tempDir, 'receipt.png'); + writeFileSync(invoicePath, 'invoice'); + writeFileSync(receiptPath, 'receipt'); + + const pages: MockPageState[] = [ + { + id: 'page-1', + title: 'Uploads', + currentUrl: 'https://example.com/uploads', + dropzones: { + '#dropzone': { accepted: true }, + }, + frames: [ + { + selector: '#upload-frame', + dropzones: { + '#frame-dropzone': { accepted: true }, + }, + }, + ], + shadowRoots: [ + { + hostSelector: 'upload-panel', + dropzones: { + '#shadow-dropzone': { accepted: true }, + }, + }, + ], + }, + ]; + + const responses = await runMcpRequests(pages, [ + { + jsonrpc: '2.0', + id: 901, + method: 'tools/call', + params: { + name: 'browser_drag_files', + arguments: { selector: '#dropzone', files: [invoicePath, receiptPath] }, + }, + }, + { + jsonrpc: '2.0', + id: 902, + method: 'tools/call', + params: { + name: 'browser_drag_files', + arguments: { + selector: '#frame-dropzone', + files: [invoicePath], + frameSelector: '#upload-frame', + }, + }, + }, + { + jsonrpc: '2.0', + id: 903, + method: 'tools/call', + params: { + name: 'browser_drag_files', + arguments: { + selector: '#shadow-dropzone', + files: [receiptPath], + pierceShadow: true, + }, + }, + }, + ]); + + expect(getResponseText(responses.find((message) => message.id === 901))).toContain('status: files-dropped'); + expect(getMockDropzoneState(pages[0]!, '#dropzone')?.receivedEventTypes).toEqual([ + 'dragenter', + 'dragover', + 'drop', + ]); + expect(getMockDropzoneState(pages[0]!, '#dropzone')?.receivedFiles).toEqual([ + { name: 'invoice.pdf', size: 7, type: 'application/pdf' }, + { name: 'receipt.png', size: 7, type: 'image/png' }, + ]); + expect( + getMockDropzoneState(pages[0]!, '#frame-dropzone', { frameSelector: '#upload-frame' })?.receivedFiles + ).toEqual([{ name: 'invoice.pdf', size: 7, type: 'application/pdf' }]); + expect( + getMockDropzoneState(pages[0]!, '#shadow-dropzone', { pierceShadow: true })?.receivedFiles + ).toEqual([{ name: 'receipt.png', size: 7, type: 'image/png' }]); + }); + + it('rejects browser_drag_files for missing files, page conflicts, missing targets, and rejected drops', async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'ccs-browser-drag-files-')); + const okPath = join(tempDir, 'ok.txt'); + const missingPath = join(tempDir, 'missing.txt'); + writeFileSync(okPath, 'ok'); + + const responses = await runMcpRequests( + [ + { + id: 'page-1', + title: 'Uploads', + currentUrl: 'https://example.com/uploads', + dropzones: { + '#reject-dropzone': { accepted: false }, + }, + }, + ], + [ + { + jsonrpc: '2.0', + id: 904, + method: 'tools/call', + params: { + name: 'browser_drag_files', + arguments: { selector: '#reject-dropzone', files: [missingPath] }, + }, + }, + { + jsonrpc: '2.0', + id: 905, + method: 'tools/call', + params: { + name: 'browser_drag_files', + arguments: { pageIndex: 0, pageId: 'page-1', selector: '#reject-dropzone', files: [okPath] }, + }, + }, + { + jsonrpc: '2.0', + id: 906, + method: 'tools/call', + params: { + name: 'browser_drag_files', + arguments: { selector: '#missing-dropzone', files: [okPath] }, + }, + }, + { + jsonrpc: '2.0', + id: 907, + method: 'tools/call', + params: { + name: 'browser_drag_files', + arguments: { selector: '#reject-dropzone', files: [okPath] }, + }, + }, + ] + ); + + expect(getResponseText(responses.find((message) => message.id === 904))).toContain('file does not exist'); + expect(getResponseText(responses.find((message) => message.id === 905))).toContain( + 'pageIndex and pageId cannot be used together' + ); + expect(getResponseText(responses.find((message) => message.id === 906))).toContain( + 'element not found for selector: #missing-dropzone' + ); + expect(getResponseText(responses.find((message) => message.id === 907))).toContain( + 'drop target rejected files' + ); + }); + + it('drags an element to another element and to explicit coordinates', async () => { + const pages: MockPageState[] = [ + { + id: 'page-1', + title: 'Drag Page', + currentUrl: 'https://example.com/drag', + query: { + '#card-a': { + exists: true, + connected: true, + rect: { + x: 20, + y: 40, + width: 100, + height: 60, + top: 40, + right: 120, + bottom: 100, + left: 20, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + '#lane-b': { + exists: true, + connected: true, + rect: { + x: 240, + y: 60, + width: 120, + height: 80, + top: 60, + right: 360, + bottom: 140, + left: 240, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + { + id: 'page-2', + title: 'Second Drag Page', + currentUrl: 'https://example.com/drag-2', + query: { + '#card-a': { + exists: true, + connected: true, + rect: { + x: 10, + y: 20, + width: 40, + height: 20, + top: 20, + right: 50, + bottom: 40, + left: 10, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + '#lane-b': { + exists: true, + connected: true, + rect: { + x: 110, + y: 80, + width: 60, + height: 30, + top: 80, + right: 170, + bottom: 110, + left: 110, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + ]; + + const responses = await runMcpRequests(pages, [ + { + jsonrpc: '2.0', + id: 908, + method: 'tools/call', + params: { + name: 'browser_drag_element', + arguments: { selector: '#card-a', targetSelector: '#lane-b', steps: 3 }, + }, + }, + { + jsonrpc: '2.0', + id: 909, + method: 'tools/call', + params: { + name: 'browser_drag_element', + arguments: { selector: '#card-a', targetX: 420, targetY: 180, steps: 2 }, + }, + }, + { + jsonrpc: '2.0', + id: 916, + method: 'tools/call', + params: { + name: 'browser_drag_element', + arguments: { pageId: 'page-2', selector: '#card-a', targetSelector: '#lane-b', steps: 1 }, + }, + }, + ]); + + expect(getResponseText(responses.find((message) => message.id === 908))).toContain('status: dragged'); + expect(getResponseText(responses.find((message) => message.id === 908))).toContain('targetSelector: #lane-b'); + expect(getResponseText(responses.find((message) => message.id === 909))).toContain('status: dragged'); + expect(getResponseText(responses.find((message) => message.id === 909))).toContain('targetX: 420'); + expect(getResponseText(responses.find((message) => message.id === 909))).toContain('targetY: 180'); + expect(getResponseText(responses.find((message) => message.id === 916))).toContain('pageIndex: 1'); + expect(getResponseText(responses.find((message) => message.id === 916))).toContain('targetSelector: #lane-b'); + expect(pages[0]!.drag?.recordedActions).toEqual([ + { type: 'mouseMoved', x: 70, y: 70, button: 'none' }, + { type: 'mousePressed', x: 70, y: 70, button: 'left' }, + { type: 'mouseMoved', x: 147, y: 80, button: 'left' }, + { type: 'mouseMoved', x: 223, y: 90, button: 'left' }, + { type: 'mouseMoved', x: 300, y: 100, button: 'left' }, + { type: 'mouseReleased', x: 300, y: 100, button: 'left' }, + { type: 'mouseMoved', x: 70, y: 70, button: 'none' }, + { type: 'mousePressed', x: 70, y: 70, button: 'left' }, + { type: 'mouseMoved', x: 245, y: 125, button: 'left' }, + { type: 'mouseMoved', x: 420, y: 180, button: 'left' }, + { type: 'mouseReleased', x: 420, y: 180, button: 'left' }, + ]); + expect(pages[1]!.drag?.recordedActions).toEqual([ + { type: 'mouseMoved', x: 30, y: 30, button: 'none' }, + { type: 'mousePressed', x: 30, y: 30, button: 'left' }, + { type: 'mouseMoved', x: 140, y: 95, button: 'left' }, + { type: 'mouseReleased', x: 140, y: 95, button: 'left' }, + ]); + }); + + it('rejects invalid browser_drag_element target combinations, page conflicts, and missing elements', async () => { + const responses = await runMcpRequests( + [ + { + id: 'page-1', + title: 'Drag Failures', + currentUrl: 'https://example.com/drag', + query: { + '#card-a': { + exists: true, + connected: true, + rect: { + x: 20, + y: 40, + width: 100, + height: 60, + top: 40, + right: 120, + bottom: 100, + left: 20, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + ], + [ + { + jsonrpc: '2.0', + id: 910, + method: 'tools/call', + params: { + name: 'browser_drag_element', + arguments: { selector: '#card-a', targetSelector: '#lane-b', targetX: 400, targetY: 200 }, + }, + }, + { + jsonrpc: '2.0', + id: 911, + method: 'tools/call', + params: { + name: 'browser_drag_element', + arguments: { selector: '#missing-source', targetX: 400, targetY: 200 }, + }, + }, + { + jsonrpc: '2.0', + id: 912, + method: 'tools/call', + params: { + name: 'browser_drag_element', + arguments: { selector: '#card-a', targetSelector: '#missing-target' }, + }, + }, + { + jsonrpc: '2.0', + id: 917, + method: 'tools/call', + params: { + name: 'browser_drag_element', + arguments: { pageIndex: 0, pageId: 'page-1', selector: '#card-a', targetX: 400, targetY: 200 }, + }, + }, + ] + ); + + expect(getResponseText(responses.find((message) => message.id === 910))).toContain( + 'targetSelector and targetX/targetY cannot be used together' + ); + expect(getResponseText(responses.find((message) => message.id === 911))).toContain( + 'source element not found' + ); + expect(getResponseText(responses.find((message) => message.id === 912))).toContain( + 'target element not found' + ); + expect(getResponseText(responses.find((message) => message.id === 917))).toContain( + 'pageIndex and pageId cannot be used together' + ); + }); + + it('runs minimal pointer sequences with browser_pointer_action', async () => { + const pages: MockPageState[] = [ + { + id: 'page-1', + title: 'Pointer Page', + currentUrl: 'https://example.com/pointer', + query: { + '#handle': { + exists: true, + connected: true, + rect: { + x: 30, + y: 50, + width: 40, + height: 20, + top: 50, + right: 70, + bottom: 70, + left: 30, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + { + id: 'page-2', + title: 'Second Pointer Page', + currentUrl: 'https://example.com/pointer-2', + query: { + '#handle': { + exists: true, + connected: true, + rect: { + x: 100, + y: 200, + width: 20, + height: 20, + top: 200, + right: 120, + bottom: 220, + left: 100, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + ]; + + const responses = await runMcpRequests(pages, [ + { + jsonrpc: '2.0', + id: 913, + method: 'tools/call', + params: { + name: 'browser_pointer_action', + arguments: { + actions: [ + { type: 'move', selector: '#handle' }, + { type: 'down' }, + { type: 'move', x: 220, y: 160 }, + { type: 'pause', durationMs: 5 }, + { type: 'up' }, + ], + }, + }, + }, + { + jsonrpc: '2.0', + id: 918, + method: 'tools/call', + params: { + name: 'browser_pointer_action', + arguments: { + pageId: 'page-2', + actions: [ + { type: 'move', selector: '#handle' }, + { type: 'down' }, + { type: 'up' }, + ], + }, + }, + }, + ]); + + expect(getResponseText(responses.find((message) => message.id === 913))).toContain('status: pointer-actions-completed'); + expect(getResponseText(responses.find((message) => message.id === 918))).toContain('pageIndex: 1'); + expect(getResponseText(responses.find((message) => message.id === 918))).toContain('status: pointer-actions-completed'); + expect(pages[0]!.drag?.recordedActions).toEqual([ + { type: 'mouseMoved', x: 50, y: 60, button: 'none' }, + { type: 'mousePressed', x: 50, y: 60, button: 'left' }, + { type: 'mouseMoved', x: 220, y: 160, button: 'left' }, + { type: 'mouseReleased', x: 220, y: 160, button: 'left' }, + ]); + expect(pages[1]!.drag?.recordedActions).toEqual([ + { type: 'mouseMoved', x: 110, y: 210, button: 'none' }, + { type: 'mousePressed', x: 110, y: 210, button: 'left' }, + { type: 'mouseReleased', x: 110, y: 210, button: 'left' }, + ]); + }); + + it('rejects invalid browser_pointer_action sequences and page conflicts', async () => { + const pages: MockPageState[] = [ + { + id: 'page-1', + title: 'Pointer Failures', + currentUrl: 'https://example.com/pointer', + query: { + '#handle': { + exists: true, + connected: true, + rect: { + x: 30, + y: 50, + width: 40, + height: 20, + top: 50, + right: 70, + bottom: 70, + left: 30, + }, + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + ]; + + const responses = await runMcpRequests(pages, [ + { + jsonrpc: '2.0', + id: 914, + method: 'tools/call', + params: { + name: 'browser_pointer_action', + arguments: { + actions: [{ type: 'up' }], + }, + }, + }, + { + jsonrpc: '2.0', + id: 915, + method: 'tools/call', + params: { + name: 'browser_pointer_action', + arguments: { + actions: [{ type: 'move', selector: '#missing-handle' }], + }, + }, + }, + { + jsonrpc: '2.0', + id: 919, + method: 'tools/call', + params: { + name: 'browser_pointer_action', + arguments: { + pageIndex: 0, + pageId: 'page-1', + actions: [{ type: 'move', x: 10, y: 10 }], + }, + }, + }, + ]); + + expect(getResponseText(responses.find((message) => message.id === 914))).toContain('pointer state error'); + expect(getResponseText(responses.find((message) => message.id === 915))).toContain('drag coordinates unavailable'); + expect(getResponseText(responses.find((message) => message.id === 919))).toContain( + 'pageIndex and pageId cannot be used together' + ); + }); + it('waits for a matching navigation event with browser_wait_for_event', async () => { const responses = await runMcpRequests( [