feat(browser): 完成 Phase 9 拖拽与指针动作能力

补齐文件拖拽、元素拖拽与受限 pointer 原语,让 Browser MCP 能覆盖常见上传与页面内拖放交互。
并补足 pageId 路由与回归测试,保持与现有 selected-page 语义一致。

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
walker1211
2026-04-19 18:41:18 +08:00
co-authored by Claude Opus 4.7
parent 492ee5b083
commit 844b5f6c42
2 changed files with 1220 additions and 15 deletions
+492 -10
View File
@@ -59,6 +59,9 @@ const TOOL_SET_DOWNLOAD_BEHAVIOR = 'browser_set_download_behavior';
const TOOL_LIST_DOWNLOADS = 'browser_list_downloads';
const TOOL_CANCEL_DOWNLOAD = 'browser_cancel_download';
const TOOL_SET_FILE_INPUT = 'browser_set_file_input';
const TOOL_DRAG_FILES = 'browser_drag_files';
const TOOL_DRAG_ELEMENT = 'browser_drag_element';
const TOOL_POINTER_ACTION = 'browser_pointer_action';
const TOOL_TAKE_SCREENSHOT = 'browser_take_screenshot';
const TOOL_WAIT_FOR = 'browser_wait_for';
const TOOL_EVAL = 'browser_eval';
@@ -87,6 +90,9 @@ const TOOL_NAMES = [
TOOL_LIST_DOWNLOADS,
TOOL_CANCEL_DOWNLOAD,
TOOL_SET_FILE_INPUT,
TOOL_DRAG_FILES,
TOOL_DRAG_ELEMENT,
TOOL_POINTER_ACTION,
TOOL_TAKE_SCREENSHOT,
TOOL_WAIT_FOR,
TOOL_EVAL,
@@ -113,6 +119,8 @@ const CDP_TIMEOUT_MS = 5000;
const NAVIGATION_POLL_INTERVAL_MS = 100;
const DEFAULT_WAIT_TIMEOUT_MS = 2000;
const DEFAULT_WAIT_POLL_INTERVAL_MS = 100;
const DEFAULT_DRAG_STEPS = 5;
const MAX_POINTER_ACTIONS = 25;
let inputBuffer = Buffer.alloc(0);
let requestCounter = 0;
@@ -624,6 +632,83 @@ function getTools() {
additionalProperties: false,
},
},
{
name: TOOL_DRAG_FILES,
description:
'Drag one or more local files onto a matched page drop target using page-side File and DataTransfer injection.',
inputSchema: {
type: 'object',
properties: {
selector: { type: 'string' },
files: {
type: 'array',
items: { type: 'string' },
},
pageIndex: { type: 'integer', minimum: 0 },
pageId: { type: 'string' },
nth: { type: 'integer', minimum: 0 },
frameSelector: { type: 'string' },
pierceShadow: { type: 'boolean' },
},
required: ['selector', 'files'],
additionalProperties: false,
},
},
{
name: TOOL_DRAG_ELEMENT,
description:
'Drag a matched element to another matched element or explicit coordinates using browser mouse events.',
inputSchema: {
type: 'object',
properties: {
selector: { type: 'string' },
targetSelector: { type: 'string' },
targetX: { type: 'number' },
targetY: { type: 'number' },
pageIndex: { type: 'integer', minimum: 0 },
pageId: { type: 'string' },
nth: { type: 'integer', minimum: 0 },
targetNth: { type: 'integer', minimum: 0 },
frameSelector: { type: 'string' },
pierceShadow: { type: 'boolean' },
steps: { type: 'integer', minimum: 1 },
},
required: ['selector'],
additionalProperties: false,
},
},
{
name: TOOL_POINTER_ACTION,
description: 'Run a limited pointer action sequence using browser mouse events.',
inputSchema: {
type: 'object',
properties: {
pageIndex: { type: 'integer', minimum: 0 },
pageId: { type: 'string' },
actions: {
type: 'array',
items: {
type: 'object',
properties: {
type: { type: 'string', enum: ['move', 'down', 'up', 'pause'] },
selector: { type: 'string' },
nth: { type: 'integer', minimum: 0 },
frameSelector: { type: 'string' },
pierceShadow: { type: 'boolean' },
x: { type: 'number' },
y: { type: 'number' },
button: { type: 'string', enum: ['left', 'middle', 'right'] },
durationMs: { type: 'integer', minimum: 0 },
},
required: ['type'],
additionalProperties: false,
},
},
},
required: ['actions'],
additionalProperties: false,
},
},
{
name: TOOL_TAKE_SCREENSHOT,
description:
@@ -2029,6 +2114,122 @@ function formatFileInputResult({ pageIndex, selector, nth, frameSelector, pierce
return lines.join('\n');
}
function inferMimeType(filePath) {
const ext = path.extname(filePath).toLowerCase();
if (ext === '.pdf') return 'application/pdf';
if (ext === '.png') return 'image/png';
if (ext === '.jpg' || ext === '.jpeg') return 'image/jpeg';
if (ext === '.txt') return 'text/plain';
if (ext === '.json') return 'application/json';
return 'application/octet-stream';
}
function readLocalFilesForDrop(files) {
return validateLocalFiles(files).map((resolvedPath) => {
const buffer = fs.readFileSync(resolvedPath);
return {
path: resolvedPath,
name: path.basename(resolvedPath),
mimeType: inferMimeType(resolvedPath),
size: buffer.length,
base64: buffer.toString('base64'),
};
});
}
function buildDropFilesExpression(selector, nth, frameSelector, pierceShadow, files) {
return `(() => {
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
const nth = ${nth === undefined ? 'undefined' : String(nth)};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
const filePayloads = JSON.parse(${JSON.stringify(JSON.stringify(files))});
const visitRoots = (root) => {
const roots = [root];
if (!pierceShadow) {
return roots;
}
const queue = [root];
while (queue.length > 0) {
const current = queue.shift();
const elements = Array.from(current.querySelectorAll('*'));
for (const element of elements) {
if (element.shadowRoot) {
roots.push(element.shadowRoot);
queue.push(element.shadowRoot);
}
}
}
return roots;
};
let root = document;
if (frameSelector) {
const frame = document.querySelector(frameSelector);
if (!frame) {
throw new Error('frame not found for selector: ' + frameSelector);
}
const frameDocument = frame.contentDocument;
if (!frameDocument) {
throw new Error('frame document is unavailable for selector: ' + frameSelector);
}
root = frameDocument;
}
const roots = visitRoots(root);
const matches = [];
for (const currentRoot of roots) {
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
}
const targetIndex = nth ?? 0;
const element = matches[targetIndex];
if (!element) {
throw new Error('element not found for selector: ' + selector);
}
const files = filePayloads.map((file) => {
const binary = atob(file.base64);
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) {
bytes[index] = binary.charCodeAt(index);
}
return new File([bytes], file.name, { type: file.mimeType });
});
const dataTransfer = new DataTransfer();
for (const file of files) {
dataTransfer.items.add(file);
}
element.dispatchEvent(new DragEvent('dragenter', { bubbles: true, cancelable: true, dataTransfer }));
element.dispatchEvent(new DragEvent('dragover', { bubbles: true, cancelable: true, dataTransfer }));
const accepted = element.dispatchEvent(
new DragEvent('drop', { bubbles: true, cancelable: true, dataTransfer })
);
return { accepted };
})()`;
}
function formatDragFilesResult({ pageIndex, selector, nth, frameSelector, pierceShadow, files }) {
const lines = [
`pageIndex: ${pageIndex}`,
`selector: ${selector}`,
`nth: ${nth}`,
`fileCount: ${files.length}`,
];
if (frameSelector) {
lines.push(`frameSelector: ${frameSelector}`);
}
if (pierceShadow) {
lines.push('pierceShadow: true');
}
lines.push('status: files-dropped');
return lines.join('\n');
}
function parseEventCondition(value) {
if (!value || typeof value !== 'object') {
throw new Error('event is required');
@@ -2613,9 +2814,10 @@ async function getElementDiagnostics(page, selector, nth, frameSelector = '', pi
return await getScopedDiagnostics(page, selector, nth, frameSelector, pierceShadow);
}
async function getScrolledElementState(page, selector, frameSelector = '', pierceShadow = false) {
async function getScrolledElementStateAt(page, selector, nth, frameSelector = '', pierceShadow = false) {
const expression = `(() => {
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
const nth = ${nth === undefined ? 'undefined' : String(nth)};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
@@ -2659,7 +2861,8 @@ async function getScrolledElementState(page, selector, frameSelector = '', pierc
for (const currentRoot of roots) {
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
}
const element = matches[0];
const targetIndex = nth ?? 0;
const element = matches[targetIndex];
if (!element) {
return JSON.stringify({ exists: false });
}
@@ -2713,6 +2916,10 @@ async function getScrolledElementState(page, selector, frameSelector = '', pierc
return JSON.parse(await evaluateExpression(page, expression));
}
async function getScrolledElementState(page, selector, frameSelector = '', pierceShadow = false) {
return await getScrolledElementStateAt(page, selector, undefined, frameSelector, pierceShadow);
}
function formatQueryValue(field, value) {
if (field === 'boundingClientRect') {
return JSON.stringify(value);
@@ -2920,6 +3127,220 @@ async function handleEval(toolArgs) {
return `pageIndex: ${pageIndex}\nmode: ${mode}\nvalue: ${JSON.stringify(value)}`;
}
function getButtonMask(button) {
if (button === 'right') {
return 2;
}
if (button === 'middle') {
return 4;
}
return 1;
}
async function dispatchMousePointerEvent(page, type, x, y, button = 'left', isPressed = false) {
await sendCdpCommand(page, 'Input.dispatchMouseEvent', {
type,
x,
y,
button: type === 'mouseMoved' && !isPressed ? 'none' : button,
buttons: isPressed ? getButtonMask(button) : 0,
pointerType: 'mouse',
clickCount: 1,
});
}
function interpolatePoints(sourcePoint, targetPoint, steps) {
const points = [];
for (let index = 1; index <= steps; index += 1) {
const ratio = index / steps;
points.push({
x: Math.round(sourcePoint.x + (targetPoint.x - sourcePoint.x) * ratio),
y: Math.round(sourcePoint.y + (targetPoint.y - sourcePoint.y) * ratio),
});
}
return points;
}
async function resolveElementCenterPoint(page, selector, nth, frameSelector, pierceShadow, missingLabel) {
const state = await getScrolledElementStateAt(page, selector, nth, frameSelector, pierceShadow);
if (!state.exists) {
throw new Error(missingLabel);
}
if (state.connected !== true) {
throw new Error(`drag coordinates unavailable for selector: ${selector}`);
}
if (state.interactable !== true || !state.centerPoint) {
throw new Error(`drag coordinates unavailable for selector: ${selector}`);
}
return state.centerPoint;
}
async function handlePointerAction(toolArgs) {
const pages = await listPageTargets();
if (pages.length === 0) {
throw new Error('Browser MCP did not find any page targets in the current Chrome session.');
}
const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId);
if (!page.webSocketDebuggerUrl) {
throw new Error(`Browser MCP page ${pageIndex} does not expose a websocket debugger URL.`);
}
const actions = Array.isArray(toolArgs.actions) ? toolArgs.actions : null;
if (!actions || actions.length === 0) {
throw new Error('actions must be a non-empty array');
}
if (actions.length > MAX_POINTER_ACTIONS) {
throw new Error(`actions exceeds maximum of ${MAX_POINTER_ACTIONS}`);
}
let pointerX = null;
let pointerY = null;
let pressedButton = null;
for (const action of actions) {
if (!action || typeof action !== 'object') {
throw new Error('each action must be an object');
}
if (!['move', 'down', 'up', 'pause'].includes(action.type)) {
throw new Error(`unsupported pointer action: ${String(action.type || '')}`);
}
if (action.type === 'pause') {
const durationMs = action.durationMs === undefined ? 0 : requireNonNegativeInteger(action.durationMs, 'durationMs');
if (durationMs > 0) {
await sleep(durationMs);
}
continue;
}
if (action.type === 'move') {
let point = null;
if (typeof action.selector === 'string' && action.selector.trim() !== '') {
point = await resolveElementCenterPoint(
page,
requireNonEmptyString(action.selector, 'actions.selector'),
requireOptionalNonNegativeInteger(action.nth, 'actions.nth'),
parseOptionalNonEmptyString(action.frameSelector),
action.pierceShadow === true,
'drag coordinates unavailable'
);
} else if (typeof action.x === 'number' && typeof action.y === 'number') {
point = {
x: requireFiniteNumber(action.x, 'actions.x'),
y: requireFiniteNumber(action.y, 'actions.y'),
};
} else {
throw new Error('drag coordinates unavailable');
}
pointerX = point.x;
pointerY = point.y;
await dispatchMousePointerEvent(page, 'mouseMoved', pointerX, pointerY, pressedButton || 'left', Boolean(pressedButton));
continue;
}
if (pointerX === null || pointerY === null) {
throw new Error('pointer state error');
}
if (action.type === 'down') {
if (pressedButton) {
throw new Error('pointer state error');
}
pressedButton = requireEnumString(action.button, 'button', ['left', 'middle', 'right']) || 'left';
await dispatchMousePointerEvent(page, 'mousePressed', pointerX, pointerY, pressedButton, true);
continue;
}
if (!pressedButton) {
throw new Error('pointer state error');
}
const releaseButton = requireEnumString(action.button, 'button', ['left', 'middle', 'right']) || pressedButton;
await dispatchMousePointerEvent(page, 'mouseReleased', pointerX, pointerY, releaseButton, false);
pressedButton = null;
}
if (pressedButton) {
throw new Error('pointer state error');
}
return `pageIndex: ${pageIndex}\nactionCount: ${actions.length}\nstatus: pointer-actions-completed`;
}
async function handleDragElement(toolArgs) {
const pages = await listPageTargets();
if (pages.length === 0) {
throw new Error('Browser MCP did not find any page targets in the current Chrome session.');
}
const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId);
if (!page.webSocketDebuggerUrl) {
throw new Error(`Browser MCP page ${pageIndex} does not expose a websocket debugger URL.`);
}
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
const nth = requireOptionalNonNegativeInteger(toolArgs.nth, 'nth');
const targetNth = requireOptionalNonNegativeInteger(toolArgs.targetNth, 'targetNth');
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const targetSelector = parseOptionalNonEmptyString(toolArgs.targetSelector);
const hasTargetX = Object.prototype.hasOwnProperty.call(toolArgs, 'targetX');
const hasTargetY = Object.prototype.hasOwnProperty.call(toolArgs, 'targetY');
const hasTargetCoordinates = hasTargetX || hasTargetY;
if (targetSelector && hasTargetCoordinates) {
throw new Error('targetSelector and targetX/targetY cannot be used together');
}
if (!targetSelector && !hasTargetCoordinates) {
throw new Error('targetSelector or targetX/targetY is required');
}
if (hasTargetX !== hasTargetY) {
throw new Error('targetX and targetY must be provided together');
}
const sourcePoint = await resolveElementCenterPoint(
page,
selector,
nth,
frameSelector,
pierceShadow,
'source element not found'
);
const targetPoint = targetSelector
? await resolveElementCenterPoint(
page,
targetSelector,
targetNth,
frameSelector,
pierceShadow,
'target element not found'
)
: {
x: requireFiniteNumber(toolArgs.targetX, 'targetX'),
y: requireFiniteNumber(toolArgs.targetY, 'targetY'),
};
const steps = toolArgs.steps === undefined ? DEFAULT_DRAG_STEPS : requirePositiveInteger(toolArgs.steps, 'steps');
await dispatchMousePointerEvent(page, 'mouseMoved', sourcePoint.x, sourcePoint.y, 'left', false);
await dispatchMousePointerEvent(page, 'mousePressed', sourcePoint.x, sourcePoint.y, 'left', true);
for (const point of interpolatePoints(sourcePoint, targetPoint, steps)) {
await dispatchMousePointerEvent(page, 'mouseMoved', point.x, point.y, 'left', true);
}
await dispatchMousePointerEvent(page, 'mouseReleased', targetPoint.x, targetPoint.y, 'left', false);
return [
`pageIndex: ${pageIndex}`,
`selector: ${selector}`,
`nth: ${nth ?? 0}`,
targetSelector ? `targetSelector: ${targetSelector}` : `targetX: ${targetPoint.x}`,
targetSelector ? `targetNth: ${targetNth ?? 0}` : `targetY: ${targetPoint.y}`,
`steps: ${steps}`,
`${frameSelector ? `frameSelector: ${frameSelector}\n` : ''}${pierceShadow ? 'pierceShadow: true\n' : ''}status: dragged`,
].join('\n');
}
async function handleHover(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
@@ -2936,14 +3357,7 @@ async function handleHover(toolArgs) {
throw new Error(`element is hidden or not interactable for selector: ${selector}`);
}
await sendCdpCommand(page, 'Input.dispatchMouseEvent', {
type: 'mouseMoved',
x: state.centerPoint.x,
y: state.centerPoint.y,
button: 'none',
buttons: 0,
pointerType: 'mouse',
});
await dispatchMousePointerEvent(page, 'mouseMoved', state.centerPoint.x, state.centerPoint.y, 'left', false);
return `pageIndex: ${pageIndex}\nselector: ${selector}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: hovered`;
}
@@ -3684,6 +4098,50 @@ async function handleSetFileInput(toolArgs) {
});
}
async function handleDragFiles(toolArgs) {
const pages = await listPageTargets();
if (pages.length === 0) {
throw new Error('Browser MCP did not find any page targets in the current Chrome session.');
}
const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId);
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
const files = readLocalFilesForDrop(requireNonEmptyStringArray(toolArgs.files, 'files'));
const nth = toolArgs.nth === undefined ? 0 : requireNonNegativeInteger(toolArgs.nth, 'nth');
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const response = await sendCdpCommand(page, 'Runtime.evaluate', {
expression: buildDropFilesExpression(selector, nth, frameSelector, pierceShadow, files),
returnByValue: true,
awaitPromise: true,
});
if (response?.exceptionDetails) {
throw new Error(response.exceptionDetails.text || 'drop target evaluation failed');
}
const result = response?.result || null;
if (!result) {
throw new Error('Browser MCP received an invalid DevTools evaluation response.');
}
if (result.subtype === 'error') {
throw new Error(result.description || 'drop target evaluation failed');
}
if (result.value?.accepted === false) {
throw new Error('drop target rejected files');
}
return formatDragFilesResult({
pageIndex,
selector,
nth,
frameSelector,
pierceShadow,
files,
});
}
async function handleToolCall(message) {
const id = message.id;
const params = message.params || {};
@@ -3848,6 +4306,30 @@ async function handleToolCall(message) {
return;
}
if (toolName === TOOL_DRAG_FILES) {
const text = await handleDragFiles(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_DRAG_ELEMENT) {
const text = await handleDragElement(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_POINTER_ACTION) {
const text = await handlePointerAction(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_TAKE_SCREENSHOT) {
const text = await handleScreenshot(toolArgs);
writeResponse(id, {
+728 -5
View File
@@ -111,17 +111,40 @@ type MockFileInputState = {
type MockFileInputPlan = MockFileInputState | MockFileInputState[];
type MockDropzoneState = {
accepted?: boolean;
requireFiles?: boolean;
receivedEventTypes?: string[];
receivedFiles?: Array<{ name: string; size: number; type: string }>;
error?: string;
};
type MockDropzonePlan = MockDropzoneState | MockDropzoneState[];
type MockPointerActionRecord = {
type: string;
x?: number;
y?: number;
button?: string;
};
type MockDragPlan = {
recordedActions?: MockPointerActionRecord[];
};
type MockFrameState = {
selector: string;
query?: Record<string, MockQueryPlan>;
visibleText?: string;
fileInputs?: Record<string, MockFileInputPlan>;
dropzones?: Record<string, MockDropzonePlan>;
};
type MockShadowRootState = {
hostSelector: string;
query?: Record<string, MockQueryPlan>;
fileInputs?: Record<string, MockFileInputPlan>;
dropzones?: Record<string, MockDropzonePlan>;
};
type MockEvalPlan = Record<
@@ -175,6 +198,8 @@ type MockPageState = {
eval?: MockEvalPlan;
frames?: MockFrameState[];
shadowRoots?: MockShadowRootState[];
dropzones?: Record<string, MockDropzonePlan>;
drag?: MockDragPlan;
events?: MockPageEventPlan;
intercept?: MockInterceptState;
screenshot?: {
@@ -378,6 +403,36 @@ function parseNumberArgument(expression: string, key: string): number | undefine
return Number.parseInt(match[1], 10);
}
function parseParsedJsonArrayArgument<T>(expression: string, key: string): T[] {
const marker = `const ${key} = JSON.parse(`;
const start = expression.indexOf(marker);
if (start === -1) {
return [];
}
const quoteStart = start + marker.length;
const quote = expression[quoteStart];
if (quote !== '"' && quote !== "'") {
return [];
}
let index = quoteStart + 1;
while (index < expression.length) {
if (expression[index] === '\\') {
index += 2;
continue;
}
if (expression[index] === quote) {
const encoded = expression.slice(quoteStart, index + 1);
const decoded = JSON.parse(encoded) as string;
return JSON.parse(decoded) as T[];
}
index += 1;
}
return [];
}
function pickMockMatch<T>(
plan: T | T[] | undefined,
nth = 0
@@ -410,6 +465,41 @@ function getMockShadowRoot(page: MockPageState): MockShadowRootState | undefined
return page.shadowRoots?.[0];
}
function getMockDropzoneState(
page: Pick<MockPageState, 'dropzones' | 'frames' | 'shadowRoots'>,
selector: string,
options: { nth?: number; frameSelector?: string; pierceShadow?: boolean } = {}
): MockDropzoneState | null {
const nth = options.nth ?? 0;
if (options.frameSelector) {
const frame = page.frames?.find((entry) => entry.selector === options.frameSelector);
const plan = frame?.dropzones?.[selector];
if (!plan) {
return null;
}
return Array.isArray(plan) ? (plan[nth] ?? null) : plan;
}
if (options.pierceShadow) {
for (const shadowRoot of page.shadowRoots || []) {
const plan = shadowRoot.dropzones?.[selector];
if (plan) {
return Array.isArray(plan) ? (plan[nth] ?? null) : plan;
}
}
}
const plan = page.dropzones?.[selector];
if (!plan) {
return null;
}
return Array.isArray(plan) ? (plan[nth] ?? null) : plan;
}
function pushPointerAction(page: MockPageState, action: MockPointerActionRecord): void {
page.drag = page.drag || {};
page.drag.recordedActions = page.drag.recordedActions || [];
page.drag.recordedActions.push(action);
}
function shiftPageText(page: MockPageState): string {
const queue = page.wait?.pageTextSequence;
if (!queue || queue.length === 0) {
@@ -443,11 +533,13 @@ function createMockBrowser(pagesInput: MockPageState[]) {
let nextPageCounter = pagesInput.length + 1;
for (const [index, page] of pagesInput.entries()) {
pageStates.set(`/devtools/page/${index + 1}`, {
visibleText: 'Hello from visible text',
domSnapshot: '<html><body>Hello from DOM snapshot</body></html>',
...page,
});
if (page.visibleText === undefined) {
page.visibleText = 'Hello from visible text';
}
if (page.domSnapshot === undefined) {
page.domSnapshot = '<html><body>Hello from DOM snapshot</body></html>';
}
pageStates.set(`/devtools/page/${index + 1}`, page);
}
async function start(options: RunMcpRequestsOptions = {}) {
@@ -688,11 +780,13 @@ function createMockBrowser(pagesInput: MockPageState[]) {
const type = typeof message.params?.type === 'string' ? message.params.type : '';
const x = Number(message.params?.x);
const y = Number(message.params?.y);
const button = typeof message.params?.button === 'string' ? message.params.button : undefined;
for (const hoverPlan of Object.values(page.hover || {})) {
if (type === 'mouseMoved') {
hoverPlan.lastMouseMove = { x, y };
}
}
pushPointerAction(page, { type, x, y, button });
reply({});
return;
}
@@ -868,6 +962,51 @@ function createMockBrowser(pagesInput: MockPageState[]) {
const expression = String(message.params?.expression || '');
if (expression.includes('new DragEvent') && expression.includes('new DataTransfer()')) {
const selector = parseJsonArgument(expression, 'selector') || '';
const nth = parseNumberArgument(expression, 'nth') ?? 0;
const frameSelector = parseJsonArgument(expression, 'frameSelector') || '';
const pierceShadow = expression.includes('const pierceShadow = true');
const filePayloads = parseParsedJsonArrayArgument<{
name: string;
size: number;
mimeType: string;
}>(expression, 'filePayloads');
const eventTypes = Array.from(
expression.matchAll(/new DragEvent\('(dragenter|dragover|drop)'/g)
).map((match) => match[1]);
const target = getMockDropzoneState(page, selector, {
nth,
frameSelector,
pierceShadow,
});
if (!target) {
replyError(`element not found for selector: ${selector}`);
return;
}
if (target.error) {
replyError(target.error);
return;
}
if (eventTypes.length === 0) {
replyError(`drag event sequence not found for selector: ${selector}`);
return;
}
if (target.requireFiles && filePayloads.length === 0) {
reply({ result: { type: 'object', value: { accepted: false } } });
return;
}
target.receivedEventTypes = eventTypes;
target.receivedFiles = filePayloads.map((file) => ({
name: file.name,
size: file.size,
type: file.mimeType,
}));
reply({ result: { type: 'object', value: { accepted: target.accepted !== false } } });
return;
}
if (page.eval?.[expression]) {
const evalPlan = page.eval[expression];
if (evalPlan.error) {
@@ -1535,6 +1674,9 @@ describe('ccs-browser MCP server', () => {
'browser_list_downloads',
'browser_cancel_download',
'browser_set_file_input',
'browser_drag_files',
'browser_drag_element',
'browser_pointer_action',
'browser_take_screenshot',
'browser_wait_for',
'browser_eval',
@@ -1624,6 +1766,25 @@ describe('ccs-browser MCP server', () => {
expect(uploadTool?.inputSchema?.properties?.frameSelector).toMatchObject({ type: 'string' });
expect(uploadTool?.inputSchema?.properties?.pierceShadow).toMatchObject({ type: 'boolean' });
const dragFilesTool = tools.find((tool) => tool.name === 'browser_drag_files');
expect(dragFilesTool?.inputSchema?.properties?.selector).toMatchObject({ type: 'string' });
expect(dragFilesTool?.inputSchema?.properties?.files).toMatchObject({ type: 'array' });
expect(dragFilesTool?.inputSchema?.properties?.pageIndex).toMatchObject({ type: 'integer' });
expect(dragFilesTool?.inputSchema?.properties?.pageId).toMatchObject({ type: 'string' });
expect(dragFilesTool?.inputSchema?.properties?.nth).toMatchObject({ type: 'integer' });
expect(dragFilesTool?.inputSchema?.properties?.frameSelector).toMatchObject({ type: 'string' });
expect(dragFilesTool?.inputSchema?.properties?.pierceShadow).toMatchObject({ type: 'boolean' });
const dragElementTool = tools.find((tool) => tool.name === 'browser_drag_element');
expect(dragElementTool?.inputSchema?.properties?.selector).toMatchObject({ type: 'string' });
expect(dragElementTool?.inputSchema?.properties?.targetSelector).toMatchObject({ type: 'string' });
expect(dragElementTool?.inputSchema?.properties?.targetX).toMatchObject({ type: 'number' });
expect(dragElementTool?.inputSchema?.properties?.targetY).toMatchObject({ type: 'number' });
expect(dragElementTool?.inputSchema?.properties?.steps).toMatchObject({ type: 'integer' });
const pointerTool = tools.find((tool) => tool.name === 'browser_pointer_action');
expect(pointerTool?.inputSchema?.properties?.actions).toMatchObject({ type: 'array' });
const queryTool = tools.find((tool) => tool.name === 'browser_query');
expect(queryTool?.inputSchema?.properties?.fields).toMatchObject({
type: 'array',
@@ -5277,6 +5438,568 @@ describe('ccs-browser MCP server', () => {
);
});
it('drags local files onto normal, frame, and shadow dropzones', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'ccs-browser-drag-files-'));
const invoicePath = join(tempDir, 'invoice.pdf');
const receiptPath = join(tempDir, 'receipt.png');
writeFileSync(invoicePath, 'invoice');
writeFileSync(receiptPath, 'receipt');
const pages: MockPageState[] = [
{
id: 'page-1',
title: 'Uploads',
currentUrl: 'https://example.com/uploads',
dropzones: {
'#dropzone': { accepted: true },
},
frames: [
{
selector: '#upload-frame',
dropzones: {
'#frame-dropzone': { accepted: true },
},
},
],
shadowRoots: [
{
hostSelector: 'upload-panel',
dropzones: {
'#shadow-dropzone': { accepted: true },
},
},
],
},
];
const responses = await runMcpRequests(pages, [
{
jsonrpc: '2.0',
id: 901,
method: 'tools/call',
params: {
name: 'browser_drag_files',
arguments: { selector: '#dropzone', files: [invoicePath, receiptPath] },
},
},
{
jsonrpc: '2.0',
id: 902,
method: 'tools/call',
params: {
name: 'browser_drag_files',
arguments: {
selector: '#frame-dropzone',
files: [invoicePath],
frameSelector: '#upload-frame',
},
},
},
{
jsonrpc: '2.0',
id: 903,
method: 'tools/call',
params: {
name: 'browser_drag_files',
arguments: {
selector: '#shadow-dropzone',
files: [receiptPath],
pierceShadow: true,
},
},
},
]);
expect(getResponseText(responses.find((message) => message.id === 901))).toContain('status: files-dropped');
expect(getMockDropzoneState(pages[0]!, '#dropzone')?.receivedEventTypes).toEqual([
'dragenter',
'dragover',
'drop',
]);
expect(getMockDropzoneState(pages[0]!, '#dropzone')?.receivedFiles).toEqual([
{ name: 'invoice.pdf', size: 7, type: 'application/pdf' },
{ name: 'receipt.png', size: 7, type: 'image/png' },
]);
expect(
getMockDropzoneState(pages[0]!, '#frame-dropzone', { frameSelector: '#upload-frame' })?.receivedFiles
).toEqual([{ name: 'invoice.pdf', size: 7, type: 'application/pdf' }]);
expect(
getMockDropzoneState(pages[0]!, '#shadow-dropzone', { pierceShadow: true })?.receivedFiles
).toEqual([{ name: 'receipt.png', size: 7, type: 'image/png' }]);
});
it('rejects browser_drag_files for missing files, page conflicts, missing targets, and rejected drops', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'ccs-browser-drag-files-'));
const okPath = join(tempDir, 'ok.txt');
const missingPath = join(tempDir, 'missing.txt');
writeFileSync(okPath, 'ok');
const responses = await runMcpRequests(
[
{
id: 'page-1',
title: 'Uploads',
currentUrl: 'https://example.com/uploads',
dropzones: {
'#reject-dropzone': { accepted: false },
},
},
],
[
{
jsonrpc: '2.0',
id: 904,
method: 'tools/call',
params: {
name: 'browser_drag_files',
arguments: { selector: '#reject-dropzone', files: [missingPath] },
},
},
{
jsonrpc: '2.0',
id: 905,
method: 'tools/call',
params: {
name: 'browser_drag_files',
arguments: { pageIndex: 0, pageId: 'page-1', selector: '#reject-dropzone', files: [okPath] },
},
},
{
jsonrpc: '2.0',
id: 906,
method: 'tools/call',
params: {
name: 'browser_drag_files',
arguments: { selector: '#missing-dropzone', files: [okPath] },
},
},
{
jsonrpc: '2.0',
id: 907,
method: 'tools/call',
params: {
name: 'browser_drag_files',
arguments: { selector: '#reject-dropzone', files: [okPath] },
},
},
]
);
expect(getResponseText(responses.find((message) => message.id === 904))).toContain('file does not exist');
expect(getResponseText(responses.find((message) => message.id === 905))).toContain(
'pageIndex and pageId cannot be used together'
);
expect(getResponseText(responses.find((message) => message.id === 906))).toContain(
'element not found for selector: #missing-dropzone'
);
expect(getResponseText(responses.find((message) => message.id === 907))).toContain(
'drop target rejected files'
);
});
it('drags an element to another element and to explicit coordinates', async () => {
const pages: MockPageState[] = [
{
id: 'page-1',
title: 'Drag Page',
currentUrl: 'https://example.com/drag',
query: {
'#card-a': {
exists: true,
connected: true,
rect: {
x: 20,
y: 40,
width: 100,
height: 60,
top: 40,
right: 120,
bottom: 100,
left: 20,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
'#lane-b': {
exists: true,
connected: true,
rect: {
x: 240,
y: 60,
width: 120,
height: 80,
top: 60,
right: 360,
bottom: 140,
left: 240,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
},
},
{
id: 'page-2',
title: 'Second Drag Page',
currentUrl: 'https://example.com/drag-2',
query: {
'#card-a': {
exists: true,
connected: true,
rect: {
x: 10,
y: 20,
width: 40,
height: 20,
top: 20,
right: 50,
bottom: 40,
left: 10,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
'#lane-b': {
exists: true,
connected: true,
rect: {
x: 110,
y: 80,
width: 60,
height: 30,
top: 80,
right: 170,
bottom: 110,
left: 110,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
},
},
];
const responses = await runMcpRequests(pages, [
{
jsonrpc: '2.0',
id: 908,
method: 'tools/call',
params: {
name: 'browser_drag_element',
arguments: { selector: '#card-a', targetSelector: '#lane-b', steps: 3 },
},
},
{
jsonrpc: '2.0',
id: 909,
method: 'tools/call',
params: {
name: 'browser_drag_element',
arguments: { selector: '#card-a', targetX: 420, targetY: 180, steps: 2 },
},
},
{
jsonrpc: '2.0',
id: 916,
method: 'tools/call',
params: {
name: 'browser_drag_element',
arguments: { pageId: 'page-2', selector: '#card-a', targetSelector: '#lane-b', steps: 1 },
},
},
]);
expect(getResponseText(responses.find((message) => message.id === 908))).toContain('status: dragged');
expect(getResponseText(responses.find((message) => message.id === 908))).toContain('targetSelector: #lane-b');
expect(getResponseText(responses.find((message) => message.id === 909))).toContain('status: dragged');
expect(getResponseText(responses.find((message) => message.id === 909))).toContain('targetX: 420');
expect(getResponseText(responses.find((message) => message.id === 909))).toContain('targetY: 180');
expect(getResponseText(responses.find((message) => message.id === 916))).toContain('pageIndex: 1');
expect(getResponseText(responses.find((message) => message.id === 916))).toContain('targetSelector: #lane-b');
expect(pages[0]!.drag?.recordedActions).toEqual([
{ type: 'mouseMoved', x: 70, y: 70, button: 'none' },
{ type: 'mousePressed', x: 70, y: 70, button: 'left' },
{ type: 'mouseMoved', x: 147, y: 80, button: 'left' },
{ type: 'mouseMoved', x: 223, y: 90, button: 'left' },
{ type: 'mouseMoved', x: 300, y: 100, button: 'left' },
{ type: 'mouseReleased', x: 300, y: 100, button: 'left' },
{ type: 'mouseMoved', x: 70, y: 70, button: 'none' },
{ type: 'mousePressed', x: 70, y: 70, button: 'left' },
{ type: 'mouseMoved', x: 245, y: 125, button: 'left' },
{ type: 'mouseMoved', x: 420, y: 180, button: 'left' },
{ type: 'mouseReleased', x: 420, y: 180, button: 'left' },
]);
expect(pages[1]!.drag?.recordedActions).toEqual([
{ type: 'mouseMoved', x: 30, y: 30, button: 'none' },
{ type: 'mousePressed', x: 30, y: 30, button: 'left' },
{ type: 'mouseMoved', x: 140, y: 95, button: 'left' },
{ type: 'mouseReleased', x: 140, y: 95, button: 'left' },
]);
});
it('rejects invalid browser_drag_element target combinations, page conflicts, and missing elements', async () => {
const responses = await runMcpRequests(
[
{
id: 'page-1',
title: 'Drag Failures',
currentUrl: 'https://example.com/drag',
query: {
'#card-a': {
exists: true,
connected: true,
rect: {
x: 20,
y: 40,
width: 100,
height: 60,
top: 40,
right: 120,
bottom: 100,
left: 20,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
},
},
],
[
{
jsonrpc: '2.0',
id: 910,
method: 'tools/call',
params: {
name: 'browser_drag_element',
arguments: { selector: '#card-a', targetSelector: '#lane-b', targetX: 400, targetY: 200 },
},
},
{
jsonrpc: '2.0',
id: 911,
method: 'tools/call',
params: {
name: 'browser_drag_element',
arguments: { selector: '#missing-source', targetX: 400, targetY: 200 },
},
},
{
jsonrpc: '2.0',
id: 912,
method: 'tools/call',
params: {
name: 'browser_drag_element',
arguments: { selector: '#card-a', targetSelector: '#missing-target' },
},
},
{
jsonrpc: '2.0',
id: 917,
method: 'tools/call',
params: {
name: 'browser_drag_element',
arguments: { pageIndex: 0, pageId: 'page-1', selector: '#card-a', targetX: 400, targetY: 200 },
},
},
]
);
expect(getResponseText(responses.find((message) => message.id === 910))).toContain(
'targetSelector and targetX/targetY cannot be used together'
);
expect(getResponseText(responses.find((message) => message.id === 911))).toContain(
'source element not found'
);
expect(getResponseText(responses.find((message) => message.id === 912))).toContain(
'target element not found'
);
expect(getResponseText(responses.find((message) => message.id === 917))).toContain(
'pageIndex and pageId cannot be used together'
);
});
it('runs minimal pointer sequences with browser_pointer_action', async () => {
const pages: MockPageState[] = [
{
id: 'page-1',
title: 'Pointer Page',
currentUrl: 'https://example.com/pointer',
query: {
'#handle': {
exists: true,
connected: true,
rect: {
x: 30,
y: 50,
width: 40,
height: 20,
top: 50,
right: 70,
bottom: 70,
left: 30,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
},
},
{
id: 'page-2',
title: 'Second Pointer Page',
currentUrl: 'https://example.com/pointer-2',
query: {
'#handle': {
exists: true,
connected: true,
rect: {
x: 100,
y: 200,
width: 20,
height: 20,
top: 200,
right: 120,
bottom: 220,
left: 100,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
},
},
];
const responses = await runMcpRequests(pages, [
{
jsonrpc: '2.0',
id: 913,
method: 'tools/call',
params: {
name: 'browser_pointer_action',
arguments: {
actions: [
{ type: 'move', selector: '#handle' },
{ type: 'down' },
{ type: 'move', x: 220, y: 160 },
{ type: 'pause', durationMs: 5 },
{ type: 'up' },
],
},
},
},
{
jsonrpc: '2.0',
id: 918,
method: 'tools/call',
params: {
name: 'browser_pointer_action',
arguments: {
pageId: 'page-2',
actions: [
{ type: 'move', selector: '#handle' },
{ type: 'down' },
{ type: 'up' },
],
},
},
},
]);
expect(getResponseText(responses.find((message) => message.id === 913))).toContain('status: pointer-actions-completed');
expect(getResponseText(responses.find((message) => message.id === 918))).toContain('pageIndex: 1');
expect(getResponseText(responses.find((message) => message.id === 918))).toContain('status: pointer-actions-completed');
expect(pages[0]!.drag?.recordedActions).toEqual([
{ type: 'mouseMoved', x: 50, y: 60, button: 'none' },
{ type: 'mousePressed', x: 50, y: 60, button: 'left' },
{ type: 'mouseMoved', x: 220, y: 160, button: 'left' },
{ type: 'mouseReleased', x: 220, y: 160, button: 'left' },
]);
expect(pages[1]!.drag?.recordedActions).toEqual([
{ type: 'mouseMoved', x: 110, y: 210, button: 'none' },
{ type: 'mousePressed', x: 110, y: 210, button: 'left' },
{ type: 'mouseReleased', x: 110, y: 210, button: 'left' },
]);
});
it('rejects invalid browser_pointer_action sequences and page conflicts', async () => {
const pages: MockPageState[] = [
{
id: 'page-1',
title: 'Pointer Failures',
currentUrl: 'https://example.com/pointer',
query: {
'#handle': {
exists: true,
connected: true,
rect: {
x: 30,
y: 50,
width: 40,
height: 20,
top: 50,
right: 70,
bottom: 70,
left: 30,
},
display: 'block',
visibility: 'visible',
opacity: '1',
},
},
},
];
const responses = await runMcpRequests(pages, [
{
jsonrpc: '2.0',
id: 914,
method: 'tools/call',
params: {
name: 'browser_pointer_action',
arguments: {
actions: [{ type: 'up' }],
},
},
},
{
jsonrpc: '2.0',
id: 915,
method: 'tools/call',
params: {
name: 'browser_pointer_action',
arguments: {
actions: [{ type: 'move', selector: '#missing-handle' }],
},
},
},
{
jsonrpc: '2.0',
id: 919,
method: 'tools/call',
params: {
name: 'browser_pointer_action',
arguments: {
pageIndex: 0,
pageId: 'page-1',
actions: [{ type: 'move', x: 10, y: 10 }],
},
},
},
]);
expect(getResponseText(responses.find((message) => message.id === 914))).toContain('pointer state error');
expect(getResponseText(responses.find((message) => message.id === 915))).toContain('drag coordinates unavailable');
expect(getResponseText(responses.find((message) => message.id === 919))).toContain(
'pageIndex and pageId cannot be used together'
);
});
it('waits for a matching navigation event with browser_wait_for_event', async () => {
const responses = await runMcpRequests(
[