mirror of
https://github.com/tiennm99/ccs.git
synced 2026-07-16 04:18:05 +00:00
feat(browser): 完成 browser MCP 第三阶段能力
补齐 iframe/shadow 作用域查询与页面事件观察能力,并同步测试与文档,确保 Phase 3 以可验证增量独立收口。 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
b90eae3892
commit
cf2375e169
@@ -20,12 +20,13 @@ that already has useful authenticated state.
|
||||
Claude Browser Attach requires a browser launched in attach mode with remote debugging
|
||||
enabled. A recent Chrome update alone is not sufficient.
|
||||
|
||||
The managed `ccs-browser` runtime currently exposes four tool groups:
|
||||
The managed `ccs-browser` runtime currently exposes five tool groups:
|
||||
|
||||
- **Session inspection**: `browser_get_session_info`, `browser_get_url_and_title`, `browser_get_visible_text`, `browser_get_dom_snapshot`
|
||||
- **Navigation and interaction**: `browser_navigate`, `browser_click`, `browser_type`, `browser_take_screenshot`
|
||||
- **Hover diagnostics**: `browser_hover`, `browser_query`, `browser_take_element_screenshot`
|
||||
- **Readiness and page evaluation**: `browser_wait_for`, `browser_eval`
|
||||
- **Event observation**: `browser_wait_for_event`
|
||||
|
||||
Notable Phase 1 capability details:
|
||||
|
||||
@@ -34,6 +35,12 @@ Notable Phase 1 capability details:
|
||||
- `browser_wait_for` can wait on page text or selector state before the next step runs
|
||||
- `browser_eval` is gated by `browser.claude.eval_mode` and supports `disabled`, `readonly`, or `readwrite`; readonly mode uses side-effect-blocked evaluation and may reject expressions that could mutate page state
|
||||
|
||||
Phase 3 capability details:
|
||||
|
||||
- `browser_click`, `browser_hover`, `browser_query`, `browser_wait_for`, and `browser_take_element_screenshot` accept optional `frameSelector` to resolve targets inside a specific same-origin iframe
|
||||
- the same scoped-selector tools accept optional `pierceShadow: true` to search open shadow roots beneath the selected root
|
||||
- `browser_wait_for_event` observes typed page or browser events for dialogs, navigation, requests, and downloads
|
||||
|
||||
A common hover-debug workflow is:
|
||||
|
||||
1. call `browser_hover` to move the browser pointer onto the card or trigger
|
||||
@@ -42,6 +49,24 @@ A common hover-debug workflow is:
|
||||
4. call `browser_take_element_screenshot` to confirm the revealed state
|
||||
5. call `browser_eval` in read-only mode when you need page-side inspection that the structured tools do not expose directly
|
||||
|
||||
Scoped selector notes:
|
||||
|
||||
- `browser_click`, `browser_hover`, `browser_query`, `browser_wait_for`, and `browser_take_element_screenshot` accept optional `frameSelector` for same-origin iframes whose `contentDocument` is accessible
|
||||
- the same selector-based tools accept optional `pierceShadow: true` for open shadow-root traversal
|
||||
- closed shadow roots, frame-index routing, request interception, and download acceptance controls are still out of scope
|
||||
|
||||
Example event wait:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "browser_wait_for_event",
|
||||
"arguments": {
|
||||
"event": { "kind": "navigation", "urlIncludes": "/checkout" },
|
||||
"timeoutMs": 2000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Codex Browser Tools
|
||||
|
||||
Codex-target CCS launches use a separate managed path: CCS injects Playwright MCP overrides
|
||||
|
||||
+546
-83
@@ -15,9 +15,12 @@ function loadWebSocketImplementation() {
|
||||
}
|
||||
|
||||
try {
|
||||
const { WebSocket } = require('ws');
|
||||
if (typeof WebSocket === 'function') {
|
||||
return WebSocket;
|
||||
const wsModule = require('ws');
|
||||
if (typeof wsModule === 'function') {
|
||||
return wsModule;
|
||||
}
|
||||
if (typeof wsModule?.WebSocket === 'function') {
|
||||
return wsModule.WebSocket;
|
||||
}
|
||||
} catch {
|
||||
// Surface a dedicated error below if no implementation is available.
|
||||
@@ -46,6 +49,7 @@ const TOOL_EVAL = 'browser_eval';
|
||||
const TOOL_HOVER = 'browser_hover';
|
||||
const TOOL_QUERY = 'browser_query';
|
||||
const TOOL_TAKE_ELEMENT_SCREENSHOT = 'browser_take_element_screenshot';
|
||||
const TOOL_WAIT_FOR_EVENT = 'browser_wait_for_event';
|
||||
const TOOL_NAMES = [
|
||||
TOOL_SESSION_INFO,
|
||||
TOOL_URL_TITLE,
|
||||
@@ -60,6 +64,7 @@ const TOOL_NAMES = [
|
||||
TOOL_HOVER,
|
||||
TOOL_QUERY,
|
||||
TOOL_TAKE_ELEMENT_SCREENSHOT,
|
||||
TOOL_WAIT_FOR_EVENT,
|
||||
];
|
||||
const SUPPORTED_QUERY_FIELDS = [
|
||||
'exists',
|
||||
@@ -265,6 +270,14 @@ function getTools() {
|
||||
minimum: 0,
|
||||
description: 'Optional zero-based match index for selectors returning multiple elements.',
|
||||
},
|
||||
frameSelector: {
|
||||
type: 'string',
|
||||
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
|
||||
},
|
||||
pierceShadow: {
|
||||
type: 'boolean',
|
||||
description: 'When true, search open shadow roots beneath the selected root.',
|
||||
},
|
||||
},
|
||||
required: ['selector'],
|
||||
additionalProperties: false,
|
||||
@@ -340,6 +353,14 @@ function getTools() {
|
||||
minimum: 0,
|
||||
description: 'Optional zero-based match index for selectors returning multiple elements.',
|
||||
},
|
||||
frameSelector: {
|
||||
type: 'string',
|
||||
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
|
||||
},
|
||||
pierceShadow: {
|
||||
type: 'boolean',
|
||||
description: 'When true, search open shadow roots beneath the selected root.',
|
||||
},
|
||||
timeoutMs: {
|
||||
type: 'integer',
|
||||
minimum: 1,
|
||||
@@ -401,6 +422,14 @@ function getTools() {
|
||||
type: 'string',
|
||||
description: 'Required CSS selector for the hover target.',
|
||||
},
|
||||
frameSelector: {
|
||||
type: 'string',
|
||||
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
|
||||
},
|
||||
pierceShadow: {
|
||||
type: 'boolean',
|
||||
description: 'When true, search open shadow roots beneath the selected root.',
|
||||
},
|
||||
},
|
||||
required: ['selector'],
|
||||
additionalProperties: false,
|
||||
@@ -427,6 +456,14 @@ function getTools() {
|
||||
minimum: 0,
|
||||
description: 'Optional zero-based match index for selectors returning multiple elements.',
|
||||
},
|
||||
frameSelector: {
|
||||
type: 'string',
|
||||
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
|
||||
},
|
||||
pierceShadow: {
|
||||
type: 'boolean',
|
||||
description: 'When true, search open shadow roots beneath the selected root.',
|
||||
},
|
||||
fields: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
@@ -453,11 +490,44 @@ function getTools() {
|
||||
type: 'string',
|
||||
description: 'Required CSS selector for the screenshot target.',
|
||||
},
|
||||
frameSelector: {
|
||||
type: 'string',
|
||||
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
|
||||
},
|
||||
pierceShadow: {
|
||||
type: 'boolean',
|
||||
description: 'When true, search open shadow roots beneath the selected root.',
|
||||
},
|
||||
},
|
||||
required: ['selector'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: TOOL_WAIT_FOR_EVENT,
|
||||
description: 'Wait until a page or browser event matching the requested filter is observed.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
pageIndex: {
|
||||
type: 'integer',
|
||||
minimum: 0,
|
||||
description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.',
|
||||
},
|
||||
timeoutMs: {
|
||||
type: 'integer',
|
||||
minimum: 1,
|
||||
description: 'Optional timeout in milliseconds.',
|
||||
},
|
||||
event: {
|
||||
type: 'object',
|
||||
description: 'Required event selector.',
|
||||
},
|
||||
},
|
||||
required: ['event'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
@@ -733,6 +803,177 @@ function getBrowserEvalMode() {
|
||||
return 'readonly';
|
||||
}
|
||||
|
||||
function parseOptionalNonEmptyString(value) {
|
||||
return typeof value === 'string' && value.trim() !== '' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function formatScopedSelectorSuffix(frameSelector, pierceShadow) {
|
||||
return `${frameSelector ? `\nframeSelector: ${frameSelector}` : ''}${pierceShadow ? '\npierceShadow: true' : ''}`;
|
||||
}
|
||||
|
||||
function buildScopedMatchesExpression(selector, nth, frameSelector, pierceShadow) {
|
||||
return `(() => {
|
||||
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
|
||||
const nth = ${nth === undefined ? 'undefined' : String(nth)};
|
||||
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
|
||||
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
|
||||
|
||||
const visitRoots = (root) => {
|
||||
const roots = [root];
|
||||
if (!pierceShadow) {
|
||||
return roots;
|
||||
}
|
||||
const queue = [root];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
const elements = Array.from(current.querySelectorAll('*'));
|
||||
for (const element of elements) {
|
||||
if (element.shadowRoot) {
|
||||
roots.push(element.shadowRoot);
|
||||
queue.push(element.shadowRoot);
|
||||
}
|
||||
}
|
||||
}
|
||||
return roots;
|
||||
};
|
||||
|
||||
let root = document;
|
||||
if (frameSelector) {
|
||||
const frame = document.querySelector(frameSelector);
|
||||
if (!frame) {
|
||||
throw new Error('frame not found for selector: ' + frameSelector);
|
||||
}
|
||||
const frameDocument = frame.contentDocument;
|
||||
if (!frameDocument) {
|
||||
throw new Error('frame document is unavailable for selector: ' + frameSelector);
|
||||
}
|
||||
root = frameDocument;
|
||||
}
|
||||
|
||||
const roots = visitRoots(root);
|
||||
const matches = [];
|
||||
for (const currentRoot of roots) {
|
||||
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
|
||||
}
|
||||
|
||||
const count = matches.length;
|
||||
const targetIndex = nth ?? 0;
|
||||
const element = matches[targetIndex];
|
||||
if (!element) {
|
||||
return JSON.stringify({
|
||||
exists: nth === undefined ? count > 0 : count > targetIndex,
|
||||
count,
|
||||
targetIndex,
|
||||
targetMissing: true,
|
||||
});
|
||||
}
|
||||
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(element);
|
||||
const text = typeof element.innerText === 'string' ? element.innerText : (element.textContent || '');
|
||||
return JSON.stringify({
|
||||
exists: true,
|
||||
count,
|
||||
targetIndex,
|
||||
connected: element.isConnected,
|
||||
text,
|
||||
innerText: typeof element.innerText === 'string' ? element.innerText : '',
|
||||
textContent: element.textContent || '',
|
||||
boundingClientRect: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
top: rect.top,
|
||||
right: rect.right,
|
||||
bottom: rect.bottom,
|
||||
left: rect.left,
|
||||
},
|
||||
display: style.display,
|
||||
visibility: style.visibility,
|
||||
opacity: style.opacity,
|
||||
href: typeof element.getAttribute === 'function' ? element.getAttribute('href') || '' : '',
|
||||
onclick: typeof element.getAttribute === 'function' ? element.getAttribute('onclick') || '' : '',
|
||||
interactable:
|
||||
element.isConnected &&
|
||||
style.display !== 'none' &&
|
||||
style.visibility !== 'hidden' &&
|
||||
rect.width > 0 &&
|
||||
rect.height > 0,
|
||||
centerPoint: {
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
},
|
||||
});
|
||||
})()`;
|
||||
}
|
||||
|
||||
async function getScopedDiagnostics(page, selector, nth, frameSelector, pierceShadow) {
|
||||
const raw = await evaluateExpression(page, buildScopedMatchesExpression(selector, nth, frameSelector, pierceShadow));
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
function parseEventCondition(value) {
|
||||
if (!value || typeof value !== 'object') {
|
||||
throw new Error('event is required');
|
||||
}
|
||||
if (value.kind === 'dialog') {
|
||||
return {
|
||||
kind: 'dialog',
|
||||
dialogType: value.dialogType ? String(value.dialogType) : undefined,
|
||||
messageIncludes: value.messageIncludes ? String(value.messageIncludes) : undefined,
|
||||
};
|
||||
}
|
||||
if (value.kind === 'navigation') {
|
||||
return {
|
||||
kind: 'navigation',
|
||||
urlIncludes: value.urlIncludes ? String(value.urlIncludes) : undefined,
|
||||
};
|
||||
}
|
||||
if (value.kind === 'request') {
|
||||
return {
|
||||
kind: 'request',
|
||||
urlIncludes: value.urlIncludes ? String(value.urlIncludes) : undefined,
|
||||
method: value.method ? String(value.method) : undefined,
|
||||
};
|
||||
}
|
||||
if (value.kind === 'download') {
|
||||
return {
|
||||
kind: 'download',
|
||||
urlIncludes: value.urlIncludes ? String(value.urlIncludes) : undefined,
|
||||
suggestedFilenameIncludes: value.suggestedFilenameIncludes
|
||||
? String(value.suggestedFilenameIncludes)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
throw new Error(`unknown event kind: ${String(value.kind || '')}`);
|
||||
}
|
||||
|
||||
function matchesObservedEvent(event, observed) {
|
||||
if (event.kind === 'dialog') {
|
||||
return (
|
||||
(!event.dialogType || observed.type === event.dialogType) &&
|
||||
(!event.messageIncludes || String(observed.message || '').includes(event.messageIncludes))
|
||||
);
|
||||
}
|
||||
if (event.kind === 'navigation') {
|
||||
return !event.urlIncludes || String(observed.url || '').includes(event.urlIncludes);
|
||||
}
|
||||
if (event.kind === 'request') {
|
||||
return (
|
||||
(!event.urlIncludes || String(observed.url || '').includes(event.urlIncludes)) &&
|
||||
(!event.method || String(observed.method || '').toUpperCase() === event.method.toUpperCase())
|
||||
);
|
||||
}
|
||||
if (event.kind === 'download') {
|
||||
return (
|
||||
(!event.urlIncludes || String(observed.url || '').includes(event.urlIncludes)) &&
|
||||
(!event.suggestedFilenameIncludes || String(observed.suggestedFilename || '').includes(event.suggestedFilenameIncludes))
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -815,11 +1056,52 @@ async function handleClick(toolArgs) {
|
||||
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
|
||||
const nth = requireOptionalNonNegativeInteger(toolArgs.nth, 'nth');
|
||||
const targetIndex = nth ?? 0;
|
||||
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
|
||||
const pierceShadow = toolArgs.pierceShadow === true;
|
||||
|
||||
const expression = `(() => {
|
||||
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
|
||||
const nth = ${nth === undefined ? 'undefined' : String(nth)};
|
||||
const matches = Array.from(document.querySelectorAll(selector));
|
||||
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
|
||||
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
|
||||
|
||||
const visitRoots = (root) => {
|
||||
const roots = [root];
|
||||
if (!pierceShadow) {
|
||||
return roots;
|
||||
}
|
||||
const queue = [root];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
const elements = Array.from(current.querySelectorAll('*'));
|
||||
for (const element of elements) {
|
||||
if (element.shadowRoot) {
|
||||
roots.push(element.shadowRoot);
|
||||
queue.push(element.shadowRoot);
|
||||
}
|
||||
}
|
||||
}
|
||||
return roots;
|
||||
};
|
||||
|
||||
let root = document;
|
||||
if (frameSelector) {
|
||||
const frame = document.querySelector(frameSelector);
|
||||
if (!frame) {
|
||||
throw new Error('frame not found for selector: ' + frameSelector);
|
||||
}
|
||||
const frameDocument = frame.contentDocument;
|
||||
if (!frameDocument) {
|
||||
throw new Error('frame document is unavailable for selector: ' + frameSelector);
|
||||
}
|
||||
root = frameDocument;
|
||||
}
|
||||
|
||||
const roots = visitRoots(root);
|
||||
const matches = [];
|
||||
for (const currentRoot of roots) {
|
||||
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
|
||||
}
|
||||
const element = matches[nth ?? 0];
|
||||
if (!element) {
|
||||
throw new Error('element index ' + (nth ?? 0) + ' is out of range for selector: ' + selector);
|
||||
@@ -876,7 +1158,7 @@ async function handleClick(toolArgs) {
|
||||
})()`;
|
||||
|
||||
await evaluateExpression(page, expression);
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}\nstatus: clicked`;
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`;
|
||||
}
|
||||
|
||||
async function handleType(toolArgs) {
|
||||
@@ -974,63 +1256,57 @@ async function handleScreenshot(toolArgs) {
|
||||
return `pageIndex: ${pageIndex}\nformat: png\nfullPage: ${fullPage ? 'true' : 'false'}\ndata: ${data}`;
|
||||
}
|
||||
|
||||
async function getElementDiagnostics(page, selector, nth) {
|
||||
const expression = `(() => {
|
||||
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
|
||||
const nth = ${nth === undefined ? 'undefined' : String(nth)};
|
||||
const matches = Array.from(document.querySelectorAll(selector));
|
||||
const count = matches.length;
|
||||
const targetIndex = nth ?? 0;
|
||||
const element = matches[targetIndex];
|
||||
if (!element) {
|
||||
return JSON.stringify({
|
||||
exists: nth === undefined ? count > 0 : count > targetIndex,
|
||||
count,
|
||||
targetIndex,
|
||||
targetMissing: true,
|
||||
});
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(element);
|
||||
const text = typeof element.innerText === 'string' ? element.innerText : (element.textContent || '');
|
||||
return JSON.stringify({
|
||||
exists: true,
|
||||
count,
|
||||
targetIndex,
|
||||
connected: element.isConnected,
|
||||
text,
|
||||
innerText: typeof element.innerText === 'string' ? element.innerText : '',
|
||||
textContent: element.textContent || '',
|
||||
boundingClientRect: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
top: rect.top,
|
||||
right: rect.right,
|
||||
bottom: rect.bottom,
|
||||
left: rect.left,
|
||||
},
|
||||
display: style.display,
|
||||
visibility: style.visibility,
|
||||
opacity: style.opacity,
|
||||
href: typeof element.getAttribute === 'function' ? element.getAttribute('href') || '' : '',
|
||||
onclick: typeof element.getAttribute === 'function' ? element.getAttribute('onclick') || '' : '',
|
||||
interactable:
|
||||
element.isConnected &&
|
||||
style.display !== 'none' &&
|
||||
style.visibility !== 'hidden' &&
|
||||
rect.width > 0 &&
|
||||
rect.height > 0,
|
||||
});
|
||||
})()`;
|
||||
return JSON.parse(await evaluateExpression(page, expression));
|
||||
async function getElementDiagnostics(page, selector, nth, frameSelector = '', pierceShadow = false) {
|
||||
return await getScopedDiagnostics(page, selector, nth, frameSelector, pierceShadow);
|
||||
}
|
||||
|
||||
async function getScrolledElementState(page, selector) {
|
||||
async function getScrolledElementState(page, selector, frameSelector = '', pierceShadow = false) {
|
||||
const expression = `(() => {
|
||||
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
|
||||
const element = document.querySelector(selector);
|
||||
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
|
||||
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
|
||||
|
||||
const visitRoots = (root) => {
|
||||
const roots = [root];
|
||||
if (!pierceShadow) {
|
||||
return roots;
|
||||
}
|
||||
const queue = [root];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
const elements = Array.from(current.querySelectorAll('*'));
|
||||
for (const element of elements) {
|
||||
if (element.shadowRoot) {
|
||||
roots.push(element.shadowRoot);
|
||||
queue.push(element.shadowRoot);
|
||||
}
|
||||
}
|
||||
}
|
||||
return roots;
|
||||
};
|
||||
|
||||
let root = document;
|
||||
let frameOffset = { left: 0, top: 0 };
|
||||
if (frameSelector) {
|
||||
const frame = document.querySelector(frameSelector);
|
||||
if (!frame) {
|
||||
throw new Error('frame not found for selector: ' + frameSelector);
|
||||
}
|
||||
const frameDocument = frame.contentDocument;
|
||||
if (!frameDocument) {
|
||||
throw new Error('frame document is unavailable for selector: ' + frameSelector);
|
||||
}
|
||||
const frameRect = frame.getBoundingClientRect();
|
||||
frameOffset = { left: frameRect.left, top: frameRect.top };
|
||||
root = frameDocument;
|
||||
}
|
||||
|
||||
const roots = visitRoots(root);
|
||||
const matches = [];
|
||||
for (const currentRoot of roots) {
|
||||
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
|
||||
}
|
||||
const element = matches[0];
|
||||
if (!element) {
|
||||
return JSON.stringify({ exists: false });
|
||||
}
|
||||
@@ -1039,13 +1315,23 @@ async function getScrolledElementState(page, selector) {
|
||||
}
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
const rect = element.getBoundingClientRect();
|
||||
const absoluteRect = {
|
||||
x: rect.x + frameOffset.left,
|
||||
y: rect.y + frameOffset.top,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
top: rect.top + frameOffset.top,
|
||||
right: rect.right + frameOffset.left,
|
||||
bottom: rect.bottom + frameOffset.top,
|
||||
left: rect.left + frameOffset.left,
|
||||
};
|
||||
const style = window.getComputedStyle(element);
|
||||
const viewportWidth = window.innerWidth || document.documentElement.clientWidth || 0;
|
||||
const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 0;
|
||||
const clipX = Math.max(0, rect.left);
|
||||
const clipY = Math.max(0, rect.top);
|
||||
const clipRight = Math.min(viewportWidth, rect.right);
|
||||
const clipBottom = Math.min(viewportHeight, rect.bottom);
|
||||
const clipX = Math.max(0, absoluteRect.left);
|
||||
const clipY = Math.max(0, absoluteRect.top);
|
||||
const clipRight = Math.min(viewportWidth, absoluteRect.right);
|
||||
const clipBottom = Math.min(viewportHeight, absoluteRect.bottom);
|
||||
return JSON.stringify({
|
||||
exists: true,
|
||||
connected: true,
|
||||
@@ -1057,19 +1343,10 @@ async function getScrolledElementState(page, selector) {
|
||||
style.visibility !== 'hidden' &&
|
||||
rect.width > 0 &&
|
||||
rect.height > 0,
|
||||
boundingClientRect: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
top: rect.top,
|
||||
right: rect.right,
|
||||
bottom: rect.bottom,
|
||||
left: rect.left,
|
||||
},
|
||||
boundingClientRect: absoluteRect,
|
||||
centerPoint: {
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
x: absoluteRect.left + absoluteRect.width / 2,
|
||||
y: absoluteRect.top + absoluteRect.height / 2,
|
||||
},
|
||||
visibleClip: {
|
||||
x: clipX,
|
||||
@@ -1218,14 +1495,16 @@ async function getWaitPageObservation(page) {
|
||||
return { text };
|
||||
}
|
||||
|
||||
async function getWaitSelectorObservation(page, selector, nth) {
|
||||
return getElementDiagnostics(page, selector, nth);
|
||||
async function getWaitSelectorObservation(page, selector, nth, frameSelector, pierceShadow) {
|
||||
return getElementDiagnostics(page, selector, nth, frameSelector, pierceShadow);
|
||||
}
|
||||
|
||||
async function handleWaitFor(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const selector = typeof toolArgs.selector === 'string' ? toolArgs.selector.trim() : '';
|
||||
const nth = requireOptionalNonNegativeInteger(toolArgs.nth, 'nth');
|
||||
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
|
||||
const pierceShadow = toolArgs.pierceShadow === true;
|
||||
const timeoutMs = requirePositiveIntegerOrDefault(toolArgs.timeoutMs, 'timeoutMs', DEFAULT_WAIT_TIMEOUT_MS);
|
||||
const pollIntervalMs = requirePositiveIntegerOrDefault(
|
||||
toolArgs.pollIntervalMs,
|
||||
@@ -1238,10 +1517,10 @@ async function handleWaitFor(toolArgs) {
|
||||
|
||||
while (Date.now() <= deadline) {
|
||||
lastObserved = selector
|
||||
? await getWaitSelectorObservation(page, selector, nth)
|
||||
? await getWaitSelectorObservation(page, selector, nth, frameSelector, pierceShadow)
|
||||
: await getWaitPageObservation(page);
|
||||
if (isWaitConditionSatisfied(lastObserved, condition)) {
|
||||
return `pageIndex: ${pageIndex}${selector ? `\nselector: ${selector}` : ''}\nstatus: satisfied`;
|
||||
return `pageIndex: ${pageIndex}${selector ? `\nselector: ${selector}` : ''}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: satisfied`;
|
||||
}
|
||||
if (Date.now() + pollIntervalMs > deadline) {
|
||||
break;
|
||||
@@ -1291,7 +1570,9 @@ async function handleEval(toolArgs) {
|
||||
async function handleHover(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
|
||||
const state = await getScrolledElementState(page, selector);
|
||||
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
|
||||
const pierceShadow = toolArgs.pierceShadow === true;
|
||||
const state = await getScrolledElementState(page, selector, frameSelector, pierceShadow);
|
||||
if (!state.exists) {
|
||||
throw new Error(`element not found for selector: ${selector}`);
|
||||
}
|
||||
@@ -1311,22 +1592,26 @@ async function handleHover(toolArgs) {
|
||||
pointerType: 'mouse',
|
||||
});
|
||||
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}\nstatus: hovered`;
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: hovered`;
|
||||
}
|
||||
|
||||
async function handleQuery(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
|
||||
const nth = requireOptionalNonNegativeInteger(toolArgs.nth, 'nth');
|
||||
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
|
||||
const pierceShadow = toolArgs.pierceShadow === true;
|
||||
const fields = parseQueryFields(toolArgs.fields);
|
||||
const diagnostics = await getElementDiagnostics(page, selector, nth);
|
||||
const diagnostics = await getElementDiagnostics(page, selector, nth, frameSelector, pierceShadow);
|
||||
return formatQueryResponse(pageIndex, selector, nth, diagnostics, fields);
|
||||
}
|
||||
|
||||
async function handleElementScreenshot(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
|
||||
const state = await getScrolledElementState(page, selector);
|
||||
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
|
||||
const pierceShadow = toolArgs.pierceShadow === true;
|
||||
const state = await getScrolledElementState(page, selector, frameSelector, pierceShadow);
|
||||
if (!state.exists) {
|
||||
throw new Error(`element not found for selector: ${selector}`);
|
||||
}
|
||||
@@ -1350,7 +1635,177 @@ async function handleElementScreenshot(toolArgs) {
|
||||
throw new Error('screenshot capture failed');
|
||||
}
|
||||
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}\nformat: png\ndata: ${data}`;
|
||||
return `pageIndex: ${pageIndex}\nselector: ${selector}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nformat: png\ndata: ${data}`;
|
||||
}
|
||||
|
||||
async function waitForPageEvent({ page, timeoutMs, event }) {
|
||||
const ws = new WebSocket(page.webSocketDebuggerUrl);
|
||||
return await new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
abortSocket(ws);
|
||||
reject(new Error(`event wait timed out for kind: ${event.kind}`));
|
||||
}, timeoutMs);
|
||||
|
||||
const settleError = (error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
abortSocket(ws);
|
||||
reject(toSocketError(error));
|
||||
};
|
||||
|
||||
const settleSuccess = (observed) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
closeSocket(ws);
|
||||
resolve(observed);
|
||||
};
|
||||
|
||||
addSocketListener(ws, 'open', () => {
|
||||
if (event.kind === 'dialog' || event.kind === 'navigation') {
|
||||
ws.send(JSON.stringify({ id: ++requestCounter, method: 'Page.enable', params: {} }));
|
||||
}
|
||||
if (event.kind === 'request') {
|
||||
ws.send(JSON.stringify({ id: ++requestCounter, method: 'Network.enable', params: {} }));
|
||||
}
|
||||
});
|
||||
|
||||
addSocketListener(ws, 'message', (data) => {
|
||||
void (async () => {
|
||||
const raw = await getSocketMessageText(data);
|
||||
let message;
|
||||
try {
|
||||
message = JSON.parse(raw);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!message || typeof message !== 'object' || typeof message.method !== 'string') {
|
||||
return;
|
||||
}
|
||||
let observed = null;
|
||||
if (message.method === 'Page.javascriptDialogOpening') {
|
||||
observed = { type: message.params?.type || '', message: message.params?.message || '' };
|
||||
} else if (message.method === 'Page.frameNavigated') {
|
||||
const frame = message.params?.frame;
|
||||
if (!frame?.parentId) {
|
||||
observed = { url: frame?.url || '' };
|
||||
}
|
||||
} else if (message.method === 'Network.requestWillBeSent') {
|
||||
observed = {
|
||||
url: message.params?.request?.url || '',
|
||||
method: message.params?.request?.method || '',
|
||||
};
|
||||
}
|
||||
if (observed && matchesObservedEvent(event, observed)) {
|
||||
settleSuccess(observed);
|
||||
}
|
||||
})().catch(settleError);
|
||||
});
|
||||
|
||||
addSocketListener(ws, 'error', settleError);
|
||||
addSocketListener(ws, 'close', () => {
|
||||
if (!settled) {
|
||||
settleError(new Error('Browser MCP lost the DevTools websocket connection.'));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForBrowserDownloadEvent(timeoutMs, event) {
|
||||
const targets = await fetchJson(`${getHttpUrl()}/json/list`);
|
||||
const browserTarget = Array.isArray(targets)
|
||||
? targets.find((target) => target && typeof target === 'object' && target.type === 'browser')
|
||||
: null;
|
||||
if (!browserTarget || typeof browserTarget.webSocketDebuggerUrl !== 'string' || !browserTarget.webSocketDebuggerUrl) {
|
||||
throw new Error('browser-level download events are unavailable');
|
||||
}
|
||||
|
||||
const ws = new WebSocket(browserTarget.webSocketDebuggerUrl);
|
||||
return await new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
abortSocket(ws);
|
||||
reject(new Error('event wait timed out for kind: download'));
|
||||
}, timeoutMs);
|
||||
|
||||
const settleError = (error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
abortSocket(ws);
|
||||
reject(toSocketError(error));
|
||||
};
|
||||
|
||||
const settleSuccess = (observed) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
closeSocket(ws);
|
||||
resolve(observed);
|
||||
};
|
||||
|
||||
addSocketListener(ws, 'message', (data) => {
|
||||
void (async () => {
|
||||
const raw = await getSocketMessageText(data);
|
||||
let message;
|
||||
try {
|
||||
message = JSON.parse(raw);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (message?.method !== 'Browser.downloadWillBegin') {
|
||||
return;
|
||||
}
|
||||
const observed = {
|
||||
url: message.params?.url || '',
|
||||
suggestedFilename: message.params?.suggestedFilename || '',
|
||||
};
|
||||
if (matchesObservedEvent(event, observed)) {
|
||||
settleSuccess(observed);
|
||||
}
|
||||
})().catch(settleError);
|
||||
});
|
||||
|
||||
addSocketListener(ws, 'error', settleError);
|
||||
addSocketListener(ws, 'close', () => {
|
||||
if (!settled) {
|
||||
settleError(new Error('Browser MCP lost the DevTools websocket connection.'));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForMatchingEvent({ page, timeoutMs, event }) {
|
||||
if (event.kind === 'download') {
|
||||
return await waitForBrowserDownloadEvent(timeoutMs, event);
|
||||
}
|
||||
return await waitForPageEvent({ page, timeoutMs, event });
|
||||
}
|
||||
|
||||
async function handleWaitForEvent(toolArgs) {
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
const timeoutMs = requirePositiveIntegerOrDefault(toolArgs.timeoutMs, 'timeoutMs', DEFAULT_WAIT_TIMEOUT_MS);
|
||||
const event = parseEventCondition(toolArgs.event);
|
||||
const observed = await waitForMatchingEvent({ page, pageIndex, timeoutMs, event });
|
||||
return `pageIndex: ${pageIndex}\nevent: ${event.kind}\nstatus: observed\ndetail: ${JSON.stringify(observed)}`;
|
||||
}
|
||||
|
||||
async function handleToolCall(message) {
|
||||
@@ -1458,6 +1913,14 @@ async function handleToolCall(message) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (toolName === TOOL_WAIT_FOR_EVENT) {
|
||||
const text = await handleWaitForEvent(toolArgs);
|
||||
writeResponse(id, {
|
||||
content: [{ type: 'text', text }],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const { page, pageIndex } = await getSelectedPage(toolArgs);
|
||||
|
||||
if (toolName === TOOL_URL_TITLE) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user