mirror of
https://github.com/tiennm99/ccs.git
synced 2026-07-15 22:21:20 +00:00
feat(browser): 增强 assert_query 断言与诊断信息
为 assert_query 增加 assertions 数组、更多比较操作和结构化失败 diagnostics。 并补充多断言、类型错误和数值比较的回归覆盖。 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
c708bde817
commit
4af967559c
@@ -2427,7 +2427,7 @@ function formatReplaySummary(session) {
|
||||
}
|
||||
|
||||
function formatOrchestrationSummary(session) {
|
||||
return [
|
||||
const lines = [
|
||||
`orchestrationId: ${session.orchestrationId}`,
|
||||
`pageId: ${session.pageId}`,
|
||||
`pageIndex: ${session.pageIndex}`,
|
||||
@@ -2437,7 +2437,15 @@ function formatOrchestrationSummary(session) {
|
||||
`failedBlockIndex: ${session.failedBlockIndex === null ? 'none' : session.failedBlockIndex}`,
|
||||
`error: ${session.error || 'none'}`,
|
||||
`status: ${session.status}`,
|
||||
].join('\n');
|
||||
];
|
||||
if (session.errorDetails?.failedAssertionIndex !== undefined) {
|
||||
lines.push(`failedAssertionIndex: ${session.errorDetails.failedAssertionIndex}`);
|
||||
lines.push(`field: ${session.errorDetails.field}`);
|
||||
lines.push(`op: ${session.errorDetails.op}`);
|
||||
lines.push(`expected: ${JSON.stringify(session.errorDetails.expected)}`);
|
||||
lines.push(`actual: ${JSON.stringify(session.errorDetails.actual)}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function formatRecordingDetail(session) {
|
||||
@@ -2517,6 +2525,133 @@ const SUPPORTED_ORCHESTRATION_BLOCK_TYPES = new Set([
|
||||
'assert_query',
|
||||
]);
|
||||
|
||||
const SUPPORTED_ASSERTION_OPERATORS = new Set(['equals', 'contains', 'gt', 'gte', 'lt', 'lte']);
|
||||
|
||||
function parseAssertionValue(value) {
|
||||
return value;
|
||||
}
|
||||
|
||||
function toComparableNumber(field, value) {
|
||||
const numeric = Number(value);
|
||||
if (Number.isNaN(numeric)) {
|
||||
throw new Error(`numeric comparison expects a number-like field: ${field}`);
|
||||
}
|
||||
return numeric;
|
||||
}
|
||||
|
||||
function parseAssertions(assertQueryArgs) {
|
||||
if (Array.isArray(assertQueryArgs.assertions) && assertQueryArgs.assertions.length > 0) {
|
||||
return assertQueryArgs.assertions.map((entry, index) => {
|
||||
if (!entry || typeof entry !== 'object') {
|
||||
throw new Error(`invalid assertion payload at index ${index}`);
|
||||
}
|
||||
const field = requireNonEmptyString(entry.field, 'assertion.field');
|
||||
const op = requireNonEmptyString(entry.op, 'assertion.op');
|
||||
if (!SUPPORTED_ASSERTION_OPERATORS.has(op)) {
|
||||
throw new Error('unsupported assertion operator');
|
||||
}
|
||||
if (!Object.prototype.hasOwnProperty.call(entry, 'value')) {
|
||||
throw new Error('assertion value is required');
|
||||
}
|
||||
return { field, op, value: parseAssertionValue(entry.value) };
|
||||
});
|
||||
}
|
||||
|
||||
if (assertQueryArgs.assert && typeof assertQueryArgs.assert === 'object') {
|
||||
return [
|
||||
{
|
||||
field: requireNonEmptyString(assertQueryArgs.assert.field, 'assert.field'),
|
||||
op: 'equals',
|
||||
value: assertQueryArgs.assert.equals,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
throw new Error('assertions must be a non-empty array');
|
||||
}
|
||||
|
||||
function parseQueryTextToMap(text) {
|
||||
const result = {};
|
||||
for (const line of String(text || '').split('\n')) {
|
||||
const separatorIndex = line.indexOf(': ');
|
||||
if (separatorIndex === -1) {
|
||||
continue;
|
||||
}
|
||||
const key = line.slice(0, separatorIndex).trim();
|
||||
const rawValue = line.slice(separatorIndex + 2).trim();
|
||||
result[key] = rawValue;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function evaluateAssertion(assertion, actualValue, assertionIndex) {
|
||||
const { field, op, value: expected } = assertion;
|
||||
|
||||
if (op === 'equals') {
|
||||
return String(actualValue) === String(expected)
|
||||
? null
|
||||
: {
|
||||
failedAssertionIndex: assertionIndex,
|
||||
field,
|
||||
op,
|
||||
expected,
|
||||
actual: actualValue,
|
||||
message: `assert_query failed: ${field} equals ${JSON.stringify(expected)} (actual: ${JSON.stringify(actualValue)})`,
|
||||
};
|
||||
}
|
||||
|
||||
if (op === 'contains') {
|
||||
if (typeof actualValue !== 'string') {
|
||||
throw new Error('contains expects a string field');
|
||||
}
|
||||
return actualValue.includes(String(expected))
|
||||
? null
|
||||
: {
|
||||
failedAssertionIndex: assertionIndex,
|
||||
field,
|
||||
op,
|
||||
expected,
|
||||
actual: actualValue,
|
||||
message: `assert_query failed: ${field} contains ${JSON.stringify(expected)} (actual: ${JSON.stringify(actualValue)})`,
|
||||
};
|
||||
}
|
||||
|
||||
const actualNumber = toComparableNumber(field, actualValue);
|
||||
const expectedNumber = toComparableNumber(field, expected);
|
||||
const passed =
|
||||
op === 'gt'
|
||||
? actualNumber > expectedNumber
|
||||
: op === 'gte'
|
||||
? actualNumber >= expectedNumber
|
||||
: op === 'lt'
|
||||
? actualNumber < expectedNumber
|
||||
: actualNumber <= expectedNumber;
|
||||
|
||||
return passed
|
||||
? null
|
||||
: {
|
||||
failedAssertionIndex: assertionIndex,
|
||||
field,
|
||||
op,
|
||||
expected: expectedNumber,
|
||||
actual: actualNumber,
|
||||
message: `assert_query failed: ${field} ${op} ${expectedNumber} (actual: ${actualNumber})`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatOrchestrationError(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
try {
|
||||
const parsed = JSON.parse(message);
|
||||
if (parsed && typeof parsed === 'object' && 'failedAssertionIndex' in parsed) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
// Keep plain-text error fallback below.
|
||||
}
|
||||
return { message };
|
||||
}
|
||||
|
||||
function requireOrchestrationBlocks(toolArgs) {
|
||||
const blocks = Array.isArray(toolArgs.blocks) ? toolArgs.blocks : null;
|
||||
if (!blocks || blocks.length === 0) {
|
||||
@@ -4965,10 +5100,18 @@ async function executeOrchestrationBlock(session, block) {
|
||||
|
||||
if (block.type === 'assert_query') {
|
||||
const queryText = await handleQuery({ pageId: session.pageId, ...block.args.query });
|
||||
const expectedField = String(block.args.assert.field || '');
|
||||
const expectedValue = String(block.args.assert.equals || '');
|
||||
if (!queryText.includes(`${expectedField}: ${expectedValue}`)) {
|
||||
throw new Error(`assert_query failed for field ${expectedField}`);
|
||||
const queryMap = parseQueryTextToMap(queryText);
|
||||
const assertions = parseAssertions(block.args);
|
||||
|
||||
for (const [index, assertion] of assertions.entries()) {
|
||||
const actualValue = queryMap[assertion.field];
|
||||
if (actualValue === undefined) {
|
||||
throw new Error(`assert_query field missing from query result: ${assertion.field}`);
|
||||
}
|
||||
const failure = evaluateAssertion(assertion, actualValue, index);
|
||||
if (failure) {
|
||||
throw new Error(JSON.stringify(failure));
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -5024,7 +5167,8 @@ async function handleStartOrchestration(toolArgs) {
|
||||
} catch (error) {
|
||||
session.status = 'failed';
|
||||
session.failedBlockIndex = session.currentBlockIndex;
|
||||
session.error = error instanceof Error ? error.message : String(error);
|
||||
session.errorDetails = formatOrchestrationError(error);
|
||||
session.error = session.errorDetails.message || (error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
activeOrchestrationSession = null;
|
||||
|
||||
@@ -6410,6 +6410,270 @@ describe('ccs-browser MCP server', () => {
|
||||
expect(text).toContain('failedBlockIndex: 0');
|
||||
});
|
||||
|
||||
it('passes assert_query with multiple structured assertions', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Assert Query Page',
|
||||
currentUrl: 'https://example.com/assert-query',
|
||||
query: {
|
||||
'#status': {
|
||||
exists: true,
|
||||
connected: true,
|
||||
innerText: 'ready state',
|
||||
textContent: 'ready state',
|
||||
display: 'block',
|
||||
visibility: 'visible',
|
||||
opacity: '0.95',
|
||||
rect: {
|
||||
x: 20,
|
||||
y: 30,
|
||||
width: 100,
|
||||
height: 40,
|
||||
top: 30,
|
||||
right: 120,
|
||||
bottom: 70,
|
||||
left: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 1301,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_start_orchestration',
|
||||
arguments: {
|
||||
blocks: [
|
||||
createOrchestrationBlock({
|
||||
type: 'assert_query',
|
||||
args: {
|
||||
query: { selector: '#status', fields: ['exists', 'innerText', 'opacity'] },
|
||||
assertions: [
|
||||
{ field: 'exists', op: 'equals', value: true },
|
||||
{ field: 'innerText', op: 'contains', value: 'ready' },
|
||||
{ field: 'opacity', op: 'gte', value: 0.9 },
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
expect(getResponseText(responses.find((message) => message.id === 1301))).toContain('status: completed');
|
||||
});
|
||||
|
||||
it('returns failedAssertionIndex, expected, and actual when assert_query fails', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Assert Failure Page',
|
||||
currentUrl: 'https://example.com/assert-failure',
|
||||
query: {
|
||||
'#status': {
|
||||
exists: true,
|
||||
connected: true,
|
||||
innerText: 'loading',
|
||||
textContent: 'loading',
|
||||
display: 'block',
|
||||
visibility: 'visible',
|
||||
opacity: '0.42',
|
||||
rect: {
|
||||
x: 20,
|
||||
y: 30,
|
||||
width: 100,
|
||||
height: 40,
|
||||
top: 30,
|
||||
right: 120,
|
||||
bottom: 70,
|
||||
left: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 1302,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_start_orchestration',
|
||||
arguments: {
|
||||
blocks: [
|
||||
createOrchestrationBlock({
|
||||
type: 'assert_query',
|
||||
args: {
|
||||
query: { selector: '#status', fields: ['innerText', 'opacity'] },
|
||||
assertions: [
|
||||
{ field: 'innerText', op: 'contains', value: 'ready' },
|
||||
{ field: 'opacity', op: 'gte', value: 0.9 },
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 1303,
|
||||
method: 'tools/call',
|
||||
params: { name: 'browser_get_orchestration', arguments: {} },
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const text = getResponseText(responses.find((message) => message.id === 1303));
|
||||
expect(text).toContain('status: failed');
|
||||
expect(text).toContain('failedBlockIndex: 0');
|
||||
expect(text).toContain('failedAssertionIndex: 0');
|
||||
expect(text).toContain('field: innerText');
|
||||
expect(text).toContain('expected: "ready"');
|
||||
expect(text).toContain('actual: "loading"');
|
||||
});
|
||||
|
||||
it('fails when numeric comparison is used on a non-number-like field or when op is unsupported', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Assert Type Error Page',
|
||||
currentUrl: 'https://example.com/assert-type-error',
|
||||
query: {
|
||||
'#status': {
|
||||
exists: true,
|
||||
connected: true,
|
||||
innerText: 'loading',
|
||||
textContent: 'loading',
|
||||
opacity: '0.42',
|
||||
display: 'block',
|
||||
visibility: 'visible',
|
||||
rect: {
|
||||
x: 20,
|
||||
y: 30,
|
||||
width: 100,
|
||||
height: 40,
|
||||
top: 30,
|
||||
right: 120,
|
||||
bottom: 70,
|
||||
left: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 1304,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_start_orchestration',
|
||||
arguments: {
|
||||
blocks: [
|
||||
createOrchestrationBlock({
|
||||
type: 'assert_query',
|
||||
args: {
|
||||
query: { selector: '#status', fields: ['innerText'] },
|
||||
assertions: [{ field: 'innerText', op: 'gte', value: 1 }],
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 1305,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_start_orchestration',
|
||||
arguments: {
|
||||
blocks: [
|
||||
createOrchestrationBlock({
|
||||
type: 'assert_query',
|
||||
args: {
|
||||
query: { selector: '#status', fields: ['opacity'] },
|
||||
assertions: [{ field: 'opacity', op: 'matches', value: '.*' }],
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
expect(getResponseText(responses.find((message) => message.id === 1304))).toContain('numeric comparison expects a number-like field: innerText');
|
||||
expect(getResponseText(responses.find((message) => message.id === 1305))).toContain('unsupported assertion operator');
|
||||
});
|
||||
|
||||
it('passes numeric gte and lt assertions', async () => {
|
||||
const responses = await runMcpRequests(
|
||||
[
|
||||
{
|
||||
id: 'page-1',
|
||||
title: 'Numeric Assertion Page',
|
||||
currentUrl: 'https://example.com/assert-numeric',
|
||||
query: {
|
||||
'#status': {
|
||||
exists: true,
|
||||
connected: true,
|
||||
opacity: '0.95',
|
||||
display: 'block',
|
||||
visibility: 'visible',
|
||||
rect: {
|
||||
x: 20,
|
||||
y: 30,
|
||||
width: 100,
|
||||
height: 40,
|
||||
top: 30,
|
||||
right: 120,
|
||||
bottom: 70,
|
||||
left: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: 1306,
|
||||
method: 'tools/call',
|
||||
params: {
|
||||
name: 'browser_start_orchestration',
|
||||
arguments: {
|
||||
blocks: [
|
||||
createOrchestrationBlock({
|
||||
type: 'assert_query',
|
||||
args: {
|
||||
query: { selector: '#status', fields: ['opacity'] },
|
||||
assertions: [
|
||||
{ field: 'opacity', op: 'gte', value: 0.9 },
|
||||
{ field: 'opacity', op: 'lt', value: 1.0 },
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
expect(getResponseText(responses.find((message) => message.id === 1306))).toContain('status: completed');
|
||||
});
|
||||
|
||||
it('drags local files onto normal, frame, and shadow dropzones', async () => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), 'ccs-browser-drag-files-'));
|
||||
const invoicePath = join(tempDir, 'invoice.pdf');
|
||||
|
||||
Reference in New Issue
Block a user