feat: respond with action and generated locator (#181)

Closes https://github.com/microsoft/playwright-mcp/issues/163
This commit is contained in:
Simon Knott 2025-04-15 19:55:20 +02:00 committed by GitHub
parent 4d59e06184
commit 4a19e18999
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 126 additions and 52 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
lib/
node_modules/
test-results/
.vscode/mcp.json

View File

@ -207,36 +207,52 @@ class Tab {
await this.page.waitForLoadState('load', { timeout: 5000 }).catch(() => {});
}
async run(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
async run(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
let actionCode: string | undefined;
try {
if (!options?.noClearFileChooser)
this._fileChooser = undefined;
if (options?.waitForCompletion)
await waitForCompletion(this.page, () => callback(this));
actionCode = await waitForCompletion(this.page, () => callback(this)) ?? undefined;
else
await callback(this);
actionCode = await callback(this) ?? undefined;
} finally {
if (options?.captureSnapshot)
this._snapshot = await PageSnapshot.create(this.page);
}
const tabList = this.context.tabs().length > 1 ? await this.context.listTabs() + '\n\nCurrent tab:' + '\n' : '';
const snapshot = this._snapshot?.text({ status: options?.status, hasFileChooser: !!this._fileChooser }) ?? options?.status ?? '';
const result: string[] = [];
if (options?.status)
result.push(options.status, '');
if (this.context.tabs().length > 1)
result.push(await this.context.listTabs(), '');
if (actionCode)
result.push('- Action: ' + actionCode, '');
if (this._snapshot) {
if (this.context.tabs().length > 1)
result.push('Current tab:');
result.push(this._snapshot.text({ hasFileChooser: !!this._fileChooser }));
}
return {
content: [{
type: 'text',
text: tabList + snapshot,
text: result.join('\n'),
}],
};
}
async runAndWait(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
async runAndWait(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
return await this.run(callback, {
waitForCompletion: true,
...options,
});
}
async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
return await this.run(tab => callback(tab.lastSnapshot()), {
captureSnapshot: true,
waitForCompletion: true,
@ -275,13 +291,9 @@ class PageSnapshot {
return snapshot;
}
text(options?: { status?: string, hasFileChooser?: boolean }): string {
text(options: { hasFileChooser: boolean }): string {
const results: string[] = [];
if (options?.status) {
results.push(options.status);
results.push('');
}
if (options?.hasFileChooser) {
if (options.hasFileChooser) {
results.push('- There is a file chooser visible that requires browser_file_upload to be called');
results.push('');
}
@ -359,3 +371,7 @@ class PageSnapshot {
return frame.locator(`aria-ref=${ref}`);
}
}
export async function generateLocator(locator: playwright.Locator): Promise<string> {
return (locator as any)._generateLocatorString();
}

53
src/javascript.ts Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// adapted from:
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts
// NOTE: this function should not be used to escape any selectors.
export function escapeWithQuotes(text: string, char: string = '\'') {
const stringified = JSON.stringify(text);
const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');
if (char === '\'')
return char + escapedText.replace(/[']/g, '\\\'') + char;
if (char === '"')
return char + escapedText.replace(/["]/g, '\\"') + char;
if (char === '`')
return char + escapedText.replace(/[`]/g, '`') + char;
throw new Error('Invalid escape char');
}
export function quote(text: string) {
return escapeWithQuotes(text, '\'');
}
export function formatObject(value: any, indent = ' '): string {
if (typeof value === 'string')
return quote(value);
if (Array.isArray(value))
return `[${value.map(o => formatObject(o)).join(', ')}]`;
if (typeof value === 'object') {
const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();
if (!keys.length)
return '{}';
const tokens: string[] = [];
for (const key of keys)
tokens.push(`${key}: ${formatObject(value[key])}`);
return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;
}
return String(value);
}

View File

@ -34,6 +34,7 @@ const pressKey: ToolFactory = captureSnapshot => ({
const validatedParams = pressKeySchema.parse(params);
return await context.currentTab().runAndWait(async tab => {
await tab.page.keyboard.press(validatedParams.key);
return `await page.keyboard.press('${validatedParams.key}');`;
}, {
status: `Pressed key ${validatedParams.key}`,
captureSnapshot,

View File

@ -35,6 +35,7 @@ const navigate: ToolFactory = captureSnapshot => ({
const currentTab = await context.ensureTab();
return await currentTab.run(async tab => {
await tab.navigate(validatedParams.url);
return `await page.goto('${validatedParams.url}');`;
}, {
status: `Navigated to ${validatedParams.url}`,
captureSnapshot,
@ -54,6 +55,7 @@ const goBack: ToolFactory = snapshot => ({
handle: async context => {
return await context.currentTab().runAndWait(async tab => {
await tab.page.goBack();
return `await page.goBack();`;
}, {
status: 'Navigated back',
captureSnapshot: snapshot,
@ -73,6 +75,7 @@ const goForward: ToolFactory = snapshot => ({
handle: async context => {
return await context.currentTab().runAndWait(async tab => {
await tab.page.goForward();
return `await page.goForward();`;
}, {
status: 'Navigated forward',
captureSnapshot: snapshot,

View File

@ -19,6 +19,8 @@ import zodToJsonSchema from 'zod-to-json-schema';
import type * as playwright from 'playwright';
import type { Tool } from './tool';
import { generateLocator } from '../context';
import * as javascript from '../javascript';
const snapshot: Tool = {
capability: 'core',
@ -51,7 +53,9 @@ const click: Tool = {
const validatedParams = elementSchema.parse(params);
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
const locator = snapshot.refLocator(validatedParams.ref);
const action = `await page.${await generateLocator(locator)}.click();`;
await locator.click();
return action;
}, {
status: `Clicked "${validatedParams.element}"`,
});
@ -78,7 +82,9 @@ const drag: Tool = {
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
const startLocator = snapshot.refLocator(validatedParams.startRef);
const endLocator = snapshot.refLocator(validatedParams.endRef);
const action = `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`;
await startLocator.dragTo(endLocator);
return action;
}, {
status: `Dragged "${validatedParams.startElement}" to "${validatedParams.endElement}"`,
});
@ -97,7 +103,9 @@ const hover: Tool = {
const validatedParams = elementSchema.parse(params);
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
const locator = snapshot.refLocator(validatedParams.ref);
const action = `await page.${await generateLocator(locator)}.hover();`;
await locator.hover();
return action;
}, {
status: `Hovered over "${validatedParams.element}"`,
});
@ -122,12 +130,20 @@ const type: Tool = {
const validatedParams = typeSchema.parse(params);
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
const locator = snapshot.refLocator(validatedParams.ref);
if (validatedParams.slowly)
let action = '';
if (validatedParams.slowly) {
action = `await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(validatedParams.text)});`;
await locator.pressSequentially(validatedParams.text);
else
} else {
action = `await page.${await generateLocator(locator)}.fill(${javascript.quote(validatedParams.text)});`;
await locator.fill(validatedParams.text);
if (validatedParams.submit)
}
if (validatedParams.submit) {
action += `\nawait page.${await generateLocator(locator)}.press('Enter');`;
await locator.press('Enter');
}
return action;
}, {
status: `Typed "${validatedParams.text}" into "${validatedParams.element}"`,
});
@ -150,7 +166,9 @@ const selectOption: Tool = {
const validatedParams = selectOptionSchema.parse(params);
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
const locator = snapshot.refLocator(validatedParams.ref);
const action = `await page.${await generateLocator(locator)}.selectOption(${javascript.formatObject(validatedParams.values)});`;
await locator.selectOption(validatedParams.values);
return action;
}, {
status: `Selected option in "${validatedParams.element}"`,
});

View File

@ -26,6 +26,8 @@ test('browser_navigate', async ({ client }) => {
})).toHaveTextContent(`
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Action: await page.goto('data:text/html,<html><title>Title</title><body>Hello, world!</body></html>');
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page Title: Title
- Page Snapshot
@ -50,7 +52,10 @@ test('browser_click', async ({ client }) => {
element: 'Submit button',
ref: 's1e3',
},
})).toHaveTextContent(`Clicked "Submit button"
})).toHaveTextContent(`
Clicked "Submit button"
- Action: await page.getByRole('button', { name: 'Submit' }).click();
- Page URL: data:text/html,<html><title>Title</title><button>Submit</button></html>
- Page Title: Title
@ -77,7 +82,10 @@ test('browser_select_option', async ({ client }) => {
ref: 's1e3',
values: ['bar'],
},
})).toHaveTextContent(`Selected option in "Select"
})).toHaveTextContent(`
Selected option in "Select"
- Action: await page.getByRole('combobox').selectOption(['bar']);
- Page URL: data:text/html,<html><title>Title</title><select><option value="foo">Foo</option><option value="bar">Bar</option></select></html>
- Page Title: Title
@ -105,7 +113,10 @@ test('browser_select_option (multiple)', async ({ client }) => {
ref: 's1e3',
values: ['bar', 'baz'],
},
})).toHaveTextContent(`Selected option in "Select"
})).toHaveTextContent(`
Selected option in "Select"
- Action: await page.getByRole('listbox').selectOption(['bar', 'baz']);
- Page URL: data:text/html,<html><title>Title</title><select multiple><option value="foo">Foo</option><option value="bar">Bar</option><option value="baz">Baz</option></select></html>
- Page Title: Title

View File

@ -23,17 +23,7 @@ test('cdp server', async ({ cdpEndpoint, startClient }) => {
arguments: {
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
},
})).toHaveTextContent(`
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page Title: Title
- Page Snapshot
\`\`\`yaml
- text: Hello, world!
\`\`\`
`
);
})).toContainTextContent(`- text: Hello, world!`);
});
test('cdp server reuse tab', async ({ cdpEndpoint, startClient }) => {

View File

@ -33,16 +33,7 @@ test('test reopen browser', async ({ client }) => {
arguments: {
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
},
})).toHaveTextContent(`
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page Title: Title
- Page Snapshot
\`\`\`yaml
- text: Hello, world!
\`\`\`
`);
})).toContainTextContent(`- text: Hello, world!`);
});
test('executable path', async ({ startClient }) => {

View File

@ -36,17 +36,7 @@ test('save as pdf', async ({ client }) => {
arguments: {
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
},
})).toHaveTextContent(`
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page Title: Title
- Page Snapshot
\`\`\`yaml
- text: Hello, world!
\`\`\`
`
);
})).toContainTextContent(`- text: Hello, world!`);
const response = await client.callTool({
name: 'browser_pdf_save',