mirror of
https://github.com/microsoft/playwright-mcp.git
synced 2025-07-25 16:02:26 +08:00
feat: respond with action and generated locator (#181)
Closes https://github.com/microsoft/playwright-mcp/issues/163
This commit is contained in:
parent
4d59e06184
commit
4a19e18999
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,3 +1,4 @@
|
||||
lib/
|
||||
node_modules/
|
||||
test-results/
|
||||
.vscode/mcp.json
|
||||
|
@ -207,36 +207,52 @@ class Tab {
|
||||
await this.page.waitForLoadState('load', { timeout: 5000 }).catch(() => {});
|
||||
}
|
||||
|
||||
async run(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
|
||||
async run(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
|
||||
let actionCode: string | undefined;
|
||||
try {
|
||||
if (!options?.noClearFileChooser)
|
||||
this._fileChooser = undefined;
|
||||
if (options?.waitForCompletion)
|
||||
await waitForCompletion(this.page, () => callback(this));
|
||||
actionCode = await waitForCompletion(this.page, () => callback(this)) ?? undefined;
|
||||
else
|
||||
await callback(this);
|
||||
actionCode = await callback(this) ?? undefined;
|
||||
} finally {
|
||||
if (options?.captureSnapshot)
|
||||
this._snapshot = await PageSnapshot.create(this.page);
|
||||
}
|
||||
const tabList = this.context.tabs().length > 1 ? await this.context.listTabs() + '\n\nCurrent tab:' + '\n' : '';
|
||||
const snapshot = this._snapshot?.text({ status: options?.status, hasFileChooser: !!this._fileChooser }) ?? options?.status ?? '';
|
||||
|
||||
const result: string[] = [];
|
||||
if (options?.status)
|
||||
result.push(options.status, '');
|
||||
|
||||
if (this.context.tabs().length > 1)
|
||||
result.push(await this.context.listTabs(), '');
|
||||
|
||||
if (actionCode)
|
||||
result.push('- Action: ' + actionCode, '');
|
||||
|
||||
if (this._snapshot) {
|
||||
if (this.context.tabs().length > 1)
|
||||
result.push('Current tab:');
|
||||
result.push(this._snapshot.text({ hasFileChooser: !!this._fileChooser }));
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: tabList + snapshot,
|
||||
text: result.join('\n'),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async runAndWait(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
|
||||
async runAndWait(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
|
||||
return await this.run(callback, {
|
||||
waitForCompletion: true,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
|
||||
async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
|
||||
return await this.run(tab => callback(tab.lastSnapshot()), {
|
||||
captureSnapshot: true,
|
||||
waitForCompletion: true,
|
||||
@ -275,13 +291,9 @@ class PageSnapshot {
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
text(options?: { status?: string, hasFileChooser?: boolean }): string {
|
||||
text(options: { hasFileChooser: boolean }): string {
|
||||
const results: string[] = [];
|
||||
if (options?.status) {
|
||||
results.push(options.status);
|
||||
results.push('');
|
||||
}
|
||||
if (options?.hasFileChooser) {
|
||||
if (options.hasFileChooser) {
|
||||
results.push('- There is a file chooser visible that requires browser_file_upload to be called');
|
||||
results.push('');
|
||||
}
|
||||
@ -359,3 +371,7 @@ class PageSnapshot {
|
||||
return frame.locator(`aria-ref=${ref}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateLocator(locator: playwright.Locator): Promise<string> {
|
||||
return (locator as any)._generateLocatorString();
|
||||
}
|
||||
|
53
src/javascript.ts
Normal file
53
src/javascript.ts
Normal file
@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// adapted from:
|
||||
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
|
||||
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts
|
||||
|
||||
// NOTE: this function should not be used to escape any selectors.
|
||||
export function escapeWithQuotes(text: string, char: string = '\'') {
|
||||
const stringified = JSON.stringify(text);
|
||||
const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');
|
||||
if (char === '\'')
|
||||
return char + escapedText.replace(/[']/g, '\\\'') + char;
|
||||
if (char === '"')
|
||||
return char + escapedText.replace(/["]/g, '\\"') + char;
|
||||
if (char === '`')
|
||||
return char + escapedText.replace(/[`]/g, '`') + char;
|
||||
throw new Error('Invalid escape char');
|
||||
}
|
||||
|
||||
export function quote(text: string) {
|
||||
return escapeWithQuotes(text, '\'');
|
||||
}
|
||||
|
||||
export function formatObject(value: any, indent = ' '): string {
|
||||
if (typeof value === 'string')
|
||||
return quote(value);
|
||||
if (Array.isArray(value))
|
||||
return `[${value.map(o => formatObject(o)).join(', ')}]`;
|
||||
if (typeof value === 'object') {
|
||||
const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();
|
||||
if (!keys.length)
|
||||
return '{}';
|
||||
const tokens: string[] = [];
|
||||
for (const key of keys)
|
||||
tokens.push(`${key}: ${formatObject(value[key])}`);
|
||||
return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;
|
||||
}
|
||||
return String(value);
|
||||
}
|
@ -34,6 +34,7 @@ const pressKey: ToolFactory = captureSnapshot => ({
|
||||
const validatedParams = pressKeySchema.parse(params);
|
||||
return await context.currentTab().runAndWait(async tab => {
|
||||
await tab.page.keyboard.press(validatedParams.key);
|
||||
return `await page.keyboard.press('${validatedParams.key}');`;
|
||||
}, {
|
||||
status: `Pressed key ${validatedParams.key}`,
|
||||
captureSnapshot,
|
||||
|
@ -35,6 +35,7 @@ const navigate: ToolFactory = captureSnapshot => ({
|
||||
const currentTab = await context.ensureTab();
|
||||
return await currentTab.run(async tab => {
|
||||
await tab.navigate(validatedParams.url);
|
||||
return `await page.goto('${validatedParams.url}');`;
|
||||
}, {
|
||||
status: `Navigated to ${validatedParams.url}`,
|
||||
captureSnapshot,
|
||||
@ -54,6 +55,7 @@ const goBack: ToolFactory = snapshot => ({
|
||||
handle: async context => {
|
||||
return await context.currentTab().runAndWait(async tab => {
|
||||
await tab.page.goBack();
|
||||
return `await page.goBack();`;
|
||||
}, {
|
||||
status: 'Navigated back',
|
||||
captureSnapshot: snapshot,
|
||||
@ -73,6 +75,7 @@ const goForward: ToolFactory = snapshot => ({
|
||||
handle: async context => {
|
||||
return await context.currentTab().runAndWait(async tab => {
|
||||
await tab.page.goForward();
|
||||
return `await page.goForward();`;
|
||||
}, {
|
||||
status: 'Navigated forward',
|
||||
captureSnapshot: snapshot,
|
||||
|
@ -19,6 +19,8 @@ import zodToJsonSchema from 'zod-to-json-schema';
|
||||
|
||||
import type * as playwright from 'playwright';
|
||||
import type { Tool } from './tool';
|
||||
import { generateLocator } from '../context';
|
||||
import * as javascript from '../javascript';
|
||||
|
||||
const snapshot: Tool = {
|
||||
capability: 'core',
|
||||
@ -51,7 +53,9 @@ const click: Tool = {
|
||||
const validatedParams = elementSchema.parse(params);
|
||||
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
|
||||
const locator = snapshot.refLocator(validatedParams.ref);
|
||||
const action = `await page.${await generateLocator(locator)}.click();`;
|
||||
await locator.click();
|
||||
return action;
|
||||
}, {
|
||||
status: `Clicked "${validatedParams.element}"`,
|
||||
});
|
||||
@ -78,7 +82,9 @@ const drag: Tool = {
|
||||
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
|
||||
const startLocator = snapshot.refLocator(validatedParams.startRef);
|
||||
const endLocator = snapshot.refLocator(validatedParams.endRef);
|
||||
const action = `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`;
|
||||
await startLocator.dragTo(endLocator);
|
||||
return action;
|
||||
}, {
|
||||
status: `Dragged "${validatedParams.startElement}" to "${validatedParams.endElement}"`,
|
||||
});
|
||||
@ -97,7 +103,9 @@ const hover: Tool = {
|
||||
const validatedParams = elementSchema.parse(params);
|
||||
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
|
||||
const locator = snapshot.refLocator(validatedParams.ref);
|
||||
const action = `await page.${await generateLocator(locator)}.hover();`;
|
||||
await locator.hover();
|
||||
return action;
|
||||
}, {
|
||||
status: `Hovered over "${validatedParams.element}"`,
|
||||
});
|
||||
@ -122,12 +130,20 @@ const type: Tool = {
|
||||
const validatedParams = typeSchema.parse(params);
|
||||
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
|
||||
const locator = snapshot.refLocator(validatedParams.ref);
|
||||
if (validatedParams.slowly)
|
||||
|
||||
let action = '';
|
||||
if (validatedParams.slowly) {
|
||||
action = `await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(validatedParams.text)});`;
|
||||
await locator.pressSequentially(validatedParams.text);
|
||||
else
|
||||
} else {
|
||||
action = `await page.${await generateLocator(locator)}.fill(${javascript.quote(validatedParams.text)});`;
|
||||
await locator.fill(validatedParams.text);
|
||||
if (validatedParams.submit)
|
||||
}
|
||||
if (validatedParams.submit) {
|
||||
action += `\nawait page.${await generateLocator(locator)}.press('Enter');`;
|
||||
await locator.press('Enter');
|
||||
}
|
||||
return action;
|
||||
}, {
|
||||
status: `Typed "${validatedParams.text}" into "${validatedParams.element}"`,
|
||||
});
|
||||
@ -150,7 +166,9 @@ const selectOption: Tool = {
|
||||
const validatedParams = selectOptionSchema.parse(params);
|
||||
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
|
||||
const locator = snapshot.refLocator(validatedParams.ref);
|
||||
const action = `await page.${await generateLocator(locator)}.selectOption(${javascript.formatObject(validatedParams.values)});`;
|
||||
await locator.selectOption(validatedParams.values);
|
||||
return action;
|
||||
}, {
|
||||
status: `Selected option in "${validatedParams.element}"`,
|
||||
});
|
||||
|
@ -26,6 +26,8 @@ test('browser_navigate', async ({ client }) => {
|
||||
})).toHaveTextContent(`
|
||||
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
|
||||
- Action: await page.goto('data:text/html,<html><title>Title</title><body>Hello, world!</body></html>');
|
||||
|
||||
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
- Page Title: Title
|
||||
- Page Snapshot
|
||||
@ -50,7 +52,10 @@ test('browser_click', async ({ client }) => {
|
||||
element: 'Submit button',
|
||||
ref: 's1e3',
|
||||
},
|
||||
})).toHaveTextContent(`Clicked "Submit button"
|
||||
})).toHaveTextContent(`
|
||||
Clicked "Submit button"
|
||||
|
||||
- Action: await page.getByRole('button', { name: 'Submit' }).click();
|
||||
|
||||
- Page URL: data:text/html,<html><title>Title</title><button>Submit</button></html>
|
||||
- Page Title: Title
|
||||
@ -77,7 +82,10 @@ test('browser_select_option', async ({ client }) => {
|
||||
ref: 's1e3',
|
||||
values: ['bar'],
|
||||
},
|
||||
})).toHaveTextContent(`Selected option in "Select"
|
||||
})).toHaveTextContent(`
|
||||
Selected option in "Select"
|
||||
|
||||
- Action: await page.getByRole('combobox').selectOption(['bar']);
|
||||
|
||||
- Page URL: data:text/html,<html><title>Title</title><select><option value="foo">Foo</option><option value="bar">Bar</option></select></html>
|
||||
- Page Title: Title
|
||||
@ -105,7 +113,10 @@ test('browser_select_option (multiple)', async ({ client }) => {
|
||||
ref: 's1e3',
|
||||
values: ['bar', 'baz'],
|
||||
},
|
||||
})).toHaveTextContent(`Selected option in "Select"
|
||||
})).toHaveTextContent(`
|
||||
Selected option in "Select"
|
||||
|
||||
- Action: await page.getByRole('listbox').selectOption(['bar', 'baz']);
|
||||
|
||||
- Page URL: data:text/html,<html><title>Title</title><select multiple><option value="foo">Foo</option><option value="bar">Bar</option><option value="baz">Baz</option></select></html>
|
||||
- Page Title: Title
|
||||
|
@ -23,17 +23,7 @@ test('cdp server', async ({ cdpEndpoint, startClient }) => {
|
||||
arguments: {
|
||||
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
|
||||
},
|
||||
})).toHaveTextContent(`
|
||||
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
|
||||
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
- Page Title: Title
|
||||
- Page Snapshot
|
||||
\`\`\`yaml
|
||||
- text: Hello, world!
|
||||
\`\`\`
|
||||
`
|
||||
);
|
||||
})).toContainTextContent(`- text: Hello, world!`);
|
||||
});
|
||||
|
||||
test('cdp server reuse tab', async ({ cdpEndpoint, startClient }) => {
|
||||
|
@ -33,16 +33,7 @@ test('test reopen browser', async ({ client }) => {
|
||||
arguments: {
|
||||
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
|
||||
},
|
||||
})).toHaveTextContent(`
|
||||
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
|
||||
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
- Page Title: Title
|
||||
- Page Snapshot
|
||||
\`\`\`yaml
|
||||
- text: Hello, world!
|
||||
\`\`\`
|
||||
`);
|
||||
})).toContainTextContent(`- text: Hello, world!`);
|
||||
});
|
||||
|
||||
test('executable path', async ({ startClient }) => {
|
||||
|
@ -36,17 +36,7 @@ test('save as pdf', async ({ client }) => {
|
||||
arguments: {
|
||||
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
|
||||
},
|
||||
})).toHaveTextContent(`
|
||||
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
|
||||
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
|
||||
- Page Title: Title
|
||||
- Page Snapshot
|
||||
\`\`\`yaml
|
||||
- text: Hello, world!
|
||||
\`\`\`
|
||||
`
|
||||
);
|
||||
})).toContainTextContent(`- text: Hello, world!`);
|
||||
|
||||
const response = await client.callTool({
|
||||
name: 'browser_pdf_save',
|
||||
|
Loading…
x
Reference in New Issue
Block a user