feat: respond with action and generated locator (#181)

Closes https://github.com/microsoft/playwright-mcp/issues/163

.gitignore

@@ -1,3 +1,4 @@
 lib/
 node_modules/
 test-results/
+.vscode/mcp.json

src/context.ts

@@ -207,36 +207,52 @@ class Tab {
     await this.page.waitForLoadState('load', { timeout: 5000 }).catch(() => {});
   }
 
-  async run(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
+  async run(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
+    let actionCode: string | undefined;
     try {
       if (!options?.noClearFileChooser)
         this._fileChooser = undefined;
       if (options?.waitForCompletion)
-        await waitForCompletion(this.page, () => callback(this));
+        actionCode = await waitForCompletion(this.page, () => callback(this)) ?? undefined;
       else
-        await callback(this);
+        actionCode = await callback(this) ?? undefined;
     } finally {
       if (options?.captureSnapshot)
         this._snapshot = await PageSnapshot.create(this.page);
     }
-    const tabList = this.context.tabs().length > 1 ? await this.context.listTabs() + '\n\nCurrent tab:' + '\n' : '';
+
-    const snapshot = this._snapshot?.text({ status: options?.status, hasFileChooser: !!this._fileChooser }) ?? options?.status ?? '';
+    const result: string[] = [];
+    if (options?.status)
+      result.push(options.status, '');
+
+    if (this.context.tabs().length > 1)
+      result.push(await this.context.listTabs(), '');
+
+    if (actionCode)
+      result.push('- Action: ' + actionCode, '');
+
+    if (this._snapshot) {
+      if (this.context.tabs().length > 1)
+        result.push('Current tab:');
+      result.push(this._snapshot.text({ hasFileChooser: !!this._fileChooser }));
+    }
+
     return {
       content: [{
         type: 'text',
-        text: tabList + snapshot,
+        text: result.join('\n'),
       }],
     };
   }
 
-  async runAndWait(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
+  async runAndWait(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
     return await this.run(callback, {
       waitForCompletion: true,
       ...options,
     });
   }
 
-  async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
+  async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
     return await this.run(tab => callback(tab.lastSnapshot()), {
       captureSnapshot: true,
       waitForCompletion: true,
@@ -275,13 +291,9 @@ class PageSnapshot {
     return snapshot;
   }
 
-  text(options?: { status?: string, hasFileChooser?: boolean }): string {
+  text(options: { hasFileChooser: boolean }): string {
     const results: string[] = [];
-    if (options?.status) {
+    if (options.hasFileChooser) {
-      results.push(options.status);
-      results.push('');
-    }
-    if (options?.hasFileChooser) {
       results.push('- There is a file chooser visible that requires browser_file_upload to be called');
       results.push('');
     }
@@ -359,3 +371,7 @@ class PageSnapshot {
     return frame.locator(`aria-ref=${ref}`);
   }
 }
+
+export async function generateLocator(locator: playwright.Locator): Promise<string> {
+  return (locator as any)._generateLocatorString();
+}

src/javascript.ts

@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// adapted from:
+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts
+
+// NOTE: this function should not be used to escape any selectors.
+export function escapeWithQuotes(text: string, char: string = '\'') {
+  const stringified = JSON.stringify(text);
+  const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');
+  if (char === '\'')
+    return char + escapedText.replace(/[']/g, '\\\'') + char;
+  if (char === '"')
+    return char + escapedText.replace(/["]/g, '\\"') + char;
+  if (char === '`')
+    return char + escapedText.replace(/[`]/g, '`') + char;
+  throw new Error('Invalid escape char');
+}
+
+export function quote(text: string) {
+  return escapeWithQuotes(text, '\'');
+}
+
+export function formatObject(value: any, indent = '  '): string {
+  if (typeof value === 'string')
+    return quote(value);
+  if (Array.isArray(value))
+    return `[${value.map(o => formatObject(o)).join(', ')}]`;
+  if (typeof value === 'object') {
+    const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();
+    if (!keys.length)
+      return '{}';
+    const tokens: string[] = [];
+    for (const key of keys)
+      tokens.push(`${key}: ${formatObject(value[key])}`);
+    return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;
+  }
+  return String(value);
+}

src/tools/keyboard.ts

@@ -34,6 +34,7 @@ const pressKey: ToolFactory = captureSnapshot => ({
     const validatedParams = pressKeySchema.parse(params);
     return await context.currentTab().runAndWait(async tab => {
       await tab.page.keyboard.press(validatedParams.key);
+      return `await page.keyboard.press('${validatedParams.key}');`;
     }, {
       status: `Pressed key ${validatedParams.key}`,
       captureSnapshot,

src/tools/navigate.ts

@@ -35,6 +35,7 @@ const navigate: ToolFactory = captureSnapshot => ({
     const currentTab = await context.ensureTab();
     return await currentTab.run(async tab => {
       await tab.navigate(validatedParams.url);
+      return `await page.goto('${validatedParams.url}');`;
     }, {
       status: `Navigated to ${validatedParams.url}`,
       captureSnapshot,
@@ -54,6 +55,7 @@ const goBack: ToolFactory = snapshot => ({
   handle: async context => {
     return await context.currentTab().runAndWait(async tab => {
       await tab.page.goBack();
+      return `await page.goBack();`;
     }, {
       status: 'Navigated back',
       captureSnapshot: snapshot,
@@ -73,6 +75,7 @@ const goForward: ToolFactory = snapshot => ({
   handle: async context => {
     return await context.currentTab().runAndWait(async tab => {
       await tab.page.goForward();
+      return `await page.goForward();`;
     }, {
       status: 'Navigated forward',
       captureSnapshot: snapshot,

src/tools/snapshot.ts

@@ -19,6 +19,8 @@ import zodToJsonSchema from 'zod-to-json-schema';
 
 import type * as playwright from 'playwright';
 import type { Tool } from './tool';
+import { generateLocator } from '../context';
+import * as javascript from '../javascript';
 
 const snapshot: Tool = {
   capability: 'core',
@@ -51,7 +53,9 @@ const click: Tool = {
     const validatedParams = elementSchema.parse(params);
     return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
       const locator = snapshot.refLocator(validatedParams.ref);
+      const action = `await page.${await generateLocator(locator)}.click();`;
       await locator.click();
+      return action;
     }, {
       status: `Clicked "${validatedParams.element}"`,
     });
@@ -78,7 +82,9 @@ const drag: Tool = {
     return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
       const startLocator = snapshot.refLocator(validatedParams.startRef);
       const endLocator = snapshot.refLocator(validatedParams.endRef);
+      const action = `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`;
       await startLocator.dragTo(endLocator);
+      return action;
     }, {
       status: `Dragged "${validatedParams.startElement}" to "${validatedParams.endElement}"`,
     });
@@ -97,7 +103,9 @@ const hover: Tool = {
     const validatedParams = elementSchema.parse(params);
     return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
       const locator = snapshot.refLocator(validatedParams.ref);
+      const action = `await page.${await generateLocator(locator)}.hover();`;
       await locator.hover();
+      return action;
     }, {
       status: `Hovered over "${validatedParams.element}"`,
     });
@@ -122,12 +130,20 @@ const type: Tool = {
     const validatedParams = typeSchema.parse(params);
     return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
       const locator = snapshot.refLocator(validatedParams.ref);
-      if (validatedParams.slowly)
+
+      let action = '';
+      if (validatedParams.slowly) {
+        action = `await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(validatedParams.text)});`;
         await locator.pressSequentially(validatedParams.text);
-      else
+      } else {
+        action = `await page.${await generateLocator(locator)}.fill(${javascript.quote(validatedParams.text)});`;
         await locator.fill(validatedParams.text);
-      if (validatedParams.submit)
+      }
+      if (validatedParams.submit) {
+        action += `\nawait page.${await generateLocator(locator)}.press('Enter');`;
         await locator.press('Enter');
+      }
+      return action;
     }, {
       status: `Typed "${validatedParams.text}" into "${validatedParams.element}"`,
     });
@@ -150,7 +166,9 @@ const selectOption: Tool = {
     const validatedParams = selectOptionSchema.parse(params);
     return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
       const locator = snapshot.refLocator(validatedParams.ref);
+      const action = `await page.${await generateLocator(locator)}.selectOption(${javascript.formatObject(validatedParams.values)});`;
       await locator.selectOption(validatedParams.values);
+      return action;
     }, {
       status: `Selected option in "${validatedParams.element}"`,
     });

tests/basic.spec.ts

@@ -26,6 +26,8 @@ test('browser_navigate', async ({ client }) => {
   })).toHaveTextContent(`
 Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
 
+- Action: await page.goto('data:text/html,<html><title>Title</title><body>Hello, world!</body></html>');
+
 - Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
 - Page Title: Title
 - Page Snapshot
@@ -50,7 +52,10 @@ test('browser_click', async ({ client }) => {
       element: 'Submit button',
       ref: 's1e3',
     },
-  })).toHaveTextContent(`Clicked "Submit button"
+  })).toHaveTextContent(`
+Clicked "Submit button"
+
+- Action: await page.getByRole('button', { name: 'Submit' }).click();
 
 - Page URL: data:text/html,<html><title>Title</title><button>Submit</button></html>
 - Page Title: Title
@@ -77,7 +82,10 @@ test('browser_select_option', async ({ client }) => {
       ref: 's1e3',
       values: ['bar'],
     },
-  })).toHaveTextContent(`Selected option in "Select"
+  })).toHaveTextContent(`
+Selected option in "Select"
+
+- Action: await page.getByRole('combobox').selectOption(['bar']);
 
 - Page URL: data:text/html,<html><title>Title</title><select><option value="foo">Foo</option><option value="bar">Bar</option></select></html>
 - Page Title: Title
@@ -105,7 +113,10 @@ test('browser_select_option (multiple)', async ({ client }) => {
       ref: 's1e3',
       values: ['bar', 'baz'],
     },
-  })).toHaveTextContent(`Selected option in "Select"
+  })).toHaveTextContent(`
+Selected option in "Select"
+
+- Action: await page.getByRole('listbox').selectOption(['bar', 'baz']);
 
 - Page URL: data:text/html,<html><title>Title</title><select multiple><option value="foo">Foo</option><option value="bar">Bar</option><option value="baz">Baz</option></select></html>
 - Page Title: Title

tests/cdp.spec.ts

@@ -23,17 +23,7 @@ test('cdp server', async ({ cdpEndpoint, startClient }) => {
     arguments: {
       url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
     },
-  })).toHaveTextContent(`
+  })).toContainTextContent(`- text: Hello, world!`);
-Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
-
-- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
-- Page Title: Title
-- Page Snapshot
-\`\`\`yaml
-- text: Hello, world!
-\`\`\`
-`
-  );
 });
 
 test('cdp server reuse tab', async ({ cdpEndpoint, startClient }) => {

tests/launch.spec.ts

@@ -33,16 +33,7 @@ test('test reopen browser', async ({ client }) => {
     arguments: {
       url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
     },
-  })).toHaveTextContent(`
+  })).toContainTextContent(`- text: Hello, world!`);
-Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
-
-- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
-- Page Title: Title
-- Page Snapshot
-\`\`\`yaml
-- text: Hello, world!
-\`\`\`
-`);
 });
 
 test('executable path', async ({ startClient }) => {

tests/pdf.spec.ts

@@ -36,17 +36,7 @@ test('save as pdf', async ({ client }) => {
     arguments: {
       url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
     },
-  })).toHaveTextContent(`
+  })).toContainTextContent(`- text: Hello, world!`);
-Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
-
-- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
-- Page Title: Title
-- Page Snapshot
-\`\`\`yaml
-- text: Hello, world!
-\`\`\`
-`
-  );
 
   const response = await client.callTool({
     name: 'browser_pdf_save',