diff --git a/src/context.ts b/src/context.ts index 77fb725..226b107 100644 --- a/src/context.ts +++ b/src/context.ts @@ -19,7 +19,8 @@ import yaml from 'yaml'; import { waitForCompletion } from './tools/utils'; -import type { ModalState, Tool, ToolResult } from './tools/tool'; +import type { ImageContent, TextContent } from '@modelcontextprotocol/sdk/types'; +import type { ModalState, Tool } from './tools/tool'; export type ContextOptions = { browserName?: 'chromium' | 'firefox' | 'webkit'; @@ -31,11 +32,6 @@ export type ContextOptions = { type PageOrFrameLocator = playwright.Page | playwright.FrameLocator; -type RunOptions = { - captureSnapshot?: boolean; - waitForCompletion?: boolean; -}; - export class Context { readonly tools: Tool[]; readonly options: ContextOptions; @@ -75,7 +71,7 @@ export class Context { return this._tabs; } - currentTab(): Tab { + currentTabOrDie(): Tab { if (!this._currentTab) throw new Error('No current snapshot available. Capture a snapshot of navigate to a new location first.'); return this._currentTab; @@ -100,7 +96,7 @@ export class Context { return this._currentTab!; } - async listTabs(): Promise { + async listTabsMarkdown(): Promise { if (!this._tabs.length) return '### No tabs open'; const lines: string[] = ['### Open tabs']; @@ -115,9 +111,75 @@ export class Context { } async closeTab(index: number | undefined) { - const tab = index === undefined ? this.currentTab() : this._tabs[index - 1]; - await tab.page.close(); - return await this.listTabs(); + const tab = index === undefined ? this._currentTab : this._tabs[index - 1]; + await tab?.page.close(); + return await this.listTabsMarkdown(); + } + + async run(tool: Tool, params: Record | undefined) { + // Tab management is done outside of the action() call. + const toolResult = await tool.handle(this, params); + const { code, action, waitForNetwork, captureSnapshot } = toolResult; + + if (!this._currentTab) { + return { + content: [{ + type: 'text', + text: 'No open pages available. Use the "browser_navigate" tool to navigate to a page first.', + }], + }; + } + + const tab = this.currentTabOrDie(); + // TODO: race against modal dialogs to resolve clicks. + let actionResult: { content?: (ImageContent | TextContent)[] }; + try { + if (waitForNetwork) + actionResult = await waitForCompletion(tab.page, () => action()) ?? undefined; + else + actionResult = await action(); + } finally { + if (captureSnapshot) + await tab.captureSnapshot(); + } + + const result: string[] = []; + result.push(`- Ran Playwright code: +\`\`\`js +${code.join('\n')} +\`\`\` +`); + + if (this.modalStates().length) { + result.push(...this.modalStatesMarkdown()); + return { + content: [{ + type: 'text', + text: result.join('\n'), + }], + }; + } + + if (this.tabs().length > 1) + result.push(await this.listTabsMarkdown(), ''); + + if (tab.hasSnapshot()) { + if (this.tabs().length > 1) + result.push('### Current tab'); + result.push(tab.snapshotOrDie().text()); + } + + const content = actionResult?.content ?? []; + + return { + content: [ + ...content, + { + type: 'text', + text: result.join('\n'), + } + ], + }; } private _onPageCreated(page: playwright.Page) { @@ -199,17 +261,7 @@ export class Context { } } -type RunResult = { - code: string[]; - images?: ImageContent[]; -}; - -type ImageContent = { - data: string; - mimeType: string; -}; - -class Tab { +export class Tab { readonly context: Context; readonly page: playwright.Page; private _console: playwright.ConsoleMessage[] = []; @@ -248,76 +300,11 @@ class Tab { await this.page.waitForLoadState('load', { timeout: 5000 }).catch(() => {}); } - async run(callback: (tab: Tab) => Promise, options?: RunOptions): Promise { - let runResult: RunResult | undefined; - try { - if (options?.waitForCompletion) - runResult = await waitForCompletion(this.page, () => callback(this)) ?? undefined; - else - runResult = await callback(this) ?? undefined; - } finally { - if (options?.captureSnapshot) - this._snapshot = await PageSnapshot.create(this.page); - } - - const result: string[] = []; - result.push(`- Ran Playwright code: -\`\`\`js -${runResult.code.join('\n')} -\`\`\` -`); - - if (this.context.modalStates().length) { - result.push(...this.context.modalStatesMarkdown()); - return { - content: [{ - type: 'text', - text: result.join('\n'), - }], - }; - } - - if (this.context.tabs().length > 1) - result.push(await this.context.listTabs(), ''); - - if (this._snapshot) { - if (this.context.tabs().length > 1) - result.push('### Current tab'); - result.push(this._snapshot.text()); - } - - const images = runResult.images?.map(image => { - return { - type: 'image' as 'image', - data: image.data, - mimeType: image.mimeType, - }; - }) ?? []; - - return { - content: [...images, { - type: 'text', - text: result.join('\n'), - }], - }; + hasSnapshot(): boolean { + return !!this._snapshot; } - async runAndWait(callback: (tab: Tab) => Promise, options?: RunOptions): Promise { - return await this.run(callback, { - waitForCompletion: true, - ...options, - }); - } - - async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise, options?: RunOptions): Promise { - return await this.run(tab => callback(tab.lastSnapshot()), { - captureSnapshot: true, - waitForCompletion: true, - ...options, - }); - } - - lastSnapshot(): PageSnapshot { + snapshotOrDie(): PageSnapshot { if (!this._snapshot) throw new Error('No snapshot available'); return this._snapshot; @@ -326,6 +313,10 @@ ${runResult.code.join('\n')} async console(): Promise { return this._console; } + + async captureSnapshot() { + this._snapshot = await PageSnapshot.create(this.page); + } } class PageSnapshot { diff --git a/src/server.ts b/src/server.ts index 2d81c8b..bc5e8e3 100644 --- a/src/server.ts +++ b/src/server.ts @@ -71,7 +71,7 @@ export function createServerWithTools(options: Options): Server { } try { - return await tool.handle(context, request.params.arguments); + return await context.run(tool, request.params.arguments); } catch (error) { return { content: [{ type: 'text', text: String(error) }], diff --git a/src/tools/common.ts b/src/tools/common.ts index 784c890..dfeb6ed 100644 --- a/src/tools/common.ts +++ b/src/tools/common.ts @@ -23,41 +23,45 @@ const waitSchema = z.object({ time: z.number().describe('The time to wait in seconds'), }); -const wait: Tool = { +const wait: ToolFactory = captureSnapshot => ({ capability: 'wait', + schema: { name: 'browser_wait', description: 'Wait for a specified time in seconds', inputSchema: zodToJsonSchema(waitSchema), }, + handle: async (context, params) => { const validatedParams = waitSchema.parse(params); await new Promise(f => setTimeout(f, Math.min(10000, validatedParams.time * 1000))); return { - content: [{ - type: 'text', - text: `Waited for ${validatedParams.time} seconds`, - }], + code: [`// Waited for ${validatedParams.time} seconds`], + action: async () => ({}), + captureSnapshot, + waitForNetwork: false, }; }, -}; +}); const closeSchema = z.object({}); const close: Tool = { capability: 'core', + schema: { name: 'browser_close', description: 'Close the page', inputSchema: zodToJsonSchema(closeSchema), }, + handle: async context => { await context.close(); return { - content: [{ - type: 'text', - text: `Page closed`, - }], + code: [`// Internal to close the page`], + action: async () => ({}), + captureSnapshot: false, + waitForNetwork: false, }; }, }; @@ -74,25 +78,33 @@ const resize: ToolFactory = captureSnapshot => ({ description: 'Resize the browser window', inputSchema: zodToJsonSchema(resizeSchema), }, + handle: async (context, params) => { const validatedParams = resizeSchema.parse(params); - const tab = context.currentTab(); - return await tab.run(async tab => { + const tab = context.currentTabOrDie(); + + const code = [ + `// Resize browser window to ${validatedParams.width}x${validatedParams.height}`, + `await page.setViewportSize({ width: ${validatedParams.width}, height: ${validatedParams.height} });` + ]; + + const action = async () => { await tab.page.setViewportSize({ width: validatedParams.width, height: validatedParams.height }); - const code = [ - `// Resize browser window to ${validatedParams.width}x${validatedParams.height}`, - `await page.setViewportSize({ width: ${validatedParams.width}, height: ${validatedParams.height} });` - ]; - return { code }; - }, { + return {}; + }; + + return { + code, + action, captureSnapshot, - }); + waitForNetwork: true + }; }, }); export default (captureSnapshot: boolean) => [ close, - wait, + wait(captureSnapshot), resize(captureSnapshot) ]; diff --git a/src/tools/console.ts b/src/tools/console.ts index 29132e5..ecc8bb1 100644 --- a/src/tools/console.ts +++ b/src/tools/console.ts @@ -29,13 +29,17 @@ const console: Tool = { inputSchema: zodToJsonSchema(consoleSchema), }, handle: async context => { - const messages = await context.currentTab().console(); + const messages = await context.currentTabOrDie().console(); const log = messages.map(message => `[${message.type().toUpperCase()}] ${message.text()}`).join('\n'); return { - content: [{ - type: 'text', - text: log - }], + code: [`// `], + action: async () => { + return { + content: [{ type: 'text', text: log }] + }; + }, + captureSnapshot: false, + waitForNetwork: false, }; }, }; diff --git a/src/tools/files.ts b/src/tools/files.ts index 13c5b6c..f866be2 100644 --- a/src/tools/files.ts +++ b/src/tools/files.ts @@ -25,27 +25,35 @@ const uploadFileSchema = z.object({ const uploadFile: ToolFactory = captureSnapshot => ({ capability: 'files', + schema: { name: 'browser_file_upload', description: 'Upload one or multiple files', inputSchema: zodToJsonSchema(uploadFileSchema), }, + handle: async (context, params) => { const validatedParams = uploadFileSchema.parse(params); - const tab = context.currentTab(); - return await tab.runAndWait(async () => { - const modalState = context.modalStates().find(state => state.type === 'fileChooser'); - if (!modalState) - throw new Error('No file chooser visible'); + const modalState = context.modalStates().find(state => state.type === 'fileChooser'); + if (!modalState) + throw new Error('No file chooser visible'); + + const code = [ + `// { await modalState.fileChooser.setFiles(validatedParams.paths); context.clearModalState(modalState); - const code = [ - `// ({}), + captureSnapshot: false, + waitForNetwork: false, }; }, }; diff --git a/src/tools/keyboard.ts b/src/tools/keyboard.ts index df38d0c..972ecc5 100644 --- a/src/tools/keyboard.ts +++ b/src/tools/keyboard.ts @@ -25,23 +25,30 @@ const pressKeySchema = z.object({ const pressKey: ToolFactory = captureSnapshot => ({ capability: 'core', + schema: { name: 'browser_press_key', description: 'Press a key on the keyboard', inputSchema: zodToJsonSchema(pressKeySchema), }, + handle: async (context, params) => { const validatedParams = pressKeySchema.parse(params); - return await context.currentTab().runAndWait(async tab => { - await tab.page.keyboard.press(validatedParams.key); - const code = [ - `// Press ${validatedParams.key}`, - `await page.keyboard.press('${validatedParams.key}');`, - ]; - return { code }; - }, { + const tab = context.currentTabOrDie(); + + const code = [ + `// Press ${validatedParams.key}`, + `await page.keyboard.press('${validatedParams.key}');`, + ]; + + const action = () => tab.page.keyboard.press(validatedParams.key).then(() => ({})); + + return { + code, + action, captureSnapshot, - }); + waitForNetwork: true + }; }, }); diff --git a/src/tools/navigate.ts b/src/tools/navigate.ts index 3ed8878..664a584 100644 --- a/src/tools/navigate.ts +++ b/src/tools/navigate.ts @@ -25,53 +25,62 @@ const navigateSchema = z.object({ const navigate: ToolFactory = captureSnapshot => ({ capability: 'core', + schema: { name: 'browser_navigate', description: 'Navigate to a URL', inputSchema: zodToJsonSchema(navigateSchema), }, + handle: async (context, params) => { const validatedParams = navigateSchema.parse(params); - const currentTab = await context.ensureTab(); - return await currentTab.run(async tab => { - await tab.navigate(validatedParams.url); - const code = [ - `// Navigate to ${validatedParams.url}`, - `await page.goto('${validatedParams.url}');`, - ]; - return { code }; - }, { + const tab = await context.ensureTab(); + await tab.navigate(validatedParams.url); + + const code = [ + `// Navigate to ${validatedParams.url}`, + `await page.goto('${validatedParams.url}');`, + ]; + + return { + code, + action: async () => ({}), captureSnapshot, - }); + waitForNetwork: false, + }; }, }); const goBackSchema = z.object({}); -const goBack: ToolFactory = snapshot => ({ +const goBack: ToolFactory = captureSnapshot => ({ capability: 'history', schema: { name: 'browser_navigate_back', description: 'Go back to the previous page', inputSchema: zodToJsonSchema(goBackSchema), }, + handle: async context => { - return await context.currentTab().runAndWait(async tab => { - await tab.page.goBack(); - const code = [ - `// Navigate back`, - `await page.goBack();`, - ]; - return { code }; - }, { - captureSnapshot: snapshot, - }); + const tab = await context.ensureTab(); + await tab.page.goBack(); + const code = [ + `// Navigate back`, + `await page.goBack();`, + ]; + + return { + code, + action: async () => ({}), + captureSnapshot, + waitForNetwork: false, + }; }, }); const goForwardSchema = z.object({}); -const goForward: ToolFactory = snapshot => ({ +const goForward: ToolFactory = captureSnapshot => ({ capability: 'history', schema: { name: 'browser_navigate_forward', @@ -79,16 +88,18 @@ const goForward: ToolFactory = snapshot => ({ inputSchema: zodToJsonSchema(goForwardSchema), }, handle: async context => { - return await context.currentTab().runAndWait(async tab => { - await tab.page.goForward(); - const code = [ - `// Navigate forward`, - `await page.goForward();`, - ]; - return { code }; - }, { - captureSnapshot: snapshot, - }); + const tab = context.currentTabOrDie(); + await tab.page.goForward(); + const code = [ + `// Navigate forward`, + `await page.goForward();`, + ]; + return { + code, + action: async () => ({}), + captureSnapshot, + waitForNetwork: false, + }; }, }); diff --git a/src/tools/pdf.ts b/src/tools/pdf.ts index e35fe5c..09e5671 100644 --- a/src/tools/pdf.ts +++ b/src/tools/pdf.ts @@ -21,6 +21,7 @@ import { z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { sanitizeForFilePath } from './utils'; +import * as javascript from '../javascript'; import type { Tool } from './tool'; @@ -28,20 +29,27 @@ const pdfSchema = z.object({}); const pdf: Tool = { capability: 'pdf', + schema: { name: 'browser_pdf_save', description: 'Save page as PDF', inputSchema: zodToJsonSchema(pdfSchema), }, + handle: async context => { - const tab = context.currentTab(); + const tab = context.currentTabOrDie(); const fileName = path.join(os.tmpdir(), sanitizeForFilePath(`page-${new Date().toISOString()}`)) + '.pdf'; - await tab.page.pdf({ path: fileName }); + + const code = [ + `// Save page as ${fileName}`, + `await page.pdf(${javascript.formatObject({ path: fileName })});`, + ]; + return { - content: [{ - type: 'text', - text: `Saved as ${fileName}`, - }], + code, + action: async () => tab.page.pdf({ path: fileName }).then(() => ({})), + captureSnapshot: false, + waitForNetwork: false, }; }, }; diff --git a/src/tools/screen.ts b/src/tools/screen.ts index 4924644..aea70f6 100644 --- a/src/tools/screen.ts +++ b/src/tools/screen.ts @@ -17,6 +17,8 @@ import { z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; +import * as javascript from '../javascript'; + import type { Tool } from './tool'; const screenshot: Tool = { @@ -29,9 +31,24 @@ const screenshot: Tool = { handle: async context => { const tab = await context.ensureTab(); - const screenshot = await tab.page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' }); + const options = { type: 'jpeg' as 'jpeg', quality: 50, scale: 'css' as 'css' }; + + const code = [ + `// Take a screenshot of the current page`, + `await page.screenshot(${javascript.formatObject(options)});`, + ]; + + const action = () => tab.page.screenshot(options).then(buffer => { + return { + content: [{ type: 'image' as 'image', data: buffer.toString('base64'), mimeType: 'image/jpeg' }], + }; + }); + return { - content: [{ type: 'image', data: screenshot.toString('base64'), mimeType: 'image/jpeg' }], + code, + action, + captureSnapshot: false, + waitForNetwork: false }; }, }; @@ -55,10 +72,17 @@ const moveMouse: Tool = { handle: async (context, params) => { const validatedParams = moveMouseSchema.parse(params); - const tab = context.currentTab(); - await tab.page.mouse.move(validatedParams.x, validatedParams.y); + const tab = context.currentTabOrDie(); + const code = [ + `// Move mouse to (${validatedParams.x}, ${validatedParams.y})`, + `await page.mouse.move(${validatedParams.x}, ${validatedParams.y});`, + ]; + const action = () => tab.page.mouse.move(validatedParams.x, validatedParams.y).then(() => ({})); return { - content: [{ type: 'text', text: `Moved mouse to (${validatedParams.x}, ${validatedParams.y})` }], + code, + action, + captureSnapshot: false, + waitForNetwork: false }; }, }; @@ -77,19 +101,26 @@ const click: Tool = { }, handle: async (context, params) => { - return await context.currentTab().runAndWait(async tab => { - const validatedParams = clickSchema.parse(params); - const code = [ - `// Click mouse at coordinates (${validatedParams.x}, ${validatedParams.y})`, - `await page.mouse.move(${validatedParams.x}, ${validatedParams.y});`, - `await page.mouse.down();`, - `await page.mouse.up();`, - ]; + const validatedParams = clickSchema.parse(params); + const tab = context.currentTabOrDie(); + const code = [ + `// Click mouse at coordinates (${validatedParams.x}, ${validatedParams.y})`, + `await page.mouse.move(${validatedParams.x}, ${validatedParams.y});`, + `await page.mouse.down();`, + `await page.mouse.up();`, + ]; + const action = async () => { await tab.page.mouse.move(validatedParams.x, validatedParams.y); await tab.page.mouse.down(); await tab.page.mouse.up(); - return { code }; - }); + return {}; + }; + return { + code, + action, + captureSnapshot: false, + waitForNetwork: true, + }; }, }; @@ -102,6 +133,7 @@ const dragSchema = elementSchema.extend({ const drag: Tool = { capability: 'core', + schema: { name: 'browser_screen_drag', description: 'Drag left mouse button', @@ -110,20 +142,30 @@ const drag: Tool = { handle: async (context, params) => { const validatedParams = dragSchema.parse(params); - return await context.currentTab().runAndWait(async tab => { + const tab = context.currentTabOrDie(); + + const code = [ + `// Drag mouse from (${validatedParams.startX}, ${validatedParams.startY}) to (${validatedParams.endX}, ${validatedParams.endY})`, + `await page.mouse.move(${validatedParams.startX}, ${validatedParams.startY});`, + `await page.mouse.down();`, + `await page.mouse.move(${validatedParams.endX}, ${validatedParams.endY});`, + `await page.mouse.up();`, + ]; + + const action = async () => { await tab.page.mouse.move(validatedParams.startX, validatedParams.startY); await tab.page.mouse.down(); await tab.page.mouse.move(validatedParams.endX, validatedParams.endY); await tab.page.mouse.up(); - const code = [ - `// Drag mouse from (${validatedParams.startX}, ${validatedParams.startY}) to (${validatedParams.endX}, ${validatedParams.endY})`, - `await page.mouse.move(${validatedParams.startX}, ${validatedParams.startY});`, - `await page.mouse.down();`, - `await page.mouse.move(${validatedParams.endX}, ${validatedParams.endY});`, - `await page.mouse.up();`, - ]; - return { code }; - }); + return {}; + }; + + return { + code, + action, + captureSnapshot: false, + waitForNetwork: true, + }; }, }; @@ -134,6 +176,7 @@ const typeSchema = z.object({ const type: Tool = { capability: 'core', + schema: { name: 'browser_screen_type', description: 'Type text', @@ -142,19 +185,31 @@ const type: Tool = { handle: async (context, params) => { const validatedParams = typeSchema.parse(params); - return await context.currentTab().runAndWait(async tab => { - const code = [ - `// Type ${validatedParams.text}`, - `await page.keyboard.type('${validatedParams.text}');`, - ]; + const tab = context.currentTabOrDie(); + + const code = [ + `// Type ${validatedParams.text}`, + `await page.keyboard.type('${validatedParams.text}');`, + ]; + + const action = async () => { await tab.page.keyboard.type(validatedParams.text); - if (validatedParams.submit) { - code.push(`// Submit text`); - code.push(`await page.keyboard.press('Enter');`); + if (validatedParams.submit) await tab.page.keyboard.press('Enter'); - } - return { code }; - }); + return {}; + }; + + if (validatedParams.submit) { + code.push(`// Submit text`); + code.push(`await page.keyboard.press('Enter');`); + } + + return { + code, + action, + captureSnapshot: false, + waitForNetwork: true, + }; }, }; diff --git a/src/tools/snapshot.ts b/src/tools/snapshot.ts index ba7ed8f..19ee28c 100644 --- a/src/tools/snapshot.ts +++ b/src/tools/snapshot.ts @@ -14,17 +14,19 @@ * limitations under the License. */ +import path from 'path'; +import os from 'os'; + import { z } from 'zod'; import zodToJsonSchema from 'zod-to-json-schema'; -import type * as playwright from 'playwright'; -import type { Tool } from './tool'; -import path from 'path'; -import os from 'os'; import { sanitizeForFilePath } from './utils'; import { generateLocator } from '../context'; import * as javascript from '../javascript'; +import type * as playwright from 'playwright'; +import type { Tool } from './tool'; + const snapshot: Tool = { capability: 'core', schema: { @@ -34,11 +36,14 @@ const snapshot: Tool = { }, handle: async context => { - const tab = await context.ensureTab(); - return await tab.run(async () => { - const code = [`// `]; - return { code }; - }, { captureSnapshot: true }); + await context.ensureTab(); + + return { + code: [`// `], + action: async () => ({}), + captureSnapshot: true, + waitForNetwork: false, + }; }, }; @@ -57,15 +62,20 @@ const click: Tool = { handle: async (context, params) => { const validatedParams = elementSchema.parse(params); - return await context.currentTab().runAndWaitWithSnapshot(async snapshot => { - const locator = snapshot.refLocator(validatedParams.ref); - const code = [ - `// Click ${validatedParams.element}`, - `await page.${await generateLocator(locator)}.click();` - ]; - await locator.click(); - return { code }; - }); + const tab = context.currentTabOrDie(); + const locator = tab.snapshotOrDie().refLocator(validatedParams.ref); + + const code = [ + `// Click ${validatedParams.element}`, + `await page.${await generateLocator(locator)}.click();` + ]; + + return { + code, + action: () => locator.click().then(() => ({})), + captureSnapshot: true, + waitForNetwork: true, + }; }, }; @@ -86,16 +96,21 @@ const drag: Tool = { handle: async (context, params) => { const validatedParams = dragSchema.parse(params); - return await context.currentTab().runAndWaitWithSnapshot(async snapshot => { - const startLocator = snapshot.refLocator(validatedParams.startRef); - const endLocator = snapshot.refLocator(validatedParams.endRef); - const code = [ - `// Drag ${validatedParams.startElement} to ${validatedParams.endElement}`, - `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});` - ]; - await startLocator.dragTo(endLocator); - return { code }; - }); + const snapshot = context.currentTabOrDie().snapshotOrDie(); + const startLocator = snapshot.refLocator(validatedParams.startRef); + const endLocator = snapshot.refLocator(validatedParams.endRef); + + const code = [ + `// Drag ${validatedParams.startElement} to ${validatedParams.endElement}`, + `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});` + ]; + + return { + code, + action: () => startLocator.dragTo(endLocator).then(() => ({})), + captureSnapshot: true, + waitForNetwork: true, + }; }, }; @@ -109,15 +124,20 @@ const hover: Tool = { handle: async (context, params) => { const validatedParams = elementSchema.parse(params); - return await context.currentTab().runAndWaitWithSnapshot(async snapshot => { - const locator = snapshot.refLocator(validatedParams.ref); - const code = [ - `// Hover over ${validatedParams.element}`, - `await page.${await generateLocator(locator)}.hover();` - ]; - await locator.hover(); - return { code }; - }); + const snapshot = context.currentTabOrDie().snapshotOrDie(); + const locator = snapshot.refLocator(validatedParams.ref); + + const code = [ + `// Hover over ${validatedParams.element}`, + `await page.${await generateLocator(locator)}.hover();` + ]; + + return { + code, + action: () => locator.hover().then(() => ({})), + captureSnapshot: true, + waitForNetwork: true, + }; }, }; @@ -137,26 +157,34 @@ const type: Tool = { handle: async (context, params) => { const validatedParams = typeSchema.parse(params); - return await context.currentTab().runAndWaitWithSnapshot(async snapshot => { - const locator = snapshot.refLocator(validatedParams.ref); + const snapshot = context.currentTabOrDie().snapshotOrDie(); + const locator = snapshot.refLocator(validatedParams.ref); - const code: string[] = []; - if (validatedParams.slowly) { - code.push(`// Press "${validatedParams.text}" sequentially into "${validatedParams.element}"`); - code.push(`await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(validatedParams.text)});`); - await locator.pressSequentially(validatedParams.text); - } else { - code.push(`// Fill "${validatedParams.text}" into "${validatedParams.element}"`); - code.push(`await page.${await generateLocator(locator)}.fill(${javascript.quote(validatedParams.text)});`); - await locator.fill(validatedParams.text); - } - if (validatedParams.submit) { - code.push(`// Submit text`); - code.push(`await page.${await generateLocator(locator)}.press('Enter');`); - await locator.press('Enter'); - } - return { code }; - }); + const code: string[] = []; + const steps: (() => Promise)[] = []; + + if (validatedParams.slowly) { + code.push(`// Press "${validatedParams.text}" sequentially into "${validatedParams.element}"`); + code.push(`await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(validatedParams.text)});`); + steps.push(() => locator.pressSequentially(validatedParams.text)); + } else { + code.push(`// Fill "${validatedParams.text}" into "${validatedParams.element}"`); + code.push(`await page.${await generateLocator(locator)}.fill(${javascript.quote(validatedParams.text)});`); + steps.push(() => locator.fill(validatedParams.text)); + } + + if (validatedParams.submit) { + code.push(`// Submit text`); + code.push(`await page.${await generateLocator(locator)}.press('Enter');`); + steps.push(() => locator.press('Enter')); + } + + return { + code, + action: () => steps.reduce((acc, step) => acc.then(step), Promise.resolve()).then(() => ({})), + captureSnapshot: true, + waitForNetwork: true, + }; }, }; @@ -174,15 +202,20 @@ const selectOption: Tool = { handle: async (context, params) => { const validatedParams = selectOptionSchema.parse(params); - return await context.currentTab().runAndWaitWithSnapshot(async snapshot => { - const locator = snapshot.refLocator(validatedParams.ref); - const code = [ - `// Select options [${validatedParams.values.join(', ')}] in ${validatedParams.element}`, - `await page.${await generateLocator(locator)}.selectOption(${javascript.formatObject(validatedParams.values)});` - ]; - await locator.selectOption(validatedParams.values); - return { code }; - }); + const snapshot = context.currentTabOrDie().snapshotOrDie(); + const locator = snapshot.refLocator(validatedParams.ref); + + const code = [ + `// Select options [${validatedParams.values.join(', ')}] in ${validatedParams.element}`, + `await page.${await generateLocator(locator)}.selectOption(${javascript.formatObject(validatedParams.values)});` + ]; + + return { + code, + action: () => locator.selectOption(validatedParams.values).then(() => ({})), + captureSnapshot: true, + waitForNetwork: true, + }; }, }; @@ -207,32 +240,41 @@ const screenshot: Tool = { handle: async (context, params) => { const validatedParams = screenshotSchema.parse(params); - const tab = context.currentTab(); + const tab = context.currentTabOrDie(); + const snapshot = tab.snapshotOrDie(); const fileType = validatedParams.raw ? 'png' : 'jpeg'; const fileName = path.join(os.tmpdir(), sanitizeForFilePath(`page-${new Date().toISOString()}`)) + `.${fileType}`; const options: playwright.PageScreenshotOptions = { type: fileType, quality: fileType === 'png' ? undefined : 50, scale: 'css', path: fileName }; const isElementScreenshot = validatedParams.element && validatedParams.ref; - return await context.currentTab().runAndWaitWithSnapshot(async snapshot => { - let screenshot: Buffer | undefined; - const code = [ - `// Screenshot ${isElementScreenshot ? validatedParams.element : 'viewport'} and save it as ${fileName}`, - ]; - if (isElementScreenshot) { - const locator = snapshot.refLocator(validatedParams.ref!); - code.push(`await page.${await generateLocator(locator)}.screenshot(${javascript.formatObject(options)});`); - screenshot = await locator.screenshot(options); - } else { - code.push(`await page.screenshot(${javascript.formatObject(options)});`); - screenshot = await tab.page.screenshot(options); - } + + const code = [ + `// Screenshot ${isElementScreenshot ? validatedParams.element : 'viewport'} and save it as ${fileName}`, + ]; + + const locator = validatedParams.ref ? snapshot.refLocator(validatedParams.ref) : null; + + if (locator) + code.push(`await page.${await generateLocator(locator)}.screenshot(${javascript.formatObject(options)});`); + else + code.push(`await page.screenshot(${javascript.formatObject(options)});`); + + const action = async () => { + const screenshot = locator ? await locator.screenshot(options) : await tab.page.screenshot(options); return { - code, - images: [{ + content: [{ + type: 'image' as 'image', data: screenshot.toString('base64'), mimeType: fileType === 'png' ? 'image/png' : 'image/jpeg', }] }; - }); + }; + + return { + code, + action, + captureSnapshot: true, + waitForNetwork: false, + }; } }; diff --git a/src/tools/tabs.ts b/src/tools/tabs.ts index 22a37e9..aaab412 100644 --- a/src/tools/tabs.ts +++ b/src/tools/tabs.ts @@ -21,17 +21,19 @@ import type { ToolFactory, Tool } from './tool'; const listTabs: Tool = { capability: 'tabs', + schema: { name: 'browser_tab_list', description: 'List browser tabs', inputSchema: zodToJsonSchema(z.object({})), }, - handle: async context => { + + handle: async () => { return { - content: [{ - type: 'text', - text: await context.listTabs(), - }], + code: [`// `], + action: async () => ({}), + captureSnapshot: false, + waitForNetwork: false, }; }, }; @@ -42,21 +44,26 @@ const selectTabSchema = z.object({ const selectTab: ToolFactory = captureSnapshot => ({ capability: 'tabs', + schema: { name: 'browser_tab_select', description: 'Select a tab by index', inputSchema: zodToJsonSchema(selectTabSchema), }, + handle: async (context, params) => { const validatedParams = selectTabSchema.parse(params); await context.selectTab(validatedParams.index); - const currentTab = await context.ensureTab(); - return await currentTab.run(async () => { - const code = [ - `// `, - ]; - return { code }; - }, { captureSnapshot }); + const code = [ + `// `, + ]; + + return { + code, + action: async () => ({}), + captureSnapshot, + waitForNetwork: false + }; }, }); @@ -64,26 +71,32 @@ const newTabSchema = z.object({ url: z.string().optional().describe('The URL to navigate to in the new tab. If not provided, the new tab will be blank.'), }); -const newTab: Tool = { +const newTab: ToolFactory = captureSnapshot => ({ capability: 'tabs', + schema: { name: 'browser_tab_new', description: 'Open a new tab', inputSchema: zodToJsonSchema(newTabSchema), }, + handle: async (context, params) => { const validatedParams = newTabSchema.parse(params); await context.newTab(); if (validatedParams.url) - await context.currentTab().navigate(validatedParams.url); - return await context.currentTab().run(async () => { - const code = [ - `// `, - ]; - return { code }; - }, { captureSnapshot: true }); + await context.currentTabOrDie().navigate(validatedParams.url); + + const code = [ + `// `, + ]; + return { + code, + action: async () => ({}), + captureSnapshot, + waitForNetwork: false + }; }, -}; +}); const closeTabSchema = z.object({ index: z.number().optional().describe('The index of the tab to close. Closes current tab if not provided.'), @@ -91,35 +104,31 @@ const closeTabSchema = z.object({ const closeTab: ToolFactory = captureSnapshot => ({ capability: 'tabs', + schema: { name: 'browser_tab_close', description: 'Close a tab', inputSchema: zodToJsonSchema(closeTabSchema), }, + handle: async (context, params) => { const validatedParams = closeTabSchema.parse(params); await context.closeTab(validatedParams.index); - const currentTab = context.currentTab(); - if (currentTab) { - return await currentTab.run(async () => { - const code = [ - `// `, - ]; - return { code }; - }, { captureSnapshot }); - } + const code = [ + `// `, + ]; return { - content: [{ - type: 'text', - text: await context.listTabs(), - }], + code, + action: async () => ({}), + captureSnapshot, + waitForNetwork: false }; }, }); export default (captureSnapshot: boolean) => [ listTabs, - newTab, + newTab(captureSnapshot), selectTab(captureSnapshot), closeTab(captureSnapshot), ]; diff --git a/src/tools/tool.ts b/src/tools/tool.ts index c37d150..a83c6c7 100644 --- a/src/tools/tool.ts +++ b/src/tools/tool.ts @@ -35,8 +35,10 @@ export type FileUploadModalState = { export type ModalState = FileUploadModalState; export type ToolResult = { - content: (ImageContent | TextContent)[]; - isError?: boolean; + code: string[]; + action: () => Promise<{ content?: (ImageContent | TextContent)[] }>; + captureSnapshot: boolean; + waitForNetwork: boolean; }; export type Tool = { diff --git a/tests/core.spec.ts b/tests/core.spec.ts index ed47e70..e741aad 100644 --- a/tests/core.spec.ts +++ b/tests/core.spec.ts @@ -69,7 +69,6 @@ await page.getByRole('button', { name: 'Submit' }).click(); `); }); - test('browser_select_option', async ({ client }) => { await client.callTool({ name: 'browser_navigate', diff --git a/tests/launch.spec.ts b/tests/launch.spec.ts index 7e82b6d..e296820 100644 --- a/tests/launch.spec.ts +++ b/tests/launch.spec.ts @@ -26,7 +26,7 @@ test('test reopen browser', async ({ client }) => { expect(await client.callTool({ name: 'browser_close', - })).toHaveTextContent('Page closed'); + })).toContainTextContent('No open pages available'); expect(await client.callTool({ name: 'browser_navigate', diff --git a/tests/pdf.spec.ts b/tests/pdf.spec.ts index 32090a1..13e8d48 100644 --- a/tests/pdf.spec.ts +++ b/tests/pdf.spec.ts @@ -42,5 +42,5 @@ test('save as pdf', async ({ client, mcpBrowser }) => { const response = await client.callTool({ name: 'browser_pdf_save', }); - expect(response).toHaveTextContent(/^Saved as.*page-[^:]+.pdf$/); + expect(response).toHaveTextContent(/Save page as.*page-[^:]+.pdf/); });