chore: refactor tests for real

This commit is contained in:
Pavel Feldman 2025-03-27 16:50:43 -07:00
parent b477b7c26f
commit 3b0e4c3b27
3 changed files with 223 additions and 273 deletions

View File

@ -79,8 +79,10 @@ export function createServerWithTools(options: Options): Server {
return { contents };
});
const oldClose = server.close.bind(server);
server.close = async () => {
await server.close();
await oldClose();
await context.close();
};

View File

@ -19,8 +19,8 @@ import { spawn } from 'node:child_process';
import path from 'node:path';
import { test, expect } from './fixtures';
test('test tool list', async ({ server, visionServer }) => {
const tools = await server.listTools();
test('test tool list', async ({ client, visionClient }) => {
const { tools } = await client.listTools();
expect(tools.map(t => t.name)).toEqual([
'browser_navigate',
'browser_go_back',
@ -38,7 +38,7 @@ test('test tool list', async ({ server, visionServer }) => {
'browser_close',
]);
const visionTools = await visionServer.listTools();
const { tools: visionTools } = await visionClient.listTools();
expect(visionTools.map(t => t.name)).toEqual([
'browser_navigate',
'browser_go_back',
@ -56,8 +56,9 @@ test('test tool list', async ({ server, visionServer }) => {
]);
});
test('test resources list', async ({ server }) => {
expect(await server.listResources()).toEqual([
test('test resources list', async ({ client }) => {
const { resources } = await client.listResources();
expect(resources).toEqual([
expect.objectContaining({
uri: 'browser://console',
mimeType: 'text/plain',
@ -65,9 +66,13 @@ test('test resources list', async ({ server }) => {
]);
});
test('test browser_navigate', async ({ server }) => {
expect(await server.callTool('browser_navigate', { url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>' })).toEqual([
`
test('test browser_navigate', async ({ client }) => {
expect(await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
},
})).toHaveTextContent(`
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page Title: Title
- Page Snapshot
@ -75,56 +80,78 @@ test('test browser_navigate', async ({ server }) => {
- document [ref=s1e2]: Hello, world!
\`\`\`
`
]);
);
});
test('test browser_click', async ({ server }) => {
await server.callTool(
'browser_navigate',
{ url: 'data:text/html,<html><title>Title</title><button>Submit</button></html>' }
);
test('test browser_click', async ({ client }) => {
await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><title>Title</title><button>Submit</button></html>',
},
});
expect(await server.callTool('browser_click', { element: 'Submit button', ref: 's1e4' })).toEqual([
`\"Submit button\" clicked
expect(await client.callTool({
name: 'browser_click',
arguments: {
element: 'Submit button',
ref: 's1e4',
},
})).toHaveTextContent(`"Submit button" clicked
- Page URL: data:text/html,<html><title>Title</title><button>Submit</button></html>
- Page Title: Title
- Page Snapshot
\`\`\`yaml
- document [ref=s2e2]:
- button \"Submit\" [ref=s2e4]
- button "Submit" [ref=s2e4]
\`\`\`
`]);
`);
});
test('test reopen browser', async ({ server }) => {
await server.callTool(
'browser_navigate',
{ url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>' }
);
test('test reopen browser', async ({ client }) => {
await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
},
});
expect(await server.callTool('browser_close')).toEqual(['Page closed']);
expect(await client.callTool({
name: 'browser_close',
})).toHaveTextContent('Page closed');
expect(await server.callTool('browser_navigate', { url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>' })).toEqual([`
expect(await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
},
})).toHaveTextContent(`
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
- Page Title: Title
- Page Snapshot
\`\`\`yaml
- document [ref=s1e2]: Hello, world!
\`\`\`
`,]);
`);
});
test.describe('test browser_select_option', () => {
test('single option', async ({ server }) => {
await server.callTool('browser_navigate', {
test('single option', async ({ client }) => {
await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><title>Title</title><select><option value="foo">Foo</option><option value="bar">Bar</option></select></html>',
});
},
});
const response = await server.callTool('browser_select_option', { element: 'Select', ref: 's1e4', values: ['bar'] });
expect(response).toEqual([
`Selected option in \"Select\"
expect(await client.callTool({
name: 'browser_select_option',
arguments: {
element: 'Select',
ref: 's1e4',
values: ['bar'],
},
})).toHaveTextContent(`Selected option in "Select"
- Page URL: data:text/html,<html><title>Title</title><select><option value="foo">Foo</option><option value="bar">Bar</option></select></html>
- Page Title: Title
@ -132,21 +159,28 @@ test.describe('test browser_select_option', () => {
\`\`\`yaml
- document [ref=s2e2]:
- combobox [ref=s2e4]:
- option \"Foo\" [ref=s2e5]
- option \"Bar\" [selected] [ref=s2e6]
- option "Foo" [ref=s2e5]
- option "Bar" [selected] [ref=s2e6]
\`\`\`
`]);
`);
});
test('multiple option', async ({ client }) => {
await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><title>Title</title><select multiple><option value="foo">Foo</option><option value="bar">Bar</option><option value="baz">Baz</option></select></html>',
},
});
test('multiple option', async ({ server }) => {
await server.callTool('browser_navigate', {
url: 'data:text/html,<html><title>Title</title><select multiple><option value="foo">Foo</option><option value="bar">Bar</option><option value="baz">Baz</option></select></html>',
});
const response = await server.callTool('browser_select_option', { element: 'Select', ref: 's1e4', values: ['bar', 'baz'] });
expect(response).toEqual([
`Selected option in \"Select\"
expect(await client.callTool({
name: 'browser_select_option',
arguments: {
element: 'Select',
ref: 's1e4',
values: ['bar', 'baz'],
},
})).toHaveTextContent(`Selected option in "Select"
- Page URL: data:text/html,<html><title>Title</title><select multiple><option value="foo">Foo</option><option value="bar">Bar</option><option value="baz">Baz</option></select></html>
- Page Title: Title
@ -154,83 +188,107 @@ test.describe('test browser_select_option', () => {
\`\`\`yaml
- document [ref=s2e2]:
- listbox [ref=s2e4]:
- option \"Foo\" [ref=s2e5]
- option \"Bar\" [selected] [ref=s2e6]
- option \"Baz\" [selected] [ref=s2e7]
- option "Foo" [ref=s2e5]
- option "Bar" [selected] [ref=s2e6]
- option "Baz" [selected] [ref=s2e7]
\`\`\`
`]);
});
`);
});
test('browser://console', async ({ server }) => {
await server.callTool('browser_navigate', {
url: 'data:text/html,<html><script>console.log("Hello, world!");console.error("Error"); </script></html>',
test('browser://console', async ({ client }) => {
await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><script>console.log("Hello, world!");console.error("Error"); </script></html>',
},
});
expect(await server.readResource('browser://console')).toEqual([{
const resource = await client.readResource({
uri: 'browser://console',
});
expect(resource.contents).toEqual([{
uri: 'browser://console',
mimeType: 'text/plain',
text: '[LOG] Hello, world!\n[ERROR] Error',
}]);
});
test('stitched aria frames', async ({ server }) => {
const response = await server.callTool('browser_navigate', {
url: 'data:text/html,<h1>Hello</h1><iframe src="data:text/html,<h1>World</h1>"></iframe><iframe src="data:text/html,<h1>Should be invisible</h1>" style="display: none;"></iframe>',
});
expect(response).toEqual([`
test('stitched aria frames', async ({ client }) => {
expect(await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<h1>Hello</h1><iframe src="data:text/html,<h1>World</h1>"></iframe><iframe src="data:text/html,<h1>Should be invisible</h1>" style="display: none;"></iframe>',
},
})).toHaveTextContent(`
- Page URL: data:text/html,<h1>Hello</h1><iframe src="data:text/html,<h1>World</h1>"></iframe><iframe src="data:text/html,<h1>Should be invisible</h1>" style="display: none;"></iframe>
- Page Title:
- Page Snapshot
\`\`\`yaml
- document [ref=s1e2]:
- heading \"Hello\" [level=1] [ref=s1e4]
- heading "Hello" [level=1] [ref=s1e4]
# iframe src=data:text/html,<h1>World</h1>
- document [ref=f0s1e2]:
- heading \"World\" [level=1] [ref=f0s1e4]
- heading "World" [level=1] [ref=f0s1e4]
\`\`\`
`
]);
`);
});
test('browser_choose_file', async ({ server }) => {
let response = await server.callTool('browser_navigate', {
url: 'data:text/html,<html><title>Title</title><input type="file" /><button>Button</button></html>',
});
test('browser_choose_file', async ({ client }) => {
expect(await client.callTool({
name: 'browser_navigate',
arguments: {
url: 'data:text/html,<html><title>Title</title><input type="file" /><button>Button</button></html>',
},
})).toContainTextContent('- textbox [ref=s1e4]');
expect(response[0]).toContain('- textbox [ref=s1e4]');
response = await server.callTool('browser_click', {
element: 'Textbox',
ref: 's1e4',
});
expect(response[0]).toContain('There is a file chooser visible that requires browser_choose_file to be called');
expect(await client.callTool({
name: 'browser_click',
arguments: {
element: 'Textbox',
ref: 's1e4',
},
})).toContainTextContent('There is a file chooser visible that requires browser_choose_file to be called');
const filePath = test.info().outputPath('test.txt');
await fs.writeFile(filePath, 'Hello, world!');
response = await server.callTool('browser_choose_file', {
paths: [filePath],
});
expect(response[0]).not.toContain('There is a file chooser visible that requires browser_choose_file to be called');
expect(response[0]).toContain('textbox [ref=s3e4]: C:\\fakepath\\test.txt');
{
const response = await client.callTool({
name: 'browser_choose_file',
arguments: {
paths: [filePath],
},
});
response = await server.callTool('browser_click', {
element: 'Textbox',
ref: 's3e4',
});
expect(response).not.toContainTextContent('There is a file chooser visible that requires browser_choose_file to be called');
expect(response).toContainTextContent('textbox [ref=s3e4]: C:\\fakepath\\test.txt');
}
expect(response[0]).toContain('There is a file chooser visible that requires browser_choose_file to be called');
expect(response[0]).toContain('button "Button" [ref=s4e5]');
{
const response = await client.callTool({
name: 'browser_click',
arguments: {
element: 'Textbox',
ref: 's3e4',
},
});
response = await server.callTool('browser_click', {
element: 'Button',
ref: 's4e5',
});
expect(response).toContainTextContent('There is a file chooser visible that requires browser_choose_file to be called');
expect(response).toContainTextContent('button "Button" [ref=s4e5]');
}
expect(response[0], 'not submitting browser_choose_file dismisses file chooser').not.toContain('There is a file chooser visible that requires browser_choose_file to be called');
{
const response = await client.callTool({
name: 'browser_click',
arguments: {
element: 'Button',
ref: 's4e5',
},
});
expect(response, 'not submitting browser_choose_file dismisses file chooser').not.toContainTextContent('There is a file chooser visible that requires browser_choose_file to be called');
}
});
test('sse transport', async () => {

View File

@ -15,206 +15,48 @@
*/
import path from 'path';
import { spawn } from 'child_process';
import EventEmitter from 'events';
import { chromium } from 'playwright';
import { test as baseTest, expect } from '@playwright/test';
import type { ChildProcess } from 'child_process';
export { expect } from '@playwright/test';
class MCPServer extends EventEmitter {
private _child: ChildProcess;
private _messageQueue: any[] = [];
private _messageResolvers: ((value: any) => void)[] = [];
private _buffer: string = '';
constructor(command: string, args: string[], options?: { env?: NodeJS.ProcessEnv }) {
super();
this._child = spawn(command, args, {
stdio: ['pipe', 'pipe', 'pipe'],
env: { ...process.env, ...options?.env },
});
this._child.stdout?.on('data', data => {
this._buffer += data.toString();
let newlineIndex: number;
while ((newlineIndex = this._buffer.indexOf('\n')) !== -1) {
const message = this._buffer.slice(0, newlineIndex).trim();
this._buffer = this._buffer.slice(newlineIndex + 1);
if (!message)
continue;
const parsed = JSON.parse(message);
if (this._messageResolvers.length > 0) {
const resolve = this._messageResolvers.shift();
resolve!(parsed);
} else {
this._messageQueue.push(parsed);
}
}
});
this._child.stderr?.on('data', data => {
throw new Error('Server stderr:', data.toString());
});
this._child.on('exit', code => {
if (code !== 0)
throw new Error(`Server exited with code ${code}`);
});
}
async send(message: any, options?: { timeout?: number }): Promise<any> {
await this.sendNoReply(message);
return this._waitForResponse(options || {});
}
async sendNoReply(message: any): Promise<void> {
const jsonMessage = JSON.stringify(message) + '\n';
await new Promise<void>((resolve, reject) => {
this._child.stdin?.write(jsonMessage, err => {
if (err)
reject(err);
else
resolve();
});
});
}
private async _waitForResponse(options: { timeout?: number }): Promise<any> {
if (this._messageQueue.length > 0)
return this._messageQueue.shift();
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
reject(new Error('Timeout waiting for message'));
}, options.timeout || 15000);
this._messageResolvers.push(message => {
clearTimeout(timeoutId);
resolve(message);
});
});
}
async close(): Promise<void> {
return new Promise(resolve => {
this._child.on('exit', () => resolve());
this._child.stdin?.end();
});
}
async listTools() {
const list = await this.send({
jsonrpc: '2.0',
id: 0,
method: 'tools/list',
});
return list.result.tools;
}
async listResources() {
const list = await this.send({
jsonrpc: '2.0',
id: 0,
method: 'resources/list',
});
return list.result.resources;
}
async callTool(name: string, args?: any) {
const result = await this.send({
jsonrpc: '2.0',
id: 0,
method: 'tools/call',
params: {
name,
arguments: args,
}
});
return result.result.content.map(c => c.text);
}
async readResource(uri: string) {
const result = await this.send({
jsonrpc: '2.0',
id: 0,
method: 'resources/read',
params: {
uri,
},
});
return result.result.contents;
}
}
import { test as baseTest, expect as baseExpect } from '@playwright/test';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
type Fixtures = {
server: MCPServer;
visionServer: MCPServer;
startServer: (options?: { env?: NodeJS.ProcessEnv, vision?: boolean }) => Promise<MCPServer>;
client: Client;
visionClient: Client;
startClient: (options?: { env?: NodeJS.ProcessEnv, vision?: boolean }) => Promise<Client>;
wsEndpoint: string;
};
export const test = baseTest.extend<Fixtures>({
server: async ({ startServer }, use) => {
await use(await startServer());
client: async ({ startClient }, use) => {
await use(await startClient());
},
visionServer: async ({ startServer }, use) => {
await use(await startServer({ vision: true }));
visionClient: async ({ startClient }, use) => {
await use(await startClient({ vision: true }));
},
startServer: async ({ }, use, testInfo) => {
let server: MCPServer | undefined;
startClient: async ({ }, use, testInfo) => {
const userDataDir = testInfo.outputPath('user-data-dir');
let client: StdioClientTransport | undefined;
use(async options => {
const args = ['--headless', '--user-data-dir', userDataDir];
if (options?.vision)
args.push('--vision');
server = new MCPServer('node', [path.join(__dirname, '../cli.js'), ...args], options);
const initialize = await server.send({
jsonrpc: '2.0',
id: 0,
method: 'initialize',
params: {
protocolVersion: '2024-11-05',
capabilities: {},
clientInfo: {
name: 'Playwright Test',
version: '0.0.0',
},
},
const transport = new StdioClientTransport({
command: 'node',
args: [path.join(__dirname, '../cli.js'), ...args],
});
expect(initialize).toEqual(expect.objectContaining({
id: 0,
result: expect.objectContaining({
protocolVersion: '2024-11-05',
capabilities: {
tools: {},
resources: {},
},
serverInfo: expect.objectContaining({
name: 'Playwright',
version: expect.any(String),
}),
}),
}));
await server.sendNoReply({
jsonrpc: '2.0',
method: 'notifications/initialized',
});
return server;
const client = new Client({ name: 'test', version: '1.0.0' });
await client.connect(transport);
await client.ping();
return client;
});
await server?.close();
await client?.close();
},
wsEndpoint: async ({ }, use) => {
@ -223,3 +65,51 @@ export const test = baseTest.extend<Fixtures>({
await browserServer.close();
},
});
type Response = Awaited<ReturnType<Client['callTool']>>;
export const expect = baseExpect.extend({
toHaveTextContent(response: Response, content: string | string[]) {
const isNot = this.isNot;
try {
content = Array.isArray(content) ? content : [content];
const texts = (response.content as any).map(c => c.text);
if (isNot)
baseExpect(texts).not.toEqual(content);
else
baseExpect(texts).toEqual(content);
} catch (e) {
return {
pass: isNot,
message: () => e.message,
};
}
return {
pass: !isNot,
message: () => ``,
};
},
toContainTextContent(response: Response, content: string | string[]) {
const isNot = this.isNot;
try {
content = Array.isArray(content) ? content : [content];
const texts = (response.content as any).map(c => c.text);
for (let i = 0; i < texts.length; i++) {
if (isNot)
expect(texts[i]).not.toContain(content[i]);
else
expect(texts[i]).toContain(content[i]);
}
} catch (e) {
return {
pass: isNot,
message: () => e.message,
};
}
return {
pass: !isNot,
message: () => ``,
};
},
});