From 09ba7989c36c29a31f135c0824a7b062e4862bfe Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Wed, 7 May 2025 18:04:20 +0200 Subject: [PATCH] test: run tests on MCP server inside Docker (#361) https://github.com/microsoft/playwright-mcp/issues/346 --- .github/workflows/ci.yml | 48 ++++++++++++++++++++++++++++++---------- playwright.config.ts | 1 + tests/config.spec.ts | 6 ++--- tests/files.spec.ts | 8 +++---- tests/fixtures.ts | 43 +++++++++++++++++++++++++++-------- tests/headed.spec.ts | 1 + tests/network.spec.ts | 2 +- tests/screenshot.spec.ts | 8 +++---- 8 files changed, 84 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7c1068..2c28167 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,32 +30,56 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} - steps: - uses: actions/checkout@v4 - - name: Use Node.js 18 uses: actions/setup-node@v4 with: # https://github.com/microsoft/playwright-mcp/issues/344 node-version: '18.19' cache: 'npm' - - name: Install dependencies run: npm ci - - name: Playwright install run: npx playwright install --with-deps - - name: Install MS Edge - if: ${{ matrix.os == 'macos-latest' }} # MS Edge is not preinstalled on macOS runners. + # MS Edge is not preinstalled on macOS runners. + if: ${{ matrix.os == 'macos-latest' }} run: npx playwright install msedge - - name: Build run: npm run build - - - name: Install Playwright browsers - run: npx playwright install --with-deps - - name: Run tests - run: npm test -- --forbid-only + run: npm test + + test_docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18 + uses: actions/setup-node@v4 + with: + node-version: '18' + cache: 'npm' + - name: Install dependencies + run: npm ci + - name: Playwright install + run: npx playwright install --with-deps chromium + - name: Build + run: npm run build + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v6 + with: + tags: playwright-mcp-dev:latest + cache-from: type=gha + cache-to: type=gha,mode=max + load: true + - name: Run tests + shell: bash + run: | + # Used for the Docker tests to share the test-results folder with the container. + umask 0000 + npm run test -- --project=chromium-docker + env: + MCP_IN_DOCKER: 1 diff --git a/playwright.config.ts b/playwright.config.ts index c98b74c..76e0344 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -29,6 +29,7 @@ export default defineConfig({ { name: 'chrome' }, { name: 'msedge', use: { mcpBrowser: 'msedge' } }, { name: 'chromium', use: { mcpBrowser: 'chromium' } }, + ...process.env.MCP_IN_DOCKER ? [{ name: 'chromium-docker', use: { mcpBrowser: 'chromium', mcpMode: 'docker' as const } }] : [], { name: 'firefox', use: { mcpBrowser: 'firefox' } }, { name: 'webkit', use: { mcpBrowser: 'webkit' } }, ], diff --git a/tests/config.spec.ts b/tests/config.spec.ts index 8c54bc2..2b09c0f 100644 --- a/tests/config.spec.ts +++ b/tests/config.spec.ts @@ -19,13 +19,13 @@ import fs from 'node:fs'; import { Config } from '../config.js'; import { test, expect } from './fixtures.js'; -test('config user data dir', async ({ startClient, mcpBrowser }, testInfo) => { +test('config user data dir', async ({ startClient, localOutputPath }) => { const config: Config = { browser: { - userDataDir: testInfo.outputPath('user-data-dir'), + userDataDir: localOutputPath('user-data-dir'), }, }; - const configPath = testInfo.outputPath('config.json'); + const configPath = localOutputPath('config.json'); await fs.promises.writeFile(configPath, JSON.stringify(config, null, 2)); const client = await startClient({ args: ['--config', configPath] }); diff --git a/tests/files.spec.ts b/tests/files.spec.ts index 10fd96c..fe3f260 100644 --- a/tests/files.spec.ts +++ b/tests/files.spec.ts @@ -18,7 +18,7 @@ import { test, expect } from './fixtures.js'; import fs from 'fs/promises'; import path from 'path'; -test('browser_file_upload', async ({ client }) => { +test('browser_file_upload', async ({ client, localOutputPath }) => { expect(await client.callTool({ name: 'browser_navigate', arguments: { @@ -50,7 +50,7 @@ The tool "browser_file_upload" can only be used when there is related modal stat })).toContainTextContent(`### Modal state - [File chooser]: can be handled by the "browser_file_upload" tool`); - const filePath = test.info().outputPath('test.txt'); + const filePath = localOutputPath('test.txt'); await fs.writeFile(filePath, 'Hello, world!'); { @@ -96,8 +96,8 @@ The tool "browser_file_upload" can only be used when there is related modal stat } }); -test('clicking on download link emits download', async ({ startClient }, testInfo) => { - const outputDir = testInfo.outputPath('output'); +test('clicking on download link emits download', async ({ startClient, localOutputPath }) => { + const outputDir = localOutputPath('output'); const client = await startClient({ config: { outputDir }, }); diff --git a/tests/fixtures.ts b/tests/fixtures.ts index 1697147..ab33940 100644 --- a/tests/fixtures.ts +++ b/tests/fixtures.ts @@ -29,6 +29,7 @@ import type { Config } from '../config'; export type TestOptions = { mcpBrowser: string | undefined; + mcpMode: 'docker' | undefined; }; type TestFixtures = { @@ -40,6 +41,7 @@ type TestFixtures = { server: TestServer; httpsServer: TestServer; mcpHeadless: boolean; + localOutputPath: (filePath: string) => string; }; type WorkerFixtures = { @@ -56,12 +58,13 @@ export const test = baseTest.extend( await use(await startClient({ args: ['--vision'] })); }, - startClient: async ({ mcpHeadless, mcpBrowser }, use, testInfo) => { + startClient: async ({ mcpHeadless, mcpBrowser, mcpMode }, use, testInfo) => { const userDataDir = testInfo.outputPath('user-data-dir'); + const configDir = path.dirname(test.info().config.configFile!); let client: Client | undefined; await use(async options => { - const args = ['--user-data-dir', userDataDir]; + const args = ['--user-data-dir', path.relative(configDir, userDataDir)]; if (mcpHeadless) args.push('--headless'); if (mcpBrowser) @@ -71,15 +74,11 @@ export const test = baseTest.extend( if (options?.config) { const configFile = testInfo.outputPath('config.json'); await fs.promises.writeFile(configFile, JSON.stringify(options.config, null, 2)); - args.push(`--config=${configFile}`); + args.push(`--config=${path.relative(configDir, configFile)}`); } - // NOTE: Can be removed when we drop Node.js 18 support and changed to import.meta.filename. - const __filename = url.fileURLToPath(import.meta.url); - const transport = new StdioClientTransport({ - command: 'node', - args: [path.join(path.dirname(__filename), '../cli.js'), ...args], - }); + client = new Client({ name: options?.clientName ?? 'test', version: '1.0.0' }); + const transport = createTransport(args, mcpMode); await client.connect(transport); await client.ping(); return client; @@ -130,6 +129,15 @@ export const test = baseTest.extend( mcpBrowser: ['chrome', { option: true }], + mcpMode: [undefined, { option: true }], + + localOutputPath: async ({ mcpMode }, use, testInfo) => { + await use(filePath => { + test.skip(mcpMode === 'docker', 'Mounting files is not supported in docker mode'); + return testInfo.outputPath(filePath); + }); + }, + _workerServers: [async ({}, use, workerInfo) => { const port = 8907 + workerInfo.workerIndex * 4; const server = await TestServer.create(port); @@ -156,6 +164,23 @@ export const test = baseTest.extend( }, }); +function createTransport(args: string[], mcpMode: TestOptions['mcpMode']) { + // NOTE: Can be removed when we drop Node.js 18 support and changed to import.meta.filename. + const __filename = url.fileURLToPath(import.meta.url); + if (mcpMode === 'docker') { + const dockerArgs = ['run', '--rm', '-i', '--network=host', '-v', `${test.info().project.outputDir}:/app/test-results`]; + return new StdioClientTransport({ + command: 'docker', + args: [...dockerArgs, 'playwright-mcp-dev:latest', ...args], + }); + } + return new StdioClientTransport({ + command: 'node', + args: [path.join(path.dirname(__filename), '../cli.js'), ...args], + cwd: path.join(path.dirname(__filename), '..'), + }); +} + type Response = Awaited>; export const expect = baseExpect.extend({ diff --git a/tests/headed.spec.ts b/tests/headed.spec.ts index 82f4a97..69202c4 100644 --- a/tests/headed.spec.ts +++ b/tests/headed.spec.ts @@ -20,6 +20,7 @@ for (const mcpHeadless of [false, true]) { test.describe(`mcpHeadless: ${mcpHeadless}`, () => { test.use({ mcpHeadless }); test.skip(process.platform === 'linux', 'Auto-detection wont let this test run on linux'); + test.skip(({ mcpMode, mcpHeadless }) => mcpMode === 'docker' && !mcpHeadless, 'Headed mode is not supported in docker'); test('browser', async ({ client, server, mcpBrowser }) => { test.skip(!['chrome', 'msedge', 'chromium'].includes(mcpBrowser ?? ''), 'Only chrome is supported for this test'); server.route('/', (req, res) => { diff --git a/tests/network.spec.ts b/tests/network.spec.ts index 3e67cec..90f4cf5 100644 --- a/tests/network.spec.ts +++ b/tests/network.spec.ts @@ -45,5 +45,5 @@ test('browser_network_requests', async ({ client, server }) => { await expect.poll(() => client.callTool({ name: 'browser_network_requests', arguments: {}, - })).toHaveTextContent(`[GET] http://localhost:${server.PORT}/json => [200] OK`); + })).toHaveTextContent(`[GET] ${`${server.PREFIX}/json`} => [200] OK`); }); diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index 5f99b79..8f25dd9 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -73,8 +73,8 @@ test('browser_take_screenshot (element)', async ({ client }) => { }); }); -test('--output-dir should work', async ({ startClient }, testInfo) => { - const outputDir = testInfo.outputPath('output'); +test('--output-dir should work', async ({ startClient, localOutputPath }) => { + const outputDir = localOutputPath('output'); const client = await startClient({ args: ['--output-dir', outputDir], }); @@ -95,8 +95,8 @@ test('--output-dir should work', async ({ startClient }, testInfo) => { }); -test('browser_take_screenshot (outputDir)', async ({ startClient }, testInfo) => { - const outputDir = testInfo.outputPath('output'); +test('browser_take_screenshot (outputDir)', async ({ startClient, localOutputPath }) => { + const outputDir = localOutputPath('output'); const client = await startClient({ config: { outputDir }, });