test: run tests on MCP server inside Docker (#361)

https://github.com/microsoft/playwright-mcp/issues/346

.github/workflows/ci.yml

@@ -30,32 +30,56 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
-
     steps:
     - uses: actions/checkout@v4
-
     - name: Use Node.js 18
       uses: actions/setup-node@v4
       with:
         # https://github.com/microsoft/playwright-mcp/issues/344
         node-version: '18.19'
         cache: 'npm'
-
     - name: Install dependencies
       run: npm ci
-
     - name: Playwright install
       run: npx playwright install --with-deps
-
     - name: Install MS Edge
-      if: ${{ matrix.os == 'macos-latest' }} # MS Edge is not preinstalled on macOS runners.
+      # MS Edge is not preinstalled on macOS runners.
+      if: ${{ matrix.os == 'macos-latest' }}
       run: npx playwright install msedge
-
     - name: Build
       run: npm run build
-
-    - name: Install Playwright browsers
-      run: npx playwright install --with-deps
-
     - name: Run tests
-      run: npm test -- --forbid-only
+      run: npm test
+
+  test_docker:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Use Node.js 18
+      uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+        cache: 'npm'
+    - name: Install dependencies
+      run: npm ci
+    - name: Playwright install
+      run: npx playwright install --with-deps chromium
+    - name: Build
+      run: npm run build
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Build and push
+      uses: docker/build-push-action@v6
+      with:
+        tags: playwright-mcp-dev:latest
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        load: true
+    - name: Run tests
+      shell: bash
+      run: |
+        # Used for the Docker tests to share the test-results folder with the container.
+        umask 0000
+        npm run test -- --project=chromium-docker
+      env:
+        MCP_IN_DOCKER: 1

playwright.config.ts

@@ -29,6 +29,7 @@ export default defineConfig<TestOptions>({
     { name: 'chrome' },
     { name: 'msedge', use: { mcpBrowser: 'msedge' } },
     { name: 'chromium', use: { mcpBrowser: 'chromium' } },
+    ...process.env.MCP_IN_DOCKER ? [{ name: 'chromium-docker', use: { mcpBrowser: 'chromium', mcpMode: 'docker' as const } }] : [],
     { name: 'firefox', use: { mcpBrowser: 'firefox' } },
     { name: 'webkit', use: { mcpBrowser: 'webkit' } },
   ],

tests/config.spec.ts

@@ -19,13 +19,13 @@ import fs from 'node:fs';
 import { Config } from '../config.js';
 import { test, expect } from './fixtures.js';
 
-test('config user data dir', async ({ startClient, mcpBrowser }, testInfo) => {
+test('config user data dir', async ({ startClient, localOutputPath }) => {
   const config: Config = {
     browser: {
-      userDataDir: testInfo.outputPath('user-data-dir'),
+      userDataDir: localOutputPath('user-data-dir'),
     },
   };
-  const configPath = testInfo.outputPath('config.json');
+  const configPath = localOutputPath('config.json');
   await fs.promises.writeFile(configPath, JSON.stringify(config, null, 2));
 
   const client = await startClient({ args: ['--config', configPath] });

tests/files.spec.ts

@@ -18,7 +18,7 @@ import { test, expect } from './fixtures.js';
 import fs from 'fs/promises';
 import path from 'path';
 
-test('browser_file_upload', async ({ client }) => {
+test('browser_file_upload', async ({ client, localOutputPath }) => {
   expect(await client.callTool({
     name: 'browser_navigate',
     arguments: {
@@ -50,7 +50,7 @@ The tool "browser_file_upload" can only be used when there is related modal stat
   })).toContainTextContent(`### Modal state
 - [File chooser]: can be handled by the "browser_file_upload" tool`);
 
-  const filePath = test.info().outputPath('test.txt');
+  const filePath = localOutputPath('test.txt');
   await fs.writeFile(filePath, 'Hello, world!');
 
   {
@@ -96,8 +96,8 @@ The tool "browser_file_upload" can only be used when there is related modal stat
   }
 });
 
-test('clicking on download link emits download', async ({ startClient }, testInfo) => {
-  const outputDir = testInfo.outputPath('output');
+test('clicking on download link emits download', async ({ startClient, localOutputPath }) => {
+  const outputDir = localOutputPath('output');
   const client = await startClient({
     config: { outputDir },
   });

tests/fixtures.ts

@@ -29,6 +29,7 @@ import type { Config } from '../config';
 
 export type TestOptions = {
   mcpBrowser: string | undefined;
+  mcpMode: 'docker' | undefined;
 };
 
 type TestFixtures = {
@@ -40,6 +41,7 @@ type TestFixtures = {
   server: TestServer;
   httpsServer: TestServer;
   mcpHeadless: boolean;
+  localOutputPath: (filePath: string) => string;
 };
 
 type WorkerFixtures = {
@@ -56,12 +58,13 @@ export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>(
     await use(await startClient({ args: ['--vision'] }));
   },
 
-  startClient: async ({ mcpHeadless, mcpBrowser }, use, testInfo) => {
+  startClient: async ({ mcpHeadless, mcpBrowser, mcpMode }, use, testInfo) => {
     const userDataDir = testInfo.outputPath('user-data-dir');
+    const configDir = path.dirname(test.info().config.configFile!);
     let client: Client | undefined;
 
     await use(async options => {
-      const args = ['--user-data-dir', userDataDir];
+      const args = ['--user-data-dir', path.relative(configDir, userDataDir)];
       if (mcpHeadless)
         args.push('--headless');
       if (mcpBrowser)
@@ -71,15 +74,11 @@ export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>(
       if (options?.config) {
         const configFile = testInfo.outputPath('config.json');
         await fs.promises.writeFile(configFile, JSON.stringify(options.config, null, 2));
-        args.push(`--config=${configFile}`);
+        args.push(`--config=${path.relative(configDir, configFile)}`);
       }
-      // NOTE: Can be removed when we drop Node.js 18 support and changed to import.meta.filename.
-      const __filename = url.fileURLToPath(import.meta.url);
-      const transport = new StdioClientTransport({
-        command: 'node',
-        args: [path.join(path.dirname(__filename), '../cli.js'), ...args],
-      });
+
       client = new Client({ name: options?.clientName ?? 'test', version: '1.0.0' });
+      const transport = createTransport(args, mcpMode);
       await client.connect(transport);
       await client.ping();
       return client;
@@ -130,6 +129,15 @@ export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>(
 
   mcpBrowser: ['chrome', { option: true }],
 
+  mcpMode: [undefined, { option: true }],
+
+  localOutputPath: async ({ mcpMode }, use, testInfo) => {
+    await use(filePath => {
+      test.skip(mcpMode === 'docker', 'Mounting files is not supported in docker mode');
+      return testInfo.outputPath(filePath);
+    });
+  },
+
   _workerServers: [async ({}, use, workerInfo) => {
     const port = 8907 + workerInfo.workerIndex * 4;
     const server = await TestServer.create(port);
@@ -156,6 +164,23 @@ export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>(
   },
 });
 
+function createTransport(args: string[], mcpMode: TestOptions['mcpMode']) {
+  // NOTE: Can be removed when we drop Node.js 18 support and changed to import.meta.filename.
+  const __filename = url.fileURLToPath(import.meta.url);
+  if (mcpMode === 'docker') {
+    const dockerArgs = ['run', '--rm', '-i', '--network=host', '-v', `${test.info().project.outputDir}:/app/test-results`];
+    return new StdioClientTransport({
+      command: 'docker',
+      args: [...dockerArgs, 'playwright-mcp-dev:latest', ...args],
+    });
+  }
+  return new StdioClientTransport({
+    command: 'node',
+    args: [path.join(path.dirname(__filename), '../cli.js'), ...args],
+    cwd: path.join(path.dirname(__filename), '..'),
+  });
+}
+
 type Response = Awaited<ReturnType<Client['callTool']>>;
 
 export const expect = baseExpect.extend({

tests/headed.spec.ts

@@ -20,6 +20,7 @@ for (const mcpHeadless of [false, true]) {
   test.describe(`mcpHeadless: ${mcpHeadless}`, () => {
     test.use({ mcpHeadless });
     test.skip(process.platform === 'linux', 'Auto-detection wont let this test run on linux');
+    test.skip(({ mcpMode, mcpHeadless }) => mcpMode === 'docker' && !mcpHeadless, 'Headed mode is not supported in docker');
     test('browser', async ({ client, server, mcpBrowser }) => {
       test.skip(!['chrome', 'msedge', 'chromium'].includes(mcpBrowser ?? ''), 'Only chrome is supported for this test');
       server.route('/', (req, res) => {

tests/network.spec.ts

@@ -45,5 +45,5 @@ test('browser_network_requests', async ({ client, server }) => {
   await expect.poll(() => client.callTool({
     name: 'browser_network_requests',
     arguments: {},
-  })).toHaveTextContent(`[GET] http://localhost:${server.PORT}/json => [200] OK`);
+  })).toHaveTextContent(`[GET] ${`${server.PREFIX}/json`} => [200] OK`);
 });

tests/screenshot.spec.ts

@@ -73,8 +73,8 @@ test('browser_take_screenshot (element)', async ({ client }) => {
   });
 });
 
-test('--output-dir should work', async ({ startClient }, testInfo) => {
-  const outputDir = testInfo.outputPath('output');
+test('--output-dir should work', async ({ startClient, localOutputPath }) => {
+  const outputDir = localOutputPath('output');
   const client = await startClient({
     args: ['--output-dir', outputDir],
   });
@@ -95,8 +95,8 @@ test('--output-dir should work', async ({ startClient }, testInfo) => {
 });
 
 
-test('browser_take_screenshot (outputDir)', async ({ startClient }, testInfo) => {
-  const outputDir = testInfo.outputPath('output');
+test('browser_take_screenshot (outputDir)', async ({ startClient, localOutputPath }) => {
+  const outputDir = localOutputPath('output');
   const client = await startClient({
     config: { outputDir },
   });