From 0d6bb2f547c40043cde3b9214dce9700a5ae96c7 Mon Sep 17 00:00:00 2001
From: Yury Semikhatsky <yurys@chromium.org>
Date: Tue, 15 Apr 2025 13:16:56 -0700
Subject: [PATCH] devops: add bots for other browsers/platforms (#174)

---
 .github/workflows/ci.yml      | 10 +++++++++-
 .github/workflows/publish.yml |  2 +-
 package.json                  |  1 +
 playwright.config.ts          | 10 +++++++++-
 tests/fixtures.ts             | 18 ++++++++++++++++--
 tests/pdf.spec.ts             |  3 ++-
 6 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3ad890d..d4164a3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,7 +8,11 @@ on:
 
 jobs:
   build-and-test:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
@@ -25,6 +29,10 @@ jobs:
     - name: Playwright install
       run: npx playwright install --with-deps
 
+    - name: Install MS Edge
+      if: ${{ matrix.os == 'macos-latest' }} # MS Edge is not preinstalled on macOS runners.
+      run: npx playwright install msedge
+
     - name: Run linting
       run: npm run lint
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index b10d4a7..bbf06a7 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -18,7 +18,7 @@ jobs:
       - run: npx playwright install --with-deps
       - run: npm run build
       - run: npm run lint
-      - run: npm run test
+      - run: npm run ctest
       - run: npm publish --provenance
         env:
           NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
diff --git a/package.json b/package.json
index 9706071..8edc404 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
     "lint": "eslint .",
     "watch": "tsc --watch",
     "test": "playwright test",
+    "ctest": "playwright test --project=chrome",
     "clean": "rm -rf lib",
     "npm-publish": "npm run clean && npm run build && npm run test && npm publish"
   },
diff --git a/playwright.config.ts b/playwright.config.ts
index dfd128d..ddb3ce7 100644
--- a/playwright.config.ts
+++ b/playwright.config.ts
@@ -16,6 +16,8 @@
 
 import { defineConfig } from '@playwright/test';
 
+import type { Project } from '@playwright/test';
+
 export default defineConfig({
   testDir: './tests',
   fullyParallel: true,
@@ -23,5 +25,11 @@ export default defineConfig({
   retries: process.env.CI ? 2 : 0,
   workers: process.env.CI ? 1 : undefined,
   reporter: 'list',
-  projects: [{ name: 'default' }],
+  projects: [
+    { name: 'chrome' },
+    { name: 'msedge', use: { mcpBrowser: 'msedge' } },
+    { name: 'chromium', use: { mcpBrowser: 'chromium' } },
+    { name: 'firefox', use: { mcpBrowser: 'firefox' } },
+    { name: 'webkit', use: { mcpBrowser: 'webkit' } },
+  ].filter(Boolean) as Project[],
 });
diff --git a/tests/fixtures.ts b/tests/fixtures.ts
index f8b4ee0..4601271 100644
--- a/tests/fixtures.ts
+++ b/tests/fixtures.ts
@@ -28,6 +28,10 @@ type Fixtures = {
   startClient: (options?: { args?: string[] }) => Promise<Client>;
   wsEndpoint: string;
   cdpEndpoint: string;
+
+  // Cli options.
+  mcpHeadless: boolean;
+  mcpBrowser: string | undefined;
 };
 
 export const test = baseTest.extend<Fixtures>({
@@ -40,12 +44,16 @@ export const test = baseTest.extend<Fixtures>({
     await use(await startClient({ args: ['--vision'] }));
   },
 
-  startClient: async ({ }, use, testInfo) => {
+  startClient: async ({ mcpHeadless, mcpBrowser }, use, testInfo) => {
     const userDataDir = testInfo.outputPath('user-data-dir');
     let client: StdioClientTransport | undefined;
 
     use(async options => {
-      const args = ['--headless', '--user-data-dir', userDataDir];
+      const args = ['--user-data-dir', userDataDir];
+      if (mcpHeadless)
+        args.push('--headless');
+      if (mcpBrowser)
+        args.push(`--browser=${mcpBrowser}`);
       if (options?.args)
         args.push(...options.args);
       const transport = new StdioClientTransport({
@@ -89,6 +97,12 @@ export const test = baseTest.extend<Fixtures>({
     await use(`http://localhost:${port}`);
     browserProcess.kill();
   },
+
+  mcpHeadless: async ({ headless }, use) => {
+    await use(headless);
+  },
+
+  mcpBrowser: ['chromium', { option: true }],
 });
 
 type Response = Awaited<ReturnType<Client['callTool']>>;
diff --git a/tests/pdf.spec.ts b/tests/pdf.spec.ts
index b9959ae..32090a1 100644
--- a/tests/pdf.spec.ts
+++ b/tests/pdf.spec.ts
@@ -30,7 +30,8 @@ test('save as pdf unavailable', async ({ startClient }) => {
   })).toHaveTextContent(/Tool \"browser_pdf_save\" not found/);
 });
 
-test('save as pdf', async ({ client }) => {
+test('save as pdf', async ({ client, mcpBrowser }) => {
+  test.skip(!!mcpBrowser && !['chromium', 'chrome', 'msedge'].includes(mcpBrowser), 'Save as PDF is only supported in Chromium.');
   expect(await client.callTool({
     name: 'browser_navigate',
     arguments: {