chore: turn vision into capability (#679)

Fixes https://github.com/microsoft/playwright-mcp/issues/420

README.md

@@ -193,9 +193,8 @@ Playwright MCP server supports following arguments. They can be provided in the
   --browser <browser>          browser or chrome channel to use, possible
                                values: chrome, firefox, webkit, msedge.
   --browser-agent <endpoint>   Use browser agent (experimental).
-  --caps <caps>                comma-separated list of capabilities to enable,
+  --caps <caps>                comma-separated list of additional capabilities
-                               possible values: tabs, pdf, history, wait, files,
+                               to enable, possible values: vision, pdf.
-                               install. Default is all.
   --cdp-endpoint <endpoint>    CDP endpoint to connect to.
   --config <path>              path to the configuration file.
   --device <device>            device to emulate, for example: "iPhone 15"
@@ -227,8 +226,6 @@ Playwright MCP server supports following arguments. They can be provided in the
                                specified, a temporary directory will be created.
   --viewport-size <size>       specify browser viewport size in pixels, for
                                example "1280, 720"
-  --vision                     Run server that uses screenshots (Aria snapshots
-                               are used by default)
 ```
 
 <!--- End of options generated section -->
@@ -329,21 +326,14 @@ npx @playwright/mcp@latest --config path/to/config.json
     host?: string;  // Host to bind to (default: localhost)
   },
 
-  // List of enabled capabilities
+  // List of additional capabilities
   capabilities?: Array<
-    'core' |    // Core browser automation
     'tabs' |    // Tab management
-    'pdf' |     // PDF generation
-    'history' | // Browser history
-    'wait' |    // Wait utilities
-    'files' |   // File handling
     'install' | // Browser installation
-    'testing'   // Testing
+    'pdf' |     // PDF generation
+    'vision' |  // Coordinate-based interactions
   >;
 
-  // Enable vision mode (screenshots instead of accessibility snapshots)
-  vision?: boolean;
-
   // Directory for output files
   outputDir?: string;
 
@@ -433,42 +423,10 @@ http.createServer(async (req, res) => {
 
 ### Tools
 
-The tools are available in two modes:
-
-1. **Snapshot Mode** (default): Uses accessibility snapshots for better performance and reliability
-2. **Vision Mode**: Uses screenshots for visual-based interactions
-
-To use Vision Mode, add the `--vision` flag when starting the server:
-
-```js
-{
-  "mcpServers": {
-    "playwright": {
-      "command": "npx",
-      "args": [
-        "@playwright/mcp@latest",
-        "--vision"
-      ]
-    }
-  }
-}
-```
-
-Vision Mode works best with the computer use models that are able to interact with elements using
-X Y coordinate space, based on the provided screenshot.
-
 <!--- Tools generated by update-readme.js -->
 
 <details>
-<summary><b>Interactions</b></summary>
+<summary><b>Core automation</b></summary>
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_snapshot**
-  - Title: Page snapshot
-  - Description: Capture accessibility snapshot of the current page, this is better than screenshot
-  - Parameters: None
-  - Read-only: **true**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
@@ -483,6 +441,22 @@ X Y coordinate space, based on the provided screenshot.
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
+- **browser_close**
+  - Title: Close browser
+  - Description: Close the page
+  - Parameters: None
+  - Read-only: **true**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
+- **browser_console_messages**
+  - Title: Get console messages
+  - Description: Returns all console messages
+  - Parameters: None
+  - Read-only: **true**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
 - **browser_drag**
   - Title: Drag mouse
   - Description: Perform drag and drop between two elements
@@ -495,60 +469,17 @@ X Y coordinate space, based on the provided screenshot.
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_hover**
+- **browser_evaluate**
-  - Title: Hover mouse
+  - Title: Evaluate JavaScript
-  - Description: Hover over element on page
+  - Description: Evaluate JavaScript expression on page or element
   - Parameters:
-    - `element` (string): Human-readable element description used to obtain permission to interact with the element
+    - `function` (string): () => { /* code */ } or (element) => { /* code */ } when element is provided
-    - `ref` (string): Exact target element reference from the page snapshot
+    - `element` (string, optional): Human-readable element description used to obtain permission to interact with the element
-  - Read-only: **true**
+    - `ref` (string, optional): Exact target element reference from the page snapshot
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_type**
-  - Title: Type text
-  - Description: Type text into editable element
-  - Parameters:
-    - `element` (string): Human-readable element description used to obtain permission to interact with the element
-    - `ref` (string): Exact target element reference from the page snapshot
-    - `text` (string): Text to type into the element
-    - `submit` (boolean, optional): Whether to submit entered text (press Enter after)
-    - `slowly` (boolean, optional): Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.
   - Read-only: **false**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_select_option**
-  - Title: Select option
-  - Description: Select an option in a dropdown
-  - Parameters:
-    - `element` (string): Human-readable element description used to obtain permission to interact with the element
-    - `ref` (string): Exact target element reference from the page snapshot
-    - `values` (array): Array of values to select in the dropdown. This can be a single value or multiple values.
-  - Read-only: **false**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_press_key**
-  - Title: Press a key
-  - Description: Press a key on the keyboard
-  - Parameters:
-    - `key` (string): Name of the key to press or a character to generate, such as `ArrowLeft` or `a`
-  - Read-only: **false**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_wait_for**
-  - Title: Wait for
-  - Description: Wait for text to appear or disappear or a specified time to pass
-  - Parameters:
-    - `time` (number, optional): The time to wait in seconds
-    - `text` (string, optional): The text to wait for
-    - `textGone` (string, optional): The text to wait for to disappear
-  - Read-only: **true**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
 - **browser_file_upload**
   - Title: Upload files
   - Description: Upload one or multiple files
@@ -566,10 +497,15 @@ X Y coordinate space, based on the provided screenshot.
     - `promptText` (string, optional): The text of the prompt in case of a prompt dialog.
   - Read-only: **false**
 
-</details>
+<!-- NOTE: This has been generated via update-readme.js -->
 
-<details>
+- **browser_hover**
-<summary><b>Navigation</b></summary>
+  - Title: Hover mouse
+  - Description: Hover over element on page
+  - Parameters:
+    - `element` (string): Human-readable element description used to obtain permission to interact with the element
+    - `ref` (string): Exact target element reference from the page snapshot
+  - Read-only: **true**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
@@ -596,26 +532,51 @@ X Y coordinate space, based on the provided screenshot.
   - Parameters: None
   - Read-only: **true**
 
-</details>
+<!-- NOTE: This has been generated via update-readme.js -->
 
-<details>
+- **browser_network_requests**
-<summary><b>Evaluation</b></summary>
+  - Title: List network requests
+  - Description: Returns all network requests since loading the page
+  - Parameters: None
+  - Read-only: **true**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_evaluate**
+- **browser_press_key**
-  - Title: Evaluate JavaScript
+  - Title: Press a key
-  - Description: Evaluate JavaScript expression on page or element
+  - Description: Press a key on the keyboard
   - Parameters:
-    - `function` (string): () => { /* code */ } or (element) => { /* code */ } when element is provided
+    - `key` (string): Name of the key to press or a character to generate, such as `ArrowLeft` or `a`
-    - `element` (string, optional): Human-readable element description used to obtain permission to interact with the element
-    - `ref` (string, optional): Exact target element reference from the page snapshot
   - Read-only: **false**
 
-</details>
+<!-- NOTE: This has been generated via update-readme.js -->
 
-<details>
+- **browser_resize**
-<summary><b>Resources</b></summary>
+  - Title: Resize browser window
+  - Description: Resize the browser window
+  - Parameters:
+    - `width` (number): Width of the browser window
+    - `height` (number): Height of the browser window
+  - Read-only: **true**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
+- **browser_select_option**
+  - Title: Select option
+  - Description: Select an option in a dropdown
+  - Parameters:
+    - `element` (string): Human-readable element description used to obtain permission to interact with the element
+    - `ref` (string): Exact target element reference from the page snapshot
+    - `values` (array): Array of values to select in the dropdown. This can be a single value or multiple values.
+  - Read-only: **false**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
+- **browser_snapshot**
+  - Title: Page snapshot
+  - Description: Capture accessibility snapshot of the current page, this is better than screenshot
+  - Parameters: None
+  - Read-only: **true**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
@@ -631,64 +592,41 @@ X Y coordinate space, based on the provided screenshot.
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_pdf_save**
+- **browser_type**
-  - Title: Save as PDF
+  - Title: Type text
-  - Description: Save page as PDF
+  - Description: Type text into editable element
   - Parameters:
-    - `filename` (string, optional): File name to save the pdf to. Defaults to `page-{timestamp}.pdf` if not specified.
+    - `element` (string): Human-readable element description used to obtain permission to interact with the element
-  - Read-only: **true**
+    - `ref` (string): Exact target element reference from the page snapshot
-
+    - `text` (string): Text to type into the element
-<!-- NOTE: This has been generated via update-readme.js -->
+    - `submit` (boolean, optional): Whether to submit entered text (press Enter after)
-
+    - `slowly` (boolean, optional): Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.
-- **browser_network_requests**
-  - Title: List network requests
-  - Description: Returns all network requests since loading the page
-  - Parameters: None
-  - Read-only: **true**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_console_messages**
-  - Title: Get console messages
-  - Description: Returns all console messages
-  - Parameters: None
-  - Read-only: **true**
-
-</details>
-
-<details>
-<summary><b>Utilities</b></summary>
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_install**
-  - Title: Install the browser specified in the config
-  - Description: Install the browser specified in the config. Call this if you get an error about the browser not being installed.
-  - Parameters: None
   - Read-only: **false**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_close**
+- **browser_wait_for**
-  - Title: Close browser
+  - Title: Wait for
-  - Description: Close the page
+  - Description: Wait for text to appear or disappear or a specified time to pass
-  - Parameters: None
-  - Read-only: **true**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_resize**
-  - Title: Resize browser window
-  - Description: Resize the browser window
   - Parameters:
-    - `width` (number): Width of the browser window
+    - `time` (number, optional): The time to wait in seconds
-    - `height` (number): Height of the browser window
+    - `text` (string, optional): The text to wait for
+    - `textGone` (string, optional): The text to wait for to disappear
   - Read-only: **true**
 
 </details>
 
 <details>
-<summary><b>Tabs</b></summary>
+<summary><b>Tab management</b></summary>
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
+- **browser_tab_close**
+  - Title: Close a tab
+  - Description: Close a tab
+  - Parameters:
+    - `index` (number, optional): The index of the tab to close. Closes current tab if not provided.
+  - Read-only: **false**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
@@ -716,44 +654,29 @@ X Y coordinate space, based on the provided screenshot.
     - `index` (number): The index of the tab to select
   - Read-only: **true**
 
+</details>
+
+<details>
+<summary><b>Browser installation</b></summary>
+
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_tab_close**
+- **browser_install**
-  - Title: Close a tab
+  - Title: Install the browser specified in the config
-  - Description: Close a tab
+  - Description: Install the browser specified in the config. Call this if you get an error about the browser not being installed.
-  - Parameters:
+  - Parameters: None
-    - `index` (number, optional): The index of the tab to close. Closes current tab if not provided.
   - Read-only: **false**
 
 </details>
 
 <details>
-<summary><b>Vision mode</b></summary>
+<summary><b>Coordinate-based (opt-in via --caps=vision)</b></summary>
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_screen_capture**
+- **browser_mouse_click_xy**
-  - Title: Take a screenshot
-  - Description: Take a screenshot of the current page
-  - Parameters: None
-  - Read-only: **true**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_screen_move_mouse**
-  - Title: Move mouse
-  - Description: Move mouse to a given position
-  - Parameters:
-    - `element` (string): Human-readable element description used to obtain permission to interact with the element
-    - `x` (number): X coordinate
-    - `y` (number): Y coordinate
-  - Read-only: **true**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_screen_click**
   - Title: Click
-  - Description: Click left mouse button
+  - Description: Click left mouse button at a given position
   - Parameters:
     - `element` (string): Human-readable element description used to obtain permission to interact with the element
     - `x` (number): X coordinate
@@ -762,9 +685,9 @@ X Y coordinate space, based on the provided screenshot.
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_screen_drag**
+- **browser_mouse_drag_xy**
   - Title: Drag mouse
-  - Description: Drag left mouse button
+  - Description: Drag left mouse button to a given position
   - Parameters:
     - `element` (string): Human-readable element description used to obtain permission to interact with the element
     - `startX` (number): Start X coordinate
@@ -775,52 +698,28 @@ X Y coordinate space, based on the provided screenshot.
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_screen_type**
+- **browser_mouse_move_xy**
-  - Title: Type text
+  - Title: Move mouse
-  - Description: Type text
+  - Description: Move mouse to a given position
   - Parameters:
-    - `text` (string): Text to type into the element
+    - `element` (string): Human-readable element description used to obtain permission to interact with the element
-    - `submit` (boolean, optional): Whether to submit entered text (press Enter after)
+    - `x` (number): X coordinate
-  - Read-only: **false**
+    - `y` (number): Y coordinate
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_press_key**
-  - Title: Press a key
-  - Description: Press a key on the keyboard
-  - Parameters:
-    - `key` (string): Name of the key to press or a character to generate, such as `ArrowLeft` or `a`
-  - Read-only: **false**
-
-<!-- NOTE: This has been generated via update-readme.js -->
-
-- **browser_wait_for**
-  - Title: Wait for
-  - Description: Wait for text to appear or disappear or a specified time to pass
-  - Parameters:
-    - `time` (number, optional): The time to wait in seconds
-    - `text` (string, optional): The text to wait for
-    - `textGone` (string, optional): The text to wait for to disappear
   - Read-only: **true**
 
-<!-- NOTE: This has been generated via update-readme.js -->
+</details>
 
-- **browser_file_upload**
+<details>
-  - Title: Upload files
+<summary><b>PDF generation (opt-in via --caps=pdf)</b></summary>
-  - Description: Upload one or multiple files
-  - Parameters:
-    - `paths` (array): The absolute paths to the files to upload. Can be a single file or multiple files.
-  - Read-only: **false**
 
 <!-- NOTE: This has been generated via update-readme.js -->
 
-- **browser_handle_dialog**
+- **browser_pdf_save**
-  - Title: Handle a dialog
+  - Title: Save as PDF
-  - Description: Handle a dialog
+  - Description: Save page as PDF
   - Parameters:
-    - `accept` (boolean): Whether to accept the dialog.
+    - `filename` (string, optional): File name to save the pdf to. Defaults to `page-{timestamp}.pdf` if not specified.
-    - `promptText` (string, optional): The text of the prompt in case of a prompt dialog.
+  - Read-only: **true**
-  - Read-only: **false**
 
 </details>
 

config.d.ts

@@ -16,7 +16,7 @@
 
 import type * as playwright from 'playwright';
 
-export type ToolCapability = 'core' | 'tabs' | 'pdf' | 'history' | 'wait' | 'files' | 'install' | 'testing';
+export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf';
 
 export type Config = {
   /**
@@ -85,20 +85,11 @@ export type Config = {
   /**
    * List of enabled tool capabilities. Possible values:
    *   - 'core': Core browser automation features.
-   *   - 'tabs': Tab management features.
    *   - 'pdf': PDF generation and manipulation.
-   *   - 'history': Browser history access.
+   *   - 'vision': Coordinate-based interactions.
-   *   - 'wait': Wait and timing utilities.
-   *   - 'files': File upload/download support.
-   *   - 'install': Browser installation utilities.
    */
   capabilities?: ToolCapability[];
 
-  /**
-   * Run server that uses screenshots (Aria snapshots are used by default).
-   */
-  vision?: boolean;
-
   /**
    * Whether to save the Playwright trace of the session into the output directory.
    */

src/config.ts

@@ -49,7 +49,6 @@ export type CLIOptions = {
   userAgent?: string;
   userDataDir?: string;
   viewportSize?: string;
-  vision?: boolean;
 };
 
 const defaultConfig: FullConfig = {
@@ -185,7 +184,6 @@ export async function configFromCLIOptions(cliOptions: CLIOptions): Promise<Conf
       host: cliOptions.host,
     },
     capabilities: cliOptions.caps?.split(',').map((c: string) => c.trim() as ToolCapability),
-    vision: !!cliOptions.vision,
     network: {
       allowedOrigins: cliOptions.allowedOrigins,
       blockedOrigins: cliOptions.blockedOrigins,

src/connection.ts

@@ -19,7 +19,7 @@ import { CallToolRequestSchema, ListToolsRequestSchema, Tool as McpTool } from '
 import { zodToJsonSchema } from 'zod-to-json-schema';
 
 import { Context } from './context.js';
-import { snapshotTools, visionTools } from './tools.js';
+import { allTools } from './tools.js';
 import { packageJSON } from './package.js';
 
 import { FullConfig } from './config.js';
@@ -27,8 +27,7 @@ import { FullConfig } from './config.js';
 import type { BrowserContextFactory } from './browserContextFactory.js';
 
 export function createConnection(config: FullConfig, browserContextFactory: BrowserContextFactory): Connection {
-  const allTools = config.vision ? visionTools : snapshotTools;
+  const tools = allTools.filter(tool => tool.capability.startsWith('core') || config.capabilities?.includes(tool.capability));
-  const tools = allTools.filter(tool => !config.capabilities || tool.capability === 'core' || config.capabilities.includes(tool.capability));
   const context = new Context(tools, config, browserContextFactory);
   const server = new McpServer({ name: 'Playwright', version: packageJSON.version }, {
     capabilities: {

src/program.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { program } from 'commander';
+import { program, Option } from 'commander';
 // @ts-ignore
 import { startTraceViewerServer } from 'playwright-core/lib/server';
 
@@ -31,7 +31,7 @@ program
     .option('--block-service-workers', 'block service workers')
     .option('--browser <browser>', 'browser or chrome channel to use, possible values: chrome, firefox, webkit, msedge.')
     .option('--browser-agent <endpoint>', 'Use browser agent (experimental).')
-    .option('--caps <caps>', 'comma-separated list of capabilities to enable, possible values: tabs, pdf, history, wait, files, install. Default is all.')
+    .option('--caps <caps>', 'comma-separated list of additional capabilities to enable, possible values: vision, pdf.')
     .option('--cdp-endpoint <endpoint>', 'CDP endpoint to connect to.')
     .option('--config <path>', 'path to the configuration file.')
     .option('--device <device>', 'device to emulate, for example: "iPhone 15"')
@@ -51,8 +51,13 @@ program
     .option('--user-agent <ua string>', 'specify user agent string')
     .option('--user-data-dir <path>', 'path to the user data directory. If not specified, a temporary directory will be created.')
     .option('--viewport-size <size>', 'specify browser viewport size in pixels, for example "1280, 720"')
-    .option('--vision', 'Run server that uses screenshots (Aria snapshots are used by default)')
+    .addOption(new Option('--vision', 'Legacy option, use --caps=vision instead').hideHelp())
     .action(async options => {
+      if (options.vision) {
+        // eslint-disable-next-line no-console
+        console.error('The --vision option is deprecated, use --caps=vision instead');
+        options.caps = 'vision';
+      }
       const config = await resolveCLIConfig(options);
       const httpServer = config.server.port !== undefined ? await startHttpServer(config.server) : undefined;
 

src/tools.ts

@@ -27,39 +27,25 @@ import pdf from './tools/pdf.js';
 import snapshot from './tools/snapshot.js';
 import tabs from './tools/tabs.js';
 import screenshot from './tools/screenshot.js';
-import vision from './tools/vision.js';
 import wait from './tools/wait.js';
+import mouse from './tools/mouse.js';
 
 import type { Tool } from './tools/tool.js';
 
-export const snapshotTools: Tool<any>[] = [
+export const allTools: Tool<any>[] = [
-  ...common(true),
+  ...common,
   ...console,
-  ...dialogs(true),
+  ...dialogs,
   ...evaluate,
-  ...files(true),
+  ...files,
   ...install,
-  ...keyboard(true),
+  ...keyboard,
-  ...navigate(true),
+  ...navigate,
   ...network,
+  ...mouse,
   ...pdf,
   ...screenshot,
   ...snapshot,
-  ...tabs(true),
+  ...tabs,
-  ...wait(true),
+  ...wait,
-];
-
-export const visionTools: Tool<any>[] = [
-  ...common(false),
-  ...console,
-  ...dialogs(false),
-  ...files(false),
-  ...install,
-  ...keyboard(false),
-  ...navigate(false),
-  ...network,
-  ...pdf,
-  ...tabs(false),
-  ...vision,
-  ...wait(false),
 ];

src/tools/common.ts

@@ -15,7 +15,7 @@
  */
 
 import { z } from 'zod';
-import { defineTool, type ToolFactory } from './tool.js';
+import { defineTool } from './tool.js';
 
 const close = defineTool({
   capability: 'core',
@@ -38,7 +38,7 @@ const close = defineTool({
   },
 });
 
-const resize: ToolFactory = captureSnapshot => defineTool({
+const resize = defineTool({
   capability: 'core',
   schema: {
     name: 'browser_resize',
@@ -66,13 +66,13 @@ const resize: ToolFactory = captureSnapshot => defineTool({
     return {
       code,
       action,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: true
     };
   },
 });
 
-export default (captureSnapshot: boolean) => [
+export default [
   close,
-  resize(captureSnapshot)
+  resize
 ];

src/tools/dialogs.ts

@@ -15,9 +15,9 @@
  */
 
 import { z } from 'zod';
-import { defineTool, type ToolFactory } from './tool.js';
+import { defineTool } from './tool.js';
 
-const handleDialog: ToolFactory = captureSnapshot => defineTool({
+const handleDialog = defineTool({
   capability: 'core',
 
   schema: {
@@ -49,7 +49,7 @@ const handleDialog: ToolFactory = captureSnapshot => defineTool({
 
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false,
     };
   },
@@ -57,6 +57,6 @@ const handleDialog: ToolFactory = captureSnapshot => defineTool({
   clearsModalState: 'dialog',
 });
 
-export default (captureSnapshot: boolean) => [
+export default [
-  handleDialog(captureSnapshot),
+  handleDialog,
 ];

src/tools/files.ts

@@ -15,10 +15,10 @@
  */
 
 import { z } from 'zod';
-import { defineTool, type ToolFactory } from './tool.js';
+import { defineTool } from './tool.js';
 
-const uploadFile: ToolFactory = captureSnapshot => defineTool({
+const uploadFile = defineTool({
-  capability: 'files',
+  capability: 'core',
 
   schema: {
     name: 'browser_file_upload',
@@ -47,13 +47,13 @@ const uploadFile: ToolFactory = captureSnapshot => defineTool({
     return {
       code,
       action,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: true,
     };
   },
   clearsModalState: 'fileChooser',
 });
 
-export default (captureSnapshot: boolean) => [
+export default [
-  uploadFile(captureSnapshot),
+  uploadFile,
 ];

src/tools/install.ts

@@ -23,7 +23,7 @@ import { defineTool } from './tool.js';
 import { fileURLToPath } from 'node:url';
 
 const install = defineTool({
-  capability: 'install',
+  capability: 'core-install',
   schema: {
     name: 'browser_install',
     title: 'Install the browser specified in the config',

src/tools/keyboard.ts

@@ -15,9 +15,13 @@
  */
 
 import { z } from 'zod';
-import { defineTool, type ToolFactory } from './tool.js';
 
-const pressKey: ToolFactory = captureSnapshot => defineTool({
+import { defineTool } from './tool.js';
+import { elementSchema } from './snapshot.js';
+import { generateLocator } from './utils.js';
+import * as javascript from '../javascript.js';
+
+const pressKey = defineTool({
   capability: 'core',
 
   schema: {
@@ -43,12 +47,61 @@ const pressKey: ToolFactory = captureSnapshot => defineTool({
     return {
       code,
       action,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: true
     };
   },
 });
 
-export default (captureSnapshot: boolean) => [
+const typeSchema = elementSchema.extend({
-  pressKey(captureSnapshot),
+  text: z.string().describe('Text to type into the element'),
+  submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),
+  slowly: z.boolean().optional().describe('Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.'),
+});
+
+const type = defineTool({
+  capability: 'core',
+  schema: {
+    name: 'browser_type',
+    title: 'Type text',
+    description: 'Type text into editable element',
+    inputSchema: typeSchema,
+    type: 'destructive',
+  },
+
+  handle: async (context, params) => {
+    const snapshot = context.currentTabOrDie().snapshotOrDie();
+    const locator = snapshot.refLocator(params);
+
+    const code: string[] = [];
+    const steps: (() => Promise<void>)[] = [];
+
+    if (params.slowly) {
+      code.push(`// Press "${params.text}" sequentially into "${params.element}"`);
+      code.push(`await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(params.text)});`);
+      steps.push(() => locator.pressSequentially(params.text));
+    } else {
+      code.push(`// Fill "${params.text}" into "${params.element}"`);
+      code.push(`await page.${await generateLocator(locator)}.fill(${javascript.quote(params.text)});`);
+      steps.push(() => locator.fill(params.text));
+    }
+
+    if (params.submit) {
+      code.push(`// Submit text`);
+      code.push(`await page.${await generateLocator(locator)}.press('Enter');`);
+      steps.push(() => locator.press('Enter'));
+    }
+
+    return {
+      code,
+      action: () => steps.reduce((acc, step) => acc.then(step), Promise.resolve()),
+      captureSnapshot: true,
+      waitForNetwork: true,
+    };
+  },
+});
+
+export default [
+  pressKey,
+  type,
 ];

src/tools/mouse.ts

@@ -17,50 +17,14 @@
 import { z } from 'zod';
 import { defineTool } from './tool.js';
 
-import * as javascript from '../javascript.js';
-
 const elementSchema = z.object({
   element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
 });
 
-const screenshot = defineTool({
+const mouseMove = defineTool({
-  capability: 'core',
+  capability: 'vision',
   schema: {
-    name: 'browser_screen_capture',
+    name: 'browser_mouse_move_xy',
-    title: 'Take a screenshot',
-    description: 'Take a screenshot of the current page',
-    inputSchema: z.object({}),
-    type: 'readOnly',
-  },
-
-  handle: async context => {
-    const tab = await context.ensureTab();
-    const options = { type: 'jpeg' as 'jpeg', quality: 50, scale: 'css' as 'css' };
-
-    const code = [
-      `// Take a screenshot of the current page`,
-      `await page.screenshot(${javascript.formatObject(options)});`,
-    ];
-
-    const action = () => tab.page.screenshot(options).then(buffer => {
-      return {
-        content: [{ type: 'image' as 'image', data: buffer.toString('base64'), mimeType: 'image/jpeg' }],
-      };
-    });
-
-    return {
-      code,
-      action,
-      captureSnapshot: false,
-      waitForNetwork: false
-    };
-  },
-});
-
-const moveMouse = defineTool({
-  capability: 'core',
-  schema: {
-    name: 'browser_screen_move_mouse',
     title: 'Move mouse',
     description: 'Move mouse to a given position',
     inputSchema: elementSchema.extend({
@@ -86,12 +50,12 @@ const moveMouse = defineTool({
   },
 });
 
-const click = defineTool({
+const mouseClick = defineTool({
-  capability: 'core',
+  capability: 'vision',
   schema: {
-    name: 'browser_screen_click',
+    name: 'browser_mouse_click_xy',
     title: 'Click',
-    description: 'Click left mouse button',
+    description: 'Click left mouse button at a given position',
     inputSchema: elementSchema.extend({
       x: z.number().describe('X coordinate'),
       y: z.number().describe('Y coordinate'),
@@ -121,12 +85,12 @@ const click = defineTool({
   },
 });
 
-const drag = defineTool({
+const mouseDrag = defineTool({
-  capability: 'core',
+  capability: 'vision',
   schema: {
-    name: 'browser_screen_drag',
+    name: 'browser_mouse_drag_xy',
     title: 'Drag mouse',
-    description: 'Drag left mouse button',
+    description: 'Drag left mouse button to a given position',
     inputSchema: elementSchema.extend({
       startX: z.number().describe('Start X coordinate'),
       startY: z.number().describe('Start Y coordinate'),
@@ -163,51 +127,8 @@ const drag = defineTool({
   },
 });
 
-const type = defineTool({
-  capability: 'core',
-  schema: {
-    name: 'browser_screen_type',
-    title: 'Type text',
-    description: 'Type text',
-    inputSchema: z.object({
-      text: z.string().describe('Text to type into the element'),
-      submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),
-    }),
-    type: 'destructive',
-  },
-
-  handle: async (context, params) => {
-    const tab = context.currentTabOrDie();
-
-    const code = [
-      `// Type ${params.text}`,
-      `await page.keyboard.type('${params.text}');`,
-    ];
-
-    const action = async () => {
-      await tab.page.keyboard.type(params.text);
-      if (params.submit)
-        await tab.page.keyboard.press('Enter');
-    };
-
-    if (params.submit) {
-      code.push(`// Submit text`);
-      code.push(`await page.keyboard.press('Enter');`);
-    }
-
-    return {
-      code,
-      action,
-      captureSnapshot: false,
-      waitForNetwork: true,
-    };
-  },
-});
-
 export default [
-  screenshot,
+  mouseMove,
-  moveMouse,
+  mouseClick,
-  click,
+  mouseDrag,
-  drag,
-  type,
 ];

src/tools/navigate.ts

@@ -15,9 +15,9 @@
  */
 
 import { z } from 'zod';
-import { defineTool, type ToolFactory } from './tool.js';
+import { defineTool } from './tool.js';
 
-const navigate: ToolFactory = captureSnapshot => defineTool({
+const navigate = defineTool({
   capability: 'core',
 
   schema: {
@@ -41,14 +41,14 @@ const navigate: ToolFactory = captureSnapshot => defineTool({
 
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false,
     };
   },
 });
 
-const goBack: ToolFactory = captureSnapshot => defineTool({
+const goBack = defineTool({
-  capability: 'history',
+  capability: 'core',
   schema: {
     name: 'browser_navigate_back',
     title: 'Go back',
@@ -67,14 +67,14 @@ const goBack: ToolFactory = captureSnapshot => defineTool({
 
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false,
     };
   },
 });
 
-const goForward: ToolFactory = captureSnapshot => defineTool({
+const goForward = defineTool({
-  capability: 'history',
+  capability: 'core',
   schema: {
     name: 'browser_navigate_forward',
     title: 'Go forward',
@@ -91,14 +91,14 @@ const goForward: ToolFactory = captureSnapshot => defineTool({
     ];
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false,
     };
   },
 });
 
-export default (captureSnapshot: boolean) => [
+export default [
-  navigate(captureSnapshot),
+  navigate,
-  goBack(captureSnapshot),
+  goBack,
-  goForward(captureSnapshot),
+  goForward,
 ];

src/tools/screenshot.ts

@@ -79,7 +79,7 @@ const screenshot = defineTool({
     return {
       code,
       action,
-      captureSnapshot: true,
+      captureSnapshot: false,
       waitForNetwork: false,
     };
   }

src/tools/snapshot.ts

@@ -41,7 +41,7 @@ const snapshot = defineTool({
   },
 });
 
-const elementSchema = z.object({
+export const elementSchema = z.object({
   element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
   ref: z.string().describe('Exact target element reference from the page snapshot'),
 });
@@ -144,54 +144,6 @@ const hover = defineTool({
   },
 });
 
-const typeSchema = elementSchema.extend({
-  text: z.string().describe('Text to type into the element'),
-  submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),
-  slowly: z.boolean().optional().describe('Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.'),
-});
-
-const type = defineTool({
-  capability: 'core',
-  schema: {
-    name: 'browser_type',
-    title: 'Type text',
-    description: 'Type text into editable element',
-    inputSchema: typeSchema,
-    type: 'destructive',
-  },
-
-  handle: async (context, params) => {
-    const snapshot = context.currentTabOrDie().snapshotOrDie();
-    const locator = snapshot.refLocator(params);
-
-    const code: string[] = [];
-    const steps: (() => Promise<void>)[] = [];
-
-    if (params.slowly) {
-      code.push(`// Press "${params.text}" sequentially into "${params.element}"`);
-      code.push(`await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(params.text)});`);
-      steps.push(() => locator.pressSequentially(params.text));
-    } else {
-      code.push(`// Fill "${params.text}" into "${params.element}"`);
-      code.push(`await page.${await generateLocator(locator)}.fill(${javascript.quote(params.text)});`);
-      steps.push(() => locator.fill(params.text));
-    }
-
-    if (params.submit) {
-      code.push(`// Submit text`);
-      code.push(`await page.${await generateLocator(locator)}.press('Enter');`);
-      steps.push(() => locator.press('Enter'));
-    }
-
-    return {
-      code,
-      action: () => steps.reduce((acc, step) => acc.then(step), Promise.resolve()),
-      captureSnapshot: true,
-      waitForNetwork: true,
-    };
-  },
-});
-
 const selectOptionSchema = elementSchema.extend({
   values: z.array(z.string()).describe('Array of values to select in the dropdown. This can be a single value or multiple values.'),
 });
@@ -229,6 +181,5 @@ export default [
   click,
   drag,
   hover,
-  type,
   selectOption,
 ];

src/tools/tabs.ts

@@ -15,10 +15,10 @@
  */
 
 import { z } from 'zod';
-import { defineTool, type ToolFactory } from './tool.js';
+import { defineTool } from './tool.js';
 
 const listTabs = defineTool({
-  capability: 'tabs',
+  capability: 'core-tabs',
 
   schema: {
     name: 'browser_tab_list',
@@ -44,8 +44,8 @@ const listTabs = defineTool({
   },
 });
 
-const selectTab: ToolFactory = captureSnapshot => defineTool({
+const selectTab = defineTool({
-  capability: 'tabs',
+  capability: 'core-tabs',
 
   schema: {
     name: 'browser_tab_select',
@@ -65,14 +65,14 @@ const selectTab: ToolFactory = captureSnapshot => defineTool({
 
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false
     };
   },
 });
 
-const newTab: ToolFactory = captureSnapshot => defineTool({
+const newTab = defineTool({
-  capability: 'tabs',
+  capability: 'core-tabs',
 
   schema: {
     name: 'browser_tab_new',
@@ -94,14 +94,14 @@ const newTab: ToolFactory = captureSnapshot => defineTool({
     ];
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false
     };
   },
 });
 
-const closeTab: ToolFactory = captureSnapshot => defineTool({
+const closeTab = defineTool({
-  capability: 'tabs',
+  capability: 'core-tabs',
 
   schema: {
     name: 'browser_tab_close',
@@ -120,15 +120,15 @@ const closeTab: ToolFactory = captureSnapshot => defineTool({
     ];
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false
     };
   },
 });
 
-export default (captureSnapshot: boolean) => [
+export default [
   listTabs,
-  newTab(captureSnapshot),
+  newTab,
-  selectTab(captureSnapshot),
+  selectTab,
-  closeTab(captureSnapshot),
+  closeTab,
 ];

src/tools/tool.ts

@@ -61,8 +61,6 @@ export type Tool<Input extends InputType = InputType> = {
   handle: (context: Context, params: z.output<Input>) => Promise<ToolResult>;
 };
 
-export type ToolFactory = (snapshot: boolean) => Tool<any>;
-
 export function defineTool<Input extends InputType>(tool: Tool<Input>): Tool<Input> {
   return tool;
 }

src/tools/wait.ts

@@ -15,10 +15,10 @@
  */
 
 import { z } from 'zod';
-import { defineTool, type ToolFactory } from './tool.js';
+import { defineTool } from './tool.js';
 
-const wait: ToolFactory = captureSnapshot => defineTool({
+const wait = defineTool({
-  capability: 'wait',
+  capability: 'core',
 
   schema: {
     name: 'browser_wait_for',
@@ -40,7 +40,7 @@ const wait: ToolFactory = captureSnapshot => defineTool({
 
     if (params.time) {
       code.push(`await new Promise(f => setTimeout(f, ${params.time!} * 1000));`);
-      await new Promise(f => setTimeout(f, Math.min(10000, params.time! * 1000)));
+      await new Promise(f => setTimeout(f, Math.min(30000, params.time! * 1000)));
     }
 
     const tab = context.currentTabOrDie();
@@ -59,12 +59,12 @@ const wait: ToolFactory = captureSnapshot => defineTool({
 
     return {
       code,
-      captureSnapshot,
+      captureSnapshot: true,
       waitForNetwork: false,
     };
   },
 });
 
-export default (captureSnapshot: boolean) => [
+export default [
-  wait(captureSnapshot),
+  wait,
 ];

tests/capabilities.spec.ts

@@ -34,7 +34,6 @@ test('test snapshot tool list', async ({ client }) => {
     'browser_navigate_forward',
     'browser_navigate',
     'browser_network_requests',
-    'browser_pdf_save',
     'browser_press_key',
     'browser_resize',
     'browser_snapshot',
@@ -47,45 +46,33 @@ test('test snapshot tool list', async ({ client }) => {
   ]));
 });
 
-test('test vision tool list', async ({ visionClient }) => {
+test('test capabilities (pdf)', async ({ startClient }) => {
-  const { tools: visionTools } = await visionClient.listTools();
-  expect(new Set(visionTools.map(t => t.name))).toEqual(new Set([
-    'browser_close',
-    'browser_console_messages',
-    'browser_file_upload',
-    'browser_handle_dialog',
-    'browser_install',
-    'browser_navigate_back',
-    'browser_navigate_forward',
-    'browser_navigate',
-    'browser_network_requests',
-    'browser_pdf_save',
-    'browser_press_key',
-    'browser_resize',
-    'browser_screen_capture',
-    'browser_screen_click',
-    'browser_screen_drag',
-    'browser_screen_move_mouse',
-    'browser_screen_type',
-    'browser_tab_close',
-    'browser_tab_list',
-    'browser_tab_new',
-    'browser_tab_select',
-    'browser_wait_for',
-  ]));
-});
-
-test('test capabilities', async ({ startClient }) => {
   const { client } = await startClient({
-    args: ['--caps="core"'],
+    args: ['--caps=pdf'],
   });
   const { tools } = await client.listTools();
   const toolNames = tools.map(t => t.name);
-  expect(toolNames).not.toContain('browser_file_upload');
+  expect(toolNames).toContain('browser_pdf_save');
-  expect(toolNames).not.toContain('browser_pdf_save');
+});
-  expect(toolNames).not.toContain('browser_screen_capture');
+
-  expect(toolNames).not.toContain('browser_screen_click');
+test('test capabilities (vision)', async ({ startClient }) => {
-  expect(toolNames).not.toContain('browser_screen_drag');
+  const { client } = await startClient({
-  expect(toolNames).not.toContain('browser_screen_move_mouse');
+    args: ['--caps=vision'],
-  expect(toolNames).not.toContain('browser_screen_type');
+  });
+  const { tools } = await client.listTools();
+  const toolNames = tools.map(t => t.name);
+  expect(toolNames).toContain('browser_mouse_move_xy');
+  expect(toolNames).toContain('browser_mouse_click_xy');
+  expect(toolNames).toContain('browser_mouse_drag_xy');
+});
+
+test('support for legacy --vision option', async ({ startClient }) => {
+  const { client } = await startClient({
+    args: ['--vision'],
+  });
+  const { tools } = await client.listTools();
+  const toolNames = tools.map(t => t.name);
+  expect(toolNames).toContain('browser_mouse_move_xy');
+  expect(toolNames).toContain('browser_mouse_click_xy');
+  expect(toolNames).toContain('browser_mouse_drag_xy');
 });

tests/evaluate.spec.ts

@@ -0,0 +1,51 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './fixtures.js';
+
+test('browser_evaluate', async ({ client, server }) => {
+  expect(await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.HELLO_WORLD },
+  })).toContainTextContent(`- Page Title: Title`);
+
+  const result = await client.callTool({
+    name: 'browser_evaluate',
+    arguments: {
+      function: '() => document.title',
+    },
+  });
+  expect(result).toContainTextContent(`"Title"`);
+});
+
+test('browser_evaluate (element)', async ({ client, server }) => {
+  server.setContent('/', `
+    <body style="background-color: red">Hello, world!</body>
+  `, 'text/html');
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_evaluate',
+    arguments: {
+      function: 'element => element.style.backgroundColor',
+      element: 'body',
+      ref: 'e1',
+    },
+  })).toContainTextContent(`- Result: "red"`);
+});

tests/fixtures.ts

@@ -41,7 +41,6 @@ type CDPServer = {
 
 type TestFixtures = {
   client: Client;
-  visionClient: Client;
   startClient: (options?: { clientName?: string, args?: string[], config?: Config }) => Promise<{ client: Client, stderr: () => string }>;
   wsEndpoint: string;
   cdpServer: CDPServer;
@@ -61,11 +60,6 @@ export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>(
     await use(client);
   },
 
-  visionClient: async ({ startClient }, use) => {
-    const { client } = await startClient({ args: ['--vision'] });
-    await use(client);
-  },
-
   startClient: async ({ mcpHeadless, mcpBrowser, mcpMode }, use, testInfo) => {
     const userDataDir = mcpMode !== 'docker' ? testInfo.outputPath('user-data-dir') : undefined;
     const configDir = path.dirname(test.info().config.configFile!);

tests/pdf.spec.ts

@@ -19,7 +19,7 @@ import fs from 'fs';
 import { test, expect } from './fixtures.js';
 
 test('save as pdf unavailable', async ({ startClient, server }) => {
-  const { client } = await startClient({ args: ['--caps="no-pdf"'] });
+  const { client } = await startClient();
   await client.callTool({
     name: 'browser_navigate',
     arguments: { url: server.HELLO_WORLD },
@@ -32,7 +32,7 @@ test('save as pdf unavailable', async ({ startClient, server }) => {
 
 test('save as pdf', async ({ startClient, mcpBrowser, server }, testInfo) => {
   const { client } = await startClient({
-    config: { outputDir: testInfo.outputPath('output') },
+    config: { outputDir: testInfo.outputPath('output'), capabilities: ['pdf'] },
   });
 
   test.skip(!!mcpBrowser && !['chromium', 'chrome', 'msedge'].includes(mcpBrowser), 'Save as PDF is only supported in Chromium.');
@@ -52,7 +52,7 @@ test('save as pdf (filename: output.pdf)', async ({ startClient, mcpBrowser, ser
   const outputDir = testInfo.outputPath('output');
   test.skip(!!mcpBrowser && !['chromium', 'chrome', 'msedge'].includes(mcpBrowser), 'Save as PDF is only supported in Chromium.');
   const { client } = await startClient({
-    config: { outputDir },
+    config: { outputDir, capabilities: ['pdf'] },
   });
 
   expect(await client.callTool({

utils/update-readme.js

@@ -20,60 +20,20 @@ import fs from 'node:fs'
 import path from 'node:path'
 import url from 'node:url'
 import zodToJsonSchema from 'zod-to-json-schema'
-
-import commonTools from '../lib/tools/common.js';
-import consoleTools from '../lib/tools/console.js';
-import dialogsTools from '../lib/tools/dialogs.js';
-import evaluateTools from '../lib/tools/evaluate.js';
-import filesTools from '../lib/tools/files.js';
-import installTools from '../lib/tools/install.js';
-import keyboardTools from '../lib/tools/keyboard.js';
-import navigateTools from '../lib/tools/navigate.js';
-import networkTools from '../lib/tools/network.js';
-import pdfTools from '../lib/tools/pdf.js';
-import snapshotTools from '../lib/tools/snapshot.js';
-import tabsTools from '../lib/tools/tabs.js';
-import screenshotTools from '../lib/tools/screenshot.js';
-import visionTools from '../lib/tools/vision.js';
-import waitTools from '../lib/tools/wait.js';
 import { execSync } from 'node:child_process';
 
-const categories = {
+import { allTools } from '../lib/tools.js';
-  'Interactions': [
+
-    ...snapshotTools,
+const capabilities = {
-    ...keyboardTools(true),
+  'core': 'Core automation',
-    ...waitTools(true),
+  'core-tabs': 'Tab management',
-    ...filesTools(true),
+  'core-install': 'Browser installation',
-    ...dialogsTools(true),
+  'vision': 'Coordinate-based (opt-in via --caps=vision)',
-  ],
+  'pdf': 'PDF generation (opt-in via --caps=pdf)',
-  'Navigation': [
-    ...navigateTools(true),
-  ],
-  'Evaluation': [
-    ...evaluateTools,
-  ],
-  'Resources': [
-    ...screenshotTools,
-    ...pdfTools,
-    ...networkTools,
-    ...consoleTools,
-  ],
-  'Utilities': [
-    ...installTools,
-    ...commonTools(true),
-  ],
-  'Tabs': [
-    ...tabsTools(true),
-  ],
-  'Vision mode': [
-    ...visionTools,
-    ...keyboardTools(),
-    ...waitTools(false),
-    ...filesTools(false),
-    ...dialogsTools(false),
-  ],
 };
 
+const toolsByCapability = Object.fromEntries(Object.entries(capabilities).map(([capability, title]) => [title, allTools.filter(tool => tool.capability === capability).sort((a, b) => a.schema.name.localeCompare(b.schema.name))]));
+
 // NOTE: Can be removed when we drop Node.js 18 support and changed to import.meta.filename.
 const __filename = url.fileURLToPath(import.meta.url);
 
@@ -139,14 +99,12 @@ async function updateSection(content, startMarker, endMarker, generatedLines) {
 async function updateTools(content) {
   console.log('Loading tool information from compiled modules...');
 
-  const totalTools = Object.values(categories).flat().length;
-  console.log(`Found ${totalTools} tools`);
-
   const generatedLines = /** @type {string[]} */ ([]);
-  for (const [category, categoryTools] of Object.entries(categories)) {
+  for (const [capability, tools] of Object.entries(toolsByCapability)) {
-    generatedLines.push(`<details>\n<summary><b>${category}</b></summary>`);
+    console.log('Updating tools for capability:', capability);
+    generatedLines.push(`<details>\n<summary><b>${capability}</b></summary>`);
     generatedLines.push('');
-    for (const tool of categoryTools)
+    for (const tool of tools)
       generatedLines.push(...formatToolForReadme(tool.schema));
     generatedLines.push(`</details>`);
     generatedLines.push('');