From d7c6431213d356577c4153c9edc7a6e1b89030da Mon Sep 17 00:00:00 2001 From: Priya Narayanaswamy Date: Wed, 16 Sep 2026 17:15:50 +0200 Subject: [PATCH 1/2] chore: add android cdp --- SKILL.md | 32 ++-- package.json | 2 +- src/cli/mm.test.ts | 56 +++++++ src/cli/mm.ts | 22 ++- .../mobile-platform-driver.hermes.test.ts | 128 +++++++++++++++- src/platform/mobile-platform-driver.test.ts | 138 ++++++++++++++++++ src/platform/mobile-platform-driver.ts | 125 ++++++++++++++-- src/tools/types/discovery.ts | 11 ++ src/tools/types/tool-inputs.ts | 11 ++ src/validation/schemas.ts | 26 +++- vitest.config.mts | 8 +- yarn.lock | 10 +- 12 files changed, 525 insertions(+), 44 deletions(-) diff --git a/SKILL.md b/SKILL.md index 90286fa..166f9c3 100644 --- a/SKILL.md +++ b/SKILL.md @@ -549,9 +549,9 @@ mm mock-network requests [--limit ] | ------------- | ------------------------------------------ | | `--limit ` | Maximum number of recent records to return | -#### `mm cdp [params-json] [--timeout ] [--metro-port

] [--app-id ]` +#### `mm cdp [params-json] [--timeout ] [--target hermes|android-webview] [--url-filter ] [--metro-port

] [--app-id ]` -Sends a raw Chrome DevTools Protocol command against the active session. This is an escape hatch for cases where structured tools are insufficient — e.g., evaluating JavaScript, enabling network tracking, or inspecting the DOM tree. It dispatches through the active platform driver, so it works on **both** browser and mobile sessions — but the target runtime and available methods differ (see the table below). +Sends a raw Chrome DevTools Protocol command against the active session. This is an escape hatch for cases where structured tools are insufficient — e.g., evaluating JavaScript, enabling network tracking, or inspecting the DOM tree. It dispatches through the active platform driver, so it works on **both** browser and mobile sessions — but the target runtime and available methods differ (see the table below). On mobile, `--target` selects between the React Native JS runtime (`hermes`, default) and a debuggable in-app Android WebView (`android-webview`). ```bash # Browser @@ -559,8 +559,11 @@ mm cdp Runtime.evaluate '{"expression":"document.title"}' mm cdp Network.enable mm cdp DOM.getDocument '{"depth":2}' --timeout 60000 -# Mobile (Hermes) — evaluate JS in the running app +# Mobile (Hermes, default) — evaluate JS in the running React Native app mm cdp Runtime.evaluate '{"expression":"1+1","returnByValue":true}' --app-id io.metamask --metro-port 8081 + +# Mobile (Android WebView) — drive the DOM of the in-app browser +mm cdp Runtime.evaluate '{"expression":"document.querySelector(\'#personalSign\').click()"}' --target android-webview ``` | Argument | Description | @@ -568,17 +571,20 @@ mm cdp Runtime.evaluate '{"expression":"1+1","returnByValue":true}' --app-id io. | `` | CDP method name (e.g., `Runtime.evaluate`, `DOM.getDocument` on browser; `Runtime.evaluate`, `Debugger.enable` on mobile) | | `[params-json]` | Optional JSON object with method-specific parameters | | `--timeout` | Per-command timeout in ms (default: 30 000, max: 30 000) | -| `--metro-port` | **Mobile only** — override the Metro inspector proxy port (default: 8081). Ignored on browser. | -| `--app-id` | **Mobile only** — override the expected app bundle identifier. Ignored on browser. | +| `--target` | **Mobile only** — `hermes` (default, RN JS runtime) or `android-webview` (in-app WebView DOM). Ignored on browser. | +| `--url-filter` | **`--target android-webview` only** — select the WebView page whose URL contains this substring. | +| `--metro-port` | **Mobile (Hermes) only** — override the Metro inspector proxy port (default: 8081). Ignored on browser. | +| `--app-id` | **Mobile (Hermes) only** — override the expected app bundle identifier. Ignored on browser. | -**Browser vs Mobile (Hermes):** the same command targets different runtimes. +**Three CDP targets:** the same command reaches different runtimes. -| Aspect | Browser (Playwright) | Mobile (React Native Hermes) | -| ----------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | -| Target | The page's Chrome DevTools session | The app's Hermes JS engine, via Metro's inspector proxy (needs a DEBUG build with Metro running) | -| Available domains | Full Chrome surface (`Runtime`, `DOM`, `Network`, `Page`, …) | JS-engine subset only (`Runtime`, `Debugger`, `Log`, `HeapProfiler`) — no `DOM`/`Page`/`Network` | -| Blocked methods | `Browser.close`, `Target.closeTarget`, `Target.disposeBrowserContext`, `Browser.crashGpuProcess` | `Runtime.terminateExecution`, `Inspector.detached` | -| Result shape | Standard CDP response | `Runtime.evaluate` nests the value at `result.result.value` | +| Aspect | Browser (Playwright) | Mobile — Hermes (`--target hermes`, default) | Mobile — Android WebView (`--target android-webview`) | +| ----------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| Target | The page's Chrome DevTools session | The app's Hermes JS engine, via Metro's inspector proxy (needs a DEBUG build with Metro running) | The web page inside a debuggable in-app Android WebView, via adb (needs `setWebContentsDebuggingEnabled(true)`) | +| Available domains | Full Chrome surface (`Runtime`, `DOM`, `Network`, `Page`, …) | JS-engine subset only (`Runtime`, `Debugger`, `Log`, `HeapProfiler`) — no `DOM`/`Page`/`Network` | Full Chrome surface (`Runtime`, `DOM`, `Network`, `Page`, `Input`) | +| Use it for | Web page DOM in the browser | The React Native app's own JavaScript | The web page DOM inside the app's in-app browser | +| Blocked methods | `Browser.close`, `Target.closeTarget`, `Target.disposeBrowserContext`, `Browser.crashGpuProcess` | `Runtime.terminateExecution`, `Inspector.detached` | `Browser.close`, `Target.closeTarget`, `Target.disposeBrowserContext`, `Browser.crashGpuProcess` | +| Result shape | Standard CDP response | `Runtime.evaluate` nests the value at `result.result.value` | `Runtime.evaluate` nests the value at `result.result.value` | Blocked methods return `MM_CDP_BLOCKED` on either platform; other failures return `MM_CDP_FAILED` (on mobile the underlying `HERMES_*` code is preserved in the message). The tool is categorized as **mutating** — run `describe-screen` afterward to re-sync if the call changed runtime/page state. @@ -635,7 +641,7 @@ Presses a hardware/system button. Only `home`, `back`, `enter`, and `lock` are a #### `mm device-context list` / `mm device-context switch ` -Lists available native/webview contexts, or switches the active context (e.g. `WEBVIEW_1`). +Lists available native/webview contexts, or switches the active context. On **iOS** WebView contexts look like `WEBVIEW_1` (Appium backend). On **Android** a single `WEBVIEW` context appears whenever a debuggable in-app WebView is open; to drive that page's DOM use `mm cdp --target android-webview` (see [`mm cdp`](#mm-cdp-method-params-json---timeout-ms---target-hermesandroid-webview---url-filter-substr---metro-port-p---app-id-id)). #### `mm device-clipboard read` / `mm device-clipboard write ` diff --git a/package.json b/package.json index 1cd5234..18534c1 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "hono": "^4.12.25" }, "dependencies": { - "@metamask/device-mcp": "^0.3.2", + "@metamask/device-mcp": "^0.4.1", "cosmiconfig": "^9.0.0", "express": "^5.2.1", "zod": "^4.4.3" diff --git a/src/cli/mm.test.ts b/src/cli/mm.test.ts index 88efa70..cb6a304 100644 --- a/src/cli/mm.test.ts +++ b/src/cli/mm.test.ts @@ -2013,6 +2013,62 @@ describe('routeCommand', () => { ); }); + it('routes cdp with --target android-webview and --url-filter', async () => { + await routeCommand( + 'cdp', + [ + 'Runtime.evaluate', + '{"expression":"1+1"}', + '--target', + 'android-webview', + '--url-filter', + 'test-dapp', + ], + 3000, + ); + expect(globalThis.fetch).toHaveBeenCalledWith( + 'http://127.0.0.1:3000/tool/cdp', + expect.objectContaining({ + body: JSON.stringify({ + method: 'Runtime.evaluate', + params: { expression: '1+1' }, + target: 'android-webview', + urlFilter: 'test-dapp', + }), + }), + ); + }); + + it('routes cdp with --target hermes', async () => { + await routeCommand( + 'cdp', + ['Runtime.evaluate', '--target', 'hermes'], + 3000, + ); + expect(globalThis.fetch).toHaveBeenCalledWith( + 'http://127.0.0.1:3000/tool/cdp', + expect.objectContaining({ + body: JSON.stringify({ + method: 'Runtime.evaluate', + target: 'hermes', + }), + }), + ); + }); + + it('exits when cdp --target is invalid', async () => { + await expect( + routeCommand( + 'cdp', + ['Runtime.evaluate', '--target', 'bogus'], + 3000, + ), + ).rejects.toThrowError('process.exit'); + expect(stderrSpy).toHaveBeenCalledWith( + expect.stringContaining('--target must be'), + ); + }); + it('routes scroll-to-element with a11y ref', async () => { await routeCommand('scroll-to-element', ['e1'], 3000); expect(globalThis.fetch).toHaveBeenCalledWith( diff --git a/src/cli/mm.ts b/src/cli/mm.ts index bb57310..d558df4 100644 --- a/src/cli/mm.ts +++ b/src/cli/mm.ts @@ -653,18 +653,32 @@ export async function routeCommand( const cdpMethod = args[0]; if (!cdpMethod) { process.stderr.write( - 'Usage: mm cdp [params-json] [--timeout ] [--metro-port ] [--app-id ]\n' + + 'Usage: mm cdp [params-json] [--timeout ] [--target hermes|android-webview] [--url-filter ] [--metro-port ] [--app-id ]\n' + ' mm cdp Runtime.evaluate \'{"expression":"document.title"}\'\n' + ' mm cdp Network.enable\n' + ' mm cdp DOM.getDocument \'{"depth":2}\' --timeout 60000\n' + ' # mobile (Hermes): --metro-port / --app-id override the Metro target\n' + - ' mm cdp Runtime.evaluate \'{"expression":"1+1","returnByValue":true}\' --app-id io.metamask\n', + ' mm cdp Runtime.evaluate \'{"expression":"1+1","returnByValue":true}\' --app-id io.metamask\n' + + ' # mobile (Android WebView): drive the in-app browser DOM\n' + + ' mm cdp Runtime.evaluate \'{"expression":"document.title"}\' --target android-webview\n', ); process.exit(1); } const cdpTimeout = parseIntFlag(args, '--timeout'); const cdpMetroPort = parseIntFlag(args, '--metro-port'); const cdpAppId = parseStringFlag(args, '--app-id'); + const cdpTarget = parseStringFlag(args, '--target'); + const cdpUrlFilter = parseStringFlag(args, '--url-filter'); + if ( + cdpTarget !== undefined && + cdpTarget !== 'hermes' && + cdpTarget !== 'android-webview' + ) { + process.stderr.write( + 'Error: --target must be "hermes" or "android-webview"\n', + ); + process.exit(1); + } const cdpParamsRaw = args[1] !== undefined && !args[1].startsWith('--') ? args[1] @@ -694,6 +708,8 @@ export async function routeCommand( ...(cdpTimeout === undefined ? {} : { timeoutMs: cdpTimeout }), ...(cdpMetroPort === undefined ? {} : { metroPort: cdpMetroPort }), ...(cdpAppId ? { appId: cdpAppId } : {}), + ...(cdpTarget ? { target: cdpTarget } : {}), + ...(cdpUrlFilter ? { urlFilter: cdpUrlFilter } : {}), }); break; } @@ -1677,7 +1693,7 @@ Advanced: mm mock-network clear mm mock-network list mm mock-network requests [--limit ] - mm cdp [params-json] [--timeout ] [--metro-port

] [--app-id ] + mm cdp [params-json] [--timeout ] [--target hermes|android-webview] [--url-filter ] [--metro-port

] [--app-id ] mm hermes-targets [--all] [--metro-port

] [--app-id ] (mobile only) Mobile (iOS/Android only): diff --git a/src/platform/mobile-platform-driver.hermes.test.ts b/src/platform/mobile-platform-driver.hermes.test.ts index b958f0e..e9cbe88 100644 --- a/src/platform/mobile-platform-driver.hermes.test.ts +++ b/src/platform/mobile-platform-driver.hermes.test.ts @@ -24,8 +24,11 @@ vi.mock('@metamask/device-mcp', () => ({ // Imported AFTER vi.mock so the driver binds to the mocked device-mcp runtime. const { MobilePlatformDriver } = await import('./mobile-platform-driver.js'); -function createBackend(platform: 'ios' | 'android' = 'android'): DeviceBackend { - return { platform } as unknown as DeviceBackend; +function createBackend( + platform: 'ios' | 'android' = 'android', + overrides: Partial = {}, +): DeviceBackend { + return { platform, ...overrides } as unknown as DeviceBackend; } describe('MobilePlatformDriver hermes delegation', () => { @@ -336,3 +339,124 @@ describe('MobilePlatformDriver hermes delegation', () => { }); }); }); + +describe('MobilePlatformDriver webview cdp routing', () => { + beforeEach(() => { + mocks.HermesSession.mockImplementation(() => ({ + resolve: mocks.resolve, + getPinnedHermesDeviceId: mocks.getPinnedHermesDeviceId, + setPinnedHermesDeviceId: mocks.setPinnedHermesDeviceId, + })); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('routes target "android-webview" to backend.webviewCdp and NOT Hermes', async () => { + const webviewCdp = vi + .fn() + .mockResolvedValue({ ok: true, result: { value: 'clicked' } }); + const driver = new MobilePlatformDriver( + createBackend('android', { webviewCdp }), + ); + + const outcome = await driver.cdp({ + method: 'Runtime.evaluate', + params: { expression: "document.querySelector('#x').click()" }, + timeoutMs: 30_000, + target: 'android-webview', + urlFilter: 'test-dapp', + }); + + expect(outcome).toStrictEqual({ ok: true, result: { value: 'clicked' } }); + expect(webviewCdp).toHaveBeenCalledWith({ + method: 'Runtime.evaluate', + params: { expression: "document.querySelector('#x').click()" }, + timeoutMs: 30_000, + urlFilter: 'test-dapp', + }); + expect(mocks.runHermesCdp).not.toHaveBeenCalled(); + }); + + it('defaults to the Hermes path when target is omitted', async () => { + mocks.resolve.mockReturnValue({ + metroPort: 8081, + appId: 'io.metamask', + pinnedDeviceId: undefined, + }); + mocks.getPinnedHermesDeviceId.mockReturnValue(undefined); + mocks.runHermesCdp.mockResolvedValue({ ok: true, result: { value: 2 } }); + const webviewCdp = vi.fn(); + const driver = new MobilePlatformDriver( + createBackend('android', { webviewCdp }), + ); + + await driver.cdp({ method: 'Runtime.evaluate', timeoutMs: 30_000 }); + + expect(mocks.runHermesCdp).toHaveBeenCalledTimes(1); + expect(webviewCdp).not.toHaveBeenCalled(); + }); + + it('maps the WebView blocked-method code to MM_CDP_BLOCKED', async () => { + const webviewCdp = vi.fn().mockResolvedValue({ + ok: false, + code: 'WEBVIEW_BLOCKED_METHOD', + message: 'blocked', + }); + const driver = new MobilePlatformDriver( + createBackend('android', { webviewCdp }), + ); + + const outcome = await driver.cdp({ + method: 'Browser.close', + timeoutMs: 30_000, + target: 'android-webview', + }); + + expect(outcome.ok).toBe(false); + if (!outcome.ok) { + expect(outcome.code).toBe('MM_CDP_BLOCKED'); + expect(outcome.message).toContain('WEBVIEW_BLOCKED_METHOD'); + } + }); + + it('maps other WebView failures to MM_CDP_FAILED', async () => { + const webviewCdp = vi.fn().mockResolvedValue({ + ok: false, + code: 'WEBVIEW_TARGET_NOT_FOUND', + message: 'no page', + }); + const driver = new MobilePlatformDriver( + createBackend('android', { webviewCdp }), + ); + + const outcome = await driver.cdp({ + method: 'Runtime.evaluate', + timeoutMs: 30_000, + target: 'android-webview', + }); + + expect(outcome.ok).toBe(false); + if (!outcome.ok) { + expect(outcome.code).toBe('MM_CDP_FAILED'); + expect(outcome.message).toContain('WEBVIEW_TARGET_NOT_FOUND'); + } + }); + + it('returns MM_CDP_FAILED when the backend lacks webviewCdp support', async () => { + const driver = new MobilePlatformDriver(createBackend('android')); + + const outcome = await driver.cdp({ + method: 'Runtime.evaluate', + timeoutMs: 30_000, + target: 'android-webview', + }); + + expect(outcome.ok).toBe(false); + if (!outcome.ok) { + expect(outcome.code).toBe('MM_CDP_FAILED'); + expect(outcome.message).toContain('WebView CDP is not available'); + } + }); +}); diff --git a/src/platform/mobile-platform-driver.test.ts b/src/platform/mobile-platform-driver.test.ts index abda0e4..d87e91c 100644 --- a/src/platform/mobile-platform-driver.test.ts +++ b/src/platform/mobile-platform-driver.test.ts @@ -465,6 +465,7 @@ describe('MobilePlatformDriver', () => { name: 'Submit', path: [], testId: 'submit-btn', + bounds: { x: 0, y: 0, width: 100, height: 44 }, }); expect(nodes[1]).toStrictEqual({ ref: 'e2', @@ -473,11 +474,34 @@ describe('MobilePlatformDriver', () => { path: [], disabled: true, textContent: 'user@test.com', + bounds: { x: 0, y: 0, width: 100, height: 44 }, }); expect(refMap.get('e1')).toBe('identifier:submit-btn'); expect(refMap.get('e2')).toBe('label:Email|type:TextField'); }); + it('omits bounds for elements with zero-size frames', async () => { + const backend = createMockBackend({ + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Container', + label: 'Frameless', + frame: { x: 0, y: 0, width: 0, height: 0 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const { nodes } = await driver.getAccessibilityTree(); + + expect(nodes[0].bounds).toBeUndefined(); + }); + it('assigns sequential refs to nested children', async () => { const backend = createMockBackend({ snapshot: vi.fn().mockResolvedValue({ @@ -651,6 +675,120 @@ describe('MobilePlatformDriver', () => { expect(items[0].testId).toBe('child-btn'); }); + it('marks elements outside the viewport as not visible', async () => { + const backend = createMockBackend({ + getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Button', + identifier: 'on-screen-btn', + label: 'On screen', + frame: { x: 0, y: 100, width: 402, height: 44 }, + }), + makeElement({ + type: 'StaticText', + identifier: 'below-fold-header', + label: 'Predictions', + frame: { x: 0, y: 1000, width: 402, height: 44 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0]).toStrictEqual({ + testId: 'on-screen-btn', + tag: 'Button', + text: 'On screen', + visible: true, + }); + expect(items[1]).toStrictEqual({ + testId: 'below-fold-header', + tag: 'StaticText', + text: 'Predictions', + visible: false, + }); + }); + + it('marks partially visible elements as visible', async () => { + const backend = createMockBackend({ + getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Button', + identifier: 'peeking-btn', + frame: { x: 0, y: 800, width: 402, height: 150 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0].visible).toBe(true); + }); + + it('assumes visibility when the viewport is unavailable', async () => { + const backend = createMockBackend({ + getWindowSize: vi + .fn() + .mockRejectedValue( + new Error('Unable to determine window size from device'), + ), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Button', + identifier: 'somewhere-btn', + frame: { x: 0, y: 5000, width: 402, height: 44 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0].visible).toBe(true); + }); + + it('assumes visibility for elements with zero-size frames', async () => { + const backend = createMockBackend({ + getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Container', + identifier: 'frameless-container', + frame: { x: 0, y: 0, width: 0, height: 0 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0].visible).toBe(true); + }); + it('respects limit', async () => { const backend = createMockBackend({ snapshot: vi.fn().mockResolvedValue({ diff --git a/src/platform/mobile-platform-driver.ts b/src/platform/mobile-platform-driver.ts index 7f9ca65..bc97f8f 100644 --- a/src/platform/mobile-platform-driver.ts +++ b/src/platform/mobile-platform-driver.ts @@ -51,6 +51,10 @@ const HERMES_MULTIPLE_DEVICES_CODE = 'HERMES_MULTIPLE_DEVICES'; // (Runtime.terminateExecution, Inspector.detached); mirror it to MM_CDP_BLOCKED // so the agent-facing contract matches the browser driver. const HERMES_BLOCKED_METHOD_CODE = 'HERMES_BLOCKED_METHOD'; +// device-mcp returns this code for the WebView destructive-method blocklist +// (Browser.close, Target.closeTarget, etc.); mirror it to MM_CDP_BLOCKED so the +// agent-facing contract matches the browser and Hermes drivers. +const WEBVIEW_BLOCKED_METHOD_CODE = 'WEBVIEW_BLOCKED_METHOD'; /** * Platform driver for mobile devices backed by @metamask/device-mcp. @@ -199,12 +203,21 @@ export class MobilePlatformDriver implements IPlatformDriver { /** * @param limit - Maximum number of test IDs to return. * @returns Array of test ID items with identifiers from the UI hierarchy. + * Visibility reflects whether each element's frame intersects the device + * viewport; elements with unknown geometry are assumed visible. */ async getTestIds(limit?: number): Promise { const snapshot = await this.#backend.snapshot(); const items: TestIdItem[] = []; const max = limit ?? OBSERVATION_TESTID_LIMIT; - collectTestIds(snapshot.hierarchy, items, max); + let viewport: { width: number; height: number } | undefined; + try { + viewport = await this.#backend.getWindowSize(); + } catch { + // Without a viewport we cannot judge visibility; report everything. + viewport = undefined; + } + collectTestIds(snapshot.hierarchy, items, max, viewport); return items; } @@ -426,18 +439,35 @@ export class MobilePlatformDriver implements IPlatformDriver { } /** - * Send a raw Chrome DevTools Protocol command to the app's React Native - * Hermes JS runtime via Metro's inspector proxy. Hermes exposes only the - * JS-engine CDP subset (Runtime, Debugger, Log, HeapProfiler) — there is no - * DOM/Page/Network domain. Works on both iOS and Android; only the default - * appId differs by platform. + * Send a raw Chrome DevTools Protocol command to a mobile CDP target. + * + * Two targets are supported, selected by `input.target`: + * - `'hermes'` (default): the app's React Native Hermes JS runtime via + * Metro's inspector proxy. Exposes only the JS-engine CDP subset (Runtime, + * Debugger, Log, HeapProfiler) — no DOM/Page/Network. Works on iOS and + * Android. This path is unchanged and is what local development relies on. + * - `'android-webview'`: a debuggable in-app Android WebView via adb. Exposes + * the full Chrome surface (Runtime, DOM, Page, Network, Input). Android + * only. * - * @param input - The CDP method, params, timeout, and optional Metro port / - * appId overrides. - * @returns A discriminated outcome carrying the raw CDP result or an error, - * with the underlying device-mcp `HERMES_*` code preserved in the message. + * @param input - The CDP method, params, timeout, and optional target / + * Metro / appId / urlFilter overrides. + * @returns A discriminated outcome carrying the raw CDP result or an error. */ async cdp(input: CdpInput): Promise { + if (input.target === 'android-webview') { + return this.#webviewCdp(input); + } + return this.#hermesCdp(input); + } + + /** + * Drives the React Native Hermes JS runtime via Metro. Unchanged behavior. + * + * @param input - The CDP command input. + * @returns A discriminated CDP outcome. + */ + async #hermesCdp(input: CdpInput): Promise { const session = this.#hermesSession; const resolved = session.resolve({ metroPort: input.metroPort, @@ -472,6 +502,43 @@ export class MobilePlatformDriver implements IPlatformDriver { }; } + /** + * Drives a debuggable in-app Android WebView via the device-mcp backend. + * + * @param input - The CDP command input, honoring an optional urlFilter. + * @returns A discriminated CDP outcome. + */ + async #webviewCdp(input: CdpInput): Promise { + if (!this.#backend.webviewCdp) { + return { + ok: false, + code: ErrorCodes.MM_CDP_FAILED, + message: + 'WebView CDP is not available on this session. It requires an ' + + 'Android (adb) device with a debuggable in-app WebView.', + }; + } + + const outcome = await this.#backend.webviewCdp({ + method: input.method, + params: input.params, + timeoutMs: input.timeoutMs, + urlFilter: input.urlFilter, + }); + + if (outcome.ok) { + return { ok: true, result: outcome.result }; + } + return { + ok: false, + code: + outcome.code === WEBVIEW_BLOCKED_METHOD_CODE + ? ErrorCodes.MM_CDP_BLOCKED + : ErrorCodes.MM_CDP_FAILED, + message: `[${outcome.code}] ${outcome.message}`, + }; + } + /** * Lists and diagnoses the debuggable Hermes targets Metro currently exposes, * reporting which target would be chosen (or why selection is ambiguous). @@ -718,6 +785,9 @@ function normalizeSnapshot(hierarchy: UIElement[]): { if (el.value && el.value !== name) { node.textContent = el.value; } + if (el.frame.width > 0 || el.frame.height > 0) { + node.bounds = { ...el.frame }; + } nodes.push(node); let stableId: string | undefined; @@ -764,11 +834,17 @@ function normalizeSnapshot(hierarchy: UIElement[]): { * @param elements - The UIElement nodes to scan. * @param items - Accumulator for discovered test ID items. * @param max - Maximum number of items to collect. + * @param viewport - Device viewport in logical points, when known. Elements + * whose frames do not intersect it are reported `visible: false`; elements + * with unknown or zero-size geometry are assumed visible. + * @param viewport.width - Viewport width in logical points. + * @param viewport.height - Viewport height in logical points. */ function collectTestIds( elements: UIElement[], items: TestIdItem[], max: number, + viewport?: { width: number; height: number }, ): void { for (const el of elements) { if (items.length >= max) { @@ -779,15 +855,40 @@ function collectTestIds( testId: el.identifier, tag: el.type || 'element', text: el.label ?? el.value, - visible: true, + visible: isWithinViewport(el.frame, viewport), }); } if (el.children?.length) { - collectTestIds(el.children, items, max); + collectTestIds(el.children, items, max, viewport); } } } +/** + * Checks whether an element frame intersects the device viewport. + * + * @param frame - Element frame in logical points. + * @param viewport - Device viewport in logical points, when known. + * @param viewport.width - Viewport width in logical points. + * @param viewport.height - Viewport height in logical points. + * @returns False only when a positive-size frame lies entirely outside the + * viewport; unknown geometry is conservatively treated as visible. + */ +function isWithinViewport( + frame: UIElement['frame'], + viewport?: { width: number; height: number }, +): boolean { + if (!viewport || viewport.width <= 0 || viewport.height <= 0) { + return true; + } + if (frame.width <= 0 || frame.height <= 0) { + return true; + } + const intersectsX = frame.x < viewport.width && frame.x + frame.width > 0; + const intersectsY = frame.y < viewport.height && frame.y + frame.height > 0; + return intersectsX && intersectsY; +} + /** * @param within - The within scope to validate. */ diff --git a/src/tools/types/discovery.ts b/src/tools/types/discovery.ts index cffe420..85ea6b6 100644 --- a/src/tools/types/discovery.ts +++ b/src/tools/types/discovery.ts @@ -41,6 +41,12 @@ export type TestIdItem = { testId: string; tag: string; text?: string; + /** + * Whether the element is expected to be on-screen. On mobile this is + * derived from the element frame vs the device viewport intersection and + * does not account for occlusion by other content; elements with unknown + * geometry are assumed visible. + */ visible: boolean; }; @@ -55,6 +61,11 @@ export type A11yNodeTrimmed = { testId?: string; textContent?: string; ambiguous?: boolean; + /** + * Element frame in logical points relative to the device viewport, when the + * platform reports geometry (mobile only). + */ + bounds?: { x: number; y: number; width: number; height: number }; }; export type RawA11yNode = { diff --git a/src/tools/types/tool-inputs.ts b/src/tools/types/tool-inputs.ts index ea79ad8..1a7294d 100644 --- a/src/tools/types/tool-inputs.ts +++ b/src/tools/types/tool-inputs.ts @@ -184,6 +184,17 @@ export type CdpInput = { * the Hermes debug target. Ignored on the browser platform. */ appId?: string; + /** + * Mobile only: which CDP target to drive. `'hermes'` (the default) targets + * the React Native JS runtime via Metro; `'android-webview'` targets a + * debuggable in-app Android WebView via adb. Ignored on the browser platform. + */ + target?: 'hermes' | 'android-webview'; + /** + * `android-webview` only: select the WebView page whose URL contains this + * substring when multiple WebView pages are open. + */ + urlFilter?: string; }; export type HermesTargetsInput = { diff --git a/src/validation/schemas.ts b/src/validation/schemas.ts index 5e1d4af..360e29b 100644 --- a/src/validation/schemas.ts +++ b/src/validation/schemas.ts @@ -714,6 +714,22 @@ export const cdpInputSchema = z.object({ 'Mobile only: override the expected app bundle identifier. Ignored on browser.', ) .optional(), + target: z + .enum(['hermes', 'android-webview']) + .default('hermes') + .describe( + 'Mobile only: CDP target. "hermes" (default) drives the React Native ' + + 'JS runtime via Metro; "android-webview" drives a debuggable in-app ' + + 'Android WebView via adb (DOM/Page/Network available). Ignored on browser.', + ), + urlFilter: z + .string() + .min(1) + .describe( + 'android-webview only: select the WebView page whose URL contains this ' + + 'substring when multiple pages are open.', + ) + .optional(), }); export const hermesTargetsInputSchema = z.object({ @@ -741,7 +757,9 @@ export const scrollToElementInputSchema = targetSelectionSchema.and( z.object({ direction: z .enum(['up', 'down']) - .describe('Scroll direction to reveal the target element') + .describe( + 'Finger swipe direction to perform while searching (swipe up scrolls content down)', + ) .optional(), maxAttempts: z .number() @@ -761,20 +779,20 @@ export const deviceSwipeInputSchema = z.object({ .number() .int() .min(0) - .describe('Start X coordinate for the swipe gesture') + .describe('Start X coordinate for the swipe gesture, in logical points') .optional(), startY: z .number() .int() .min(0) - .describe('Start Y coordinate for the swipe gesture') + .describe('Start Y coordinate for the swipe gesture, in logical points') .optional(), distance: z .number() .int() .min(1) .max(10000) - .describe('Swipe distance in pixels') + .describe('Swipe distance in logical points') .optional(), }); diff --git a/vitest.config.mts b/vitest.config.mts index c1849d7..83ec411 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -35,10 +35,10 @@ export default defineConfig({ // Auto-update the coverage thresholds when running locally. // Disabled in CI to prevent non-deterministic config changes. autoUpdate: !process.env.CI, - branches: 90.77, - functions: 93.52, - lines: 96.24, - statements: 95.99, + branches: 90.94, + functions: 93.54, + lines: 96.27, + statements: 96.02, }, }, diff --git a/yarn.lock b/yarn.lock index 224f16b..bbd5b4c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -926,7 +926,7 @@ __metadata: "@lavamoat/allow-scripts": "npm:^3.0.4" "@lavamoat/preinstall-always-fail": "npm:^2.0.0" "@metamask/auto-changelog": "npm:^5.3.0" - "@metamask/device-mcp": "npm:^0.3.2" + "@metamask/device-mcp": "npm:^0.4.1" "@metamask/eslint-config": "npm:^15.0.0" "@metamask/eslint-config-nodejs": "npm:^15.0.0" "@metamask/eslint-config-typescript": "npm:^15.0.0" @@ -968,15 +968,15 @@ __metadata: languageName: unknown linkType: soft -"@metamask/device-mcp@npm:^0.3.2": - version: 0.3.2 - resolution: "@metamask/device-mcp@npm:0.3.2" +"@metamask/device-mcp@npm:^0.4.1": + version: 0.4.1 + resolution: "@metamask/device-mcp@npm:0.4.1" dependencies: "@modelcontextprotocol/sdk": "npm:^1.12.1" zod: "npm:^4.4.3" bin: device-mcp: ./dist/cli/device-mcp.mjs - checksum: 10/62bd0e0f4138f66c161b827dca5fa076b8d14237764c8ee62f9ccda4711ac495f1508802bbdaf5b28c9afec7118c3ff6b1c59969be0b4e3957709d1d591d36c2 + checksum: 10/e1453ad2de0405c2b09c10492d7bdb308c43abfdc5b47d85cdf39def20c18f94e7823a83333235a116718c6eaf5db4b693e9f5bfb6a2ccb9c56325e03a279dd2 languageName: node linkType: hard From a60441db371f8dcc64443565d656c772ecabf65b Mon Sep 17 00:00:00 2001 From: Priya Narayanaswamy Date: Thu, 17 Sep 2026 14:08:48 +0200 Subject: [PATCH 2/2] chore: bump @metamask/device-mcp to ^0.4.2 for webview_cdp The published 0.4.2 adds the optional webviewCdp method on the exported DeviceBackend, which the mobile CDP driver's --target android-webview path consumes. attw --pack / prepack tsc now resolve the type against the published package. --- SKILL.md | 20 ++++++++++---------- package.json | 2 +- src/cli/mm.test.ts | 12 ++---------- yarn.lock | 10 +++++----- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/SKILL.md b/SKILL.md index 166f9c3..68e918b 100644 --- a/SKILL.md +++ b/SKILL.md @@ -571,20 +571,20 @@ mm cdp Runtime.evaluate '{"expression":"document.querySelector(\'#personalSign\' | `` | CDP method name (e.g., `Runtime.evaluate`, `DOM.getDocument` on browser; `Runtime.evaluate`, `Debugger.enable` on mobile) | | `[params-json]` | Optional JSON object with method-specific parameters | | `--timeout` | Per-command timeout in ms (default: 30 000, max: 30 000) | -| `--target` | **Mobile only** — `hermes` (default, RN JS runtime) or `android-webview` (in-app WebView DOM). Ignored on browser. | -| `--url-filter` | **`--target android-webview` only** — select the WebView page whose URL contains this substring. | -| `--metro-port` | **Mobile (Hermes) only** — override the Metro inspector proxy port (default: 8081). Ignored on browser. | -| `--app-id` | **Mobile (Hermes) only** — override the expected app bundle identifier. Ignored on browser. | +| `--target` | **Mobile only** — `hermes` (default, RN JS runtime) or `android-webview` (in-app WebView DOM). Ignored on browser. | +| `--url-filter` | **`--target android-webview` only** — select the WebView page whose URL contains this substring. | +| `--metro-port` | **Mobile (Hermes) only** — override the Metro inspector proxy port (default: 8081). Ignored on browser. | +| `--app-id` | **Mobile (Hermes) only** — override the expected app bundle identifier. Ignored on browser. | **Three CDP targets:** the same command reaches different runtimes. -| Aspect | Browser (Playwright) | Mobile — Hermes (`--target hermes`, default) | Mobile — Android WebView (`--target android-webview`) | -| ----------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| Aspect | Browser (Playwright) | Mobile — Hermes (`--target hermes`, default) | Mobile — Android WebView (`--target android-webview`) | +| ----------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------- | | Target | The page's Chrome DevTools session | The app's Hermes JS engine, via Metro's inspector proxy (needs a DEBUG build with Metro running) | The web page inside a debuggable in-app Android WebView, via adb (needs `setWebContentsDebuggingEnabled(true)`) | -| Available domains | Full Chrome surface (`Runtime`, `DOM`, `Network`, `Page`, …) | JS-engine subset only (`Runtime`, `Debugger`, `Log`, `HeapProfiler`) — no `DOM`/`Page`/`Network` | Full Chrome surface (`Runtime`, `DOM`, `Network`, `Page`, `Input`) | -| Use it for | Web page DOM in the browser | The React Native app's own JavaScript | The web page DOM inside the app's in-app browser | -| Blocked methods | `Browser.close`, `Target.closeTarget`, `Target.disposeBrowserContext`, `Browser.crashGpuProcess` | `Runtime.terminateExecution`, `Inspector.detached` | `Browser.close`, `Target.closeTarget`, `Target.disposeBrowserContext`, `Browser.crashGpuProcess` | -| Result shape | Standard CDP response | `Runtime.evaluate` nests the value at `result.result.value` | `Runtime.evaluate` nests the value at `result.result.value` | +| Available domains | Full Chrome surface (`Runtime`, `DOM`, `Network`, `Page`, …) | JS-engine subset only (`Runtime`, `Debugger`, `Log`, `HeapProfiler`) — no `DOM`/`Page`/`Network` | Full Chrome surface (`Runtime`, `DOM`, `Network`, `Page`, `Input`) | +| Use it for | Web page DOM in the browser | The React Native app's own JavaScript | The web page DOM inside the app's in-app browser | +| Blocked methods | `Browser.close`, `Target.closeTarget`, `Target.disposeBrowserContext`, `Browser.crashGpuProcess` | `Runtime.terminateExecution`, `Inspector.detached` | `Browser.close`, `Target.closeTarget`, `Target.disposeBrowserContext`, `Browser.crashGpuProcess` | +| Result shape | Standard CDP response | `Runtime.evaluate` nests the value at `result.result.value` | `Runtime.evaluate` nests the value at `result.result.value` | Blocked methods return `MM_CDP_BLOCKED` on either platform; other failures return `MM_CDP_FAILED` (on mobile the underlying `HERMES_*` code is preserved in the message). The tool is categorized as **mutating** — run `describe-screen` afterward to re-sync if the call changed runtime/page state. diff --git a/package.json b/package.json index 18534c1..acfcd8e 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "hono": "^4.12.25" }, "dependencies": { - "@metamask/device-mcp": "^0.4.1", + "@metamask/device-mcp": "^0.4.2", "cosmiconfig": "^9.0.0", "express": "^5.2.1", "zod": "^4.4.3" diff --git a/src/cli/mm.test.ts b/src/cli/mm.test.ts index cb6a304..c1da26a 100644 --- a/src/cli/mm.test.ts +++ b/src/cli/mm.test.ts @@ -2040,11 +2040,7 @@ describe('routeCommand', () => { }); it('routes cdp with --target hermes', async () => { - await routeCommand( - 'cdp', - ['Runtime.evaluate', '--target', 'hermes'], - 3000, - ); + await routeCommand('cdp', ['Runtime.evaluate', '--target', 'hermes'], 3000); expect(globalThis.fetch).toHaveBeenCalledWith( 'http://127.0.0.1:3000/tool/cdp', expect.objectContaining({ @@ -2058,11 +2054,7 @@ describe('routeCommand', () => { it('exits when cdp --target is invalid', async () => { await expect( - routeCommand( - 'cdp', - ['Runtime.evaluate', '--target', 'bogus'], - 3000, - ), + routeCommand('cdp', ['Runtime.evaluate', '--target', 'bogus'], 3000), ).rejects.toThrowError('process.exit'); expect(stderrSpy).toHaveBeenCalledWith( expect.stringContaining('--target must be'), diff --git a/yarn.lock b/yarn.lock index bbd5b4c..7b19763 100644 --- a/yarn.lock +++ b/yarn.lock @@ -926,7 +926,7 @@ __metadata: "@lavamoat/allow-scripts": "npm:^3.0.4" "@lavamoat/preinstall-always-fail": "npm:^2.0.0" "@metamask/auto-changelog": "npm:^5.3.0" - "@metamask/device-mcp": "npm:^0.4.1" + "@metamask/device-mcp": "npm:^0.4.2" "@metamask/eslint-config": "npm:^15.0.0" "@metamask/eslint-config-nodejs": "npm:^15.0.0" "@metamask/eslint-config-typescript": "npm:^15.0.0" @@ -968,15 +968,15 @@ __metadata: languageName: unknown linkType: soft -"@metamask/device-mcp@npm:^0.4.1": - version: 0.4.1 - resolution: "@metamask/device-mcp@npm:0.4.1" +"@metamask/device-mcp@npm:^0.4.2": + version: 0.4.2 + resolution: "@metamask/device-mcp@npm:0.4.2" dependencies: "@modelcontextprotocol/sdk": "npm:^1.12.1" zod: "npm:^4.4.3" bin: device-mcp: ./dist/cli/device-mcp.mjs - checksum: 10/e1453ad2de0405c2b09c10492d7bdb308c43abfdc5b47d85cdf39def20c18f94e7823a83333235a116718c6eaf5db4b693e9f5bfb6a2ccb9c56325e03a279dd2 + checksum: 10/e6132f78ef90ae51460a9d88d23c0f5fb74ddde27ea10e3abcc95f01c0a8d1eaa57b5e542b00a7db5f9fbff86a7976fc6880d9fe6c1bd3f5f66b7fc6de475a02 languageName: node linkType: hard