diff --git a/package.json b/package.json index 1cd5234..18534c1 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "hono": "^4.12.25" }, "dependencies": { - "@metamask/device-mcp": "^0.3.2", + "@metamask/device-mcp": "^0.4.1", "cosmiconfig": "^9.0.0", "express": "^5.2.1", "zod": "^4.4.3" diff --git a/src/platform/mobile-platform-driver.test.ts b/src/platform/mobile-platform-driver.test.ts index abda0e4..d87e91c 100644 --- a/src/platform/mobile-platform-driver.test.ts +++ b/src/platform/mobile-platform-driver.test.ts @@ -465,6 +465,7 @@ describe('MobilePlatformDriver', () => { name: 'Submit', path: [], testId: 'submit-btn', + bounds: { x: 0, y: 0, width: 100, height: 44 }, }); expect(nodes[1]).toStrictEqual({ ref: 'e2', @@ -473,11 +474,34 @@ describe('MobilePlatformDriver', () => { path: [], disabled: true, textContent: 'user@test.com', + bounds: { x: 0, y: 0, width: 100, height: 44 }, }); expect(refMap.get('e1')).toBe('identifier:submit-btn'); expect(refMap.get('e2')).toBe('label:Email|type:TextField'); }); + it('omits bounds for elements with zero-size frames', async () => { + const backend = createMockBackend({ + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Container', + label: 'Frameless', + frame: { x: 0, y: 0, width: 0, height: 0 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const { nodes } = await driver.getAccessibilityTree(); + + expect(nodes[0].bounds).toBeUndefined(); + }); + it('assigns sequential refs to nested children', async () => { const backend = createMockBackend({ snapshot: vi.fn().mockResolvedValue({ @@ -651,6 +675,120 @@ describe('MobilePlatformDriver', () => { expect(items[0].testId).toBe('child-btn'); }); + it('marks elements outside the viewport as not visible', async () => { + const backend = createMockBackend({ + getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Button', + identifier: 'on-screen-btn', + label: 'On screen', + frame: { x: 0, y: 100, width: 402, height: 44 }, + }), + makeElement({ + type: 'StaticText', + identifier: 'below-fold-header', + label: 'Predictions', + frame: { x: 0, y: 1000, width: 402, height: 44 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0]).toStrictEqual({ + testId: 'on-screen-btn', + tag: 'Button', + text: 'On screen', + visible: true, + }); + expect(items[1]).toStrictEqual({ + testId: 'below-fold-header', + tag: 'StaticText', + text: 'Predictions', + visible: false, + }); + }); + + it('marks partially visible elements as visible', async () => { + const backend = createMockBackend({ + getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Button', + identifier: 'peeking-btn', + frame: { x: 0, y: 800, width: 402, height: 150 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0].visible).toBe(true); + }); + + it('assumes visibility when the viewport is unavailable', async () => { + const backend = createMockBackend({ + getWindowSize: vi + .fn() + .mockRejectedValue( + new Error('Unable to determine window size from device'), + ), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Button', + identifier: 'somewhere-btn', + frame: { x: 0, y: 5000, width: 402, height: 44 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0].visible).toBe(true); + }); + + it('assumes visibility for elements with zero-size frames', async () => { + const backend = createMockBackend({ + getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }), + snapshot: vi.fn().mockResolvedValue({ + platform: 'ios', + hierarchy: [ + makeElement({ + type: 'Container', + identifier: 'frameless-container', + frame: { x: 0, y: 0, width: 0, height: 0 }, + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0].visible).toBe(true); + }); + it('respects limit', async () => { const backend = createMockBackend({ snapshot: vi.fn().mockResolvedValue({ diff --git a/src/platform/mobile-platform-driver.ts b/src/platform/mobile-platform-driver.ts index 7f9ca65..754f8ad 100644 --- a/src/platform/mobile-platform-driver.ts +++ b/src/platform/mobile-platform-driver.ts @@ -199,12 +199,21 @@ export class MobilePlatformDriver implements IPlatformDriver { /** * @param limit - Maximum number of test IDs to return. * @returns Array of test ID items with identifiers from the UI hierarchy. + * Visibility reflects whether each element's frame intersects the device + * viewport; elements with unknown geometry are assumed visible. */ async getTestIds(limit?: number): Promise { const snapshot = await this.#backend.snapshot(); const items: TestIdItem[] = []; const max = limit ?? OBSERVATION_TESTID_LIMIT; - collectTestIds(snapshot.hierarchy, items, max); + let viewport: { width: number; height: number } | undefined; + try { + viewport = await this.#backend.getWindowSize(); + } catch { + // Without a viewport we cannot judge visibility; report everything. + viewport = undefined; + } + collectTestIds(snapshot.hierarchy, items, max, viewport); return items; } @@ -718,6 +727,9 @@ function normalizeSnapshot(hierarchy: UIElement[]): { if (el.value && el.value !== name) { node.textContent = el.value; } + if (el.frame.width > 0 || el.frame.height > 0) { + node.bounds = { ...el.frame }; + } nodes.push(node); let stableId: string | undefined; @@ -764,11 +776,17 @@ function normalizeSnapshot(hierarchy: UIElement[]): { * @param elements - The UIElement nodes to scan. * @param items - Accumulator for discovered test ID items. * @param max - Maximum number of items to collect. + * @param viewport - Device viewport in logical points, when known. Elements + * whose frames do not intersect it are reported `visible: false`; elements + * with unknown or zero-size geometry are assumed visible. + * @param viewport.width - Viewport width in logical points. + * @param viewport.height - Viewport height in logical points. */ function collectTestIds( elements: UIElement[], items: TestIdItem[], max: number, + viewport?: { width: number; height: number }, ): void { for (const el of elements) { if (items.length >= max) { @@ -779,15 +797,40 @@ function collectTestIds( testId: el.identifier, tag: el.type || 'element', text: el.label ?? el.value, - visible: true, + visible: isWithinViewport(el.frame, viewport), }); } if (el.children?.length) { - collectTestIds(el.children, items, max); + collectTestIds(el.children, items, max, viewport); } } } +/** + * Checks whether an element frame intersects the device viewport. + * + * @param frame - Element frame in logical points. + * @param viewport - Device viewport in logical points, when known. + * @param viewport.width - Viewport width in logical points. + * @param viewport.height - Viewport height in logical points. + * @returns False only when a positive-size frame lies entirely outside the + * viewport; unknown geometry is conservatively treated as visible. + */ +function isWithinViewport( + frame: UIElement['frame'], + viewport?: { width: number; height: number }, +): boolean { + if (!viewport || viewport.width <= 0 || viewport.height <= 0) { + return true; + } + if (frame.width <= 0 || frame.height <= 0) { + return true; + } + const intersectsX = frame.x < viewport.width && frame.x + frame.width > 0; + const intersectsY = frame.y < viewport.height && frame.y + frame.height > 0; + return intersectsX && intersectsY; +} + /** * @param within - The within scope to validate. */ diff --git a/src/tools/types/discovery.ts b/src/tools/types/discovery.ts index cffe420..85ea6b6 100644 --- a/src/tools/types/discovery.ts +++ b/src/tools/types/discovery.ts @@ -41,6 +41,12 @@ export type TestIdItem = { testId: string; tag: string; text?: string; + /** + * Whether the element is expected to be on-screen. On mobile this is + * derived from the element frame vs the device viewport intersection and + * does not account for occlusion by other content; elements with unknown + * geometry are assumed visible. + */ visible: boolean; }; @@ -55,6 +61,11 @@ export type A11yNodeTrimmed = { testId?: string; textContent?: string; ambiguous?: boolean; + /** + * Element frame in logical points relative to the device viewport, when the + * platform reports geometry (mobile only). + */ + bounds?: { x: number; y: number; width: number; height: number }; }; export type RawA11yNode = { diff --git a/src/validation/schemas.ts b/src/validation/schemas.ts index 5e1d4af..b3a3cea 100644 --- a/src/validation/schemas.ts +++ b/src/validation/schemas.ts @@ -741,7 +741,9 @@ export const scrollToElementInputSchema = targetSelectionSchema.and( z.object({ direction: z .enum(['up', 'down']) - .describe('Scroll direction to reveal the target element') + .describe( + 'Finger swipe direction to perform while searching (swipe up scrolls content down)', + ) .optional(), maxAttempts: z .number() @@ -761,20 +763,20 @@ export const deviceSwipeInputSchema = z.object({ .number() .int() .min(0) - .describe('Start X coordinate for the swipe gesture') + .describe('Start X coordinate for the swipe gesture, in logical points') .optional(), startY: z .number() .int() .min(0) - .describe('Start Y coordinate for the swipe gesture') + .describe('Start Y coordinate for the swipe gesture, in logical points') .optional(), distance: z .number() .int() .min(1) .max(10000) - .describe('Swipe distance in pixels') + .describe('Swipe distance in logical points') .optional(), }); diff --git a/vitest.config.mts b/vitest.config.mts index c1849d7..69984bc 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -35,10 +35,10 @@ export default defineConfig({ // Auto-update the coverage thresholds when running locally. // Disabled in CI to prevent non-deterministic config changes. autoUpdate: !process.env.CI, - branches: 90.77, - functions: 93.52, - lines: 96.24, - statements: 95.99, + branches: 90.86, + functions: 93.54, + lines: 96.25, + statements: 96, }, }, diff --git a/yarn.lock b/yarn.lock index 224f16b..bbd5b4c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -926,7 +926,7 @@ __metadata: "@lavamoat/allow-scripts": "npm:^3.0.4" "@lavamoat/preinstall-always-fail": "npm:^2.0.0" "@metamask/auto-changelog": "npm:^5.3.0" - "@metamask/device-mcp": "npm:^0.3.2" + "@metamask/device-mcp": "npm:^0.4.1" "@metamask/eslint-config": "npm:^15.0.0" "@metamask/eslint-config-nodejs": "npm:^15.0.0" "@metamask/eslint-config-typescript": "npm:^15.0.0" @@ -968,15 +968,15 @@ __metadata: languageName: unknown linkType: soft -"@metamask/device-mcp@npm:^0.3.2": - version: 0.3.2 - resolution: "@metamask/device-mcp@npm:0.3.2" +"@metamask/device-mcp@npm:^0.4.1": + version: 0.4.1 + resolution: "@metamask/device-mcp@npm:0.4.1" dependencies: "@modelcontextprotocol/sdk": "npm:^1.12.1" zod: "npm:^4.4.3" bin: device-mcp: ./dist/cli/device-mcp.mjs - checksum: 10/62bd0e0f4138f66c161b827dca5fa076b8d14237764c8ee62f9ccda4711ac495f1508802bbdaf5b28c9afec7118c3ff6b1c59969be0b4e3957709d1d591d36c2 + checksum: 10/e1453ad2de0405c2b09c10492d7bdb308c43abfdc5b47d85cdf39def20c18f94e7823a83333235a116718c6eaf5db4b693e9f5bfb6a2ccb9c56325e03a279dd2 languageName: node linkType: hard