fix: report mobile element geometry and viewport visibility

package.json

@@ -62,7 +62,7 @@
     "hono": "^4.12.25"
   },
   "dependencies": {
-    "@metamask/device-mcp": "^0.3.2",
+    "@metamask/device-mcp": "^0.4.1",
     "cosmiconfig": "^9.0.0",
     "express": "^5.2.1",
     "zod": "^4.4.3"

src/platform/mobile-platform-driver.test.ts

@@ -465,6 +465,7 @@ describe('MobilePlatformDriver', () => {
         name: 'Submit',
         path: [],
         testId: 'submit-btn',
+        bounds: { x: 0, y: 0, width: 100, height: 44 },
       });
       expect(nodes[1]).toStrictEqual({
         ref: 'e2',
@@ -473,11 +474,34 @@ describe('MobilePlatformDriver', () => {
         path: [],
         disabled: true,
         textContent: 'user@test.com',
+        bounds: { x: 0, y: 0, width: 100, height: 44 },
       });
       expect(refMap.get('e1')).toBe('identifier:submit-btn');
       expect(refMap.get('e2')).toBe('label:Email|type:TextField');
     });
 
+    it('omits bounds for elements with zero-size frames', async () => {
+      const backend = createMockBackend({
+        snapshot: vi.fn().mockResolvedValue({
+          platform: 'ios',
+          hierarchy: [
+            makeElement({
+              type: 'Container',
+              label: 'Frameless',
+              frame: { x: 0, y: 0, width: 0, height: 0 },
+            }),
+          ],
+          raw: '[]',
+          timestamp: Date.now(),
+        }),
+      });
+      const driver = new MobilePlatformDriver(backend);
+
+      const { nodes } = await driver.getAccessibilityTree();
+
+      expect(nodes[0].bounds).toBeUndefined();
+    });
+
     it('assigns sequential refs to nested children', async () => {
       const backend = createMockBackend({
         snapshot: vi.fn().mockResolvedValue({
@@ -651,6 +675,120 @@ describe('MobilePlatformDriver', () => {
       expect(items[0].testId).toBe('child-btn');
     });
 
+    it('marks elements outside the viewport as not visible', async () => {
+      const backend = createMockBackend({
+        getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }),
+        snapshot: vi.fn().mockResolvedValue({
+          platform: 'ios',
+          hierarchy: [
+            makeElement({
+              type: 'Button',
+              identifier: 'on-screen-btn',
+              label: 'On screen',
+              frame: { x: 0, y: 100, width: 402, height: 44 },
+            }),
+            makeElement({
+              type: 'StaticText',
+              identifier: 'below-fold-header',
+              label: 'Predictions',
+              frame: { x: 0, y: 1000, width: 402, height: 44 },
+            }),
+          ],
+          raw: '[]',
+          timestamp: Date.now(),
+        }),
+      });
+      const driver = new MobilePlatformDriver(backend);
+
+      const items = await driver.getTestIds();
+
+      expect(items[0]).toStrictEqual({
+        testId: 'on-screen-btn',
+        tag: 'Button',
+        text: 'On screen',
+        visible: true,
+      });
+      expect(items[1]).toStrictEqual({
+        testId: 'below-fold-header',
+        tag: 'StaticText',
+        text: 'Predictions',
+        visible: false,
+      });
+    });
+
+    it('marks partially visible elements as visible', async () => {
+      const backend = createMockBackend({
+        getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }),
+        snapshot: vi.fn().mockResolvedValue({
+          platform: 'ios',
+          hierarchy: [
+            makeElement({
+              type: 'Button',
+              identifier: 'peeking-btn',
+              frame: { x: 0, y: 800, width: 402, height: 150 },
+            }),
+          ],
+          raw: '[]',
+          timestamp: Date.now(),
+        }),
+      });
+      const driver = new MobilePlatformDriver(backend);
+
+      const items = await driver.getTestIds();
+
+      expect(items[0].visible).toBe(true);
+    });
+
+    it('assumes visibility when the viewport is unavailable', async () => {
+      const backend = createMockBackend({
+        getWindowSize: vi
+          .fn()
+          .mockRejectedValue(
+            new Error('Unable to determine window size from device'),
+          ),
+        snapshot: vi.fn().mockResolvedValue({
+          platform: 'ios',
+          hierarchy: [
+            makeElement({
+              type: 'Button',
+              identifier: 'somewhere-btn',
+              frame: { x: 0, y: 5000, width: 402, height: 44 },
+            }),
+          ],
+          raw: '[]',
+          timestamp: Date.now(),
+        }),
+      });
+      const driver = new MobilePlatformDriver(backend);
+
+      const items = await driver.getTestIds();
+
+      expect(items[0].visible).toBe(true);
+    });
+
+    it('assumes visibility for elements with zero-size frames', async () => {
+      const backend = createMockBackend({
+        getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }),
+        snapshot: vi.fn().mockResolvedValue({
+          platform: 'ios',
+          hierarchy: [
+            makeElement({
+              type: 'Container',
+              identifier: 'frameless-container',
+              frame: { x: 0, y: 0, width: 0, height: 0 },
+            }),
+          ],
+          raw: '[]',
+          timestamp: Date.now(),
+        }),
+      });
+      const driver = new MobilePlatformDriver(backend);
+
+      const items = await driver.getTestIds();
+
+      expect(items[0].visible).toBe(true);
+    });
+
     it('respects limit', async () => {
       const backend = createMockBackend({
         snapshot: vi.fn().mockResolvedValue({

src/platform/mobile-platform-driver.ts

@@ -199,12 +199,21 @@ export class MobilePlatformDriver implements IPlatformDriver {
   /**
    * @param limit - Maximum number of test IDs to return.
    * @returns Array of test ID items with identifiers from the UI hierarchy.
+   * Visibility reflects whether each element's frame intersects the device
+   * viewport; elements with unknown geometry are assumed visible.
    */
   async getTestIds(limit?: number): Promise<TestIdItem[]> {
     const snapshot = await this.#backend.snapshot();
     const items: TestIdItem[] = [];
     const max = limit ?? OBSERVATION_TESTID_LIMIT;
-    collectTestIds(snapshot.hierarchy, items, max);
+    let viewport: { width: number; height: number } | undefined;
+    try {
+      viewport = await this.#backend.getWindowSize();
+    } catch {
+      // Without a viewport we cannot judge visibility; report everything.
+      viewport = undefined;
+    }
+    collectTestIds(snapshot.hierarchy, items, max, viewport);
     return items;
   }
 
@@ -718,6 +727,9 @@ function normalizeSnapshot(hierarchy: UIElement[]): {
       if (el.value && el.value !== name) {
         node.textContent = el.value;
       }
+      if (el.frame.width > 0 || el.frame.height > 0) {
+        node.bounds = { ...el.frame };
+      }
       nodes.push(node);
 
       let stableId: string | undefined;
@@ -764,11 +776,17 @@ function normalizeSnapshot(hierarchy: UIElement[]): {
  * @param elements - The UIElement nodes to scan.
  * @param items - Accumulator for discovered test ID items.
  * @param max - Maximum number of items to collect.
+ * @param viewport - Device viewport in logical points, when known. Elements
+ * whose frames do not intersect it are reported `visible: false`; elements
+ * with unknown or zero-size geometry are assumed visible.
+ * @param viewport.width - Viewport width in logical points.
+ * @param viewport.height - Viewport height in logical points.
  */
 function collectTestIds(
   elements: UIElement[],
   items: TestIdItem[],
   max: number,
+  viewport?: { width: number; height: number },
 ): void {
   for (const el of elements) {
     if (items.length >= max) {
@@ -779,15 +797,40 @@ function collectTestIds(
         testId: el.identifier,
         tag: el.type || 'element',
         text: el.label ?? el.value,
-        visible: true,
+        visible: isWithinViewport(el.frame, viewport),
       });
     }
     if (el.children?.length) {
-      collectTestIds(el.children, items, max);
+      collectTestIds(el.children, items, max, viewport);
     }
   }
 }
 
+/**
+ * Checks whether an element frame intersects the device viewport.
+ *
+ * @param frame - Element frame in logical points.
+ * @param viewport - Device viewport in logical points, when known.
+ * @param viewport.width - Viewport width in logical points.
+ * @param viewport.height - Viewport height in logical points.
+ * @returns False only when a positive-size frame lies entirely outside the
+ * viewport; unknown geometry is conservatively treated as visible.
+ */
+function isWithinViewport(
+  frame: UIElement['frame'],
+  viewport?: { width: number; height: number },
+): boolean {
+  if (!viewport || viewport.width <= 0 || viewport.height <= 0) {
+    return true;
+  }
+  if (frame.width <= 0 || frame.height <= 0) {
+    return true;
+  }
+  const intersectsX = frame.x < viewport.width && frame.x + frame.width > 0;
+  const intersectsY = frame.y < viewport.height && frame.y + frame.height > 0;
+  return intersectsX && intersectsY;
+}
+
 /**
  * @param within - The within scope to validate.
  */

src/tools/types/discovery.ts

@@ -41,6 +41,12 @@ export type TestIdItem = {
   testId: string;
   tag: string;
   text?: string;
+  /**
+   * Whether the element is expected to be on-screen. On mobile this is
+   * derived from the element frame vs the device viewport intersection and
+   * does not account for occlusion by other content; elements with unknown
+   * geometry are assumed visible.
+   */
   visible: boolean;
 };
 
@@ -55,6 +61,11 @@ export type A11yNodeTrimmed = {
   testId?: string;
   textContent?: string;
   ambiguous?: boolean;
+  /**
+   * Element frame in logical points relative to the device viewport, when the
+   * platform reports geometry (mobile only).
+   */
+  bounds?: { x: number; y: number; width: number; height: number };
 };
 
 export type RawA11yNode = {

src/validation/schemas.ts

@@ -741,7 +741,9 @@ export const scrollToElementInputSchema = targetSelectionSchema.and(
   z.object({
     direction: z
       .enum(['up', 'down'])
-      .describe('Scroll direction to reveal the target element')
+      .describe(
+        'Finger swipe direction to perform while searching (swipe up scrolls content down)',
+      )
       .optional(),
     maxAttempts: z
       .number()
@@ -761,20 +763,20 @@ export const deviceSwipeInputSchema = z.object({
     .number()
     .int()
     .min(0)
-    .describe('Start X coordinate for the swipe gesture')
+    .describe('Start X coordinate for the swipe gesture, in logical points')
     .optional(),
   startY: z
     .number()
     .int()
     .min(0)
-    .describe('Start Y coordinate for the swipe gesture')
+    .describe('Start Y coordinate for the swipe gesture, in logical points')
     .optional(),
   distance: z
     .number()
     .int()
     .min(1)
     .max(10000)
-    .describe('Swipe distance in pixels')
+    .describe('Swipe distance in logical points')
     .optional(),
 });
 

vitest.config.mts

@@ -35,10 +35,10 @@ export default defineConfig({
         // Auto-update the coverage thresholds when running locally.
         // Disabled in CI to prevent non-deterministic config changes.
         autoUpdate: !process.env.CI,
-        branches: 90.77,
-        functions: 93.52,
-        lines: 96.24,
-        statements: 95.99,
+        branches: 90.86,
+        functions: 93.54,
+        lines: 96.25,
+        statements: 96,
       },
     },
 

yarn.lock

@@ -926,7 +926,7 @@ __metadata:
     "@lavamoat/allow-scripts": "npm:^3.0.4"
     "@lavamoat/preinstall-always-fail": "npm:^2.0.0"
     "@metamask/auto-changelog": "npm:^5.3.0"
-    "@metamask/device-mcp": "npm:^0.3.2"
+    "@metamask/device-mcp": "npm:^0.4.1"
     "@metamask/eslint-config": "npm:^15.0.0"
     "@metamask/eslint-config-nodejs": "npm:^15.0.0"
     "@metamask/eslint-config-typescript": "npm:^15.0.0"
@@ -968,15 +968,15 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@metamask/device-mcp@npm:^0.3.2":
-  version: 0.3.2
-  resolution: "@metamask/device-mcp@npm:0.3.2"
+"@metamask/device-mcp@npm:^0.4.1":
+  version: 0.4.1
+  resolution: "@metamask/device-mcp@npm:0.4.1"
   dependencies:
     "@modelcontextprotocol/sdk": "npm:^1.12.1"
     zod: "npm:^4.4.3"
   bin:
     device-mcp: ./dist/cli/device-mcp.mjs
-  checksum: 10/62bd0e0f4138f66c161b827dca5fa076b8d14237764c8ee62f9ccda4711ac495f1508802bbdaf5b28c9afec7118c3ff6b1c59969be0b4e3957709d1d591d36c2
+  checksum: 10/e1453ad2de0405c2b09c10492d7bdb308c43abfdc5b47d85cdf39def20c18f94e7823a83333235a116718c6eaf5db4b693e9f5bfb6a2ccb9c56325e03a279dd2
   languageName: node
   linkType: hard