feat(cli): expose simulator accessibility trees

README.md

@@ -79,7 +79,8 @@ simulators.
 - **Debug helpers**: slow animations, shake, simulated memory warning, the system log and app data
   in the Finder, and a click to frame latency overlay.
 - **A command line tool**: `odhub` drives a simulator from a script: tap, swipe, pinch, type,
-  buttons, rotation. One menu item puts it on your `PATH`.
+  buttons, rotation. `odhub ui-tree` prints the accessibility tree of the app in front, each element
+  with the coordinates `tap` takes. One menu item puts it on your `PATH`.
 
 <details>
 <summary>Keyboard shortcuts</summary>
@@ -126,12 +127,11 @@ options.
 | `rotate` | `odhub rotate A1B2C3D4 landscapeLeft` |
 | `home-swipe` | `odhub home-swipe A1B2C3D4` |
 | `app-switcher` | `odhub app-switcher A1B2C3D4` |
+| `ui-tree` | `odhub ui-tree A1B2C3D4 --labelled` |
 
 </details>
 
 ## Current limitations
-- **The windows are invisible to accessibility**: VoiceOver, window managers and scripting tools see
-  the app and its menus but not its windows.
 - **A device taller than your display** keeps its bottom edge off screen, where the pointer cannot
   reach it. Use Fit.
 - **The compact layout below 340 points wide** is written and unit tested but has never been used by

engine/Sources/OpenDeviceHubEngine/Adapter/AccessibilityTree.swift

@@ -0,0 +1,216 @@
+import CoreGraphics
+import Foundation
+import OpenDeviceHubPrivate
+
+/// One element of a guest app's accessibility tree.
+public struct AccessibilityElement: Codable, Sendable, Hashable {
+    public var role: String?
+    public var roleDescription: String?
+    public var label: String?
+    public var title: String?
+    public var identifier: String?
+    public var value: String?
+    /// In the guest's points.
+    public var frame: CGRect
+    public var children: [AccessibilityElement]
+
+    /// The centre in the portrait native coordinates `tap` takes, undoing the guest's rotation.
+    /// Nil for an empty frame or a centre off screen.
+    public func normalizedCenter(in screen: CGRect, orientation: DeviceOrientation = .portrait) -> CGPoint? {
+        guard screen.width > 0, screen.height > 0, !frame.isEmpty else { return nil }
+        let point = CGPoint(
+            x: (frame.midX - screen.minX) / screen.width,
+            y: (frame.midY - screen.minY) / screen.height
+        )
+        guard (0...1).contains(point.x), (0...1).contains(point.y) else { return nil }
+        return CoordinateMapper.portraitNativePoint(from: point, orientation: orientation)
+    }
+}
+
+// On Xcode 27 (27A266a), AXPTranslator traps on the main thread and when supportsDelegateTokens or
+// accessibilityEnabled is set. The work queue keeps reads off main and serializes the singleton.
+final class AccessibilityTreeReader: @unchecked Sendable {
+    private static let maximumDepth = 64
+    private static let workQueue = DispatchQueue(label: "\(Brand.commandName).accessibility.work")
+
+    private let device: any ODHSimDevice
+    private let timeout: DispatchTimeInterval
+
+    init(device: any ODHSimDevice, timeout: DispatchTimeInterval = .seconds(2)) {
+        self.device = device
+        self.timeout = timeout
+    }
+
+    /// `async` and a wait, not `sync`, which may run the block on the calling main thread.
+    func read() throws -> AccessibilityElement {
+        let handoff = Handoff<Result<AccessibilityElement, any Error>>()
+        Self.workQueue.async { [self] in
+            handoff.set(Result { try readOnWorkQueue() })
+        }
+        guard let result = handoff.wait(timeout: .distantFuture) else {
+            throw EngineError.privateCall(symbol: "AXPTranslator", message: "the read ended without a result")
+        }
+        return try result.get()
+    }
+
+    private func readOnWorkQueue() throws -> AccessibilityElement {
+        let translator = try Self.translator()
+        let token = UUID().uuidString
+        // Held weakly by the translator, so this frame keeps it alive.
+        let delegate = AccessibilityBridgeDelegate(device: device, timeout: timeout)
+        translator.setBridgeTokenDelegate(delegate)
+        defer { translator.setBridgeTokenDelegate(nil) }
+
+        guard let translation = translator.frontmostApplication(withDisplayId: 0, bridgeDelegateToken: token) else {
+            throw EngineError.capabilityUnavailable(
+                name: "the accessibility tree, no app answered and the guest may still be starting"
+            )
+        }
+        try Self.setToken(token, on: translation)
+        guard let root = translator.macPlatformElement(fromTranslation: translation) else {
+            throw EngineError.privateCall(
+                symbol: "-[AXPTranslator macPlatformElementFromTranslation:]",
+                message: "no element for the frontmost app"
+            )
+        }
+        return try element(from: root, token: token, depth: 0)
+    }
+
+    private func element(from object: Any, token: String, depth: Int) throws -> AccessibilityElement {
+        try AccessibilitySelectorCheck.require(on: object as AnyObject, selectors: [
+            "translation", "accessibilityChildren", "accessibilityRole", "accessibilityRoleDescription",
+            "accessibilityLabel", "accessibilityTitle", "accessibilityIdentifier", "accessibilityValue",
+            "accessibilityFrame",
+        ])
+        // Bit cast, not `as?`: the private classes do not formally adopt these protocols.
+        let platform = unsafeBitCast(object as AnyObject, to: (any ODHAXPMacPlatformElement).self)
+        // Without the token, the element's requests reach no delegate.
+        if let translation = platform.translation {
+            try Self.setToken(token, on: translation)
+        }
+        let children = try depth < Self.maximumDepth
+            ? (platform.accessibilityChildren ?? []).map { try element(from: $0, token: token, depth: depth + 1) }
+            : []
+
+        return AccessibilityElement(
+            role: platform.accessibilityRole,
+            roleDescription: platform.accessibilityRoleDescription,
+            label: platform.accessibilityLabel.nonEmpty,
+            title: platform.accessibilityTitle.nonEmpty,
+            identifier: platform.accessibilityIdentifier.nonEmpty,
+            value: platform.accessibilityValue.map { "\($0)" }.nonEmpty,
+            frame: platform.accessibilityFrame,
+            children: children
+        )
+    }
+
+    private static func setToken(_ token: String, on translation: Any) throws {
+        try AccessibilitySelectorCheck.require(on: translation as AnyObject, selectors: ["setBridgeDelegateToken:"])
+        unsafeBitCast(translation as AnyObject, to: (any ODHAXPTranslationObject).self)
+            .setBridgeDelegateToken(token)
+    }
+
+    private static func translator() throws -> any ODHAXPTranslator {
+        guard let classObject = NSClassFromString("AXPTranslator").map({ $0 as AnyObject }),
+              classObject.responds(to: NSSelectorFromString("sharedInstance")),
+              let shared = unsafeBitCast(classObject, to: (any ODHAXPTranslatorClass).self).sharedInstance() else {
+            throw EngineError.symbolNotFound(
+                name: "+[AXPTranslator sharedInstance]",
+                framework: PrivateFramework.accessibilityPlatformTranslation.rawValue
+            )
+        }
+        try AccessibilitySelectorCheck.require(on: shared as AnyObject, selectors: [
+            "setBridgeTokenDelegate:", "frontmostApplicationWithDisplayId:bridgeDelegateToken:",
+            "macPlatformElementFromTranslation:",
+        ])
+        return unsafeBitCast(shared as AnyObject, to: (any ODHAXPTranslator).self)
+    }
+}
+
+enum AccessibilitySelectorCheck {
+    static func require(on object: AnyObject, selectors: [String]) throws {
+        for name in selectors where !object.responds(to: NSSelectorFromString(name)) {
+            throw EngineError.symbolNotFound(
+                name: "-[\(type(of: object)) \(name)]",
+                framework: PrivateFramework.accessibilityPlatformTranslation.rawValue
+            )
+        }
+    }
+}
+
+/// The three required methods of `AXPTranslationTokenDelegateHelper`, a protocol only visible at runtime.
+final class AccessibilityBridgeDelegate: NSObject {
+    typealias BridgeCallback = @convention(block) (AnyObject?) -> AnyObject?
+
+    private let device: any ODHSimDevice
+    private let timeout: DispatchTimeInterval
+    /// Answers land here because the caller blocks while it waits.
+    private let queue = DispatchQueue(label: "\(Brand.commandName).accessibility")
+
+    private static let conformance: Void = {
+        if let helper = objc_getProtocol("AXPTranslationTokenDelegateHelper") {
+            class_addProtocol(AccessibilityBridgeDelegate.self, helper)
+        }
+    }()
+
+    init(device: any ODHSimDevice, timeout: DispatchTimeInterval) {
+        _ = Self.conformance
+        self.device = device
+        self.timeout = timeout
+        super.init()
+    }
+
+    @objc(accessibilityTranslationDelegateBridgeCallbackWithToken:)
+    func bridgeCallback(withToken token: String) -> BridgeCallback {
+        { [device, queue, timeout] request in
+            guard let request else { return Self.emptyResponse() }
+            let handoff = Handoff<AnyObject?>()
+            device.sendAccessibilityRequestAsync(request, completionQueue: queue) { response in
+                handoff.set(response as AnyObject?)
+            }
+            return handoff.wait(timeout: .now() + timeout).flatMap { $0 } ?? Self.emptyResponse()
+        }
+    }
+
+    /// Frames already come back in the guest's points.
+    @objc(accessibilityTranslationConvertPlatformFrameToSystem:withToken:)
+    func convertPlatformFrameToSystem(_ frame: CGRect, withToken token: String) -> CGRect {
+        frame
+    }
+
+    @objc(accessibilityTranslationRootParentWithToken:)
+    func rootParent(withToken token: String) -> AnyObject? {
+        nil
+    }
+
+    /// The translator does not expect nil from the callback.
+    private static func emptyResponse() -> AnyObject? {
+        let selector = NSSelectorFromString("emptyResponse")
+        guard let responseClass = NSClassFromString("AXPTranslatorResponse").map({ $0 as AnyObject }),
+              responseClass.responds(to: selector) else { return nil }
+        return responseClass.perform(selector)?.takeUnretainedValue()
+    }
+}
+
+private final class Handoff<Value>: @unchecked Sendable {
+    private let lock = NSLock()
+    private let semaphore = DispatchSemaphore(value: 0)
+    private var value: Value?
+
+    func set(_ value: Value) {
+        lock.withLock { self.value = value }
+        semaphore.signal()
+    }
+
+    func wait(timeout: DispatchTime) -> Value? {
+        guard semaphore.wait(timeout: timeout) == .success else { return nil }
+        return lock.withLock { value }
+    }
+}
+
+private extension Optional<String> {
+    var nonEmpty: String? {
+        guard let self, !self.isEmpty else { return nil }
+        return self
+    }
+}

engine/Sources/OpenDeviceHubEngine/Adapter/CoreSimulatorAdapter.swift

@@ -361,6 +361,25 @@ public final class CoreSimulatorAdapter: SimulatorAdapter, @unchecked Sendable {
         return try PasteboardBridge(port: port)
     }
 
+    public func accessibilityTree(_ udid: String) throws -> AccessibilityElement {
+        lock.lock()
+        defer { lock.unlock() }
+
+        let device = try rawDevice(udid)
+        guard DeviceState.from(state: device.state, stateString: device.stateString ?? "") == .booted else {
+            throw EngineError.deviceNotBooted(udid: udid)
+        }
+        let selector = NSSelectorFromString("sendAccessibilityRequestAsync:completionQueue:completionHandler:")
+        guard (device as AnyObject).responds(to: selector) else {
+            throw EngineError.symbolNotFound(
+                name: "-[SimDevice \(NSStringFromSelector(selector))]",
+                framework: PrivateFramework.coreSimulator.rawValue
+            )
+        }
+        _ = try FrameworkLoader.load(.accessibilityPlatformTranslation, from: xcode)
+        return try AccessibilityTreeReader(device: device).read()
+    }
+
     public func setOrientation(_ orientation: DeviceOrientation, udid: String) throws {
         lock.lock()
         defer { lock.unlock() }

engine/Sources/OpenDeviceHubEngine/Adapter/PrivateSymbolProbe.swift

@@ -60,12 +60,21 @@ public enum PrivateSymbolProbe {
         (.classSymbol, "SimPasteboardInterfaceListener"),
     ]
 
+    public static let accessibilityPlatformTranslationSymbols: [(PrivateSymbolKind, String)] = [
+        (.classSymbol, "AXPTranslator"),
+        (.classSymbol, "AXPTranslationObject"),
+        (.classSymbol, "AXPMacPlatformElement"),
+        (.classSymbol, "AXPTranslatorResponse"),
+        (.protocolSymbol, "AXPTranslationTokenDelegateHelper"),
+    ]
+
     public static func symbols(for framework: PrivateFramework) -> [(PrivateSymbolKind, String)] {
         switch framework {
         case .coreSimulator: coreSimulatorSymbols
         case .coreSimDeviceIO: coreSimDeviceIOSymbols
         case .simulatorKit: simulatorKitSymbols
         case .simPasteboardPlus: simPasteboardPlusSymbols
+        case .accessibilityPlatformTranslation: accessibilityPlatformTranslationSymbols
         }
     }
 

engine/Sources/OpenDeviceHubEngine/Adapter/SimulatorAdapter.swift

@@ -205,6 +205,8 @@ public protocol SimulatorAdapter: Sendable {
     func setKeyboardLanguage(_ language: String, udid: String) throws
     /// Opens the clipboard link to a booted device.
     func openPasteboard(_ udid: String) throws -> any PasteboardSession
+    /// Reads the accessibility tree of the app in front on a booted device.
+    func accessibilityTree(_ udid: String) throws -> AccessibilityElement
     /// Watches every device in the set, so a window learns that its device has gone or come back
     /// without asking.
     func watchDeviceStates() throws -> any DeviceNotifier
@@ -242,6 +244,10 @@ extension SimulatorAdapter {
 
     public func forgetConnections(_ udid: String) {}
 
+    public func accessibilityTree(_ udid: String) throws -> AccessibilityElement {
+        throw EngineError.capabilityUnavailable(name: "the accessibility tree of \(udid)")
+    }
+
     public func displayReport(_ udid: String) async throws -> DisplayReport {
         let feature = try openCoreDevice(udid, service: CoreDeviceFeature.displayInfoService)
         return try DisplayReport.parse(try await feature.perform(action: CoreDeviceFeature.displayInfoAction))

engine/Sources/OpenDeviceHubEngine/Xcode/FrameworkLoader.swift

@@ -5,6 +5,12 @@ public enum PrivateFramework: String, Sendable, Hashable, CaseIterable {
     case coreSimDeviceIO = "CoreSimDeviceIO"
     case simulatorKit = "SimulatorKit"
     case simPasteboardPlus = "SimPasteboardPlus"
+    case accessibilityPlatformTranslation = "AccessibilityPlatformTranslation"
+
+    /// Shared-cache binaries can resolve through `dlopen` without existing on disk.
+    var isInSharedCache: Bool {
+        self == .accessibilityPlatformTranslation
+    }
 
     /// Probed in order. CoreSimulator lives outside Xcode; SimulatorKit moved from the developer
     /// directory into the app bundle's SharedFrameworks in Xcode 27.
@@ -35,6 +41,10 @@ public enum PrivateFramework: String, Sendable, Hashable, CaseIterable {
                     .appending(path: "Library/PrivateFrameworks/\(suffix)")
                     .path(percentEncoded: false),
             ]
+        case .accessibilityPlatformTranslation:
+            return [
+                "/System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/AccessibilityPlatformTranslation",
+            ]
         case .simulatorKit:
             return [
                 install.appRoot
@@ -62,7 +72,7 @@ public enum FrameworkLoader {
         var attempts: [String] = []
 
         for path in framework.candidatePaths(for: install) {
-            guard FileManager.default.fileExists(atPath: path) else {
+            guard framework.isInSharedCache || FileManager.default.fileExists(atPath: path) else {
                 attempts.append("\(path) (not present)")
                 continue
             }

engine/Sources/OpenDeviceHubPrivate/include/OpenDeviceHubPrivate.h

@@ -54,6 +54,11 @@
 /// Sets the guest's keyboard language, for example "en-US". Verified present on Xcode 27 (27A266a).
 - (BOOL)setKeyboardLanguage:(NSString *_Nonnull)language
                       error:(NSError *_Nullable *_Nullable)error;
+/// Request and response are AXPTranslatorRequest and AXPTranslatorResponse from the dynamically
+/// loaded AccessibilityPlatformTranslation framework. Verified on Xcode 27 (27A266a).
+- (void)sendAccessibilityRequestAsync:(id _Nonnull)request
+                      completionQueue:(dispatch_queue_t _Nonnull)queue
+                    completionHandler:(void (^_Nonnull)(id _Nullable))handler;
 @end
 
 @protocol ODHSimDeviceSet <NSObject>
@@ -176,6 +181,38 @@
 - (void)disableRemoteAutosync;
 @end
 
+/// AccessibilityPlatformTranslation selectors and type encodings verified on macOS 27,
+/// Xcode 27 (27A266a).
+
+@protocol ODHAXPTranslatorClass <NSObject>
+- (nullable id)sharedInstance;
+@end
+
+@protocol ODHAXPTranslator <NSObject>
+- (void)setBridgeTokenDelegate:(id _Nullable)delegate;
+/// An `AXPTranslationObject` for the app in front on that display, or nil while nothing answers.
+- (nullable id)frontmostApplicationWithDisplayId:(unsigned int)displayId
+                             bridgeDelegateToken:(NSString *_Nonnull)token;
+/// Returns an AXPMacPlatformElement.
+- (nullable id)macPlatformElementFromTranslation:(id _Nonnull)translation;
+@end
+
+@protocol ODHAXPTranslationObject <NSObject>
+- (void)setBridgeDelegateToken:(NSString *_Nullable)token;
+@end
+
+@protocol ODHAXPMacPlatformElement <NSObject>
+@property (nonatomic, readonly, nullable) id translation;
+@property (nonatomic, readonly, nullable) NSArray *accessibilityChildren;
+@property (nonatomic, readonly, nullable) NSString *accessibilityRole;
+@property (nonatomic, readonly, nullable) NSString *accessibilityRoleDescription;
+@property (nonatomic, readonly, nullable) NSString *accessibilityLabel;
+@property (nonatomic, readonly, nullable) NSString *accessibilityTitle;
+@property (nonatomic, readonly, nullable) NSString *accessibilityIdentifier;
+@property (nonatomic, readonly, nullable) id accessibilityValue;
+@property (nonatomic, readonly) CGRect accessibilityFrame;
+@end
+
 @protocol ODHSimServiceContext <NSObject>
 - (nullable id<ODHSimDeviceSet>)defaultDeviceSetWithError:(NSError *_Nullable *_Nullable)error;
 @end