From 69887697a6e445e8f024c4bf17cb26200a4c4148 Mon Sep 17 00:00:00 2001 From: eksdev007 <278202555+eksdev007@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:32:21 +0000 Subject: [PATCH] feat(cli): expose simulator accessibility trees --- README.md | 6 +- .../Adapter/AccessibilityTree.swift | 216 ++++++++++++++++++ .../Adapter/CoreSimulatorAdapter.swift | 19 ++ .../Adapter/PrivateSymbolProbe.swift | 9 + .../Adapter/SimulatorAdapter.swift | 6 + .../Xcode/FrameworkLoader.swift | 12 +- .../include/OpenDeviceHubPrivate.h | 37 +++ engine/Sources/odhub/ODHubCommand.swift | 2 +- engine/Sources/odhub/UITreeCommand.swift | 75 ++++++ .../AccessibilityElementTests.swift | 116 ++++++++++ .../FrameworkLoaderTests.swift | 23 +- .../AccessibilityTreeTests.swift | 41 ++++ 12 files changed, 555 insertions(+), 7 deletions(-) create mode 100644 engine/Sources/OpenDeviceHubEngine/Adapter/AccessibilityTree.swift create mode 100644 engine/Sources/odhub/UITreeCommand.swift create mode 100644 engine/Tests/OpenDeviceHubEngineTests/AccessibilityElementTests.swift create mode 100644 engine/Tests/OpenDeviceHubIntegrationTests/AccessibilityTreeTests.swift diff --git a/README.md b/README.md index b186aca..d7742ca 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,8 @@ simulators. - **Debug helpers**: slow animations, shake, simulated memory warning, the system log and app data in the Finder, and a click to frame latency overlay. - **A command line tool**: `odhub` drives a simulator from a script: tap, swipe, pinch, type, - buttons, rotation. One menu item puts it on your `PATH`. + buttons, rotation. `odhub ui-tree` prints the accessibility tree of the app in front, each element + with the coordinates `tap` takes. One menu item puts it on your `PATH`.
Keyboard shortcuts @@ -126,12 +127,11 @@ options. | `rotate` | `odhub rotate A1B2C3D4 landscapeLeft` | | `home-swipe` | `odhub home-swipe A1B2C3D4` | | `app-switcher` | `odhub app-switcher A1B2C3D4` | +| `ui-tree` | `odhub ui-tree A1B2C3D4 --labelled` |
## Current limitations -- **The windows are invisible to accessibility**: VoiceOver, window managers and scripting tools see - the app and its menus but not its windows. - **A device taller than your display** keeps its bottom edge off screen, where the pointer cannot reach it. Use Fit. - **The compact layout below 340 points wide** is written and unit tested but has never been used by diff --git a/engine/Sources/OpenDeviceHubEngine/Adapter/AccessibilityTree.swift b/engine/Sources/OpenDeviceHubEngine/Adapter/AccessibilityTree.swift new file mode 100644 index 0000000..488a913 --- /dev/null +++ b/engine/Sources/OpenDeviceHubEngine/Adapter/AccessibilityTree.swift @@ -0,0 +1,216 @@ +import CoreGraphics +import Foundation +import OpenDeviceHubPrivate + +/// One element of a guest app's accessibility tree. +public struct AccessibilityElement: Codable, Sendable, Hashable { + public var role: String? + public var roleDescription: String? + public var label: String? + public var title: String? + public var identifier: String? + public var value: String? + /// In the guest's points. + public var frame: CGRect + public var children: [AccessibilityElement] + + /// The centre in the portrait native coordinates `tap` takes, undoing the guest's rotation. + /// Nil for an empty frame or a centre off screen. + public func normalizedCenter(in screen: CGRect, orientation: DeviceOrientation = .portrait) -> CGPoint? { + guard screen.width > 0, screen.height > 0, !frame.isEmpty else { return nil } + let point = CGPoint( + x: (frame.midX - screen.minX) / screen.width, + y: (frame.midY - screen.minY) / screen.height + ) + guard (0...1).contains(point.x), (0...1).contains(point.y) else { return nil } + return CoordinateMapper.portraitNativePoint(from: point, orientation: orientation) + } +} + +// On Xcode 27 (27A266a), AXPTranslator traps on the main thread and when supportsDelegateTokens or +// accessibilityEnabled is set. The work queue keeps reads off main and serializes the singleton. +final class AccessibilityTreeReader: @unchecked Sendable { + private static let maximumDepth = 64 + private static let workQueue = DispatchQueue(label: "\(Brand.commandName).accessibility.work") + + private let device: any ODHSimDevice + private let timeout: DispatchTimeInterval + + init(device: any ODHSimDevice, timeout: DispatchTimeInterval = .seconds(2)) { + self.device = device + self.timeout = timeout + } + + /// `async` and a wait, not `sync`, which may run the block on the calling main thread. + func read() throws -> AccessibilityElement { + let handoff = Handoff>() + Self.workQueue.async { [self] in + handoff.set(Result { try readOnWorkQueue() }) + } + guard let result = handoff.wait(timeout: .distantFuture) else { + throw EngineError.privateCall(symbol: "AXPTranslator", message: "the read ended without a result") + } + return try result.get() + } + + private func readOnWorkQueue() throws -> AccessibilityElement { + let translator = try Self.translator() + let token = UUID().uuidString + // Held weakly by the translator, so this frame keeps it alive. + let delegate = AccessibilityBridgeDelegate(device: device, timeout: timeout) + translator.setBridgeTokenDelegate(delegate) + defer { translator.setBridgeTokenDelegate(nil) } + + guard let translation = translator.frontmostApplication(withDisplayId: 0, bridgeDelegateToken: token) else { + throw EngineError.capabilityUnavailable( + name: "the accessibility tree, no app answered and the guest may still be starting" + ) + } + try Self.setToken(token, on: translation) + guard let root = translator.macPlatformElement(fromTranslation: translation) else { + throw EngineError.privateCall( + symbol: "-[AXPTranslator macPlatformElementFromTranslation:]", + message: "no element for the frontmost app" + ) + } + return try element(from: root, token: token, depth: 0) + } + + private func element(from object: Any, token: String, depth: Int) throws -> AccessibilityElement { + try AccessibilitySelectorCheck.require(on: object as AnyObject, selectors: [ + "translation", "accessibilityChildren", "accessibilityRole", "accessibilityRoleDescription", + "accessibilityLabel", "accessibilityTitle", "accessibilityIdentifier", "accessibilityValue", + "accessibilityFrame", + ]) + // Bit cast, not `as?`: the private classes do not formally adopt these protocols. + let platform = unsafeBitCast(object as AnyObject, to: (any ODHAXPMacPlatformElement).self) + // Without the token, the element's requests reach no delegate. + if let translation = platform.translation { + try Self.setToken(token, on: translation) + } + let children = try depth < Self.maximumDepth + ? (platform.accessibilityChildren ?? []).map { try element(from: $0, token: token, depth: depth + 1) } + : [] + + return AccessibilityElement( + role: platform.accessibilityRole, + roleDescription: platform.accessibilityRoleDescription, + label: platform.accessibilityLabel.nonEmpty, + title: platform.accessibilityTitle.nonEmpty, + identifier: platform.accessibilityIdentifier.nonEmpty, + value: platform.accessibilityValue.map { "\($0)" }.nonEmpty, + frame: platform.accessibilityFrame, + children: children + ) + } + + private static func setToken(_ token: String, on translation: Any) throws { + try AccessibilitySelectorCheck.require(on: translation as AnyObject, selectors: ["setBridgeDelegateToken:"]) + unsafeBitCast(translation as AnyObject, to: (any ODHAXPTranslationObject).self) + .setBridgeDelegateToken(token) + } + + private static func translator() throws -> any ODHAXPTranslator { + guard let classObject = NSClassFromString("AXPTranslator").map({ $0 as AnyObject }), + classObject.responds(to: NSSelectorFromString("sharedInstance")), + let shared = unsafeBitCast(classObject, to: (any ODHAXPTranslatorClass).self).sharedInstance() else { + throw EngineError.symbolNotFound( + name: "+[AXPTranslator sharedInstance]", + framework: PrivateFramework.accessibilityPlatformTranslation.rawValue + ) + } + try AccessibilitySelectorCheck.require(on: shared as AnyObject, selectors: [ + "setBridgeTokenDelegate:", "frontmostApplicationWithDisplayId:bridgeDelegateToken:", + "macPlatformElementFromTranslation:", + ]) + return unsafeBitCast(shared as AnyObject, to: (any ODHAXPTranslator).self) + } +} + +enum AccessibilitySelectorCheck { + static func require(on object: AnyObject, selectors: [String]) throws { + for name in selectors where !object.responds(to: NSSelectorFromString(name)) { + throw EngineError.symbolNotFound( + name: "-[\(type(of: object)) \(name)]", + framework: PrivateFramework.accessibilityPlatformTranslation.rawValue + ) + } + } +} + +/// The three required methods of `AXPTranslationTokenDelegateHelper`, a protocol only visible at runtime. +final class AccessibilityBridgeDelegate: NSObject { + typealias BridgeCallback = @convention(block) (AnyObject?) -> AnyObject? + + private let device: any ODHSimDevice + private let timeout: DispatchTimeInterval + /// Answers land here because the caller blocks while it waits. + private let queue = DispatchQueue(label: "\(Brand.commandName).accessibility") + + private static let conformance: Void = { + if let helper = objc_getProtocol("AXPTranslationTokenDelegateHelper") { + class_addProtocol(AccessibilityBridgeDelegate.self, helper) + } + }() + + init(device: any ODHSimDevice, timeout: DispatchTimeInterval) { + _ = Self.conformance + self.device = device + self.timeout = timeout + super.init() + } + + @objc(accessibilityTranslationDelegateBridgeCallbackWithToken:) + func bridgeCallback(withToken token: String) -> BridgeCallback { + { [device, queue, timeout] request in + guard let request else { return Self.emptyResponse() } + let handoff = Handoff() + device.sendAccessibilityRequestAsync(request, completionQueue: queue) { response in + handoff.set(response as AnyObject?) + } + return handoff.wait(timeout: .now() + timeout).flatMap { $0 } ?? Self.emptyResponse() + } + } + + /// Frames already come back in the guest's points. + @objc(accessibilityTranslationConvertPlatformFrameToSystem:withToken:) + func convertPlatformFrameToSystem(_ frame: CGRect, withToken token: String) -> CGRect { + frame + } + + @objc(accessibilityTranslationRootParentWithToken:) + func rootParent(withToken token: String) -> AnyObject? { + nil + } + + /// The translator does not expect nil from the callback. + private static func emptyResponse() -> AnyObject? { + let selector = NSSelectorFromString("emptyResponse") + guard let responseClass = NSClassFromString("AXPTranslatorResponse").map({ $0 as AnyObject }), + responseClass.responds(to: selector) else { return nil } + return responseClass.perform(selector)?.takeUnretainedValue() + } +} + +private final class Handoff: @unchecked Sendable { + private let lock = NSLock() + private let semaphore = DispatchSemaphore(value: 0) + private var value: Value? + + func set(_ value: Value) { + lock.withLock { self.value = value } + semaphore.signal() + } + + func wait(timeout: DispatchTime) -> Value? { + guard semaphore.wait(timeout: timeout) == .success else { return nil } + return lock.withLock { value } + } +} + +private extension Optional { + var nonEmpty: String? { + guard let self, !self.isEmpty else { return nil } + return self + } +} diff --git a/engine/Sources/OpenDeviceHubEngine/Adapter/CoreSimulatorAdapter.swift b/engine/Sources/OpenDeviceHubEngine/Adapter/CoreSimulatorAdapter.swift index 50613af..528942d 100644 --- a/engine/Sources/OpenDeviceHubEngine/Adapter/CoreSimulatorAdapter.swift +++ b/engine/Sources/OpenDeviceHubEngine/Adapter/CoreSimulatorAdapter.swift @@ -361,6 +361,25 @@ public final class CoreSimulatorAdapter: SimulatorAdapter, @unchecked Sendable { return try PasteboardBridge(port: port) } + public func accessibilityTree(_ udid: String) throws -> AccessibilityElement { + lock.lock() + defer { lock.unlock() } + + let device = try rawDevice(udid) + guard DeviceState.from(state: device.state, stateString: device.stateString ?? "") == .booted else { + throw EngineError.deviceNotBooted(udid: udid) + } + let selector = NSSelectorFromString("sendAccessibilityRequestAsync:completionQueue:completionHandler:") + guard (device as AnyObject).responds(to: selector) else { + throw EngineError.symbolNotFound( + name: "-[SimDevice \(NSStringFromSelector(selector))]", + framework: PrivateFramework.coreSimulator.rawValue + ) + } + _ = try FrameworkLoader.load(.accessibilityPlatformTranslation, from: xcode) + return try AccessibilityTreeReader(device: device).read() + } + public func setOrientation(_ orientation: DeviceOrientation, udid: String) throws { lock.lock() defer { lock.unlock() } diff --git a/engine/Sources/OpenDeviceHubEngine/Adapter/PrivateSymbolProbe.swift b/engine/Sources/OpenDeviceHubEngine/Adapter/PrivateSymbolProbe.swift index 04993b9..a48a0da 100644 --- a/engine/Sources/OpenDeviceHubEngine/Adapter/PrivateSymbolProbe.swift +++ b/engine/Sources/OpenDeviceHubEngine/Adapter/PrivateSymbolProbe.swift @@ -60,12 +60,21 @@ public enum PrivateSymbolProbe { (.classSymbol, "SimPasteboardInterfaceListener"), ] + public static let accessibilityPlatformTranslationSymbols: [(PrivateSymbolKind, String)] = [ + (.classSymbol, "AXPTranslator"), + (.classSymbol, "AXPTranslationObject"), + (.classSymbol, "AXPMacPlatformElement"), + (.classSymbol, "AXPTranslatorResponse"), + (.protocolSymbol, "AXPTranslationTokenDelegateHelper"), + ] + public static func symbols(for framework: PrivateFramework) -> [(PrivateSymbolKind, String)] { switch framework { case .coreSimulator: coreSimulatorSymbols case .coreSimDeviceIO: coreSimDeviceIOSymbols case .simulatorKit: simulatorKitSymbols case .simPasteboardPlus: simPasteboardPlusSymbols + case .accessibilityPlatformTranslation: accessibilityPlatformTranslationSymbols } } diff --git a/engine/Sources/OpenDeviceHubEngine/Adapter/SimulatorAdapter.swift b/engine/Sources/OpenDeviceHubEngine/Adapter/SimulatorAdapter.swift index c724e7d..141a36e 100644 --- a/engine/Sources/OpenDeviceHubEngine/Adapter/SimulatorAdapter.swift +++ b/engine/Sources/OpenDeviceHubEngine/Adapter/SimulatorAdapter.swift @@ -205,6 +205,8 @@ public protocol SimulatorAdapter: Sendable { func setKeyboardLanguage(_ language: String, udid: String) throws /// Opens the clipboard link to a booted device. func openPasteboard(_ udid: String) throws -> any PasteboardSession + /// Reads the accessibility tree of the app in front on a booted device. + func accessibilityTree(_ udid: String) throws -> AccessibilityElement /// Watches every device in the set, so a window learns that its device has gone or come back /// without asking. func watchDeviceStates() throws -> any DeviceNotifier @@ -242,6 +244,10 @@ extension SimulatorAdapter { public func forgetConnections(_ udid: String) {} + public func accessibilityTree(_ udid: String) throws -> AccessibilityElement { + throw EngineError.capabilityUnavailable(name: "the accessibility tree of \(udid)") + } + public func displayReport(_ udid: String) async throws -> DisplayReport { let feature = try openCoreDevice(udid, service: CoreDeviceFeature.displayInfoService) return try DisplayReport.parse(try await feature.perform(action: CoreDeviceFeature.displayInfoAction)) diff --git a/engine/Sources/OpenDeviceHubEngine/Xcode/FrameworkLoader.swift b/engine/Sources/OpenDeviceHubEngine/Xcode/FrameworkLoader.swift index 2c3be4e..d8b9665 100644 --- a/engine/Sources/OpenDeviceHubEngine/Xcode/FrameworkLoader.swift +++ b/engine/Sources/OpenDeviceHubEngine/Xcode/FrameworkLoader.swift @@ -5,6 +5,12 @@ public enum PrivateFramework: String, Sendable, Hashable, CaseIterable { case coreSimDeviceIO = "CoreSimDeviceIO" case simulatorKit = "SimulatorKit" case simPasteboardPlus = "SimPasteboardPlus" + case accessibilityPlatformTranslation = "AccessibilityPlatformTranslation" + + /// Shared-cache binaries can resolve through `dlopen` without existing on disk. + var isInSharedCache: Bool { + self == .accessibilityPlatformTranslation + } /// Probed in order. CoreSimulator lives outside Xcode; SimulatorKit moved from the developer /// directory into the app bundle's SharedFrameworks in Xcode 27. @@ -35,6 +41,10 @@ public enum PrivateFramework: String, Sendable, Hashable, CaseIterable { .appending(path: "Library/PrivateFrameworks/\(suffix)") .path(percentEncoded: false), ] + case .accessibilityPlatformTranslation: + return [ + "/System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/AccessibilityPlatformTranslation", + ] case .simulatorKit: return [ install.appRoot @@ -62,7 +72,7 @@ public enum FrameworkLoader { var attempts: [String] = [] for path in framework.candidatePaths(for: install) { - guard FileManager.default.fileExists(atPath: path) else { + guard framework.isInSharedCache || FileManager.default.fileExists(atPath: path) else { attempts.append("\(path) (not present)") continue } diff --git a/engine/Sources/OpenDeviceHubPrivate/include/OpenDeviceHubPrivate.h b/engine/Sources/OpenDeviceHubPrivate/include/OpenDeviceHubPrivate.h index 33195be..d9f5b24 100644 --- a/engine/Sources/OpenDeviceHubPrivate/include/OpenDeviceHubPrivate.h +++ b/engine/Sources/OpenDeviceHubPrivate/include/OpenDeviceHubPrivate.h @@ -54,6 +54,11 @@ /// Sets the guest's keyboard language, for example "en-US". Verified present on Xcode 27 (27A266a). - (BOOL)setKeyboardLanguage:(NSString *_Nonnull)language error:(NSError *_Nullable *_Nullable)error; +/// Request and response are AXPTranslatorRequest and AXPTranslatorResponse from the dynamically +/// loaded AccessibilityPlatformTranslation framework. Verified on Xcode 27 (27A266a). +- (void)sendAccessibilityRequestAsync:(id _Nonnull)request + completionQueue:(dispatch_queue_t _Nonnull)queue + completionHandler:(void (^_Nonnull)(id _Nullable))handler; @end @protocol ODHSimDeviceSet @@ -176,6 +181,38 @@ - (void)disableRemoteAutosync; @end +/// AccessibilityPlatformTranslation selectors and type encodings verified on macOS 27, +/// Xcode 27 (27A266a). + +@protocol ODHAXPTranslatorClass +- (nullable id)sharedInstance; +@end + +@protocol ODHAXPTranslator +- (void)setBridgeTokenDelegate:(id _Nullable)delegate; +/// An `AXPTranslationObject` for the app in front on that display, or nil while nothing answers. +- (nullable id)frontmostApplicationWithDisplayId:(unsigned int)displayId + bridgeDelegateToken:(NSString *_Nonnull)token; +/// Returns an AXPMacPlatformElement. +- (nullable id)macPlatformElementFromTranslation:(id _Nonnull)translation; +@end + +@protocol ODHAXPTranslationObject +- (void)setBridgeDelegateToken:(NSString *_Nullable)token; +@end + +@protocol ODHAXPMacPlatformElement +@property (nonatomic, readonly, nullable) id translation; +@property (nonatomic, readonly, nullable) NSArray *accessibilityChildren; +@property (nonatomic, readonly, nullable) NSString *accessibilityRole; +@property (nonatomic, readonly, nullable) NSString *accessibilityRoleDescription; +@property (nonatomic, readonly, nullable) NSString *accessibilityLabel; +@property (nonatomic, readonly, nullable) NSString *accessibilityTitle; +@property (nonatomic, readonly, nullable) NSString *accessibilityIdentifier; +@property (nonatomic, readonly, nullable) id accessibilityValue; +@property (nonatomic, readonly) CGRect accessibilityFrame; +@end + @protocol ODHSimServiceContext - (nullable id)defaultDeviceSetWithError:(NSError *_Nullable *_Nullable)error; @end diff --git a/engine/Sources/odhub/ODHubCommand.swift b/engine/Sources/odhub/ODHubCommand.swift index 3550182..7f5012b 100644 --- a/engine/Sources/odhub/ODHubCommand.swift +++ b/engine/Sources/odhub/ODHubCommand.swift @@ -7,6 +7,6 @@ struct ODHub: AsyncParsableCommand { commandName: Brand.commandName, abstract: "Inspect, view and drive iOS simulators from the active Xcode.", version: Brand.version, - subcommands: [Doctor.self, List.self, View.self, Tap.self, Swipe.self, Pinch.self, Type.self, Paste.self, Copy.self, Button.self, Rotate.self, HomeSwipe.self, AppSwitcher.self] + subcommands: [Doctor.self, List.self, View.self, Tap.self, Swipe.self, Pinch.self, Type.self, Paste.self, Copy.self, Button.self, Rotate.self, HomeSwipe.self, AppSwitcher.self, UITree.self] ) } diff --git a/engine/Sources/odhub/UITreeCommand.swift b/engine/Sources/odhub/UITreeCommand.swift new file mode 100644 index 0000000..c0f88ed --- /dev/null +++ b/engine/Sources/odhub/UITreeCommand.swift @@ -0,0 +1,75 @@ +import ArgumentParser +import CoreGraphics +import Foundation +import OpenDeviceHubEngine + +struct UITree: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "ui-tree", + abstract: "Print the accessibility tree of the app in front on a booted simulator.", + discussion: """ + Each line ends with the element's centre in the normalized coordinates `tap` takes, so a \ + line such as `Button "Save" @ 0.50,0.91` is tapped with `--x 0.50 --y 0.91`. Elements \ + scrolled off screen have no centre. + """ + ) + + @Argument(help: "The UDID of the simulator.") + var udid: String + + @Flag(help: "Print the whole tree as JSON, frames in points.") + var json = false + + @Flag(help: "Only print elements that carry a label, title, identifier or value.") + var labelled = false + + func run() async throws { + let adapter = try AdapterFactory.make(for: XcodeLocator.locate()) + let root = try adapter.accessibilityTree(udid) + + if json { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + print(String(decoding: try encoder.encode(root), as: UTF8.self)) + return + } + let report = try await adapter.displayReport(udid) + // Accessibility's display 0 denotes the main screen; CoreDevice uses its own display IDs. + guard report.isSettled, let display = report.activeIntegrated, + let orientation = DeviceOrientation.allCases.first(where: { $0.degrees == display.currentRotation }) else { + throw EngineError.capabilityUnavailable(name: "the accessibility display's rotation") + } + printTree(root, screen: root.frame, orientation: orientation, depth: 0) + } + + private func printTree(_ element: AccessibilityElement, screen: CGRect, orientation: DeviceOrientation, depth: Int) { + let isLabelled = element.label != nil || element.title != nil + || element.identifier != nil || element.value != nil + if !labelled || isLabelled { + let indent = String(repeating: " ", count: labelled ? 0 : depth) + print(indent + describe(element, screen: screen, orientation: orientation)) + } + for child in element.children { + printTree(child, screen: screen, orientation: orientation, depth: depth + 1) + } + } + + private func describe(_ element: AccessibilityElement, screen: CGRect, orientation: DeviceOrientation) -> String { + var parts = [role(of: element)] + if let label = element.label { parts.append("\"\(label)\"") } + if let title = element.title, title != element.label { parts.append("title=\"\(title)\"") } + if let value = element.value { parts.append("value=\"\(value)\"") } + if let identifier = element.identifier { parts.append("id=\(identifier)") } + if let center = element.normalizedCenter(in: screen, orientation: orientation) { + parts.append(String(format: "@ %.2f,%.2f", center.x, center.y)) + } + return parts.joined(separator: " ") + } + + private func role(of element: AccessibilityElement) -> String { + guard let role = element.role, !role.isEmpty else { + return element.roleDescription ?? "Element" + } + return role.hasPrefix("AX") ? String(role.dropFirst(2)) : role + } +} diff --git a/engine/Tests/OpenDeviceHubEngineTests/AccessibilityElementTests.swift b/engine/Tests/OpenDeviceHubEngineTests/AccessibilityElementTests.swift new file mode 100644 index 0000000..c5e8605 --- /dev/null +++ b/engine/Tests/OpenDeviceHubEngineTests/AccessibilityElementTests.swift @@ -0,0 +1,116 @@ +import CoreGraphics +import Foundation +import XCTest +import Testing +@testable import OpenDeviceHubEngine + +private func makeElement(frame: CGRect, children: [AccessibilityElement] = []) -> AccessibilityElement { + AccessibilityElement( + role: "AXButton", + roleDescription: "button", + label: "Save", + title: nil, + identifier: "save", + value: nil, + frame: frame, + children: children + ) +} + +private let screen = CGRect(x: 0, y: 0, width: 402, height: 874) + +final class AccessibilityElementCenterTests: XCTestCase { + func testCenterIsAFractionOfTheScreen() throws { + let element = makeElement(frame: CGRect(x: 0, y: 0, width: 402, height: 874)) + let center = try XCTUnwrap(element.normalizedCenter(in: screen)) + XCTAssertEqual(center.x, 0.5, accuracy: 0.0001) + XCTAssertEqual(center.y, 0.5, accuracy: 0.0001) + } + + func testCenterOfASmallElement() throws { + let element = makeElement(frame: CGRect(x: 20, y: 60, width: 40, height: 40)) + let center = try XCTUnwrap(element.normalizedCenter(in: screen)) + XCTAssertEqual(center.x, 40.0 / 402.0, accuracy: 0.0001) + XCTAssertEqual(center.y, 80.0 / 874.0, accuracy: 0.0001) + } + + func testCenterFollowsAScreenThatDoesNotStartAtTheOrigin() throws { + let offset = CGRect(x: 100, y: 200, width: 402, height: 874) + let element = makeElement(frame: CGRect(x: 100, y: 200, width: 402, height: 874)) + let center = try XCTUnwrap(element.normalizedCenter(in: offset)) + XCTAssertEqual(center.x, 0.5, accuracy: 0.0001) + XCTAssertEqual(center.y, 0.5, accuracy: 0.0001) + } + + func testAnElementScrolledOffScreenHasNoCenter() { + let element = makeElement(frame: CGRect(x: 20, y: 1_200, width: 40, height: 40)) + XCTAssertNil(element.normalizedCenter(in: screen)) + } + + func testAScreenWithNoSizeGivesNoCenter() { + let element = makeElement(frame: CGRect(x: 0, y: 0, width: 10, height: 10)) + XCTAssertNil(element.normalizedCenter(in: .zero)) + } +} + +final class AccessibilityElementCodingTests: XCTestCase { + func testRoundTripsThroughJSON() throws { + let tree = makeElement( + frame: screen, + children: [makeElement(frame: CGRect(x: 20, y: 60, width: 40, height: 40))] + ) + let data = try JSONEncoder().encode(tree) + XCTAssertEqual(try JSONDecoder().decode(AccessibilityElement.self, from: data), tree) + } +} + +struct AccessibilityElementCoordinateTests { + @Test(arguments: [ + CGRect.zero, + CGRect(x: 20, y: 60, width: 0, height: 40), + CGRect(x: 20, y: 60, width: 40, height: 0), + ]) + func emptyFramesHaveNoCenter(frame: CGRect) { + let element = makeElement(frame: frame) + #expect(element.normalizedCenter(in: screen) == nil) + } + + @Test(arguments: [ + (DeviceOrientation.portrait, CGPoint(x: 0.2, y: 0.3)), + (.landscapeLeft, CGPoint(x: 0.3, y: 0.8)), + (.portraitUpsideDown, CGPoint(x: 0.8, y: 0.7)), + (.landscapeRight, CGPoint(x: 0.7, y: 0.2)), + ]) + func centersUsePortraitNativeCoordinates(orientation: DeviceOrientation, expected: CGPoint) throws { + let displayedSize = orientation.displayedSize(portraitNative: screen.size) + let displayedScreen = CGRect(origin: .zero, size: displayedSize) + let element = makeElement(frame: CGRect( + x: displayedSize.width * 0.2 - 10, + y: displayedSize.height * 0.3 - 10, + width: 20, height: 20 + )) + + let center = try #require(element.normalizedCenter(in: displayedScreen, orientation: orientation)) + + #expect(abs(center.x - expected.x) < 0.0001) + #expect(abs(center.y - expected.y) < 0.0001) + } +} + +struct AccessibilitySelectorCheckTests { + @Test func availableSelectorsAreAccepted() throws { + try AccessibilitySelectorCheck.require(on: NSObject(), selectors: ["description", "respondsToSelector:"]) + } + + @Test func missingSelectorsReportTheUnsupportedCapability() { + let object = NSObject() + let error = EngineError.symbolNotFound( + name: "-[NSObject setBridgeDelegateToken:]", + framework: PrivateFramework.accessibilityPlatformTranslation.rawValue + ) + + #expect(throws: error) { + try AccessibilitySelectorCheck.require(on: object, selectors: ["description", "setBridgeDelegateToken:"]) + } + } +} diff --git a/engine/Tests/OpenDeviceHubEngineTests/FrameworkLoaderTests.swift b/engine/Tests/OpenDeviceHubEngineTests/FrameworkLoaderTests.swift index 662b57e..e585a8b 100644 --- a/engine/Tests/OpenDeviceHubEngineTests/FrameworkLoaderTests.swift +++ b/engine/Tests/OpenDeviceHubEngineTests/FrameworkLoaderTests.swift @@ -36,8 +36,8 @@ final class FrameworkCandidatePathTests: XCTestCase { XCTAssertTrue(paths[1].hasPrefix("/Applications/Xcode-27.app/Contents/Developer/")) } - func testEveryFrameworkOffersAtLeastTwoCandidates() { - for framework in PrivateFramework.allCases { + func testEveryXcodeFrameworkOffersAtLeastTwoCandidates() { + for framework in PrivateFramework.allCases where !framework.isInSharedCache { XCTAssertGreaterThanOrEqual(framework.candidatePaths(for: makeInstall()).count, 2) } } @@ -57,6 +57,25 @@ final class CoreSimDeviceIOPathTests: XCTestCase { } } +final class AccessibilityPlatformTranslationPathTests: XCTestCase { + func testIsLoadedFromMacOSWhateverTheXcode() { + for install in [makeInstall(), makeInstall(appPath: "/Applications/Xcode-27.app")] { + XCTAssertEqual(PrivateFramework.accessibilityPlatformTranslation.candidatePaths(for: install), [ + "/System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/AccessibilityPlatformTranslation", + ]) + } + } + + func testIsTheOnlyFrameworkInTheSharedCache() { + XCTAssertEqual(PrivateFramework.allCases.filter(\.isInSharedCache), [.accessibilityPlatformTranslation]) + } + + func testLoadsAlthoughItsPathIsNotOnDisk() throws { + let loaded = try FrameworkLoader.load(.accessibilityPlatformTranslation, from: makeInstall()) + XCTAssertTrue(loaded.path.hasPrefix("/System/Library/PrivateFrameworks/")) + } +} + final class PrivateSymbolInventoryTests: XCTestCase { func testEveryFrameworkDeclaresSymbolsToProbe() { for framework in PrivateFramework.allCases { diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/AccessibilityTreeTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/AccessibilityTreeTests.swift new file mode 100644 index 0000000..612d794 --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/AccessibilityTreeTests.swift @@ -0,0 +1,41 @@ +import XCTest +import OpenDeviceHubEngine + +final class AccessibilityTreeTests: XCTestCase { + private func makeAdapter() throws -> any SimulatorAdapter { + try AdapterFactory.make(for: XcodeLocator.locate()) + } + + func testReadsTheFrontmostAppOfABootedDevice() throws { + try IntegrationGate.requireEnabled() + let adapter = try makeAdapter() + guard let booted = try adapter.devices().first(where: { $0.state == .booted }) else { + throw XCTSkip("no booted simulator, boot one to run this test") + } + let root = try adapter.accessibilityTree(booted.udid) + XCTAssertEqual(root.role, "AXApplication") + XCTAssertGreaterThan(root.frame.width, 0) + XCTAssertFalse(root.children.isEmpty) + } + + func testReadsFromTheMainThread() throws { + try IntegrationGate.requireEnabled() + XCTAssertTrue(Thread.isMainThread) + let adapter = try makeAdapter() + guard let booted = try adapter.devices().first(where: { $0.state == .booted }) else { + throw XCTSkip("no booted simulator, boot one to run this test") + } + XCTAssertNoThrow(try adapter.accessibilityTree(booted.udid)) + } + + func testRefusesAShutdownDevice() throws { + try IntegrationGate.requireEnabled() + let adapter = try makeAdapter() + guard let shutdown = try adapter.devices().first(where: { $0.state == .shutdown }) else { + throw XCTSkip("every simulator is booted") + } + XCTAssertThrowsError(try adapter.accessibilityTree(shutdown.udid)) { error in + XCTAssertEqual(error as? EngineError, .deviceNotBooted(udid: shutdown.udid)) + } + } +}