diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf435a..ca8f4ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to the AXe iOS testing framework will be documented in this The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Fixed coordinate taps (`tap -x/-y`) silently no-oping on SwiftUI `Toggle` controls while reporting success; with the default `automatic` tap style, AXe now inspects the element at the tapped point and uses physical touch when it reports a switch-like role. `UISwitch` internals report as plain groups at the tapped point, so coordinate taps on them still need `--tap-style physical` (or a selector tap). + ## [v1.8.0] - 2026-07-20 ### Added diff --git a/Skills/CLI/axe/SKILL.md b/Skills/CLI/axe/SKILL.md index 58a3fa2..8d65585 100644 --- a/Skills/CLI/axe/SKILL.md +++ b/Skills/CLI/axe/SKILL.md @@ -7,7 +7,7 @@ description: Provides agent-ready AXe CLI usage guidance for iOS Simulator autom 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. 3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. -4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on elements that report a switch/toggle role (such as SwiftUI `Toggle`) use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. UIKit `UISwitch` internals report as plain groups at the tapped point, so coordinate taps on them still need `--tap-style physical` or a selector tap. ## Step 2: Choose the right command diff --git a/Sources/AXe/Commands/Tap.swift b/Sources/AXe/Commands/Tap.swift index 06f7184..3c6cbea 100644 --- a/Sources/AXe/Commands/Tap.swift +++ b/Sources/AXe/Commands/Tap.swift @@ -106,7 +106,13 @@ struct Tap: AsyncParsableCommand { let resolvedDescription: String if let pointX, let pointY { - resolution = TapResolution(point: (x: pointX, y: pointY), isSwitchLikeControl: false) + resolution = await Self.resolveCoordinateTap( + x: pointX, + y: pointY, + requestedStyle: tapStyle ?? .automatic, + simulatorUDID: simulatorUDID, + logger: logger + ) resolvedDescription = "(\(pointX), \(pointY))" } else { let query: AccessibilityQuery @@ -176,6 +182,32 @@ struct Tap: AsyncParsableCommand { print("✓ Tap at \(resolvedDescription) completed successfully") } + @MainActor + static func resolveCoordinateTap( + x: Double, + y: Double, + requestedStyle: TapStyle, + simulatorUDID: String, + logger: AxeLogger + ) async -> TapResolution { + guard requestedStyle == .automatic else { + return TapResolution(point: (x: x, y: y), isSwitchLikeControl: false) + } + let isSwitchLikeControl: Bool + do { + let element = try await AccessibilityFetcher.fetchAccessibilityElement( + at: AccessibilityPoint(x: x, y: y), + simulatorUDID: simulatorUDID, + logger: logger + ) + isSwitchLikeControl = element?.isSwitchLikeControl ?? false + } catch { + logger.info().log("Could not inspect the element at (\(x), \(y)) for automatic tap style; using simulator tap: \(error.localizedDescription)") + isSwitchLikeControl = false + } + return TapResolution(point: (x: x, y: y), isSwitchLikeControl: isSwitchLikeControl) + } + private func resolvedTapStyle(for resolution: TapResolution) -> TapStyle { switch tapStyle ?? .automatic { case .automatic: diff --git a/Sources/AXe/Resources/skills/axe/SKILL.md b/Sources/AXe/Resources/skills/axe/SKILL.md index e2aa89b..607422c 100644 --- a/Sources/AXe/Resources/skills/axe/SKILL.md +++ b/Sources/AXe/Resources/skills/axe/SKILL.md @@ -7,7 +7,7 @@ description: Provides agent-ready AXe CLI usage guidance for iOS Simulator autom 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. 3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. -4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on elements that report a switch/toggle role (such as SwiftUI `Toggle`) use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. UIKit `UISwitch` internals report as plain groups at the tapped point, so coordinate taps on them still need `--tap-style physical` or a selector tap. ## Step 2: Choose the right command diff --git a/Sources/AXe/Utilities/AccessibilityFetcher.swift b/Sources/AXe/Utilities/AccessibilityFetcher.swift index 3f21140..a247d26 100644 --- a/Sources/AXe/Utilities/AccessibilityFetcher.swift +++ b/Sources/AXe/Utilities/AccessibilityFetcher.swift @@ -55,6 +55,31 @@ struct AccessibilityFetcher { } } + static func fetchAccessibilityElement( + at point: AccessibilityPoint, + simulatorUDID: String, + logger: AxeLogger + ) async throws -> AccessibilityElement? { + let jsonData = try await fetchAccessibilityInfoJSONData( + for: simulatorUDID, + point: point, + logger: logger + ) + return decodeSingleAccessibilityElement(from: jsonData) + } + + static func decodeSingleAccessibilityElement(from data: Data) -> AccessibilityElement? { + let decoder = JSONDecoder() + if let element = try? decoder.decode(AccessibilityElement.self, from: data) { + return element + } + if let elements = try? decoder.decode([AccessibilityElement].self, from: data), + elements.count == 1 { + return elements.first + } + return nil + } + private static func fetchAccessibilityInfoJSONData( from target: FBSimulator, at point: AccessibilityPoint diff --git a/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift b/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift index 95acc51..819be09 100644 --- a/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift +++ b/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift @@ -80,7 +80,13 @@ extension Tap: BatchConvertible { let resolvedRoots: [AccessibilityElement]? if let pointX, let pointY { - resolution = TapResolution(point: (x: pointX, y: pointY), isSwitchLikeControl: false) + resolution = await Tap.resolveCoordinateTap( + x: pointX, + y: pointY, + requestedStyle: tapStyle ?? context.tapStyle, + simulatorUDID: context.simulatorUDID, + logger: logger + ) resolvedRoots = nil } else { let query: AccessibilityQuery diff --git a/Tests/AccessibilityFetcherTests.swift b/Tests/AccessibilityFetcherTests.swift index 47dc327..f35920b 100644 --- a/Tests/AccessibilityFetcherTests.swift +++ b/Tests/AccessibilityFetcherTests.swift @@ -271,6 +271,27 @@ struct AccessibilityFetcherTests { } } + @Test("Decodes single point-lookup elements from dictionary and singleton array payloads") + func decodesSinglePointLookupElements() throws { + let toggle: [String: Any] = [ + "type": "CheckBox", + "role": "AXCheckBox", + "role_description": "switch", + "AXValue": "0", + "frame": ["x": 309, "y": 220, "width": 63, "height": 28], + ] + let dictionaryData = try JSONSerialization.data(withJSONObject: toggle) + let singletonArrayData = try JSONSerialization.data(withJSONObject: [toggle]) + let multipleData = try JSONSerialization.data(withJSONObject: [toggle, toggle]) + + let fromDictionary = try #require(AccessibilityFetcher.decodeSingleAccessibilityElement(from: dictionaryData)) + let fromArray = try #require(AccessibilityFetcher.decodeSingleAccessibilityElement(from: singletonArrayData)) + + #expect(fromDictionary.isSwitchLikeControl) + #expect(fromArray.isSwitchLikeControl) + #expect(AccessibilityFetcher.decodeSingleAccessibilityElement(from: multipleData) == nil) + } + @Test("Restarts the canonical testmanagerd service with direct simctl arguments") func restartsCanonicalTestManagerService() async throws { var executableURL: URL?