From a191aa8a067a4a782c9425f10a5cb008c2447ba4 Mon Sep 17 00:00:00 2001 From: BariBariGood Date: Tue, 11 Aug 2026 18:54:47 +0000 Subject: [PATCH 1/2] Make automatic tap style detect switches for coordinate taps Coordinate taps (tap -x/-y) with the default automatic tap style always used the simulator tapAt event, which SwiftUI Toggle and UIKit UISwitch controls ignore. The tap printed a success message while the toggle never changed state; only --tap-style physical worked. Selector taps already resolve the target element and switch to physical touch for switch-like controls. Extend the same behavior to coordinate taps: when the style is automatic, look up the accessibility element at the requested point and use physical touch if it is a switch-like control. Explicit --tap-style simulator and --tap-style physical are unchanged, and if the point lookup fails the tap falls back to the simulator event as before. Batch coordinate taps get the same handling. --- CHANGELOG.md | 6 ++++ Skills/CLI/axe/SKILL.md | 2 +- Sources/AXe/Commands/Tap.swift | 34 ++++++++++++++++++- Sources/AXe/Resources/skills/axe/SKILL.md | 2 +- .../AXe/Utilities/AccessibilityFetcher.swift | 25 ++++++++++++++ .../Batch/Command+BatchConvertible.swift | 8 ++++- Tests/AccessibilityFetcherTests.swift | 21 ++++++++++++ 7 files changed, 94 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf435a..467509d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to the AXe iOS testing framework will be documented in this The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Fixed coordinate taps (`tap -x/-y`) silently no-oping on SwiftUI `Toggle` and `UISwitch` controls while reporting success; with the default `automatic` tap style, AXe now inspects the element at the tapped point and uses physical touch for switch-like controls. + ## [v1.8.0] - 2026-07-20 ### Added diff --git a/Skills/CLI/axe/SKILL.md b/Skills/CLI/axe/SKILL.md index 58a3fa2..8da9a8b 100644 --- a/Skills/CLI/axe/SKILL.md +++ b/Skills/CLI/axe/SKILL.md @@ -7,7 +7,7 @@ description: Provides agent-ready AXe CLI usage guidance for iOS Simulator autom 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. 3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. -4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on switch/toggle frames use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. ## Step 2: Choose the right command diff --git a/Sources/AXe/Commands/Tap.swift b/Sources/AXe/Commands/Tap.swift index 06f7184..3c6cbea 100644 --- a/Sources/AXe/Commands/Tap.swift +++ b/Sources/AXe/Commands/Tap.swift @@ -106,7 +106,13 @@ struct Tap: AsyncParsableCommand { let resolvedDescription: String if let pointX, let pointY { - resolution = TapResolution(point: (x: pointX, y: pointY), isSwitchLikeControl: false) + resolution = await Self.resolveCoordinateTap( + x: pointX, + y: pointY, + requestedStyle: tapStyle ?? .automatic, + simulatorUDID: simulatorUDID, + logger: logger + ) resolvedDescription = "(\(pointX), \(pointY))" } else { let query: AccessibilityQuery @@ -176,6 +182,32 @@ struct Tap: AsyncParsableCommand { print("✓ Tap at \(resolvedDescription) completed successfully") } + @MainActor + static func resolveCoordinateTap( + x: Double, + y: Double, + requestedStyle: TapStyle, + simulatorUDID: String, + logger: AxeLogger + ) async -> TapResolution { + guard requestedStyle == .automatic else { + return TapResolution(point: (x: x, y: y), isSwitchLikeControl: false) + } + let isSwitchLikeControl: Bool + do { + let element = try await AccessibilityFetcher.fetchAccessibilityElement( + at: AccessibilityPoint(x: x, y: y), + simulatorUDID: simulatorUDID, + logger: logger + ) + isSwitchLikeControl = element?.isSwitchLikeControl ?? false + } catch { + logger.info().log("Could not inspect the element at (\(x), \(y)) for automatic tap style; using simulator tap: \(error.localizedDescription)") + isSwitchLikeControl = false + } + return TapResolution(point: (x: x, y: y), isSwitchLikeControl: isSwitchLikeControl) + } + private func resolvedTapStyle(for resolution: TapResolution) -> TapStyle { switch tapStyle ?? .automatic { case .automatic: diff --git a/Sources/AXe/Resources/skills/axe/SKILL.md b/Sources/AXe/Resources/skills/axe/SKILL.md index e2aa89b..964ebaf 100644 --- a/Sources/AXe/Resources/skills/axe/SKILL.md +++ b/Sources/AXe/Resources/skills/axe/SKILL.md @@ -7,7 +7,7 @@ description: Provides agent-ready AXe CLI usage guidance for iOS Simulator autom 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. 3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. -4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on switch/toggle frames use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. ## Step 2: Choose the right command diff --git a/Sources/AXe/Utilities/AccessibilityFetcher.swift b/Sources/AXe/Utilities/AccessibilityFetcher.swift index 3f21140..a247d26 100644 --- a/Sources/AXe/Utilities/AccessibilityFetcher.swift +++ b/Sources/AXe/Utilities/AccessibilityFetcher.swift @@ -55,6 +55,31 @@ struct AccessibilityFetcher { } } + static func fetchAccessibilityElement( + at point: AccessibilityPoint, + simulatorUDID: String, + logger: AxeLogger + ) async throws -> AccessibilityElement? { + let jsonData = try await fetchAccessibilityInfoJSONData( + for: simulatorUDID, + point: point, + logger: logger + ) + return decodeSingleAccessibilityElement(from: jsonData) + } + + static func decodeSingleAccessibilityElement(from data: Data) -> AccessibilityElement? { + let decoder = JSONDecoder() + if let element = try? decoder.decode(AccessibilityElement.self, from: data) { + return element + } + if let elements = try? decoder.decode([AccessibilityElement].self, from: data), + elements.count == 1 { + return elements.first + } + return nil + } + private static func fetchAccessibilityInfoJSONData( from target: FBSimulator, at point: AccessibilityPoint diff --git a/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift b/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift index 95acc51..819be09 100644 --- a/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift +++ b/Sources/AXe/Utilities/Batch/Command+BatchConvertible.swift @@ -80,7 +80,13 @@ extension Tap: BatchConvertible { let resolvedRoots: [AccessibilityElement]? if let pointX, let pointY { - resolution = TapResolution(point: (x: pointX, y: pointY), isSwitchLikeControl: false) + resolution = await Tap.resolveCoordinateTap( + x: pointX, + y: pointY, + requestedStyle: tapStyle ?? context.tapStyle, + simulatorUDID: context.simulatorUDID, + logger: logger + ) resolvedRoots = nil } else { let query: AccessibilityQuery diff --git a/Tests/AccessibilityFetcherTests.swift b/Tests/AccessibilityFetcherTests.swift index 47dc327..f35920b 100644 --- a/Tests/AccessibilityFetcherTests.swift +++ b/Tests/AccessibilityFetcherTests.swift @@ -271,6 +271,27 @@ struct AccessibilityFetcherTests { } } + @Test("Decodes single point-lookup elements from dictionary and singleton array payloads") + func decodesSinglePointLookupElements() throws { + let toggle: [String: Any] = [ + "type": "CheckBox", + "role": "AXCheckBox", + "role_description": "switch", + "AXValue": "0", + "frame": ["x": 309, "y": 220, "width": 63, "height": 28], + ] + let dictionaryData = try JSONSerialization.data(withJSONObject: toggle) + let singletonArrayData = try JSONSerialization.data(withJSONObject: [toggle]) + let multipleData = try JSONSerialization.data(withJSONObject: [toggle, toggle]) + + let fromDictionary = try #require(AccessibilityFetcher.decodeSingleAccessibilityElement(from: dictionaryData)) + let fromArray = try #require(AccessibilityFetcher.decodeSingleAccessibilityElement(from: singletonArrayData)) + + #expect(fromDictionary.isSwitchLikeControl) + #expect(fromArray.isSwitchLikeControl) + #expect(AccessibilityFetcher.decodeSingleAccessibilityElement(from: multipleData) == nil) + } + @Test("Restarts the canonical testmanagerd service with direct simctl arguments") func restartsCanonicalTestManagerService() async throws { var executableURL: URL? From 4e4fe656db1371f6b3e04e1597140f0dad19a933 Mon Sep 17 00:00:00 2001 From: BariBariGood Date: Wed, 12 Aug 2026 20:42:27 +0000 Subject: [PATCH 2/2] Scope the coordinate-tap switch detection docs to point-reported roles Point lookup only sees the accessibility role that the element exposes at the tapped coordinate. SwiftUI Toggle reports a switch role there, but UISwitch internals report as plain groups, so coordinate taps on a UISwitch still need --tap-style physical or a selector tap. Say so in the changelog entry and skill docs instead of implying both are covered. --- CHANGELOG.md | 2 +- Skills/CLI/axe/SKILL.md | 2 +- Sources/AXe/Resources/skills/axe/SKILL.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 467509d..ca8f4ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed coordinate taps (`tap -x/-y`) silently no-oping on SwiftUI `Toggle` and `UISwitch` controls while reporting success; with the default `automatic` tap style, AXe now inspects the element at the tapped point and uses physical touch for switch-like controls. +- Fixed coordinate taps (`tap -x/-y`) silently no-oping on SwiftUI `Toggle` controls while reporting success; with the default `automatic` tap style, AXe now inspects the element at the tapped point and uses physical touch when it reports a switch-like role. `UISwitch` internals report as plain groups at the tapped point, so coordinate taps on them still need `--tap-style physical` (or a selector tap). ## [v1.8.0] - 2026-07-20 diff --git a/Skills/CLI/axe/SKILL.md b/Skills/CLI/axe/SKILL.md index 8da9a8b..8d65585 100644 --- a/Skills/CLI/axe/SKILL.md +++ b/Skills/CLI/axe/SKILL.md @@ -7,7 +7,7 @@ description: Provides agent-ready AXe CLI usage guidance for iOS Simulator autom 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. 3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. -4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on switch/toggle frames use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on elements that report a switch/toggle role (such as SwiftUI `Toggle`) use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. UIKit `UISwitch` internals report as plain groups at the tapped point, so coordinate taps on them still need `--tap-style physical` or a selector tap. ## Step 2: Choose the right command diff --git a/Sources/AXe/Resources/skills/axe/SKILL.md b/Sources/AXe/Resources/skills/axe/SKILL.md index 964ebaf..607422c 100644 --- a/Sources/AXe/Resources/skills/axe/SKILL.md +++ b/Sources/AXe/Resources/skills/axe/SKILL.md @@ -7,7 +7,7 @@ description: Provides agent-ready AXe CLI usage guidance for iOS Simulator autom 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. 3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. -4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on switch/toggle frames use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. Coordinate taps (`-x`/`-y`) also inspect the element at the point when the style is `automatic`, so taps on elements that report a switch/toggle role (such as SwiftUI `Toggle`) use physical touch; pass `--tap-style simulator` to skip the inspection or `--tap-style physical` to force physical touch. UIKit `UISwitch` internals report as plain groups at the tapped point, so coordinate taps on them still need `--tap-style physical` or a selector tap. ## Step 2: Choose the right command