From 13a8ca3ac101b0c5b527928c90011a14596dd33d Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 17:29:48 +0100 Subject: [PATCH 1/2] feat(engine): watch which way a device's screen is turned Read from the display report after the screens announce a change: the active panel's layout rotation plus the angle its device type builds it in at. A reading has to hold before it counts, since a foldable's report goes through wrong turns on the way through a fold, and a lagging report must not count as a turn back. --- .../Adapter/ScreenOrientationWatcher.swift | 177 +++++++++++++ .../Input/DeviceOrientation.swift | 7 + .../ScreenOrientationWatcherTests.swift | 239 ++++++++++++++++++ 3 files changed, 423 insertions(+) create mode 100644 engine/Sources/OpenDeviceHubEngine/Adapter/ScreenOrientationWatcher.swift create mode 100644 engine/Tests/OpenDeviceHubEngineTests/ScreenOrientationWatcherTests.swift diff --git a/engine/Sources/OpenDeviceHubEngine/Adapter/ScreenOrientationWatcher.swift b/engine/Sources/OpenDeviceHubEngine/Adapter/ScreenOrientationWatcher.swift new file mode 100644 index 0000000..70379f7 --- /dev/null +++ b/engine/Sources/OpenDeviceHubEngine/Adapter/ScreenOrientationWatcher.swift @@ -0,0 +1,177 @@ +import Foundation + +/// How the guest's screen is turned, as it changes, whoever turned the device: the viewer, Device +/// Hub or the guest itself. It is the screen's own layout that counts, not how the device is held, +/// since a screen that does not turn (an iPhone's home screen, or upside down on a Face ID phone) +/// keeps showing its picture the way it was. +public final class ScreenOrientationWatcher: @unchecked Sendable { + public enum Event: Sendable, Equatable { + /// The screen turned, or was read for the first time. + case turned(DeviceOrientation) + /// Looked at again after a turn was asked for, the screen had not moved. + case stillAt(DeviceOrientation) + } + + public let changes: AsyncStream + + static let burstInterval: Duration = .milliseconds(100) + static let burstReads = 25 + /// While a foldable hands its layout from one panel to the other, the report still names the old + /// panel for about a fifth of a second but already turns it by the new one's angle (27A9269), so + /// a reading has to hold this long to count. + static let settleTime: Duration = .milliseconds(400) + /// A fold never turns a foldable, but its report says otherwise for a while on the way, longer + /// than `settleTime` at times (27A9269), so nothing read while the hinge moves counts. + static let foldQuiet: Duration = .seconds(1) + /// How long after asking for a turn the screen is looked at again. A turn the screen took was + /// believed at most 1.2 seconds after it was asked for (27A9269). + static let confirmDelay: Duration = .seconds(2) + + private let continuation: AsyncStream.Continuation + private let read: @Sendable () async throws -> DisplayReport + private let nativeRotation: @Sendable (Int) -> Int + private var pokes: AsyncStream.Continuation? + private var task: Task? + /// Kept from one look to the next: the report catches up with a turn half a second or more after + /// the screens announce it (27A9269), so a look begins by reading the orientation before the + /// turn, which must not count as a change back to it. Only the one task that looks touches it. + private var settler = OrientationSettler() + private let lock = NSLock() + private var quietUntil: ContinuousClock.Instant? + private var confirmAt: ContinuousClock.Instant? + + /// `nativeRotation` is the angle the panel with that screen ID is built into the body at, from + /// the device type. The report's own figure reads 0 for both of a foldable's panels (27A9269). + public init( + read: @escaping @Sendable () async throws -> DisplayReport, + nativeRotation: @escaping @Sendable (Int) -> Int + ) { + var escaping: AsyncStream.Continuation! + changes = AsyncStream(bufferingPolicy: .bufferingNewest(4)) { escaping = $0 } + continuation = escaping + self.read = read + self.nativeRotation = nativeRotation + + var pokeContinuation: AsyncStream.Continuation! + let pokeStream = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { pokeContinuation = $0 } + pokes = pokeContinuation + task = Task { [weak self] in + for await _ in pokeStream { + await self?.burst() + } + } + } + + /// Looks at the screen now and for a couple of seconds after, since the report catches up with + /// the screens' own announcement a little later. + public func poke() { + pokes?.yield(()) + } + + /// The hinge moved: what the report says until a second after is not believed, and it is looked + /// at again after that. + public func holdStill() { + lock.lock() + quietUntil = .now + Self.foldQuiet + lock.unlock() + poke() + } + + /// A turn was asked for, which the screen may refuse (upside down on a Face ID phone) or never + /// hear of: a foldable's turn is sent without an answer. Once the screen has had time to follow, + /// it is looked at again and said to be still where it was if it has not moved. + public func confirm() { + lock.lock() + confirmAt = .now + Self.confirmDelay + lock.unlock() + poke() + } + + // Swift 6 will not let a lock be taken directly in an async function. + private func isQuiet(at now: ContinuousClock.Instant) -> Bool { + lock.lock() + defer { lock.unlock() } + return quietUntil.map { now < $0 } ?? false + } + + private func takeConfirmation(at now: ContinuousClock.Instant) -> Bool { + lock.lock() + defer { lock.unlock() } + guard let due = confirmAt, now >= due else { return false } + confirmAt = nil + return true + } + + public func close() { + task?.cancel() + task = nil + pokes?.finish() + continuation.finish() + } + + private func burst() async { + for _ in 0.. Int + ) -> OrientationSettler.Reading? { + guard report.isSettled, let active = report.activeIntegrated, + let orientation = DeviceOrientation(degrees: active.currentRotation + nativeRotation(active.displayID)) + else { return nil } + return OrientationSettler.Reading(orientation: orientation, panel: active.displayID) + } +} + +/// Believes a reading once it has held, on the same panel, for a while. +public struct OrientationSettler: Sendable { + public struct Reading: Sendable, Equatable { + public let orientation: DeviceOrientation + public let panel: Int + + public init(orientation: DeviceOrientation, panel: Int) { + self.orientation = orientation + self.panel = panel + } + } + + private var candidate: (reading: Reading, since: ContinuousClock.Instant)? + public private(set) var believed: DeviceOrientation? + + public init() {} + + /// The orientation once it is believed, and again each time a different one is. + public mutating func observe( + _ reading: Reading?, + at now: ContinuousClock.Instant, + holdingFor settle: Duration + ) -> DeviceOrientation? { + guard let reading else { + candidate = nil + return nil + } + if candidate?.reading != reading { + candidate = (reading, now) + } + guard let candidate, now - candidate.since >= settle, + candidate.reading.orientation != believed else { return nil } + believed = candidate.reading.orientation + return believed + } +} diff --git a/engine/Sources/OpenDeviceHubEngine/Input/DeviceOrientation.swift b/engine/Sources/OpenDeviceHubEngine/Input/DeviceOrientation.swift index eb8eac7..ebec2cb 100644 --- a/engine/Sources/OpenDeviceHubEngine/Input/DeviceOrientation.swift +++ b/engine/Sources/OpenDeviceHubEngine/Input/DeviceOrientation.swift @@ -17,6 +17,13 @@ public enum DeviceOrientation: String, Sendable, Hashable, CaseIterable, Codable } } + /// The orientation turned this many degrees clockwise, in any multiple of a quarter turn. + public init?(degrees: Int) { + let turned = (degrees % 360 + 360) % 360 + guard let found = Self.allCases.first(where: { $0.degrees == turned }) else { return nil } + self = found + } + public var isLandscape: Bool { self == .landscapeLeft || self == .landscapeRight } diff --git a/engine/Tests/OpenDeviceHubEngineTests/ScreenOrientationWatcherTests.swift b/engine/Tests/OpenDeviceHubEngineTests/ScreenOrientationWatcherTests.swift new file mode 100644 index 0000000..8acbdaf --- /dev/null +++ b/engine/Tests/OpenDeviceHubEngineTests/ScreenOrientationWatcherTests.swift @@ -0,0 +1,239 @@ +import CoreGraphics +import Foundation +import os +import XCTest +@testable import OpenDeviceHubEngine + +final class ScreenOrientationWatcherTests: XCTestCase { + /// The iPhone Duo's panels as its device type builds them in: the cover upright, the inner panel + /// a quarter turn round. + private static let duoAngles: @Sendable (Int) -> Int = { $0 == 3 ? 270 : 0 } + + private static func report(active displayID: Int, rotation: Int, settled: Bool = true) -> DisplayReport { + DisplayReport(displays: [1, 3].map { id in + DisplayReport.Display( + uniqueID: "panel-\(id)", name: id == 1 ? "LCD" : "LCD-1", displayID: id, + isActive: id == displayID, + backlight: id == displayID ? (settled ? .activeOn : .inactiveOn) : .off, + isPrimary: id == 1, isIntegrated: true, + pixelSize: CGSize(width: 1000, height: 2000), pointScale: 3, + currentRotation: id == displayID ? rotation : 0, nativeRotation: 0, chromeIdentifier: nil + ) + }) + } + + func testAnOrientationFromDegreesWrapsRound() { + XCTAssertEqual(DeviceOrientation(degrees: 0), .portrait) + XCTAssertEqual(DeviceOrientation(degrees: 90), .landscapeLeft) + XCTAssertEqual(DeviceOrientation(degrees: 450), .landscapeLeft) + XCTAssertEqual(DeviceOrientation(degrees: -90), .landscapeRight) + XCTAssertNil(DeviceOrientation(degrees: 45)) + } + + /// Measured on the Duo (27A9269): turned through its own control, the inner panel's layout read + /// 0, 180, 90 and 270 degrees for landscape right, landscape left, portrait and upside down. + func testTheInnerPanelReadsItsLayoutTurnedByItsAngle() { + let measured: [(Int, DeviceOrientation)] = [ + (0, .landscapeRight), (180, .landscapeLeft), (90, .portrait), (270, .portraitUpsideDown), + ] + for (rotation, expected) in measured { + let reading = ScreenOrientationWatcher.reading(from: Self.report(active: 3, rotation: rotation), nativeRotation: Self.duoAngles) + XCTAssertEqual(reading, OrientationSettler.Reading(orientation: expected, panel: 3), "inner panel at \(rotation)") + } + } + + /// The cover, and an iPhone's only panel, sit upright, so the layout is the orientation. + func testAnUprightPanelReadsItsLayoutAsItIs() { + let reading = ScreenOrientationWatcher.reading(from: Self.report(active: 1, rotation: 270), nativeRotation: Self.duoAngles) + XCTAssertEqual(reading, OrientationSettler.Reading(orientation: .landscapeRight, panel: 1)) + } + + func testAReportStillChangingPanelsIsNoReading() { + XCTAssertNil(ScreenOrientationWatcher.reading(from: Self.report(active: 1, rotation: 0, settled: false), nativeRotation: Self.duoAngles)) + } + + /// Unfolding, the report went on naming the cover for about a fifth of a second while already + /// turning it by the inner panel's angle (27A9269): that is landscape for a moment, not a turn. + func testAMomentOnTheWayThroughAFoldIsNotBelieved() { + var settler = OrientationSettler() + let start = ContinuousClock.now + let hold = Duration.milliseconds(400) + let portraitOnCover = OrientationSettler.Reading(orientation: .portrait, panel: 1) + let passing = OrientationSettler.Reading(orientation: .landscapeLeft, panel: 1) + let portraitInside = OrientationSettler.Reading(orientation: .portrait, panel: 3) + XCTAssertNil(settler.observe(portraitOnCover, at: start, holdingFor: hold)) + XCTAssertEqual(settler.observe(portraitOnCover, at: start + .milliseconds(400), holdingFor: hold), .portrait) + XCTAssertNil(settler.observe(passing, at: start + .milliseconds(500), holdingFor: hold)) + XCTAssertNil(settler.observe(passing, at: start + .milliseconds(700), holdingFor: hold)) + XCTAssertNil(settler.observe(portraitInside, at: start + .milliseconds(800), holdingFor: hold)) + XCTAssertNil(settler.observe(portraitInside, at: start + .milliseconds(1300), holdingFor: hold), "still portrait, nothing new to say") + } + + func testATurnThatHoldsIsBelievedOnceAndAnotherAfterIt() { + var settler = OrientationSettler() + let start = ContinuousClock.now + let hold = Duration.milliseconds(400) + let right = OrientationSettler.Reading(orientation: .landscapeRight, panel: 3) + let left = OrientationSettler.Reading(orientation: .landscapeLeft, panel: 3) + XCTAssertNil(settler.observe(right, at: start, holdingFor: hold)) + XCTAssertEqual(settler.observe(right, at: start + .milliseconds(450), holdingFor: hold), .landscapeRight) + XCTAssertNil(settler.observe(right, at: start + .milliseconds(900), holdingFor: hold)) + XCTAssertNil(settler.observe(nil, at: start + .milliseconds(1000), holdingFor: hold)) + XCTAssertNil(settler.observe(left, at: start + .milliseconds(1100), holdingFor: hold)) + XCTAssertEqual(settler.observe(left, at: start + .milliseconds(1500), holdingFor: hold), .landscapeLeft) + } + + func testAPokeReportsTheScreenOnceItHasSettled() async throws { + let rotation = OSAllocatedUnfairLock(initialState: 90) + let watcher = ScreenOrientationWatcher( + read: { Self.report(active: 3, rotation: rotation.withLock { $0 }) }, + nativeRotation: Self.duoAngles + ) + defer { watcher.close() } + watcher.poke() + let first = await Self.firstChange(of: watcher, within: 2) + XCTAssertEqual(first, .turned(.portrait)) + + rotation.withLock { $0 = 180 } + watcher.poke() + let second = await Self.firstChange(of: watcher, within: 3) + XCTAssertEqual(second, .turned(.landscapeLeft)) + } + + /// A Face ID phone refuses upside down, and a foldable's turn can go unheard: either way the + /// screen stays put and the watcher says so once it has had time to move. + func testATurnTheScreenDidNotTakeIsSaidToBeStillWhereItWas() async throws { + let watcher = ScreenOrientationWatcher( + read: { Self.report(active: 1, rotation: 270) }, + nativeRotation: Self.duoAngles + ) + defer { watcher.close() } + watcher.poke() + let first = await Self.firstChange(of: watcher, within: 2) + XCTAssertEqual(first, .turned(.landscapeRight)) + try await Task.sleep(for: .seconds(3)) + + let asked = ContinuousClock.now + watcher.confirm() + let answer = await Self.firstChange(of: watcher, within: 4) + XCTAssertEqual(answer, .stillAt(.landscapeRight)) + XCTAssertGreaterThanOrEqual(ContinuousClock.now - asked, ScreenOrientationWatcher.confirmDelay) + } + + /// The report catches up with a turn a second or so after it was asked for; checking must not + /// say the screen stayed put in the meantime. + func testATurnTheScreenTakesLateIsATurnAndNothingElse() async throws { + let reads = OSAllocatedUnfairLock(initialState: 0) + let turnedAfter = OSAllocatedUnfairLock(initialState: Int.max) + let watcher = ScreenOrientationWatcher( + read: { + let count = reads.withLock { $0 += 1; return $0 } + return Self.report(active: 3, rotation: count >= turnedAfter.withLock { $0 } ? 180 : 90) + }, + nativeRotation: Self.duoAngles + ) + defer { watcher.close() } + watcher.poke() + _ = await Self.firstChange(of: watcher, within: 2) + try await Task.sleep(for: .seconds(3)) + + turnedAfter.withLock { $0 = reads.withLock { $0 } + 12 } + watcher.confirm() + var seen: [ScreenOrientationWatcher.Event] = [] + let collector = Task { + for await event in watcher.changes { seen.append(event) } + return seen + } + try await Task.sleep(for: .seconds(3.5)) + watcher.close() + let after = await collector.value + XCTAssertEqual(after.first, .turned(.landscapeLeft)) + XCTAssertFalse(after.contains(.stillAt(.portrait)), "said to be still where it was while turning") + } + + /// The report lags the screens' announcement, so a look after a turn first reads the orientation + /// from before it, which must not turn the window back. + func testAReportThatHasNotCaughtUpDoesNotTurnBack() async throws { + let reads = OSAllocatedUnfairLock(initialState: 0) + let turnedAfter = OSAllocatedUnfairLock(initialState: Int.max) + let watcher = ScreenOrientationWatcher( + read: { + let count = reads.withLock { $0 += 1; return $0 } + return Self.report(active: 3, rotation: count >= turnedAfter.withLock { $0 } ? 180 : 90) + }, + nativeRotation: Self.duoAngles + ) + defer { watcher.close() } + watcher.poke() + let first = await Self.firstChange(of: watcher, within: 2) + XCTAssertEqual(first, .turned(.portrait)) + try await Task.sleep(for: .seconds(3)) + + turnedAfter.withLock { $0 = reads.withLock { $0 } + 7 } + watcher.poke() + var seen: [ScreenOrientationWatcher.Event] = [] + let collector = Task { + for await orientation in watcher.changes { seen.append(orientation) } + return seen + } + try await Task.sleep(for: .seconds(3)) + watcher.close() + let after = await collector.value + XCTAssertEqual(after, [.turned(.landscapeLeft)], "only the turn, never back to where it was") + } + + /// Opening, the report named the cover for about a third of a second while it read upside down + /// and then upright (27A9269), and at times longer than a reading takes to settle. + func testWhatTheReportSaysWhileTheHingeMovesIsNotBelieved() async throws { + let rotation = OSAllocatedUnfairLock(initialState: 90) + let watcher = ScreenOrientationWatcher( + read: { Self.report(active: 3, rotation: rotation.withLock { $0 }) }, + nativeRotation: Self.duoAngles + ) + defer { watcher.close() } + watcher.poke() + let first = await Self.firstChange(of: watcher, within: 2) + XCTAssertEqual(first, .turned(.portrait)) + try await Task.sleep(for: .seconds(3)) + + watcher.holdStill() + rotation.withLock { $0 = 180 } + try await Task.sleep(for: .milliseconds(700)) + rotation.withLock { $0 = 90 } + let during = await Self.firstChange(of: watcher, within: 3) + XCTAssertNil(during, "a fold is not a turn") + } + + func testATurnDuringAFoldStillComesThroughOnceTheHingeIsStill() async throws { + let rotation = OSAllocatedUnfairLock(initialState: 90) + let watcher = ScreenOrientationWatcher( + read: { Self.report(active: 3, rotation: rotation.withLock { $0 }) }, + nativeRotation: Self.duoAngles + ) + defer { watcher.close() } + watcher.poke() + _ = await Self.firstChange(of: watcher, within: 2) + try await Task.sleep(for: .seconds(3)) + + watcher.holdStill() + rotation.withLock { $0 = 180 } + let turned = await Self.firstChange(of: watcher, within: 3) + XCTAssertEqual(turned, .turned(.landscapeLeft)) + } + + private static func firstChange(of watcher: ScreenOrientationWatcher, within seconds: Double) async -> ScreenOrientationWatcher.Event? { + await withTaskGroup(of: ScreenOrientationWatcher.Event?.self) { group in + group.addTask { + for await event in watcher.changes { return event } + return nil + } + group.addTask { + try? await Task.sleep(for: .seconds(seconds)) + return nil + } + let first = await group.next() ?? nil + group.cancelAll() + return first + } + } +} From 3b07db7557408f64129c1561967d7ebe044ed829 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 17:29:48 +0100 Subject: [PATCH 2/2] fix(viewer): keep a window turned the way its device's screen is A window only turned when its own rotate asked, so a turn from Device Hub or the guest left it sideways, and a turn the screen refused or a foldable never heard left it turned alone. It now follows the screen whoever turns the device, takes back a turn of its own the screen did not follow, and a rotate counts on from the turn last asked for so upside down on a Face ID phone is passed rather than asked for again. --- .../Sources/ODHubViewerApp/ViewerMain.swift | 49 +-- .../FoldableController.swift | 14 +- .../ScreenOrientationFollower.swift | 99 ++++++ .../ScreenOrientationFollowTests.swift | 305 ++++++++++++++++++ 4 files changed, 438 insertions(+), 29 deletions(-) create mode 100644 engine/Sources/OpenDeviceHubViewer/ScreenOrientationFollower.swift create mode 100644 engine/Tests/OpenDeviceHubIntegrationTests/ScreenOrientationFollowTests.swift diff --git a/engine/Sources/ODHubViewerApp/ViewerMain.swift b/engine/Sources/ODHubViewerApp/ViewerMain.swift index 2efb26f..06ca559 100644 --- a/engine/Sources/ODHubViewerApp/ViewerMain.swift +++ b/engine/Sources/ODHubViewerApp/ViewerMain.swift @@ -96,17 +96,25 @@ struct ODHubViewer: ParsableCommand { displayReport: { try await adapter.displayReport($0) } ) foldables.report = { print($0) } + let orientations = ScreenOrientationFollower( + read: { try await adapter.displayReport($0) }, + panels: { (try? adapter.panels($0)) ?? [] } + ) + orientations.onScreenChange = { foldables.nudge($0) } foldables.onMove = { udid, event in guard let controller = manager.controller(for: udid) else { return } switch event { case .began(let target): controller.beginFold(to: target) - case .angle(let angle): controller.showHingeAngle(angle) + case .angle(let angle): + controller.showHingeAngle(angle) + orientations.hingeMoved(udid, to: angle) case .ended: controller.endFold() } } manager.onDeviceClosed = { pasteboard.forget($0) foldables.forget($0) + orientations.forget($0) } let previews = CapturePreviewPresenter(report: { print($0) }) @@ -132,10 +140,16 @@ struct ODHubViewer: ParsableCommand { try adapter.setOrientation(orientation, udid: udid) } controller.setOrientation(orientation) + orientations.turned(udid, to: orientation) } catch { print("rotate failed: \(error.localizedDescription)") } } + let rotate: @MainActor (Bool, String) -> Void = { left, udid in + guard let controller = manager.controller(for: udid) else { return } + let from = orientations.lastTurn(udid) ?? controller.currentOrientation + turn(left ? from.rotatedLeft : from.rotatedRight, udid) + } // Opens unfolded unless told otherwise, which is where the device's own tooling starts. let followFold: @MainActor (String, DeviceWindowController, Double?) -> Void = { udid, controller, angle in @@ -146,12 +160,12 @@ struct ODHubViewer: ParsableCommand { foldables.setAngle(angle, for: udid) foldables.follow( udid, - nudges: controller.screenChanges, onPanel: { [weak controller] panel in controller?.setActivePanel(screenID: panel.displayID) }, onHinge: { [weak controller] degrees in controller?.showHingeAngle(degrees) + orientations.hingeMoved(udid, to: degrees) } ) } @@ -164,12 +178,14 @@ struct ODHubViewer: ParsableCommand { adapter: adapter, manager: manager, allowBoot: allowBoot, - rotate: turn, + rotate: rotate, present: present ) recent.remember(udid) pasteboard.adopt(udid) - if let controller = manager.controller(for: udid), controller.foldsAtHinge { + guard let controller = manager.controller(for: udid) else { return } + orientations.follow(udid, controller: controller) + if controller.foldsAtHinge { followFold(udid, controller, foldables.angle(for: udid)) } } @@ -179,7 +195,9 @@ struct ODHubViewer: ParsableCommand { manager.onReattached = { udid in pasteboard.forget(udid) pasteboard.adopt(udid) - guard let controller = manager.controller(for: udid), controller.foldsAtHinge else { return } + guard let controller = manager.controller(for: udid) else { return } + orientations.follow(udid, controller: controller) + guard controller.foldsAtHinge else { return } let angle = foldables.angle(for: udid) foldables.forget(udid) followFold(udid, controller, angle) @@ -343,12 +361,8 @@ struct ODHubViewer: ParsableCommand { manager.frontmostUDID.map { manager.hasCameraControl($0) } ?? false }, rotate: { left in - guard let udid = manager.frontmostUDID, - let controller = manager.controller(for: udid) else { return } - let next = left - ? controller.currentOrientation.rotatedLeft - : controller.currentOrientation.rotatedRight - turn(next, udid) + guard let udid = manager.frontmostUDID else { return } + rotate(left, udid) }, restart: { guard let udid = manager.frontmostUDID else { return } @@ -613,7 +627,7 @@ struct ODHubViewer: ParsableCommand { adapter: any SimulatorAdapter, manager: DeviceWindowManager, allowBoot: Bool, - rotate: @escaping @MainActor (DeviceOrientation, String) -> Void, + rotate: @escaping @MainActor (Bool, String) -> Void, present: @escaping @MainActor ([URL]) -> Void ) throws { guard var device = devices.first(where: { @@ -787,7 +801,7 @@ private func installToolbar( manager: DeviceWindowManager, adapter: any SimulatorAdapter, saveScreenshotsToClipboard: @escaping @MainActor () -> Bool, - rotate: @escaping @MainActor (DeviceOrientation, String) -> Void, + rotate: @escaping @MainActor (Bool, String) -> Void, present: @escaping @MainActor ([URL]) -> Void ) { guard let controller = manager.controller(for: udid) else { return } @@ -821,13 +835,8 @@ private func installToolbar( stopRecording: { present(manager.toggleRecording(into: CaptureStaging.directory())) }, - rotate: { [weak controller] toLeft in - guard let controller else { return } - rotate( - toLeft ? controller.currentOrientation.rotatedLeft - : controller.currentOrientation.rotatedRight, - udid - ) + rotate: { toLeft in + rotate(toLeft, udid) } )) } diff --git a/engine/Sources/OpenDeviceHubViewer/FoldableController.swift b/engine/Sources/OpenDeviceHubViewer/FoldableController.swift index eb42399..bf2ec72 100644 --- a/engine/Sources/OpenDeviceHubViewer/FoldableController.swift +++ b/engine/Sources/OpenDeviceHubViewer/FoldableController.swift @@ -130,10 +130,8 @@ public final class FoldableController { followers[udid]?.watcher.poke() } - /// `nudges` are the panels' own change announcements; each makes the watcher look at once. public func follow( _ udid: String, - nudges: [AsyncStream] = [], onPanel: @escaping @MainActor (DisplayReport.Display) -> Void, onHinge: @escaping @MainActor (Double) -> Void ) { @@ -162,16 +160,14 @@ public final class FoldableController { onPanel(panel) } }] - for nudge in nudges { - tasks.append(Task { - for await _ in nudge { - watcher.poke() - } - }) - } followers[udid] = Follower(watcher: watcher, hinge: hinge, tasks: tasks) } + /// The device's screens announced a change, which on a fold comes before the report has one. + public func nudge(_ udid: String) { + followers[udid]?.watcher.poke() + } + public func activePanel(for udid: String) -> DisplayReport.Display? { followers[udid]?.watcher.activePanel } diff --git a/engine/Sources/OpenDeviceHubViewer/ScreenOrientationFollower.swift b/engine/Sources/OpenDeviceHubViewer/ScreenOrientationFollower.swift new file mode 100644 index 0000000..83bf948 --- /dev/null +++ b/engine/Sources/OpenDeviceHubViewer/ScreenOrientationFollower.swift @@ -0,0 +1,99 @@ +import Foundation +import OpenDeviceHubEngine + +/// Keeps each window turned the way its device's screen is, whoever turned the device. +/// +/// A turn from the window itself is shown at once and taken back if the screen does not follow, so +/// the window shows what the screen shows whoever turned the device, and a window opened on a device +/// already turned shows it turned. +@MainActor +public final class ScreenOrientationFollower { + private struct Following { + let watcher: ScreenOrientationWatcher + let tasks: [Task] + } + + private let read: @Sendable (String) async throws -> DisplayReport + private let panels: (String) -> [DevicePanel] + private var following: [String: Following] = [:] + private var hinges: [String: Double] = [:] + private var asked: [String: DeviceOrientation] = [:] + /// Each change a device's screens announce, for anything else that watches them, such as a + /// foldable's panel watcher. A screen's changes can only be listened to once. + public var onScreenChange: ((String) -> Void)? + + public init( + read: @escaping @Sendable (String) async throws -> DisplayReport, + panels: @escaping (String) -> [DevicePanel] + ) { + self.read = read + self.panels = panels + } + + /// Again after the device comes back, since the window then has new screens to listen to. + public func follow(_ udid: String, controller: DeviceWindowController) { + forget(udid) + let angles = Dictionary(panels(udid).map { ($0.screenID, $0.nativeRotation) }, uniquingKeysWith: { first, _ in first }) + let read = read + let watcher = ScreenOrientationWatcher( + read: { try await read(udid) }, + nativeRotation: { angles[$0] ?? 0 } + ) + var tasks = controller.screenChanges.map { changes in + Task { [weak self] in + for await _ in changes { + watcher.poke() + self?.onScreenChange?(udid) + } + } + } + tasks.append(Task { [weak self, weak controller] in + for await event in watcher.changes { + guard let controller else { return } + let orientation: DeviceOrientation + switch event { + case .turned(let turned): + orientation = turned + self?.asked[udid] = turned + case .stillAt(let still): + orientation = still + } + if controller.currentOrientation != orientation { + controller.setOrientation(orientation) + } + } + }) + following[udid] = Following(watcher: watcher, tasks: tasks) + watcher.poke() + } + + /// Where a rotate counts on from: the turn last asked for, or the screen's last turn, rather than + /// what the window shows, so a turn the screen refuses (upside down on a Face ID phone) is passed + /// on the next press instead of asked for again. + public func lastTurn(_ udid: String) -> DeviceOrientation? { + asked[udid] + } + + /// The window asked the device to turn and shows it turned; the screen is checked once it has + /// had time to follow. + public func turned(_ udid: String, to orientation: DeviceOrientation) { + asked[udid] = orientation + following[udid]?.watcher.confirm() + } + + /// A foldable's hinge is where it is shown: a move of a degree or more keeps the window from + /// turning on what the report says while the fold goes through. + public func hingeMoved(_ udid: String, to degrees: Double) { + if let last = hinges[udid], abs(last - degrees) < 1 { return } + hinges[udid] = degrees + following[udid]?.watcher.holdStill() + } + + public func forget(_ udid: String) { + hinges[udid] = nil + asked[udid] = nil + guard let gone = following.removeValue(forKey: udid) else { return } + for task in gone.tasks { task.cancel() } + gone.watcher.close() + } +} diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/ScreenOrientationFollowTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/ScreenOrientationFollowTests.swift new file mode 100644 index 0000000..b8ebfbb --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/ScreenOrientationFollowTests.swift @@ -0,0 +1,305 @@ +import AppKit +import XCTest +import OpenDeviceHubEngine +@testable import OpenDeviceHubViewer + +/// A window keeps up with its device's screen whoever turns the device. Device Hub turns an +/// ordinary device the way `devicectl device orientation set` does and a foldable through the +/// foldable's own control, so those stand in for it here. +@MainActor +final class ScreenOrientationFollowTests: XCTestCase { + func testTheFoldableFollowsTurnsFromOutsideAndKeepsItsOwnThroughFolds() async throws { + try IntegrationGate.requireEnabled() + let adapter = try AdapterFactory.make(for: XcodeLocator.locate()) + guard let device = try adapter.devices().first(where: { + $0.state == .booted && $0.deviceTypeIdentifier.contains("Duo") + }) else { throw XCTSkip("boot an iPhone Duo to run this test") } + let udid = device.udid + try IntegrationHost.install(on: udid) + _ = try run(["simctl", "launch", udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(3)) + + let panels = try adapter.panels(udid) + let unfolded = try XCTUnwrap(panels.first { $0.name == "Unfolded" }) + let cover = try XCTUnwrap(panels.first { $0.name == "Cover" }) + let found = DevicectlService().orientation(udid: udid) ?? .portrait + let input = try adapter.openInput(udid, screenID: unfolded.screenID) + let targeted = try XCTUnwrap(input as? PanelInputSession) + let manager = DeviceWindowManager(shutdown: { _ in }) + let controller = try manager.open( + device: device, + session: try adapter.openDisplay(udid, panel: unfolded), + input: input, + scaleMode: .fit, bezelEnabled: true, keepOnTop: false, showFPS: false, + foldsAtHinge: true, + chrome: unfolded.chromeIdentifier.flatMap { ChromeLocator.chrome(identifier: $0) }, + panelNativeRotation: unfolded.nativeRotation, + unfoldedPanel: unfolded, + cover: FoldableCover(panel: cover, session: try adapter.openDisplay(udid, panel: cover)), + retarget: { targeted.setTarget(screenID: $0) }, + orientation: found + ) + let shared = try await IntegrationFoldable.shared.control(for: udid, adapter: adapter) + let foldables = FoldableController( + open: { _ in shared }, + hingeStream: { try adapter.openHingeStream($0) }, + displayReport: { try await adapter.displayReport($0) } + ) + let orientations = ScreenOrientationFollower( + read: { try await adapter.displayReport($0) }, + panels: { (try? adapter.panels($0)) ?? [] } + ) + orientations.onScreenChange = { foldables.nudge($0) } + orientations.follow(udid, controller: controller) + foldables.onMove = { [weak controller] _, event in + switch event { + case .began(let target): controller?.beginFold(to: target) + case .angle(let angle): + controller?.showHingeAngle(angle) + orientations.hingeMoved(udid, to: angle) + case .ended: controller?.endFold() + } + } + foldables.follow( + udid, + onPanel: { [weak controller] panel in controller?.setActivePanel(screenID: panel.displayID) }, + onHinge: { [weak controller] degrees in + controller?.showHingeAngle(degrees) + orientations.hingeMoved(udid, to: degrees) + } + ) + let open = DeviceControlBar.FoldMode.fullyOpen.angle + controller.showHingeAngle(open) + foldables.setAngle(open, for: udid) + defer { + foldables.setAngle(open, for: udid) + try? shared.setOrientation(found) + orientations.forget(udid) + foldables.forget(udid) + manager.close(udid) + } + try await Task.sleep(for: .seconds(3)) + + for orientation in [DeviceOrientation.landscapeRight, .landscapeLeft, .portraitUpsideDown, .portrait] { + let started = ContinuousClock.now + try shared.setOrientation(orientation) + let followed = await Self.wait(for: orientation, in: controller, within: 4) + print("RESULT turned \(orientation.rawValue) from outside, window \(controller.currentOrientation.rawValue) after \(Self.milliseconds(since: started)) ms") + XCTAssertTrue(followed, "the window did not follow a turn to \(orientation.rawValue) from outside") + } + + for orientation in [DeviceOrientation.landscapeLeft, .portrait, .landscapeRight, .landscapeLeft] { + try foldables.setOrientation(orientation, for: udid) + controller.setOrientation(orientation) + orientations.turned(udid, to: orientation) + let shown = await Self.samples(of: controller, for: 3) + let screen = await Self.screen(of: udid, adapter: adapter, panels: panels) + print("RESULT turned \(orientation.rawValue) from the window, screen \(screen?.rawValue ?? "nothing"), window \(shown.map(\.rawValue).reduce(into: [String]()) { if $0.last != $1 { $0.append($1) } })") + XCTAssertEqual(controller.currentOrientation, screen, "the window and the screen disagree after the window's own turn") + if screen == orientation { + XCTAssertEqual(Set(shown), [orientation], "the window moved off a turn the screen took") + } + } + if controller.currentOrientation != .landscapeLeft { + try foldables.setOrientation(.landscapeLeft, for: udid) + controller.setOrientation(.landscapeLeft) + orientations.turned(udid, to: .landscapeLeft) + } + try await Task.sleep(for: .seconds(3)) + XCTAssertEqual(controller.currentOrientation, .landscapeLeft) + for (name, angle) in [("shut", 0.0), ("open", open)] { + foldables.setAngle(angle, for: udid, eased: true) + let shown = await Self.samples(of: controller, for: 5) + print("RESULT \(name) while held landscape left, window \(Set(shown).map(\.rawValue)) over \(shown.count) samples") + XCTAssertEqual(Set(shown), [.landscapeLeft], "the window turned while the device was \(name == "shut" ? "shutting" : "opening")") + } + } + + func testAnIPhoneFollowsTurnsFromOutsideAndLeavesARefusedOneAlone() async throws { + try IntegrationGate.requireEnabled() + let adapter = try AdapterFactory.make(for: XcodeLocator.locate()) + guard let device = try adapter.devices().first(where: { + $0.state == .booted && $0.name.hasPrefix("iPhone") && ((try? adapter.panels($0.udid).count) ?? 0) < 2 + }) else { throw XCTSkip("boot an ordinary iPhone to run this test") } + let udid = device.udid + try IntegrationHost.install(on: udid) + _ = try run(["simctl", "launch", udid, IntegrationHost.bundleID]) + let log = URL(fileURLWithPath: try run([ + "simctl", "get_app_container", udid, IntegrationHost.bundleID, "data", + ]).trimmingCharacters(in: .whitespacesAndNewlines)) + .appending(path: "Documents").appending(path: "events.txt") + _ = try run(["devicectl", "device", "orientation", "set", "--device", udid, "portrait", "--timeout", "10", "--quiet"]) + try await Task.sleep(for: .seconds(3)) + let start = await Self.screen(of: udid, adapter: adapter, panels: []) + XCTAssertEqual(start, .portrait, "the device did not start upright") + + let manager = DeviceWindowManager(shutdown: { _ in }) + let controller = try manager.open( + device: device, + session: try adapter.openDisplay(udid, panel: nil), + input: try adapter.openInput(udid, screenID: 0), + scaleMode: .fit, bezelEnabled: true, keepOnTop: false, showFPS: false, + orientation: DevicectlService().orientation(udid: udid) ?? .portrait + ) + let orientations = ScreenOrientationFollower( + read: { try await adapter.displayReport($0) }, + panels: { (try? adapter.panels($0)) ?? [] } + ) + orientations.follow(udid, controller: controller) + defer { + _ = try? run(["devicectl", "device", "orientation", "set", "--device", udid, "portrait", "--timeout", "10", "--quiet"]) + _ = try? run(["simctl", "launch", udid, IntegrationHost.bundleID]) + orientations.forget(udid) + manager.close(udid) + } + let window = try XCTUnwrap(controller.window) + let screen = try XCTUnwrap(Self.screenView(in: window)) + let upright = await Self.wait(for: .portrait, in: controller, within: 3) + XCTAssertTrue(upright, "the window did not open upright") + + // devicectl names the side the device is turned towards; the viewer names the way the + // picture turns, so the landscapes swap. + for (asked, shown) in [("landscapeLeft", DeviceOrientation.landscapeRight), ("landscapeRight", .landscapeLeft)] { + let started = ContinuousClock.now + _ = try run(["devicectl", "device", "orientation", "set", "--device", udid, asked, "--timeout", "10", "--quiet"]) + let followed = await Self.wait(for: shown, in: controller, within: 4) + window.layoutIfNeeded() + let size = window.contentLayoutRect.size + print("RESULT turned \(asked) from outside, window \(controller.currentOrientation.rawValue) \(Int(size.width))x\(Int(size.height)) after \(Self.milliseconds(since: started)) ms") + XCTAssertTrue(followed, "the window did not follow a turn to \(asked) from outside") + XCTAssertGreaterThan(size.width, size.height, "the window stayed upright around a sideways screen") + try await Task.sleep(for: .seconds(1)) + let before = Self.taps(log).count + try await Self.click(screen, in: window, at: CGPoint(x: screen.bounds.midX - screen.bounds.width * 0.2, y: screen.bounds.midY + screen.bounds.height * 0.2)) + try await Task.sleep(for: .seconds(1.5)) + let taps = Self.taps(log) + let landed = taps.last ?? "nothing" + print("RESULT the upper left of the window after \(asked): \(landed)") + XCTAssertGreaterThan(taps.count, before, "the click reached nothing") + if let point = Self.point(from: landed) { + XCTAssertLessThan(point.x, 0.5, "the upper left of the window landed at \(point)") + XCTAssertLessThan(point.y, 0.5, "the upper left of the window landed at \(point)") + } + } + + _ = try run(["devicectl", "device", "orientation", "set", "--device", udid, "portrait", "--timeout", "10", "--quiet"]) + let back = await Self.wait(for: .portrait, in: controller, within: 4) + XCTAssertTrue(back, "the window did not come back upright") + + _ = try run(["devicectl", "device", "orientation", "set", "--device", udid, "portraitUpsideDown", "--timeout", "10", "--quiet"]) + let upsideDown = await Self.samples(of: controller, for: 3) + print("RESULT held upside down from outside, which the screen refuses: window \(Set(upsideDown).map(\.rawValue))") + XCTAssertEqual(Set(upsideDown), [.portrait], "the window turned although the screen did not") + + _ = try run(["devicectl", "device", "orientation", "set", "--device", udid, "portrait", "--timeout", "10", "--quiet"]) + let uprightAgain = await Self.wait(for: .portrait, in: controller, within: 4) + XCTAssertTrue(uprightAgain) + + // The window's own turn to upside down, as its rotate asks for it after landscape right. + try adapter.setOrientation(.portraitUpsideDown, udid: udid) + controller.setOrientation(.portraitUpsideDown) + orientations.turned(udid, to: .portraitUpsideDown) + let refused = await Self.samples(of: controller, for: 3.5) + print("RESULT the window turned upside down itself, which the screen refuses: window \(refused.map(\.rawValue).reduce(into: [String]()) { if $0.last != $1 { $0.append($1) } })") + XCTAssertEqual(refused.last, .portrait, "the window stayed upside down around an upright screen") + let next = try XCTUnwrap(orientations.lastTurn(udid)).rotatedLeft + XCTAssertEqual(next, .landscapeLeft, "the next rotate asks for the refused turn again") + try adapter.setOrientation(next, udid: udid) + controller.setOrientation(next) + orientations.turned(udid, to: next) + let passed = await Self.samples(of: controller, for: 3) + print("RESULT the next rotate after it: window \(passed.map(\.rawValue).reduce(into: [String]()) { if $0.last != $1 { $0.append($1) } })") + XCTAssertEqual(Set(passed), [.landscapeLeft], "the next rotate did not turn past upside down") + + _ = try run(["devicectl", "device", "orientation", "set", "--device", udid, "portrait", "--timeout", "10", "--quiet"]) + _ = try run(["simctl", "terminate", udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(2)) + _ = try run(["devicectl", "device", "orientation", "set", "--device", udid, "landscapeLeft", "--timeout", "10", "--quiet"]) + let home = await Self.samples(of: controller, for: 3) + print("RESULT the home screen, held sideways from outside: window \(Set(home).map(\.rawValue))") + XCTAssertEqual(Set(home), [.portrait], "the window turned around a home screen that stays upright") + } + + private static func screen(of udid: String, adapter: any SimulatorAdapter, panels: [DevicePanel]) async -> DeviceOrientation? { + let angles = Dictionary(panels.map { ($0.screenID, $0.nativeRotation) }, uniquingKeysWith: { first, _ in first }) + guard let report = try? await adapter.displayReport(udid) else { return nil } + return ScreenOrientationWatcher.reading(from: report, nativeRotation: { angles[$0] ?? 0 })?.orientation + } + + private static func wait(for expected: DeviceOrientation, in controller: DeviceWindowController, within seconds: Double) async -> Bool { + let deadline = ContinuousClock.now + .seconds(seconds) + while ContinuousClock.now < deadline { + if controller.currentOrientation == expected { return true } + try? await Task.sleep(for: .milliseconds(50)) + } + return controller.currentOrientation == expected + } + + private static func samples(of controller: DeviceWindowController, for seconds: Double) async -> [DeviceOrientation] { + var shown: [DeviceOrientation] = [] + let deadline = ContinuousClock.now + .seconds(seconds) + while ContinuousClock.now < deadline { + shown.append(controller.currentOrientation) + try? await Task.sleep(for: .milliseconds(50)) + } + return shown + } + + private static func milliseconds(since start: ContinuousClock.Instant) -> Int { + let elapsed = ContinuousClock.now - start + return Int(elapsed.components.seconds * 1000 + elapsed.components.attoseconds / 1_000_000_000_000_000) + } + + private static func click(_ view: NSView, in window: NSWindow, at spot: CGPoint) async throws { + let inWindow = view.convert(spot, to: nil) + view.mouseDown(with: try mouse(.leftMouseDown, at: inWindow, in: window)) + try await Task.sleep(for: .milliseconds(80)) + view.mouseUp(with: try mouse(.leftMouseUp, at: inWindow, in: window)) + } + + private static func mouse(_ type: NSEvent.EventType, at point: CGPoint, in window: NSWindow) throws -> NSEvent { + try XCTUnwrap(NSEvent.mouseEvent( + with: type, location: point, modifierFlags: [], + timestamp: ProcessInfo.processInfo.systemUptime, + windowNumber: window.windowNumber, context: nil, + eventNumber: 0, clickCount: 1, pressure: 1 + )) + } + + private static func taps(_ log: URL) -> [String] { + ((try? String(contentsOf: log, encoding: .utf8)) ?? "") + .split(separator: "\n").filter { $0.hasPrefix("TAP") }.map(String.init) + } + + private static func point(from line: String) -> CGPoint? { + let parts = line.split(separator: " ") + guard parts.count == 3, let x = Double(parts[1]), let y = Double(parts[2]) else { return nil } + return CGPoint(x: x, y: y) + } + + private static func screenView(in window: NSWindow) -> DeviceScreenView? { + func search(_ view: NSView) -> DeviceScreenView? { + if let found = view as? DeviceScreenView { return found } + for subview in view.subviews { + if let found = search(subview) { return found } + } + return nil + } + return window.contentView.flatMap(search) + } + + @discardableResult + private func run(_ arguments: [String]) throws -> String { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") + process.arguments = arguments + process.environment = ProcessInfo.processInfo.environment + let output = Pipe() + process.standardOutput = output + process.standardError = Pipe() + try process.run() + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + return String(decoding: data, as: UTF8.self) + } +}