diff --git a/RELEASING.md b/RELEASING.md index 3c3be9e..83ae822 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -51,21 +51,28 @@ unsigned build that reaches a release page is worse than a run that stops. 1. Add the version's section to `CHANGELOG.md`. The release notes on GitHub and the notes inside the update feed both come from it, so they cannot drift apart. 2. Update `VERSION`. -3. Run the **Release** workflow from the Actions tab with `publish` left **false**. That is a dry +3. Run `scripts/performance-check.sh` on the Mac the baselines in `performance/` were recorded on, + with only an iPhone and an iPhone Duo booted and Low Power Mode off. Every booted device joins + the windows it opens at once, and the baseline names them. It takes about eight minutes, compares + with `baseline-normal-power.json`, or `baseline-low-power.json` when Low Power Mode is on, and + exits 1 if a window got slower to open, heavier, less responsive or used more energy or GPU than + the baseline allows, or if memory kept growing. Record a new baseline with `--record-baseline` + only when a change is meant to move the numbers, and commit it with that change. +4. Run the **Release** workflow from the Actions tab with `publish` left **false**. That is a dry run: it builds, signs, notarizes, staples, packages, checksums and generates the feed, and creates no tag, no release and no commit anywhere. The artifacts are attached to the run under a short retention name so you can download and check them. -4. Install the dry run's DMG and use it. `stapler validate` on the app and on the DMG, and +5. Install the dry run's DMG and use it. `stapler validate` on the app and on the DMG, and `shasum -c SHA256SUMS` from a different directory. -5. Run the workflow again with `publish` **true**. It creates the tag and a release marked as the +6. Run the workflow again with `publish` **true**. It creates the tag and a release marked as the latest, carrying the DMG under its versioned name, the same DMG as `OpenDeviceHub.dmg`, `SHA256SUMS` and the signed `appcast.xml`. The README's download button and its Install link fetch `OpenDeviceHub.dmg` from the latest release, so they need no edit. -6. **Only once that release page exists**, commit the release's `appcast.xml` to the feed +7. **Only once that release page exists**, commit the release's `appcast.xml` to the feed repository. Installed copies read the feed from there, never from the release, which is why the feed URL can never move. A feed pointing at a download that is not there yet breaks updates for everyone who reads it in the meantime. -7. Confirm an already installed older copy finds the update, verifies it and installs it, ending on +8. Confirm an already installed older copy finds the update, verifies it and installs it, ending on the new version. Do this before announcing anything: a broken updater cannot be fixed by a later release. diff --git a/engine/Sources/ODHubViewerApp/ViewerMain.swift b/engine/Sources/ODHubViewerApp/ViewerMain.swift index 2efb26f..3df6d4a 100644 --- a/engine/Sources/ODHubViewerApp/ViewerMain.swift +++ b/engine/Sources/ODHubViewerApp/ViewerMain.swift @@ -637,7 +637,7 @@ struct ODHubViewer: ParsableCommand { } let panel = opened.unfolded let session = opened.session - let orientation = DevicectlService().orientation(udid: device.udid) ?? .portrait + let orientation = DevicectlService().orientation(udid: device.udid, foldable: opened.foldsAtHinge) ?? .portrait let controller = try manager.open( device: device, diff --git a/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift b/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift index 1f472c2..f5155ff 100644 --- a/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift +++ b/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift @@ -27,4 +27,11 @@ public struct HingeMove: Sendable, Hashable { } public static let stepInterval: Duration = .milliseconds(16) + + /// When the next step is due, counted from the start of the move: the first tick after + /// `elapsed`, so time taken between steps shortens the wait instead of adding to it, and a late + /// step is followed by the next tick rather than a burst. + public static func nextStep(after elapsed: Duration) -> Duration { + stepInterval * (Int(max(elapsed, .zero) / stepInterval) + 1) + } } diff --git a/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift b/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift index 3b5fa89..c15a61e 100644 --- a/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift +++ b/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift @@ -24,10 +24,12 @@ public struct DevicectlService: Sendable { } /// On a foldable the first answer after a turn repeats the answer given before it, however long - /// ago the turn was (27A266a), so the question is asked twice and the second answer kept. - public func orientation(udid: String) -> DeviceOrientation? { + /// ago the turn was (27A266a, 27A9269), so a foldable is asked twice and the second answer kept. + /// An ordinary iPhone answers right the first time (27A9269), and each ask costs 150 to 400 ms. + public func orientation(udid: String, foldable: Bool) -> DeviceOrientation? { let arguments = Self.orientationArguments(udid: udid) - guard run(arguments) != nil, let output = run(arguments) else { return nil } + if foldable { guard run(arguments) != nil else { return nil } } + guard let output = run(arguments) else { return nil } return Self.parseOrientation(Data(output.utf8)) } diff --git a/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift b/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift index 92e9248..de7d520 100644 --- a/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift +++ b/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift @@ -578,6 +578,11 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate public var latencyReading: LatencyMeter.Reading? { latency.reading } + /// Frames the device has sent this window, and pictures the window has drawn, for measuring + /// what a window costs while nothing on screen changes. + public private(set) var framesReceived = 0 + public var drawCount: Int { modelView?.renderCount ?? renderer.drawCount } + private func updateTitle() { guard let window else { return } let base = baseTitle ?? window.title @@ -793,7 +798,6 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate private func installClickToTap() { let pixelSize = session.pixelSize - let screenView = screenView // The model's hit test already yields the guest's own coordinates; nothing is turned here. modelView?.onTouch = { [weak self] point, phase in @@ -812,7 +816,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate let orientation = self.orientation guard let primary = CoordinateMapper.normalize( viewPoint: point, - viewSize: screenView.bounds.size, + viewSize: self.screenView.bounds.size, pixelSize: orientation.displayedSize(portraitNative: pixelSize) ) else { return } @@ -860,7 +864,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate screenView.onGesture = { [weak self] phase, spread, angle in guard let self else { return } - let size = screenView.bounds.size + let size = self.screenView.bounds.size guard size.width > 0, size.height > 0 else { return } let normalizedSpread = spread / min(size.width, size.height) @@ -891,6 +895,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate renderer.accept(frame) recording.current?.append(frame.surface, from: screenID, turnedBy: turns) await MainActor.run { [weak self] in + self?.framesReceived += 1 // The flat view sits under the model and is not seen; drawing it costs the // main thread a draw per frame. if self?.modelView == nil { screenView.needsDisplay = true } @@ -919,6 +924,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate if Task.isCancelled { return } recording.current?.append(frame.surface, from: screenID, turnedBy: turns) await MainActor.run { [weak self] in + self?.framesReceived += 1 self?.modelView?.setScreen(frame.surface, onCover: true, nativeRotation: rotation) } } diff --git a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift index 66f42e7..e6262c2 100644 --- a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift +++ b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift @@ -1,13 +1,19 @@ import AppKit import IOSurface import Metal +import MetalKit import OpenDeviceHubEngine +import QuartzCore import SceneKit import simd /// A foldable drawn as the physical device, bent by the hinge, with the guest's screen on it. +/// +/// Drawn by its own renderer once for each change. An `SCNView` drawing on demand draws about 16 +/// more times after every change, a new frame from the device included, which at 2 frames a second +/// from the device was 35 draws and 9% of the CPU at rest (Xcode 27.1, 27A9269). @MainActor -public final class DuoModelView: SCNView { +public final class DuoModelView: MTKView { /// Times in the asset's closing clip. Its other flat-to-shut stretches also turn the hardware. enum Pose { static let open: TimeInterval = 260.0 / 24 @@ -83,6 +89,13 @@ public final class DuoModelView: SCNView { /// How many probe renders have been made, for a test that expects a move to make none. var probeRenders = 0 + public private(set) var renderCount = 0 + private let sceneRenderer: SCNRenderer + private let commandQueue: MTLCommandQueue + /// Drawing runs on a timer until then, for a lift that SceneKit animates. + private var animatingUntil: CFTimeInterval = 0 + public var scene: SCNScene? { sceneRenderer.scene } + public var pointOfView: SCNNode? { sceneRenderer.pointOfView } private let cameraNode = SCNNode() private let innerScreen: SCNNode private let coverScreen: SCNNode @@ -129,6 +142,9 @@ public final class DuoModelView: SCNView { let cover = Self.closest(to: CGSize(width: 11.230, height: 7.739), among: faces), inner !== cover else { return nil } + guard let queue = metalDevice.makeCommandQueue() else { return nil } + commandQueue = queue + sceneRenderer = SCNRenderer(device: metalDevice, options: nil) self.metalDevice = metalDevice self.nativeQuarterTurns = ((-nativeRotation / 90) % 4 + 4) % 4 innerScreen = inner @@ -137,8 +153,8 @@ public final class DuoModelView: SCNView { lifts = Self.lifts(for: hardwareButtons, in: scene.rootNode) activeScreen = showingCover ? cover : inner content = scene.rootNode - // No animation players: a player churn per pose blocks behind the render thread for most - // of a frame, while setting thirty transforms by hand is free. + // No animation players: under an SCNView a player churn per pose blocked behind its render + // thread for most of a frame, while setting thirty transforms by hand is free. tracks = Self.freeze(scene.rootNode).flatMap { PoseTrack.tracks(of: $0.animation, on: $0.node) } // A source hands out its clips once, so the twin comes from a source of its own. guard let twinSource = SCNSceneSource(url: asset, options: nil), @@ -164,9 +180,7 @@ public final class DuoModelView: SCNView { tracks: twinTracks ) - super.init(frame: .zero, options: [ - SCNView.Option.preferredRenderingAPI.rawValue: SCNRenderingAPI.metal.rawValue, - ]) + super.init(frame: .zero, device: metalDevice) let camera = SCNCamera() camera.fieldOfView = 31 @@ -178,25 +192,24 @@ public final class DuoModelView: SCNView { scene.rootNode.addChildNode(cameraNode) installLighting(in: scene) - self.scene = scene - pointOfView = cameraNode - backgroundColor = .clear - wantsLayer = true + sceneRenderer.scene = scene + sceneRenderer.pointOfView = cameraNode + // What the SCNView this replaced drew into (27A9269), so the device looks the same. + colorPixelFormat = .bgra8Unorm_srgb + depthStencilPixelFormat = .depth32Float + sampleCount = 4 + clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) layer?.isOpaque = false - antialiasingMode = .multisampling4X - allowsCameraControl = false - rendersContinuously = true - // Asked for 30, SceneKit draws at 24 on a 120 Hz display, and the rate cannot be changed - // once the view exists; a fold needs 60 to look like one. + (layer as? CAMetalLayer)?.colorspace = CGColorSpace(name: CGColorSpace.sRGB) + isPaused = true + enableSetNeedsDisplay = true preferredFramesPerSecond = 60 - isPlaying = false - loops = false setHingeAngle(180) } @available(*, unavailable) - required init?(coder: NSCoder) { + required init(coder: NSCoder) { fatalError("not supported") } @@ -207,7 +220,7 @@ public final class DuoModelView: SCNView { settleLifts() if let move, let fraction = move.fraction(of: hingeAngle) { let correction = move.start + (move.end - move.start) * Float(fraction) - // The twin's bones, not this scene's: a read here waits behind the render thread. + // The twin's bones, not this scene's, which under an SCNView waited behind its render thread. place( cameraNode, angle: hingeAngle, centre: twinCentre(at: hingeAngle), correction: correction, distanceScale: move.scale(at: hingeAngle) @@ -215,6 +228,7 @@ public final class DuoModelView: SCNView { } else { frameCamera() } + needsDisplay = true } /// Framed from both ends, measured on the twin now; a probe render costs more than a frame, so @@ -283,6 +297,50 @@ public final class DuoModelView: SCNView { needsLayout = true } + public override func viewDidMoveToWindow() { + super.viewDidMoveToWindow() + needsDisplay = true + } + + public override func viewDidChangeBackingProperties() { + super.viewDidChangeBackingProperties() + needsDisplay = true + } + + public override func draw(_ dirtyRect: NSRect) { + let now = CACurrentMediaTime() + defer { + if !isPaused, now >= animatingUntil { + isPaused = true + enableSetNeedsDisplay = true + } + } + guard let pass = currentRenderPassDescriptor, let drawable = currentDrawable, + let buffer = commandQueue.makeCommandBuffer() else { return } + sceneRenderer.render( + atTime: now, + viewport: CGRect(origin: .zero, size: drawableSize), + commandBuffer: buffer, + passDescriptor: pass + ) + buffer.present(drawable) + buffer.commit() + renderCount += 1 + } + + /// Drawn on a timer for as long as SceneKit animates, then on demand again. + private func animate(for duration: TimeInterval) { + animatingUntil = max(animatingUntil, CACurrentMediaTime() + duration) + enableSetNeedsDisplay = false + isPaused = false + } + + /// The picture as it is drawn now, at the drawable's size, and empty before the view has one. + public func snapshot() -> NSImage { + guard drawableSize.width > 0, drawableSize.height > 0 else { return NSImage(size: .zero) } + return sceneRenderer.snapshot(atTime: CACurrentMediaTime(), with: drawableSize, antialiasingMode: .multisampling4X) + } + /// The hardware turns too: the guest has already turned its picture inside the panel. public func setOrientation(_ orientation: DeviceOrientation) { let turns = ((orientation.degrees / 90) % 4 + 4) % 4 @@ -347,6 +405,7 @@ public final class DuoModelView: SCNView { material.diffuse.wrapT = .clamp material.lightingModel = .constant } + needsDisplay = true } /// Frozen players rather than the view's scene time, which the view advances itself. @@ -448,6 +507,7 @@ public final class DuoModelView: SCNView { ) silhouette = framing.silhouette projectButtons() + needsDisplay = true } /// Measurements belong to a window size and a way up; either changing throws them away. @@ -873,17 +933,19 @@ public final class DuoModelView: SCNView { let amount = restLift(of: button) + rise let position = lift.outward * amount let scale = SIMD3(repeating: 1) + abs(lift.outward) * (amount / lift.depth) - // An explicit transaction commits behind the render thread, which costs most of a frame - // in a live window; a plain set costs nothing and the run loop commits it. + // Under an SCNView an explicit transaction committed behind its render thread, which cost + // most of a frame in a live window; a plain set costs nothing. if liftDuration > 0 { SCNTransaction.begin() SCNTransaction.animationDuration = liftDuration lift.node.simdPosition = position lift.node.simdScale = scale SCNTransaction.commit() + animate(for: liftDuration) } else { lift.node.simdPosition = position lift.node.simdScale = scale + needsDisplay = true } liftedReach = rise > 0 ? reach(of: button, lifted: amount) : nil } @@ -916,6 +978,7 @@ public final class DuoModelView: SCNView { for material in node.geometry?.materials ?? [] { material.emission.contents = lit ? NSColor(white: 0.35, alpha: 1) : NSColor.black } + needsDisplay = true } func liftOffset(of button: HardwareButton) -> Float { @@ -1269,4 +1332,3 @@ extension DuoModelView { return (SCNVector3(origin), SCNVector3(far)) } } - diff --git a/engine/Sources/OpenDeviceHubViewer/FoldableController.swift b/engine/Sources/OpenDeviceHubViewer/FoldableController.swift index eb42399..bb999e0 100644 --- a/engine/Sources/OpenDeviceHubViewer/FoldableController.swift +++ b/engine/Sources/OpenDeviceHubViewer/FoldableController.swift @@ -102,7 +102,10 @@ public final class FoldableController { send(angle, to: control, for: udid, poke: false) onMove?(udid, .angle(angle)) if progress >= 1 { break } - try? await Task.sleep(for: HingeMove.stepInterval) + try? await Task.sleep( + until: started + HingeMove.nextStep(after: ContinuousClock.now - started), + clock: .continuous + ) } guard let self, !Task.isCancelled else { return } moves[udid] = nil diff --git a/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift b/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift index 0894e0a..28423ea 100644 --- a/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift +++ b/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift @@ -38,6 +38,7 @@ fragment float4 odhFragment(VertexOut in [[stage_in]], texture2d screen [ /// `IOSurface`, so a frame is never copied through the CPU. public final class FrameRenderer: NSObject, MTKViewDelegate { public var onFrameDrawn: (() -> Void)? + public private(set) var drawCount = 0 private let device: MTLDevice private let commandQueue: MTLCommandQueue @@ -141,6 +142,7 @@ public final class FrameRenderer: NSObject, MTKViewDelegate { buffer.present(drawable) buffer.commit() + drawCount += 1 onFrameDrawn?() } diff --git a/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift b/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift index fbcb70c..22dd709 100644 --- a/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift +++ b/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift @@ -37,13 +37,21 @@ final class DevicectlServiceTests: XCTestCase { func testKeepsTheSecondAnswerSinceAFoldableRepeatsItsLastOne() { let answers = Answers([report("portrait"), report("landscapeRight")]) let service = DevicectlService { _ in answers.next() } - XCTAssertEqual(service.orientation(udid: "ABC"), .landscapeLeft) + XCTAssertEqual(service.orientation(udid: "ABC", foldable: true), .landscapeLeft) XCTAssertEqual(answers.asked, 2) } + func testAsksAnOrdinaryDeviceOnce() { + let answers = Answers([report("landscapeRight"), report("portrait")]) + let service = DevicectlService { _ in answers.next() } + XCTAssertEqual(service.orientation(udid: "ABC", foldable: false), .landscapeLeft) + XCTAssertEqual(answers.asked, 1) + } + func testNoAnswerWhenDevicectlCannotSeeTheDevice() { let service = DevicectlService { _ in nil } - XCTAssertNil(service.orientation(udid: "ABC")) + XCTAssertNil(service.orientation(udid: "ABC", foldable: true)) + XCTAssertNil(service.orientation(udid: "ABC", foldable: false)) } func testAnythingElseIsNoAnswer() { diff --git a/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift b/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift index fee22ef..5286d2a 100644 --- a/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift +++ b/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift @@ -33,4 +33,13 @@ import Testing #expect(HingeMove(start: 0, target: 90).duration == 0.5) #expect(HingeMove(start: 180, target: 0).duration == 1) } + + @Test func stepsOnTheNextTickWhateverTheWorkBetweenTook() { + #expect(HingeMove.nextStep(after: .zero) == .milliseconds(16)) + #expect(HingeMove.nextStep(after: .milliseconds(6)) == .milliseconds(16), "work between steps shortens the wait") + #expect(HingeMove.nextStep(after: .milliseconds(16)) == .milliseconds(32)) + #expect(HingeMove.nextStep(after: .milliseconds(22)) == .milliseconds(32)) + #expect(HingeMove.nextStep(after: .milliseconds(55)) == .milliseconds(64), "a late step skips to the next tick, not a burst") + #expect(HingeMove.nextStep(after: .milliseconds(-3)) == .milliseconds(16)) + } } diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift index cd4e762..3112844 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift @@ -25,7 +25,7 @@ final class OpenSidewaysDriveTests: XCTestCase { defer { try? adapter.setOrientation(.portrait, udid: udid) } try await Task.sleep(for: .seconds(3)) - let orientation = try XCTUnwrap(DevicectlService().orientation(udid: udid), "devicectl did not say how the device is turned") + let orientation = try XCTUnwrap(DevicectlService().orientation(udid: udid, foldable: false), "devicectl did not say how the device is turned") XCTAssertEqual(orientation, .landscapeLeft) let manager = DeviceWindowManager(shutdown: { _ in }) @@ -64,6 +64,29 @@ final class OpenSidewaysDriveTests: XCTestCase { try await checkQuadrants(screen, in: window, log: log, shown: "upright") } + /// One ask, straight after each turn, twice round, so a first answer that repeats the last one + /// would show. + func testAnOrdinaryIPhoneSaysHowItIsTurnedAtTheFirstAsk() async throws { + try IntegrationGate.requireEnabled() + let adapter = try AdapterFactory.make(for: XcodeLocator.locate()) + guard let device = try adapter.devices().first(where: { + $0.state == .booted && $0.name.hasPrefix("iPhone") && ((try? adapter.panels($0.udid).count) ?? 0) < 2 + }) else { throw XCTSkip("boot an ordinary iPhone to run this test") } + let udid = device.udid + try IntegrationHost.install(on: udid) + _ = try run(["simctl", "launch", udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(3)) + defer { try? adapter.setOrientation(.portrait, udid: udid) } + + for orientation in [DeviceOrientation.landscapeLeft, .landscapeRight, .portrait, .landscapeRight, .portrait, .landscapeLeft, .portrait] { + try adapter.setOrientation(orientation, udid: udid) + try await Task.sleep(for: .seconds(2)) + let read = DevicectlService().orientation(udid: udid, foldable: false) + print("RESULT iPhone turned \(orientation.rawValue), first answer \(read?.rawValue ?? "nothing")") + XCTAssertEqual(read, orientation) + } + } + func testAFoldableSaysHowItIsTurned() async throws { try IntegrationGate.requireEnabled() let adapter = try AdapterFactory.make(for: XcodeLocator.locate()) @@ -76,7 +99,7 @@ final class OpenSidewaysDriveTests: XCTestCase { for orientation in [DeviceOrientation.landscapeRight, .landscapeLeft, .portrait] { try control.setOrientation(orientation) try await Task.sleep(for: .seconds(3)) - let read = DevicectlService().orientation(udid: device.udid) + let read = DevicectlService().orientation(udid: device.udid, foldable: true) print("RESULT foldable turned \(orientation.rawValue), read back \(read?.rawValue ?? "nothing")") XCTAssertEqual(read, orientation) } diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift new file mode 100644 index 0000000..7f07dde --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift @@ -0,0 +1,651 @@ +import AppKit +import IOKit +import XCTest +import OpenDeviceHubEngine +@testable import OpenDeviceHubViewer + +/// Measures what a device window costs: how long it takes to open, CPU, energy, GPU and drawing +/// while nothing on screen changes, memory, click to frame latency and, on a foldable, how evenly a +/// fold is drawn. Then every booted device's window is opened at once and kept busy to see whether +/// memory keeps growing. It runs only through `scripts/performance-check.sh`, which says where the +/// report goes and what to compare it with. +@MainActor +final class PerformanceCheckTests: XCTestCase { + private var adapter: (any SimulatorAdapter)! + private var manager: DeviceWindowManager! + private var stepTimes: [ContinuousClock.Instant] = [] + + static let settleSeconds = 8 + static let idleRounds = 10 + static let idleRoundSeconds = 2 + static let clicks = 20 + static let hostSettleSeconds = 2 + static let closeSettleSeconds = 3 + static let reopens = 3 + static let soakSampleSeconds = 5.0 + static let soakFoldEverySeconds = 20.0 + static let soakClickSeconds = 0.5 + + func testMeasureEachKindOfWindow() async throws { + try IntegrationGate.requireEnabled() + let environment = ProcessInfo.processInfo.environment + guard let output = environment["ODH_PERFORMANCE_REPORT"] else { + throw XCTSkip("run scripts/performance-check.sh, which says where the report goes") + } + adapter = try AdapterFactory.make(for: XcodeLocator.locate()) + manager = DeviceWindowManager(frameStore: WindowFrameStore(storage: DiscardingStorage()), shutdown: { _ in }) + + let soakSeconds = Double(environment["ODH_PERFORMANCE_BUSY_SECONDS"] ?? "") ?? 180 + let processStart = try Self.footprintMB() + let booted = try adapter.devices().filter { $0.state == .booted } + var windows: [WindowMeasurement] = [] + if let phone = booted.first(where: { + $0.name.hasPrefix("iPhone") && ((try? adapter.panels($0.udid).count) ?? 0) < 2 + }) { + windows.append(try await measurePhone(phone)) + } + if let foldable = booted.first(where: { $0.deviceTypeIdentifier.contains("Duo") }) { + windows.append(try await measureFoldable(foldable)) + } + guard !windows.isEmpty else { throw XCTSkip("boot an iPhone and an iPhone Duo to measure them") } + let together = booted.count > 1 ? try await measureTogether(booted, soakSeconds: soakSeconds, processStart: processStart) : nil + + let report = PerformanceReport( + createdAt: ISO8601DateFormatter().string(from: Date()), + environment: try Self.environment(), + windows: windows, + together: together + ) + let directory = URL(fileURLWithPath: output, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try Self.json(report).write(to: directory.appending(path: "performance.json")) + + var findings: [PerformanceComparison.Finding]? + if let baselinePath = environment["ODH_PERFORMANCE_BASELINE"] { + let baseline = try JSONDecoder().decode( + PerformanceReport.self, + from: Data(contentsOf: URL(fileURLWithPath: baselinePath)) + ) + do { + let found = try PerformanceComparison.regressions(current: report, baseline: baseline) + findings = found + try Self.json(found).write(to: directory.appending(path: "comparison.json")) + for finding in found { + XCTFail("\(finding.window) \(finding.metric) is \(finding.current), over its limit of \(finding.limit) from a baseline of \(finding.baseline)") + } + } catch { + XCTFail("the baseline cannot be compared with this run: \(error)") + } + } + if let recordPath = environment["ODH_PERFORMANCE_RECORD"] { + try Self.json(report).write(to: URL(fileURLWithPath: recordPath)) + } + let text = PerformanceComparison.text(report, findings: findings) + try Data(text.utf8).write(to: directory.appending(path: "performance.txt")) + print(text) + } + + private func measurePhone(_ device: DeviceInfo) async throws -> WindowMeasurement { + try IntegrationHost.install(on: device.udid) + _ = try run(["simctl", "launch", device.udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(Self.hostSettleSeconds)) + let before = try Self.footprintMB() + var measurement = try await measurePhoneWindow(device, before: before) + measurement.reopen = try await measureReopen(device, before: before) + return measurement + } + + /// Apart from the reopen, so nothing here still holds the window once it closes. + private func measurePhoneWindow(_ device: DeviceInfo, before: Double) async throws -> WindowMeasurement { + let opened = try await openWindow(device) + let controller = opened.controller + defer { manager.close(device.udid) } + let window = try XCTUnwrap(controller.window) + let screen = try XCTUnwrap(Self.find(DeviceScreenView.self, in: window)) + try await Task.sleep(for: .seconds(Self.settleSeconds)) + + let idle = try await measureIdle([controller]) + let windowMemory = idle.memory - before + let box = screen.bounds + let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) + let latency = try await measureClicks(controller, on: screen, spots: spots) + return WindowMeasurement( + name: "phone", + device: "\(device.name) (\(device.runtimeName))", + idleCPUPercent: idle.cpu, + idleDrawsPerSecond: idle.draws, + idleFramesPerSecond: idle.frames, + windowMemoryMB: windowMemory, + processMemoryMB: try Self.footprintMB(), + clickLatencyMs: latency, + foldStepGapMs: nil, + foldCPUPercent: nil, + idleCPUEnergyMilliwatts: idle.energy, + idleGPUPercent: idle.gpu, + open: opened.time + ) + } + + private func measureFoldable(_ device: DeviceInfo) async throws -> WindowMeasurement { + try IntegrationHost.install(on: device.udid) + _ = try run(["simctl", "launch", device.udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(Self.hostSettleSeconds)) + let before = try Self.footprintMB() + var measurement = try await measureFoldableWindow(device, before: before) + measurement.reopen = try await measureReopen(device, before: before) + return measurement + } + + private func measureFoldableWindow(_ device: DeviceInfo, before: Double) async throws -> WindowMeasurement { + let opened = try await openWindow(device) + let controller = opened.controller + let udid = device.udid + XCTAssertNotNil(opened.input as? PanelInputSession, "touches must be panel targeted") + let foldables = try await followFolds(of: controller, udid: udid) + defer { + foldables.setAngle(DeviceControlBar.FoldMode.fullyOpen.angle, for: udid) + foldables.forget(udid) + manager.close(udid) + } + let window = try XCTUnwrap(controller.window) + let model = try XCTUnwrap(Self.find(DuoModelView.self, in: window)) + try await Task.sleep(for: .seconds(Self.settleSeconds)) + + let idle = try await measureIdle([controller]) + let windowMemory = idle.memory - before + let box = model.bounds + let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) + .filter { model.screenPoint(at: $0) != nil } + XCTAssertFalse(spots.isEmpty, "no click lands on the open screen") + let latency = try await measureClicks(controller, on: model, spots: spots) + + let control = try XCTUnwrap(Self.find(NSSegmentedControl.self, in: window), "the window's fold positions") + stepTimes.removeAll() + var gaps: [Double] = [] + let cpuStart = Self.cpuSeconds() + let wallStart = ContinuousClock.now + for mode in [DeviceControlBar.FoldMode.cover, .fullyOpen] { + stepTimes.removeAll() + control.selectedSegment = mode.rawValue + control.sendAction(control.action, to: control.target) + try await Task.sleep(for: .seconds(3)) + gaps += zip(stepTimes.dropFirst(), stepTimes).map { later, earlier in Self.milliseconds(later - earlier) } + } + let foldCPU = (Self.cpuSeconds() - cpuStart) / Self.seconds(ContinuousClock.now - wallStart) * 100 + XCTAssertGreaterThan(gaps.count, 10, "the window drew no run of fold steps") + + return WindowMeasurement( + name: "foldable", + device: "\(device.name) (\(device.runtimeName))", + idleCPUPercent: idle.cpu, + idleDrawsPerSecond: idle.draws, + idleFramesPerSecond: idle.frames, + windowMemoryMB: windowMemory, + processMemoryMB: try Self.footprintMB(), + clickLatencyMs: latency, + foldStepGapMs: gaps.isEmpty ? nil : Spread(gaps), + foldCPUPercent: foldCPU, + idleCPUEnergyMilliwatts: idle.energy, + idleGPUPercent: idle.gpu, + open: opened.time + ) + } + + /// Called once the first window has closed. The reopened windows are not wired for folding: the + /// point is what opening and closing costs. + private func measureReopen(_ device: DeviceInfo, before: Double) async throws -> Reopen { + try await Task.sleep(for: .seconds(Self.closeSettleSeconds)) + let afterFirst = try Self.footprintMB() + var opens: [OpenTime] = [] + var added: [Double] = [] + var freed = true + var previous = afterFirst + for _ in 0.. TogetherMeasurement { + var controllers: [DeviceWindowController] = [] + var foldables: [(controller: FoldableController, udid: String)] = [] + defer { + for foldable in foldables { + foldable.controller.setAngle(DeviceControlBar.FoldMode.fullyOpen.angle, for: foldable.udid) + foldable.controller.forget(foldable.udid) + } + for device in devices { manager.close(device.udid) } + } + for device in devices { + let opened = try await openWindow(device) + controllers.append(opened.controller) + if opened.controller.foldsAtHinge { + foldables.append((try await followFolds(of: opened.controller, udid: device.udid), device.udid)) + } + } + try await Task.sleep(for: .seconds(Self.settleSeconds)) + let idle = try await measureIdle(controllers) + let windowsMemory = idle.memory - processStart + let soak = try await keepBusy(controllers, seconds: soakSeconds) + return TogetherMeasurement( + devices: devices.map { "\($0.name) (\($0.runtimeName))" }, + idleCPUPercent: idle.cpu, + idleCPUEnergyMilliwatts: idle.energy, + idleGPUPercent: idle.gpu, + idleDrawsPerSecond: idle.draws, + idleFramesPerSecond: idle.frames, + windowsMemoryMB: windowsMemory, + processMemoryMB: try Self.footprintMB(), + soak: soak + ) + } + + private struct OpenedWindow { + let controller: DeviceWindowController + let input: any InputSession + let time: OpenTime + } + + /// In the app's own order: panels, display and input sessions, the orientation from the guest, + /// then the window. The first picture is the first draw after the first frame from the device. + private func openWindow(_ device: DeviceInfo) async throws -> OpenedWindow { + let started = ContinuousClock.now + let panels = (try? adapter.panels(device.udid)) ?? [] + let foldsAtHinge = panels.count > 1 + let unfolded = foldsAtHinge ? panels.first { $0.name == "Unfolded" } : nil + let coverPanel = foldsAtHinge ? panels.first { $0.name == "Cover" } : nil + let session = try adapter.openDisplay(device.udid, panel: unfolded) + session.setBezelEnabled(true) + var cover: FoldableCover? + if let coverPanel { + let coverSession = try adapter.openDisplay(device.udid, panel: coverPanel) + coverSession.setBezelEnabled(true) + cover = FoldableCover(panel: coverPanel, session: coverSession) + } + let input = try adapter.openInput(device.udid, screenID: unfolded?.screenID ?? 0) + let retarget: ((Int) -> Void)? = (input as? PanelInputSession).map { targeted in + { targeted.setTarget(screenID: $0) } + } + let sessionsOpened = ContinuousClock.now + let orientation = DevicectlService().orientation(udid: device.udid, foldable: foldsAtHinge) ?? .portrait + let oriented = ContinuousClock.now + let controller = try manager.open( + device: device, + session: session, + input: input, + scaleMode: .fit, + bezelEnabled: true, + keepOnTop: false, + showFPS: false, + foldsAtHinge: foldsAtHinge, + chrome: unfolded?.chromeIdentifier.flatMap { ChromeLocator.chrome(identifier: $0) }, + panelNativeRotation: unfolded?.nativeRotation ?? 0, + unfoldedPanel: unfolded, + cover: cover, + retarget: retarget, + orientation: orientation + ) + if let unfolded, !controller.foldsAtHinge { + controller.nativeRotation = unfolded.nativeRotation + } + _ = controller.applyScaleMode(.fit) + controller.window?.orderFront(nil) + let shown = ContinuousClock.now + var drawsBeforeFrame = controller.drawCount + while controller.framesReceived == 0 || controller.drawCount <= drawsBeforeFrame { + if controller.framesReceived == 0 { drawsBeforeFrame = controller.drawCount } + guard ContinuousClock.now - shown < .seconds(10) else { + manager.close(device.udid) + throw MeasurementUnavailable(what: "\(device.name)'s first picture after 10 seconds") + } + try await Task.sleep(for: .milliseconds(2)) + } + let drawn = ContinuousClock.now + return OpenedWindow( + controller: controller, + input: input, + time: OpenTime( + sessionsMs: Self.milliseconds(sessionsOpened - started), + orientationMs: Self.milliseconds(oriented - sessionsOpened), + windowMs: Self.milliseconds(shown - oriented), + firstPictureMs: Self.milliseconds(drawn - shown) + ) + ) + } + + /// Wired as the app wires a foldable's window, and left fully open. + private func followFolds(of controller: DeviceWindowController, udid: String) async throws -> FoldableController { + let adapter = self.adapter! + let shared = try await IntegrationFoldable.shared.control(for: udid, adapter: adapter) + let foldables = FoldableController( + open: { _ in shared }, + hingeStream: { try adapter.openHingeStream($0) }, + displayReport: { try await adapter.displayReport($0) } + ) + controller.onHingeAngle = { angle in foldables.setAngle(angle, for: udid) } + controller.onFoldPreset = { angle in foldables.setAngle(angle, for: udid, eased: true) } + foldables.onMove = { [weak controller, weak self] _, event in + switch event { + case .began(let target): controller?.beginFold(to: target) + case .angle(let angle): + self?.stepTimes.append(ContinuousClock.now) + controller?.showHingeAngle(angle) + case .ended: controller?.endFold() + } + } + foldables.follow( + udid, + onPanel: { [weak controller] panel in controller?.setActivePanel(screenID: panel.displayID) }, + onHinge: { [weak controller] degrees in controller?.showHingeAngle(degrees) } + ) + let open = DeviceControlBar.FoldMode.fullyOpen.angle + controller.showHingeAngle(open) + foldables.setAngle(open, for: udid) + return foldables + } + + private struct Idle { + let cpu: Spread + let draws: Double + let frames: Double + let energy: Double + let gpu: Double + let memory: Double + } + + /// The whole process's CPU, energy, GPU and memory, which are the open windows': the test runner + /// itself sits idle while it waits. Memory is the median of a reading each round, so one passing + /// spike does not decide it. + private func measureIdle(_ controllers: [DeviceWindowController]) async throws -> Idle { + func draws() -> Int { controllers.map(\.drawCount).reduce(0, +) } + func frames() -> Int { controllers.map(\.framesReceived).reduce(0, +) } + var cpu: [Double] = [] + var memory: [Double] = [] + let drawsStart = draws() + let framesStart = frames() + let energyStart = try Self.cpuEnergyJoules() + let gpuStart = try Self.gpuSeconds() + let started = ContinuousClock.now + for _ in 0.. MemorySoak { + var targets: [(controller: DeviceWindowController, view: NSView, spots: [CGPoint])] = [] + var folds: [NSSegmentedControl] = [] + for controller in controllers { + let window = try XCTUnwrap(controller.window) + if let model = Self.find(DuoModelView.self, in: window) { + let box = model.bounds + let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) + .filter { model.screenPoint(at: $0) != nil } + if !spots.isEmpty { targets.append((controller, model, spots)) } + folds.append(try XCTUnwrap(Self.find(NSSegmentedControl.self, in: window), "the window's fold positions")) + } else { + let screen = try XCTUnwrap(Self.find(DeviceScreenView.self, in: window)) + let box = screen.bounds + targets.append((controller, screen, Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size))) + } + } + XCTAssertFalse(targets.isEmpty, "no window has a spot to click") + + let started = ContinuousClock.now + func elapsed() -> Double { Self.seconds(ContinuousClock.now - started) } + var samples = [(seconds: 0.0, megabytes: try Self.footprintMB())] + var nextSample = Self.soakSampleSeconds + var nextFold = Self.soakFoldEverySeconds + var clicks = 0 + var foldCount = 0 + while elapsed() < seconds { + if !folds.isEmpty, elapsed() >= nextFold { + for mode in [DeviceControlBar.FoldMode.cover, .fullyOpen] { + for control in folds { + control.selectedSegment = mode.rawValue + control.sendAction(control.action, to: control.target) + } + try await Task.sleep(for: .seconds(3)) + } + foldCount += folds.count + nextFold += Self.soakFoldEverySeconds + } + if !targets.isEmpty { + let target = targets[clicks % targets.count] + let spot = target.spots[(clicks / targets.count) % target.spots.count] + let window = try XCTUnwrap(target.view.window) + let inWindow = target.view.convert(spot, to: nil) + target.view.mouseDown(with: try Self.mouse(.leftMouseDown, at: inWindow, in: window)) + try await Task.sleep(for: .milliseconds(60)) + target.view.mouseUp(with: try Self.mouse(.leftMouseUp, at: inWindow, in: window)) + clicks += 1 + } + try await Task.sleep(for: .seconds(Self.soakClickSeconds)) + while elapsed() >= nextSample { + samples.append((elapsed(), try Self.footprintMB())) + nextSample += Self.soakSampleSeconds + } + } + let warmUp = min(30, seconds / 4) + return MemorySoak( + seconds: seconds, + clicks: clicks, + folds: foldCount, + startMB: samples[0].megabytes, + endMB: samples[samples.count - 1].megabytes, + growthMBPerMinute: MemorySoak.slope(samples.filter { $0.seconds >= warmUp }) + ) + } + + /// Each click lands on the test host, which writes where it was touched and so always redraws. + /// The first contact pays for turning the input feature on, so it is not one of the readings. + private func measureClicks( + _ controller: DeviceWindowController, + on view: NSView, + spots: [CGPoint] + ) async throws -> Spread { + let window = try XCTUnwrap(view.window) + var samples: [Double] = [] + for index in -1..= 0, let reading = controller.latencyReading, reading.sampleCount > counted { + samples.append(reading.lastMilliseconds) + } + try await Task.sleep(for: .milliseconds(300)) + } + XCTAssertGreaterThanOrEqual(samples.count, Self.clicks * 2 / 3, "most clicks drew no frame") + guard !samples.isEmpty else { throw XCTSkip("no click drew a frame, so latency could not be measured") } + return Spread(samples) + } + + private static func spots(around centre: CGPoint, size: CGSize) -> [CGPoint] { + let step = CGSize(width: size.width * 0.15, height: size.height * 0.15) + return [(-1, -1), (1, -1), (0, 0), (-1, 1), (1, 1)].map { dx, dy in + CGPoint(x: centre.x + CGFloat(dx) * step.width, y: centre.y + CGFloat(dy) * step.height) + } + } + + static func environment() throws -> PerformanceEnvironment { + var size = 0 + sysctlbyname("hw.model", nil, &size, nil, 0) + var bytes = [CChar](repeating: 0, count: size) + sysctlbyname("hw.model", &bytes, &size, nil, 0) + let model = String(decoding: bytes.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self) + let system = ProcessInfo.processInfo.operatingSystemVersion + let xcode = try XcodeLocator.locate() + #if DEBUG + let configuration = "debug" + #else + let configuration = "release" + #endif + return PerformanceEnvironment( + hardwareModel: model, + macOS: "\(system.majorVersion).\(system.minorVersion).\(system.patchVersion)", + xcode: "Xcode \(xcode.version) (\(xcode.build))", + lowPowerMode: ProcessInfo.processInfo.isLowPowerModeEnabled, + buildConfiguration: configuration + ) + } + + private static func cpuSeconds() -> Double { + var usage = rusage() + getrusage(RUSAGE_SELF, &usage) + func seconds(_ time: timeval) -> Double { Double(time.tv_sec) + Double(time.tv_usec) / 1_000_000 } + return seconds(usage.ru_utime) + seconds(usage.ru_stime) + } + + private static func footprintMB() throws -> Double { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + guard result == KERN_SUCCESS else { throw MeasurementUnavailable(what: "the process's memory footprint") } + return Double(info.phys_footprint) / 1_048_576 + } + + /// The CPU's energy alone: a GPU bound Metal loop barely moves it (macOS 27.0, Mac15,6). + private static func cpuEnergyJoules() throws -> Double { + var info = task_power_info_v2() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_POWER_INFO_V2), $0, &count) + } + } + guard result == KERN_SUCCESS else { throw MeasurementUnavailable(what: "the process's energy") } + return Double(info.task_energy) / 1e9 + } + + /// The GPU driver's own undocumented count of each process's GPU time, kept on its user clients + /// in the I/O Registry. For frame sized work it read within a third of Metal's own timing, and + /// no closer (macOS 27.0, Mac15,6). + private static func gpuSeconds() throws -> Double { + func property(_ entry: io_registry_entry_t, _ key: String) -> Any? { + IORegistryEntryCreateCFProperty(entry, key as CFString, kCFAllocatorDefault, 0)?.takeRetainedValue() + } + let creator = "pid \(getpid())," + var accelerators: io_iterator_t = 0 + guard IOServiceGetMatchingServices(kIOMainPortDefault, IOServiceMatching("IOAccelerator"), &accelerators) == KERN_SUCCESS else { + throw MeasurementUnavailable(what: "the GPU") + } + defer { IOObjectRelease(accelerators) } + var nanoseconds: UInt64 = 0 + var found = false + while case let accelerator = IOIteratorNext(accelerators), accelerator != 0 { + defer { IOObjectRelease(accelerator) } + var clients: io_iterator_t = 0 + guard IORegistryEntryGetChildIterator(accelerator, kIOServicePlane, &clients) == KERN_SUCCESS else { continue } + defer { IOObjectRelease(clients) } + while case let client = IOIteratorNext(clients), client != 0 { + defer { IOObjectRelease(client) } + guard let owner = property(client, "IOUserClientCreator") as? String, owner.hasPrefix(creator), + let usage = property(client, "AppUsage") as? [[String: Any]] else { continue } + found = true + nanoseconds += usage.compactMap { ($0["accumulatedGPUTime"] as? NSNumber)?.uint64Value }.reduce(0, +) + } + } + guard found else { throw MeasurementUnavailable(what: "this process's GPU time") } + return Double(nanoseconds) / 1e9 + } + + private static func seconds(_ duration: Duration) -> Double { + Double(duration.components.seconds) + Double(duration.components.attoseconds) / 1e18 + } + + private static func milliseconds(_ duration: Duration) -> Double { + seconds(duration) * 1000 + } + + private static func json(_ value: T) throws -> Data { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + return try encoder.encode(value) + } + + private static func find(_ type: T.Type, in window: NSWindow) -> T? { + func search(_ view: NSView) -> T? { + if let found = view as? T { return found } + for subview in view.subviews { + if let found = search(subview) { return found } + } + return nil + } + return window.contentView.flatMap(search) + } + + private static func mouse(_ type: NSEvent.EventType, at point: CGPoint, in window: NSWindow) throws -> NSEvent { + try XCTUnwrap(NSEvent.mouseEvent( + with: type, location: point, modifierFlags: [], + timestamp: ProcessInfo.processInfo.systemUptime, + windowNumber: window.windowNumber, context: nil, + eventNumber: 0, clickCount: 1, pressure: 1 + )) + } + + @discardableResult + private func run(_ arguments: [String]) throws -> String { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") + process.arguments = arguments + process.environment = ProcessInfo.processInfo.environment + let output = Pipe() + process.standardOutput = output + process.standardError = Pipe() + try process.run() + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + return String(decoding: data, as: UTF8.self) + } +} + +private struct MeasurementUnavailable: Error, CustomStringConvertible { + let what: String + var description: String { "could not read \(what)" } +} + +private final class DiscardingStorage: PreferenceStorage, @unchecked Sendable { + func text(forKey key: String) -> String? { nil } + func setText(_ text: String, forKey key: String) {} + func removeText(forKey key: String) {} + func keys(withPrefix prefix: String) -> [String] { [] } +} diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift new file mode 100644 index 0000000..fc4e5a2 --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift @@ -0,0 +1,291 @@ +import XCTest + +final class PerformanceComparisonTests: XCTestCase { + private let environment = PerformanceEnvironment( + hardwareModel: "Mac16,7", macOS: "27.0", xcode: "Xcode 27.0 (27A266a)", + lowPowerMode: false, buildConfiguration: "release" + ) + + private func window( + _ name: String = "foldable", + cpu: Double = 10, + draws: Double = 60, + memory: Double = 350, + latency: Double = 20, + gap: Double? = 34, + energy: Double? = nil, + gpu: Double? = nil, + open: Double? = nil, + kept: Double? = nil, + device: String = "iPhone Duo (iOS 27.1)" + ) -> WindowMeasurement { + WindowMeasurement( + name: name, + device: device, + idleCPUPercent: Spread(median: cpu, p95: cpu * 2, samples: 10), + idleDrawsPerSecond: draws, + idleFramesPerSecond: 0.1, + windowMemoryMB: memory, + processMemoryMB: memory + 60, + clickLatencyMs: Spread(median: latency, p95: latency * 1.5, samples: 15), + foldStepGapMs: gap.map { Spread(median: $0 / 2, p95: $0, samples: 60) }, + foldCPUPercent: gap == nil ? nil : 15, + idleCPUEnergyMilliwatts: energy, + idleGPUPercent: gpu, + open: open.map(openTime), + reopen: kept.map { + Reopen( + opens: [openTime(420), openTime(380), openTime(400)], + keptAfterCloseMB: 250, + addedByEachReopenMB: [$0 * 3, $0, 0], + freedOnClose: true + ) + } + ) + } + + private func openTime(_ total: Double) -> OpenTime { + OpenTime(sessionsMs: total / 4, orientationMs: total / 2, windowMs: total / 8, firstPictureMs: total / 8) + } + + private func together( + cpu: Double = 12, + growth: Double = 1, + seconds: Double = 180, + devices: [String] = ["iPhone 17 (iOS 27.1)", "iPhone Duo (iOS 27.1)"] + ) -> TogetherMeasurement { + TogetherMeasurement( + devices: devices, + idleCPUPercent: Spread(median: cpu, p95: cpu * 2, samples: 10), + idleCPUEnergyMilliwatts: 150, + idleGPUPercent: 3, + idleDrawsPerSecond: 30, + idleFramesPerSecond: 0.2, + windowsMemoryMB: 400, + processMemoryMB: 520, + soak: MemorySoak(seconds: seconds, clicks: 300, folds: 8, startMB: 520, endMB: 530, growthMBPerMinute: growth) + ) + } + + private func report( + _ windows: [WindowMeasurement], + together: TogetherMeasurement? = nil, + environment: PerformanceEnvironment? = nil + ) -> PerformanceReport { + PerformanceReport(createdAt: "now", environment: environment ?? self.environment, windows: windows, together: together) + } + + func testTheLimitIsTheLargerOfAShareAndAFixedAmount() { + let tolerance = PerformanceComparison.Tolerance(relative: 0.5, absolute: 3) + XCTAssertEqual(tolerance.limit(for: 1), 4, "near zero the fixed amount decides") + XCTAssertEqual(tolerance.limit(for: 20), 30, "further up the share decides") + } + + func testTheFoldablesReopensHaveTheirOwnWiderLimit() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(kept: 40), window("phone", gap: nil, kept: 40)]), + baseline: report([window(kept: 0), window("phone", gap: nil, kept: 0)]) + ) + XCTAssertEqual(findings.map(\.window), ["phone"], "the model's memory swings; the phone's does not") + XCTAssertEqual(findings.first?.limit, 10) + } + + func testAReopenIsJudgedByItsMedians() { + let reopen = Reopen( + opens: [openTime(420), openTime(380), openTime(400)], + keptAfterCloseMB: 250, + addedByEachReopenMB: [24, 0.5, -1], + freedOnClose: true + ) + XCTAssertEqual(reopen.medianOpen, openTime(400)) + XCTAssertEqual(reopen.medianAddedMB, 0.5, "only the first reopen growing is a cache, not a leak") + } + + func testABaselineBelowZeroIsHeldToZeroPlusTheFixedAmount() { + let tolerance = PerformanceComparison.Tolerance(relative: 0.5, absolute: 5) + XCTAssertEqual(tolerance.limit(for: -6.8), 5, "memory that shrank is not a limit below zero") + } + + func testMemoryKeptByEachReopenBeyondItsLimitIsReported() throws { + let findings = try PerformanceComparison.regressions( + current: report([window("phone", gap: nil, kept: 40)]), + baseline: report([window("phone", gap: nil, kept: 2)]) + ) + XCTAssertEqual(findings.map(\.metric), ["memory kept by each reopen (MB)"]) + XCTAssertEqual(findings.first?.limit, 12) + } + + func testTheSpreadTakesTheSampleAtEachPercentile() { + let spread = Spread([5, 1, 4, 2, 3, 9, 8, 7, 6, 10]) + XCTAssertEqual(spread.median, 5) + XCTAssertEqual(spread.p95, 10) + XCTAssertEqual(spread.samples, 10) + } + + func testARunWithinToleranceHasNoRegressions() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 12, memory: 380, latency: 24)]), + baseline: report([window()]) + ) + XCTAssertEqual(findings, []) + } + + func testAnImprovementIsNotARegression() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 0.5, draws: 0.2, memory: 120)]), + baseline: report([window()]) + ) + XCTAssertEqual(findings, []) + } + + func testEachMetricBeyondItsLimitIsReported() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 16, draws: 60, memory: 450, latency: 40, gap: 60)]), + baseline: report([window()]) + ) + XCTAssertEqual(Set(findings.map(\.metric)), [ + "idle CPU median (%)", "window memory (MB)", "click latency median (ms)", + "click latency p95 (ms)", "fold step gap p95 (ms)", + ]) + let cpu = try XCTUnwrap(findings.first { $0.metric == "idle CPU median (%)" }) + XCTAssertEqual(cpu.baseline, 10) + XCTAssertEqual(cpu.current, 16) + XCTAssertEqual(cpu.limit, 15) + } + + func testAWindowWithoutFoldMetricsIsComparedOnTheRest() throws { + let findings = try PerformanceComparison.regressions( + current: report([window("phone", cpu: 0, draws: 0, memory: 60, gap: nil)]), + baseline: report([window("phone", cpu: 0, draws: 0, memory: 60, gap: nil)]) + ) + XCTAssertEqual(findings, []) + } + + func testABaselineFromAnotherMachineIsRefused() { + var other = environment + other = PerformanceEnvironment( + hardwareModel: other.hardwareModel, macOS: other.macOS, xcode: "Xcode 26.5 (17F42)", + lowPowerMode: other.lowPowerMode, buildConfiguration: other.buildConfiguration + ) + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()]), + baseline: report([window()], environment: other) + )) { error in + guard case PerformanceComparison.Incompatible.environment = error else { + return XCTFail("wrong error: \(error)") + } + } + } + + func testAWindowTheBaselineMeasuredMustBeMeasuredAgain() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window("phone", gap: nil)]), + baseline: report([window("phone", gap: nil), window("foldable")]) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .missingWindow("foldable")) + } + } + + func testTheReportRoundTripsThroughJSON() throws { + let original = report( + [window(energy: 120, gpu: 4, open: 900, kept: 3), window("phone", gap: nil)], + together: together() + ) + let decoded = try JSONDecoder().decode(PerformanceReport.self, from: JSONEncoder().encode(original)) + XCTAssertEqual(decoded, original) + } + + func testEnergyGPUAndOpenTimeBeyondTheirLimitsAreReported() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(energy: 200, gpu: 7, open: 1700)]), + baseline: report([window(energy: 100, gpu: 4, open: 800)]) + ) + XCTAssertEqual(Set(findings.map(\.metric)), [ + "idle CPU energy (mW)", "idle GPU (%)", "open to first picture (ms)", + ]) + let open = try XCTUnwrap(findings.first { $0.metric == "open to first picture (ms)" }) + XCTAssertEqual(open.limit, 1600, "the first open fails only once it doubles") + } + + func testABaselineFromBeforeTheNewerMetricsStillChecksTheRest() throws { + let json = String(decoding: try JSONEncoder().encode(report([window()])), as: UTF8.self) + XCTAssertFalse(json.contains("idleGPUPercent")) + XCTAssertFalse(json.contains("together")) + let baseline = try JSONDecoder().decode(PerformanceReport.self, from: Data(json.utf8)) + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 16, energy: 900, gpu: 50, open: 9000)], together: together()), + baseline: baseline + ) + XCTAssertEqual(findings.map(\.metric), ["idle CPU median (%)"]) + } + + func testTheWindowsOpenedTogetherAreComparedUnderTheirOwnName() throws { + let findings = try PerformanceComparison.regressions( + current: report([window()], together: together(cpu: 20, growth: 17)), + baseline: report([window()], together: together()) + ) + XCTAssertEqual(findings.map(\.window), ["together", "together"]) + XCTAssertEqual(Set(findings.map(\.metric)), ["idle CPU median (%)", "memory growth (MB per minute)"]) + } + + func testTheWindowsOpenedTogetherMustBeMeasuredAgain() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()]), + baseline: report([window()], together: together()) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .missingWindow("together")) + } + } + + func testOtherBootedDevicesAreRefused() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()], together: together(devices: ["iPhone 17 (iOS 27.1)"])), + baseline: report([window()], together: together()) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .differentDevices( + window: "together", + baseline: "iPhone 17 (iOS 27.1), iPhone Duo (iOS 27.1)", + current: "iPhone 17 (iOS 27.1)" + )) + } + } + + func testAWindowOnAnotherDeviceIsRefused() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window(device: "iPhone Duo (iOS 27.0)")]), + baseline: report([window()]) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .differentDevices( + window: "foldable", baseline: "iPhone Duo (iOS 27.1)", current: "iPhone Duo (iOS 27.0)" + )) + } + } + + func testAnotherBusyTimeIsRefused() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()], together: together(seconds: 60)), + baseline: report([window()], together: together()) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .differentSoak(baseline: 180, current: 60)) + } + } + + func testTheGrowthIsTheSlopeInMegabytesAMinute() { + XCTAssertEqual(MemorySoak.slope([(0, 100), (30, 101), (60, 102)]), 2, accuracy: 1e-9) + XCTAssertEqual(MemorySoak.slope([(0, 100), (5, 104), (10, 100), (15, 104)]), 9.6, accuracy: 1e-9) + XCTAssertEqual(MemorySoak.slope([(10, 100), (20, 100)]), 0) + XCTAssertEqual(MemorySoak.slope([(10, 100)]), 0, "one sample has no slope") + } + + func testTheOpenTimeIsTheSumOfItsParts() { + let open = OpenTime(sessionsMs: 40, orientationMs: 700, windowMs: 60, firstPictureMs: 25) + XCTAssertEqual(open.totalMs, 825) + } + + func testTheReportSetsTheWindowsTogetherBesideTheirSumAlone() { + let phone = window("phone", cpu: 0.5, gap: nil, device: "iPhone 17 (iOS 27.1)") + let text = PerformanceComparison.text(report([phone, window(cpu: 6)], together: together(cpu: 7))) + XCTAssertTrue(text.contains("All 2 windows together: iPhone 17 (iOS 27.1), iPhone Duo (iOS 27.1)"), text) + XCTAssertTrue(text.contains("median 7.0%, p95 14.0%, the windows alone add up to 6.5%"), text) + } +} diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift new file mode 100644 index 0000000..8103477 --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift @@ -0,0 +1,359 @@ +import Foundation + +/// What one run of the performance check measured, and the machine it measured it on. Numbers +/// only compare with a baseline taken on the same machine, OS, Xcode and build configuration. +struct PerformanceReport: Codable, Equatable { + static let schemaVersion = 1 + + var schemaVersion = Self.schemaVersion + let createdAt: String + let environment: PerformanceEnvironment + let windows: [WindowMeasurement] + /// Absent from reports taken before every window was also measured open at once. + var together: TogetherMeasurement? = nil +} + +struct PerformanceEnvironment: Codable, Equatable { + let hardwareModel: String + let macOS: String + let xcode: String + let lowPowerMode: Bool + let buildConfiguration: String +} + +/// The median and the nearest rank 95th percentile: the sample at or above that share of the +/// sorted run. +struct Spread: Codable, Equatable { + let median: Double + let p95: Double + let samples: Int + + init(median: Double, p95: Double, samples: Int) { + self.median = median + self.p95 = p95 + self.samples = samples + } + + init(_ values: [Double]) { + precondition(!values.isEmpty, "a spread needs at least one sample") + median = Self.percentile(values, 0.5) + p95 = Self.percentile(values, 0.95) + samples = values.count + } + + static func percentile(_ values: [Double], _ fraction: Double) -> Double { + let sorted = values.sorted() + return sorted[max(0, Int((Double(sorted.count) * fraction).rounded(.up)) - 1)] + } +} + +struct WindowMeasurement: Codable, Equatable { + let name: String + let device: String + let idleCPUPercent: Spread + let idleDrawsPerSecond: Double + let idleFramesPerSecond: Double + let windowMemoryMB: Double + let processMemoryMB: Double + let clickLatencyMs: Spread + let foldStepGapMs: Spread? + let foldCPUPercent: Double? + /// These are absent from reports taken before they were measured. + var idleCPUEnergyMilliwatts: Double? = nil + var idleGPUPercent: Double? = nil + var open: OpenTime? = nil + var reopen: Reopen? = nil +} + +/// The window opened and closed a few more times after the first closed. What the first left behind +/// can be a cache the next reuses; what grows again with every reopen is kept by every window. +struct Reopen: Codable, Equatable { + let opens: [OpenTime] + let keptAfterCloseMB: Double + /// For each reopen, memory once it closed less memory once the window before it closed. + let addedByEachReopenMB: [Double] + /// Whether every reopened window's controller was freed once it closed. + let freedOnClose: Bool + + var medianOpen: OpenTime { + let median = Spread(opens.map(\.totalMs)).median + return opens.first { $0.totalMs == median } ?? opens[0] + } + + var medianAddedMB: Double { Spread(addedByEachReopenMB).median } +} + +/// From starting to open a window, as the app does, to the device's first picture drawn in it. +struct OpenTime: Codable, Equatable { + /// The panels, the display sessions and the input session. + let sessionsMs: Double + let orientationMs: Double + let windowMs: Double + let firstPictureMs: Double + let totalMs: Double + + init(sessionsMs: Double, orientationMs: Double, windowMs: Double, firstPictureMs: Double) { + self.sessionsMs = sessionsMs + self.orientationMs = orientationMs + self.windowMs = windowMs + self.firstPictureMs = firstPictureMs + totalMs = sessionsMs + orientationMs + windowMs + firstPictureMs + } +} + +/// Every booted device's window open at once, then kept busy for a while to see whether memory +/// keeps growing. +struct TogetherMeasurement: Codable, Equatable { + let devices: [String] + let idleCPUPercent: Spread + let idleCPUEnergyMilliwatts: Double + let idleGPUPercent: Double + let idleDrawsPerSecond: Double + let idleFramesPerSecond: Double + /// Over the process before any window opened, since what the windows measured alone kept after + /// closing is reused by these and would otherwise be left out. + let windowsMemoryMB: Double + let processMemoryMB: Double + let soak: MemorySoak +} + +struct MemorySoak: Codable, Equatable { + let seconds: Double + let clicks: Int + let folds: Int + let startMB: Double + let endMB: Double + /// The least squares slope of the samples taken after the first quarter, at most 30 seconds, + /// which is when caches are still filling. + let growthMBPerMinute: Double + + static func slope(_ samples: [(seconds: Double, megabytes: Double)]) -> Double { + guard samples.count > 1 else { return 0 } + let count = Double(samples.count) + let meanTime = samples.map(\.seconds).reduce(0, +) / count + let meanSize = samples.map(\.megabytes).reduce(0, +) / count + let covariance = samples.map { ($0.seconds - meanTime) * ($0.megabytes - meanSize) }.reduce(0, +) + let variance = samples.map { ($0.seconds - meanTime) * ($0.seconds - meanTime) }.reduce(0, +) + return variance > 0 ? covariance / variance * 60 : 0 + } +} + +enum PerformanceComparison { + /// A limit of the baseline plus the larger of a share of it and a fixed amount, so a metric + /// near zero, idle CPU above all, is not failed for noise. A baseline below zero, memory that + /// shrank, is held to zero plus the fixed amount for the same reason. + struct Tolerance: Equatable { + let relative: Double + let absolute: Double + + func limit(for baseline: Double) -> Double { + let base = max(baseline, 0) + return base + max(base * relative, absolute) + } + } + + struct Finding: Codable, Equatable { + let window: String + let metric: String + let baseline: Double + let current: Double + let limit: Double + } + + enum Incompatible: Error, Equatable, CustomStringConvertible { + case schema(baseline: Int, current: Int) + case environment(baseline: PerformanceEnvironment, current: PerformanceEnvironment) + case missingWindow(String) + case differentDevices(window: String, baseline: String, current: String) + case differentSoak(baseline: Double, current: Double) + + var description: String { + switch self { + case .schema(let baseline, let current): + "the baseline is schema \(baseline) and this run is schema \(current)" + case .environment(let baseline, let current): + "the baseline was measured on \(baseline) and this run on \(current)" + case .missingWindow(let name): + "the baseline measured a \(name) window and this run did not" + case .differentDevices(let window, let baseline, let current): + "the baseline measured the \(window) window on \(baseline) and this run on \(current)" + case .differentSoak(let baseline, let current): + "the baseline kept the windows busy for \(Int(baseline)) seconds and this run for \(Int(current))" + } + } + } + + /// Every metric is one where less is better. + static let tolerances: [String: Tolerance] = [ + "idle CPU median (%)": Tolerance(relative: 0.5, absolute: 3), + "idle draws per second": Tolerance(relative: 0.5, absolute: 5), + "window memory (MB)": Tolerance(relative: 0.2, absolute: 30), + "click latency median (ms)": Tolerance(relative: 0.3, absolute: 8), + "click latency p95 (ms)": Tolerance(relative: 0.3, absolute: 12), + "fold step gap p95 (ms)": Tolerance(relative: 0.3, absolute: 10), + "fold CPU (%)": Tolerance(relative: 0.5, absolute: 5), + "idle CPU energy (mW)": Tolerance(relative: 0.5, absolute: 10), + "idle GPU (%)": Tolerance(relative: 0.5, absolute: 2), + // The first open in a process pays for loading what the rest reuse and moved by more than + // half between runs, so only a doubling fails it. + "open to first picture (ms)": Tolerance(relative: 1, absolute: 300), + "open again to first picture (ms)": Tolerance(relative: 0.3, absolute: 150), + "memory kept by each reopen (MB)": Tolerance(relative: 0.5, absolute: 10), + // Three busy minutes with nothing leaking read from -9 to 13 MB a minute over nine runs, as + // the model's memory dips and comes back by tens of MB. + "memory growth (MB per minute)": Tolerance(relative: 0.5, absolute: 15), + ] + + /// Where one kind of window is noisier than the rest, keyed by window and then metric. + static let windowTolerances: [String: [String: Tolerance]] = [ + "foldable": [ + // The model's memory moves by up to 50 MB from one reopen to the next with nothing + // leaking (27A9269), while a whole window is about 300 MB. + "memory kept by each reopen (MB)": Tolerance(relative: 0.5, absolute: 60), + ], + ] + + static func metrics(of window: WindowMeasurement) -> [String: Double] { + var values = [ + "idle CPU median (%)": window.idleCPUPercent.median, + "idle draws per second": window.idleDrawsPerSecond, + "window memory (MB)": window.windowMemoryMB, + "click latency median (ms)": window.clickLatencyMs.median, + "click latency p95 (ms)": window.clickLatencyMs.p95, + ] + if let gaps = window.foldStepGapMs { values["fold step gap p95 (ms)"] = gaps.p95 } + if let cpu = window.foldCPUPercent { values["fold CPU (%)"] = cpu } + if let energy = window.idleCPUEnergyMilliwatts { values["idle CPU energy (mW)"] = energy } + if let gpu = window.idleGPUPercent { values["idle GPU (%)"] = gpu } + if let open = window.open { values["open to first picture (ms)"] = open.totalMs } + if let reopen = window.reopen { + values["open again to first picture (ms)"] = reopen.medianOpen.totalMs + values["memory kept by each reopen (MB)"] = reopen.medianAddedMB + } + return values + } + + static func metrics(of together: TogetherMeasurement) -> [String: Double] { + [ + "idle CPU median (%)": together.idleCPUPercent.median, + "idle draws per second": together.idleDrawsPerSecond, + "window memory (MB)": together.windowsMemoryMB, + "idle CPU energy (mW)": together.idleCPUEnergyMilliwatts, + "idle GPU (%)": together.idleGPUPercent, + "memory growth (MB per minute)": together.soak.growthMBPerMinute, + ] + } + + static func regressions( + current: PerformanceReport, + baseline: PerformanceReport + ) throws(Incompatible) -> [Finding] { + guard baseline.schemaVersion == current.schemaVersion else { + throw .schema(baseline: baseline.schemaVersion, current: current.schemaVersion) + } + guard baseline.environment == current.environment else { + throw .environment(baseline: baseline.environment, current: current.environment) + } + var findings: [Finding] = [] + for base in baseline.windows { + guard let now = current.windows.first(where: { $0.name == base.name }) else { + throw .missingWindow(base.name) + } + guard now.device == base.device else { + throw .differentDevices(window: base.name, baseline: base.device, current: now.device) + } + findings += compare(base.name, metrics(of: base), metrics(of: now)) + } + if let base = baseline.together { + guard let now = current.together else { throw .missingWindow(togetherName) } + guard now.devices == base.devices else { + throw .differentDevices( + window: togetherName, + baseline: base.devices.joined(separator: ", "), + current: now.devices.joined(separator: ", ") + ) + } + guard now.soak.seconds == base.soak.seconds else { + throw .differentSoak(baseline: base.soak.seconds, current: now.soak.seconds) + } + findings += compare(togetherName, metrics(of: base), metrics(of: now)) + } + return findings + } + + static let togetherName = "together" + + /// A metric only the baseline or only this run has is skipped, so a baseline taken before a + /// metric existed still checks the rest. + private static func compare(_ window: String, _ before: [String: Double], _ after: [String: Double]) -> [Finding] { + before.sorted(by: { $0.key < $1.key }).compactMap { metric, value in + guard let measured = after[metric], + let tolerance = windowTolerances[window]?[metric] ?? tolerances[metric] else { return nil } + let limit = tolerance.limit(for: value) + guard measured > limit else { return nil } + return Finding(window: window, metric: metric, baseline: value, current: measured, limit: limit) + } + } + + /// The report as a person reads it, one window after another. + static func text(_ report: PerformanceReport, findings: [Finding]? = nil) -> String { + func number(_ value: Double) -> String { String(format: "%.1f", value) } + var lines = [ + "Performance check \(report.createdAt)", + "\(report.environment.hardwareModel), macOS \(report.environment.macOS), \(report.environment.xcode), \(report.environment.buildConfiguration)\(report.environment.lowPowerMode ? ", Low Power Mode" : "")", + "", + ] + for window in report.windows { + lines.append("\(window.name): \(window.device)") + lines.append(" idle CPU median \(number(window.idleCPUPercent.median))%, p95 \(number(window.idleCPUPercent.p95))% over \(window.idleCPUPercent.samples) samples") + lines.append(" idle drawing \(number(window.idleDrawsPerSecond)) pictures a second for \(number(window.idleFramesPerSecond)) frames a second from the device") + lines.append(" memory \(number(window.windowMemoryMB)) MB for the window, \(number(window.processMemoryMB)) MB for the process") + lines.append(" click to frame median \(number(window.clickLatencyMs.median)) ms, p95 \(number(window.clickLatencyMs.p95)) ms over \(window.clickLatencyMs.samples) clicks") + if let gaps = window.foldStepGapMs { + lines.append(" fold steps median gap \(number(gaps.median)) ms, p95 \(number(gaps.p95)) ms over \(gaps.samples) steps") + } + if let cpu = window.foldCPUPercent { + lines.append(" fold CPU \(number(cpu))%") + } + if let energy = window.idleCPUEnergyMilliwatts, let gpu = window.idleGPUPercent { + lines.append(" idle energy \(number(energy)) mW of CPU, GPU busy \(number(gpu))% of the time") + } + func parts(_ open: OpenTime) -> String { + "\(number(open.totalMs)) ms to the first picture: \(number(open.sessionsMs)) ms sessions, \(number(open.orientationMs)) ms orientation, \(number(open.windowMs)) ms window, \(number(open.firstPictureMs)) ms first picture" + } + if let open = window.open { + lines.append(" open \(parts(open))") + } + if let reopen = window.reopen { + lines.append(" open again median of \(reopen.opens.count), \(parts(reopen.medianOpen))") + lines.append(" after closing \(number(reopen.keptAfterCloseMB)) MB kept by the first window, then \(reopen.addedByEachReopenMB.map(number).joined(separator: ", ")) MB by each reopen; \(reopen.freedOnClose ? "every reopened window was freed" : "a reopened window outlived its window")") + } + lines.append("") + } + if let together = report.together { + let alone = together.devices.compactMap { device in report.windows.first { $0.device == device } } + let sum = alone.count == together.devices.count + ? ", the windows alone add up to \(number(alone.map(\.idleCPUPercent.median).reduce(0, +)))%" + : "" + lines.append("All \(together.devices.count) windows together: \(together.devices.joined(separator: ", "))") + lines.append(" idle CPU median \(number(together.idleCPUPercent.median))%, p95 \(number(together.idleCPUPercent.p95))%\(sum)") + lines.append(" idle drawing \(number(together.idleDrawsPerSecond)) pictures a second for \(number(together.idleFramesPerSecond)) frames a second from the devices") + lines.append(" idle energy \(number(together.idleCPUEnergyMilliwatts)) mW of CPU, GPU busy \(number(together.idleGPUPercent))% of the time") + lines.append(" memory \(number(together.windowsMemoryMB)) MB over the process before any window opened, \(number(together.processMemoryMB)) MB for the process") + let soak = together.soak + lines.append(" kept busy \(Int(soak.seconds)) s, \(soak.clicks) clicks, \(soak.folds) folds: \(number(soak.startMB)) MB to \(number(soak.endMB)) MB, growing \(number(soak.growthMBPerMinute)) MB a minute") + lines.append("") + } + if let findings { + if findings.isEmpty { + lines.append("No regressions against the baseline.") + } else { + lines.append("Regressions against the baseline:") + for finding in findings { + lines.append(" \(finding.window) \(finding.metric): \(number(finding.current)), baseline \(number(finding.baseline)), limit \(number(finding.limit))") + } + } + } + return lines.joined(separator: "\n") + "\n" + } +} diff --git a/engine/Tests/OpenDeviceHubViewerTests/WindowReleaseTests.swift b/engine/Tests/OpenDeviceHubViewerTests/WindowReleaseTests.swift new file mode 100644 index 0000000..4117483 --- /dev/null +++ b/engine/Tests/OpenDeviceHubViewerTests/WindowReleaseTests.swift @@ -0,0 +1,80 @@ +import AppKit +import XCTest +@testable import OpenDeviceHubEngine +@testable import OpenDeviceHubViewer + +/// A view that outlives its window keeps its drawables and its device's input session with it, +/// about 4 MB for every window opened. +@MainActor +final class WindowReleaseTests: XCTestCase { + func testAClosedWindowFreesItsControllerWindowAndScreen() async throws { + let manager = DeviceWindowManager( + frameStore: WindowFrameStore(storage: InMemoryPreferences(), prefix: "test."), + settings: ViewerSettings(storage: InMemoryPreferences(), prefix: "test."), + shutdown: { _ in } + ) + weak var controller: DeviceWindowController? + weak var window: NSWindow? + weak var screen: DeviceScreenView? + do { + let opened = try manager.open( + device: DeviceInfo( + udid: "released", + name: "iPhone 17", + deviceTypeIdentifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-17", + runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-27-0", + runtimeName: "iOS 27.0", + state: .booted, + isAvailable: true + ), + session: EmptySession(), + input: NoInput(), + scaleMode: .fit, + bezelEnabled: true, + keepOnTop: false, + showFPS: false + ) + controller = opened + window = opened.window + screen = opened.window.flatMap { Self.screen(in: $0) } + XCTAssertNotNil(screen) + manager.close("released") + } + for _ in 0..<20 where controller != nil || window != nil || screen != nil { + try await Task.sleep(for: .milliseconds(50)) + } + XCTAssertNil(controller, "the controller outlived its window") + XCTAssertNil(window, "the window outlived being closed") + XCTAssertNil(screen, "the screen view outlived its window") + } + + private static func screen(in window: NSWindow) -> DeviceScreenView? { + func search(_ view: NSView) -> DeviceScreenView? { + if let found = view as? DeviceScreenView { return found } + for subview in view.subviews { + if let found = search(subview) { return found } + } + return nil + } + return window.contentView.flatMap(search) + } +} + +/// Clicks are only wired up for a window that can send them. +private final class NoInput: InputSession, @unchecked Sendable { + func touch(_ event: TouchEvent) async throws {} + func key(_ event: KeyEvent) async throws {} + func button(_ button: HardwareButton, phase: ButtonPhase) async throws {} + func close() {} +} + +private final class EmptySession: DisplaySession, @unchecked Sendable { + let frames = AsyncStream { $0.finish() } + let screenChanges = AsyncStream { $0.finish() } + let pixelSize = CGSize(width: 1206, height: 2622) + let pointScale: CGFloat = 3 + let pixelsPerInch: CGFloat? = 460 + let supportsBezel = false + func setBezelEnabled(_ enabled: Bool) {} + func close() {} +} diff --git a/performance/baseline-low-power.json b/performance/baseline-low-power.json new file mode 100644 index 0000000..365503f --- /dev/null +++ b/performance/baseline-low-power.json @@ -0,0 +1,55 @@ +{ + "createdAt" : "2026-09-28T07:38:43Z", + "environment" : { + "buildConfiguration" : "release", + "hardwareModel" : "Mac15,6", + "lowPowerMode" : true, + "macOS" : "27.0.0", + "xcode" : "Xcode 27.1 (27A9269)" + }, + "schemaVersion" : 1, + "windows" : [ + { + "clickLatencyMs" : { + "median" : 13.091291666666667, + "p95" : 15.483583333333334, + "samples" : 20 + }, + "device" : "iPhone 17 (iOS 27.0)", + "idleCPUPercent" : { + "median" : 0.23452933463056475, + "p95" : 0.3247799798450037, + "samples" : 10 + }, + "idleDrawsPerSecond" : 0, + "idleFramesPerSecond" : 0, + "name" : "phone", + "processMemoryMB" : 152.17279052734375, + "windowMemoryMB" : 47.82826232910156 + }, + { + "clickLatencyMs" : { + "median" : 13.330625, + "p95" : 16.869375, + "samples" : 20 + }, + "device" : "iPhone Duo (iOS 27.1)", + "foldCPUPercent" : 23.654378541557307, + "foldStepGapMs" : { + "median" : 32.90175, + "p95" : 34.264084, + "samples" : 70 + }, + "idleCPUPercent" : { + "median" : 5.571381041923167, + "p95" : 8.18479697207644, + "samples" : 10 + }, + "idleDrawsPerSecond" : 18.98328772731046, + "idleFramesPerSecond" : 2.152743969076444, + "name" : "foldable", + "processMemoryMB" : 357.6105422973633, + "windowMemoryMB" : 295.1877746582031 + } + ] +} \ No newline at end of file diff --git a/performance/baseline-normal-power.json b/performance/baseline-normal-power.json new file mode 100644 index 0000000..180c063 --- /dev/null +++ b/performance/baseline-normal-power.json @@ -0,0 +1,162 @@ +{ + "createdAt" : "2026-09-28T14:22:18Z", + "environment" : { + "buildConfiguration" : "release", + "hardwareModel" : "Mac15,6", + "lowPowerMode" : false, + "macOS" : "27.0.0", + "xcode" : "Xcode 27.1 (27A9269)" + }, + "schemaVersion" : 1, + "together" : { + "devices" : [ + "iPhone 17 (iOS 27.0)", + "iPhone Duo (iOS 27.1)" + ], + "idleCPUEnergyMilliwatts" : 1.653480535325553, + "idleCPUPercent" : { + "median" : 1.2256303924930867, + "p95" : 1.6418494785014763, + "samples" : 10 + }, + "idleDrawsPerSecond" : 2.356308363298762, + "idleFramesPerSecond" : 2.356308363298762, + "idleGPUPercent" : 0.30492614919984484, + "processMemoryMB" : 402.1261215209961, + "soak" : { + "clicks" : 228, + "endMB" : 402.1261215209961, + "folds" : 8, + "growthMBPerMinute" : 2.3454191976695675, + "seconds" : 180, + "startMB" : 391.4698715209961 + }, + "windowsMemoryMB" : 380.3284912109375 + }, + "windows" : [ + { + "clickLatencyMs" : { + "median" : 12.212416666666666, + "p95" : 20.951125, + "samples" : 20 + }, + "device" : "iPhone 17 (iOS 27.0)", + "idleCPUEnergyMilliwatts" : 1.7159928379051423, + "idleCPUPercent" : { + "median" : 0.2203408604497964, + "p95" : 0.874295276459982, + "samples" : 10 + }, + "idleDrawsPerSecond" : 0.09745445393163028, + "idleFramesPerSecond" : 0.09745445393163028, + "idleGPUPercent" : 0.008464528414299162, + "name" : "phone", + "open" : { + "firstPictureMs" : 75.851209, + "orientationMs" : 303.730458, + "sessionsMs" : 21.275917, + "totalMs" : 688.528167, + "windowMs" : 287.67058299999997 + }, + "processMemoryMB" : 149.95404052734375, + "reopen" : { + "addedByEachReopenMB" : [ + 3.5, + 0.171875, + 0 + ], + "freedOnClose" : true, + "keptAfterCloseMB" : 11.562591552734375, + "opens" : [ + { + "firstPictureMs" : 17.954167, + "orientationMs" : 158.828875, + "sessionsMs" : 18.226499999999998, + "totalMs" : 213.850917, + "windowMs" : 18.841375 + }, + { + "firstPictureMs" : 17.68375, + "orientationMs" : 166.909625, + "sessionsMs" : 31.184917, + "totalMs" : 230.490542, + "windowMs" : 14.71225 + }, + { + "firstPictureMs" : 17.194417, + "orientationMs" : 177.183541, + "sessionsMs" : 32.812792, + "totalMs" : 239.63904200000002, + "windowMs" : 12.448292 + } + ] + }, + "windowMemoryMB" : 36.23448944091797 + }, + { + "clickLatencyMs" : { + "median" : 9.792833333333334, + "p95" : 17.169291666666666, + "samples" : 20 + }, + "device" : "iPhone Duo (iOS 27.1)", + "foldCPUPercent" : 21.496221395481253, + "foldStepGapMs" : { + "median" : 16.396542, + "p95" : 19.593458000000002, + "samples" : 126 + }, + "idleCPUEnergyMilliwatts" : 4.596634097579818, + "idleCPUPercent" : { + "median" : 0.6751292239530251, + "p95" : 1.126026700718238, + "samples" : 10 + }, + "idleDrawsPerSecond" : 2.210415008503586, + "idleFramesPerSecond" : 2.161294674981284, + "idleGPUPercent" : 0.2093037894564363, + "name" : "foldable", + "open" : { + "firstPictureMs" : 122.354041, + "orientationMs" : 693.304166, + "sessionsMs" : 174.235959, + "totalMs" : 2182.395, + "windowMs" : 1192.5008340000002 + }, + "processMemoryMB" : 348.6104736328125, + "reopen" : { + "addedByEachReopenMB" : [ + 32.390647888183594, + -49.140625, + 48.21875 + ], + "freedOnClose" : true, + "keptAfterCloseMB" : 70.48458099365234, + "opens" : [ + { + "firstPictureMs" : 94.341583, + "orientationMs" : 386.609916, + "sessionsMs" : 76.84329199999999, + "totalMs" : 1182.4217079999999, + "windowMs" : 624.6269169999999 + }, + { + "firstPictureMs" : 77.06837499999999, + "orientationMs" : 333.732709, + "sessionsMs" : 59.454791, + "totalMs" : 875.5543749999999, + "windowMs" : 405.2985 + }, + { + "firstPictureMs" : 79.98979200000001, + "orientationMs" : 382.287416, + "sessionsMs" : 39.989584, + "totalMs" : 948.8692089999998, + "windowMs" : 446.60241699999995 + } + ] + }, + "windowMemoryMB" : 318.92205810546875 + } + ] +} \ No newline at end of file diff --git a/scripts/performance-check.sh b/scripts/performance-check.sh new file mode 100755 index 0000000..4093158 --- /dev/null +++ b/scripts/performance-check.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Measures what a device window costs on this Mac and compares it with a baseline taken here +# before: how long it takes to open, CPU, energy, GPU and drawing while nothing on screen changes, +# memory, click to frame latency and, on a foldable, how evenly a fold is drawn. Then every booted +# device's window is opened at once, measured the same way and kept busy with clicks and folds to +# see whether memory keeps growing. +# +# Needs a booted iPhone and a booted iPhone Duo; a device that is not booted is not measured, and a +# baseline that measured it then fails the comparison. Every other booted device joins the windows +# opened at once. Keep the Mac otherwise quiet while it runs. +# +# A baseline only compares with a run on the same Mac model, macOS, Xcode, power mode, build +# configuration, booted devices and busy time. Low Power Mode caps drawing at 60 a second, so there +# is one baseline for each: performance/baseline-low-power.json and +# performance/baseline-normal-power.json, and the one that matches the Mac right now is used unless +# --baseline names another. +# +# Usage: scripts/performance-check.sh [--baseline FILE] [--busy-minutes N] +# [--record-baseline | --measure-only] +# (default) compare with the baseline and exit 1 on a regression +# --busy-minutes N how long to keep the windows busy, 3 unless given +# --record-baseline measure and write the baseline, accepting what was measured +# --measure-only measure and report, comparing with nothing +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +if [ "$(pmset -g | awk '/lowpowermode/ {print $2}')" = "1" ]; then + baseline="${repo_root}/performance/baseline-low-power.json" +else + baseline="${repo_root}/performance/baseline-normal-power.json" +fi +mode="compare" +busy_minutes=3 + +while [ $# -gt 0 ]; do + case "$1" in + --baseline) baseline="$2"; shift ;; + --busy-minutes) busy_minutes="$2"; shift ;; + --record-baseline) mode="record" ;; + --measure-only) mode="measure" ;; + -h|--help) sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown option $1" >&2; exit 2 ;; + esac + shift +done + +if ! [[ "${busy_minutes}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + echo "--busy-minutes takes a number of minutes, not ${busy_minutes}" >&2 + exit 2 +fi +busy_seconds="$(awk -v minutes="${busy_minutes}" 'BEGIN { printf "%d", minutes * 60 }')" + +if [ "${mode}" = "compare" ] && [ ! -f "${baseline}" ]; then + echo "No baseline at ${baseline}. Record one with --record-baseline first." >&2 + exit 2 +fi + +output="${repo_root}/build/performance/$(date +%Y%m%d-%H%M%S)" +mkdir -p "${output}" + +variables=(ODH_INTEGRATION=1 "ODH_PERFORMANCE_REPORT=${output}" "ODH_PERFORMANCE_BUSY_SECONDS=${busy_seconds}") +case "${mode}" in + compare) variables+=("ODH_PERFORMANCE_BASELINE=${baseline}") ;; + record) variables+=("ODH_PERFORMANCE_RECORD=${baseline}") ;; +esac + +status=0 +env "${variables[@]}" swift test --package-path "${repo_root}/engine" \ + -c release -Xswiftc -enable-testing --filter PerformanceCheckTests \ + > "${output}/test.log" 2>&1 || status=$? + +if [ -f "${output}/performance.txt" ]; then + cat "${output}/performance.txt" +else + echo "The check produced no report. The test log is at ${output}/test.log." >&2 + grep -E "error:|skipped" "${output}/test.log" | head -10 >&2 || true + exit 2 +fi +echo "Results are in ${output}" +[ "${mode}" = "record" ] && [ "${status}" -eq 0 ] && echo "Recorded the baseline at ${baseline}" +exit $(( status == 0 ? 0 : 1 ))