From e463481797f5f696a4f62e554475961f84779b54 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 06:51:45 +0100 Subject: [PATCH 01/17] feat(viewer): count the frames a window receives and the pictures it draws A window that draws while its device sends nothing costs CPU and GPU for no change on screen. The counts make that measurable. --- .../OpenDeviceHubViewer/DeviceWindowController.swift | 7 +++++++ engine/Sources/OpenDeviceHubViewer/DuoModelView.swift | 10 ++++++++++ engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift | 2 ++ 3 files changed, 19 insertions(+) diff --git a/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift b/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift index 92e9248..ee6875f 100644 --- a/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift +++ b/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift @@ -578,6 +578,11 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate public var latencyReading: LatencyMeter.Reading? { latency.reading } + /// Frames the device has sent this window, and pictures the window has drawn, for measuring + /// what a window costs while nothing on screen changes. + public private(set) var framesReceived = 0 + public var drawCount: Int { modelView?.renderCount ?? renderer.drawCount } + private func updateTitle() { guard let window else { return } let base = baseTitle ?? window.title @@ -891,6 +896,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate renderer.accept(frame) recording.current?.append(frame.surface, from: screenID, turnedBy: turns) await MainActor.run { [weak self] in + self?.framesReceived += 1 // The flat view sits under the model and is not seen; drawing it costs the // main thread a draw per frame. if self?.modelView == nil { screenView.needsDisplay = true } @@ -919,6 +925,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate if Task.isCancelled { return } recording.current?.append(frame.surface, from: screenID, turnedBy: turns) await MainActor.run { [weak self] in + self?.framesReceived += 1 self?.modelView?.setScreen(frame.surface, onCover: true, nativeRotation: rotation) } } diff --git a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift index 66f42e7..46da5aa 100644 --- a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift +++ b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift @@ -2,6 +2,7 @@ import AppKit import IOSurface import Metal import OpenDeviceHubEngine +import os import SceneKit import simd @@ -83,6 +84,9 @@ public final class DuoModelView: SCNView { /// How many probe renders have been made, for a test that expects a move to make none. var probeRenders = 0 + /// SceneKit calls back on its render thread, so the count is kept behind a lock. + private let renders = OSAllocatedUnfairLock(initialState: 0) + public var renderCount: Int { renders.withLock { $0 } } private let cameraNode = SCNNode() private let innerScreen: SCNNode private let coverScreen: SCNNode @@ -191,6 +195,7 @@ public final class DuoModelView: SCNView { preferredFramesPerSecond = 60 isPlaying = false loops = false + delegate = self setHingeAngle(180) } @@ -1270,3 +1275,8 @@ extension DuoModelView { } } +extension DuoModelView: SCNSceneRendererDelegate { + public nonisolated func renderer(_ renderer: any SCNSceneRenderer, didRenderScene scene: SCNScene, atTime time: TimeInterval) { + renders.withLock { $0 += 1 } + } +} diff --git a/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift b/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift index 0894e0a..28423ea 100644 --- a/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift +++ b/engine/Sources/OpenDeviceHubViewer/FrameRenderer.swift @@ -38,6 +38,7 @@ fragment float4 odhFragment(VertexOut in [[stage_in]], texture2d screen [ /// `IOSurface`, so a frame is never copied through the CPU. public final class FrameRenderer: NSObject, MTKViewDelegate { public var onFrameDrawn: (() -> Void)? + public private(set) var drawCount = 0 private let device: MTLDevice private let commandQueue: MTLCommandQueue @@ -141,6 +142,7 @@ public final class FrameRenderer: NSObject, MTKViewDelegate { buffer.present(drawable) buffer.commit() + drawCount += 1 onFrameDrawn?() } From 163e3f29782a377a657f94698d279823fc625652 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 07:50:51 +0100 Subject: [PATCH 02/17] test(viewer): measure each window's idle cost, memory, latency and fold pacing --- .../PerformanceCheckTests.swift | 369 ++++++++++++++++++ .../PerformanceComparisonTests.swift | 117 ++++++ .../PerformanceReport.swift | 184 +++++++++ 3 files changed, 670 insertions(+) create mode 100644 engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift create mode 100644 engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift create mode 100644 engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift new file mode 100644 index 0000000..ce44a83 --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift @@ -0,0 +1,369 @@ +import AppKit +import XCTest +import OpenDeviceHubEngine +@testable import OpenDeviceHubViewer + +/// Measures what a device window costs: CPU and drawing while nothing on screen changes, memory, +/// click to frame latency and, on a foldable, how evenly a fold is drawn. It runs only through +/// `scripts/performance-check.sh`, which says where the report goes and what to compare it with. +@MainActor +final class PerformanceCheckTests: XCTestCase { + private var adapter: (any SimulatorAdapter)! + private var manager: DeviceWindowManager! + private var stepTimes: [ContinuousClock.Instant] = [] + + static let settleSeconds = 8 + static let idleRounds = 10 + static let idleRoundSeconds = 2 + static let clicks = 20 + + func testMeasureEachKindOfWindow() async throws { + try IntegrationGate.requireEnabled() + let environment = ProcessInfo.processInfo.environment + guard let output = environment["ODH_PERFORMANCE_REPORT"] else { + throw XCTSkip("run scripts/performance-check.sh, which says where the report goes") + } + adapter = try AdapterFactory.make(for: XcodeLocator.locate()) + manager = DeviceWindowManager(frameStore: WindowFrameStore(storage: DiscardingStorage()), shutdown: { _ in }) + + let booted = try adapter.devices().filter { $0.state == .booted } + var windows: [WindowMeasurement] = [] + if let phone = booted.first(where: { + $0.name.hasPrefix("iPhone") && ((try? adapter.panels($0.udid).count) ?? 0) < 2 + }) { + windows.append(try await measurePhone(phone)) + } + if let foldable = booted.first(where: { $0.deviceTypeIdentifier.contains("Duo") }) { + windows.append(try await measureFoldable(foldable)) + } + guard !windows.isEmpty else { throw XCTSkip("boot an iPhone and an iPhone Duo to measure them") } + + let report = PerformanceReport( + createdAt: ISO8601DateFormatter().string(from: Date()), + environment: try Self.environment(), + windows: windows + ) + let directory = URL(fileURLWithPath: output, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try Self.json(report).write(to: directory.appending(path: "performance.json")) + + var findings: [PerformanceComparison.Finding]? + if let baselinePath = environment["ODH_PERFORMANCE_BASELINE"] { + let baseline = try JSONDecoder().decode( + PerformanceReport.self, + from: Data(contentsOf: URL(fileURLWithPath: baselinePath)) + ) + do { + let found = try PerformanceComparison.regressions(current: report, baseline: baseline) + findings = found + try Self.json(found).write(to: directory.appending(path: "comparison.json")) + for finding in found { + XCTFail("\(finding.window) \(finding.metric) is \(finding.current), over its limit of \(finding.limit) from a baseline of \(finding.baseline)") + } + } catch { + XCTFail("the baseline cannot be compared with this run: \(error)") + } + } + if let recordPath = environment["ODH_PERFORMANCE_RECORD"] { + try Self.json(report).write(to: URL(fileURLWithPath: recordPath)) + } + let text = PerformanceComparison.text(report, findings: findings) + try Data(text.utf8).write(to: directory.appending(path: "performance.txt")) + print(text) + } + + private func measurePhone(_ device: DeviceInfo) async throws -> WindowMeasurement { + try IntegrationHost.install(on: device.udid) + _ = try run(["simctl", "launch", device.udid, IntegrationHost.bundleID]) + let before = Self.footprintMB() + let controller = try manager.open( + device: device, + session: try adapter.openDisplay(device.udid, panel: nil), + input: try adapter.openInput(device.udid), + scaleMode: .fit, + bezelEnabled: true, + keepOnTop: false, + showFPS: false + ) + defer { manager.close(device.udid) } + let window = try XCTUnwrap(controller.window) + window.orderFront(nil) + let screen = try XCTUnwrap(Self.find(DeviceScreenView.self, in: window)) + try await Task.sleep(for: .seconds(Self.settleSeconds)) + let windowMemory = Self.footprintMB() - before + + let idle = try await measureIdle(controller) + let box = screen.bounds + let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) + let latency = try await measureClicks(controller, on: screen, spots: spots) + return WindowMeasurement( + name: "phone", + device: "\(device.name) (\(device.runtimeName))", + idleCPUPercent: idle.cpu, + idleDrawsPerSecond: idle.draws, + idleFramesPerSecond: idle.frames, + windowMemoryMB: windowMemory, + processMemoryMB: Self.footprintMB(), + clickLatencyMs: latency, + foldStepGapMs: nil, + foldCPUPercent: nil + ) + } + + private func measureFoldable(_ device: DeviceInfo) async throws -> WindowMeasurement { + try IntegrationHost.install(on: device.udid) + let panels = try adapter.panels(device.udid) + let unfolded = try XCTUnwrap(panels.first { $0.name == "Unfolded" }) + let cover = try XCTUnwrap(panels.first { $0.name == "Cover" }) + let input = try adapter.openInput(device.udid, screenID: unfolded.screenID) + let targeted = try XCTUnwrap(input as? PanelInputSession, "touches must be panel targeted") + let before = Self.footprintMB() + let controller = try manager.open( + device: device, + session: try adapter.openDisplay(device.udid, panel: unfolded), + input: input, + scaleMode: .fit, + bezelEnabled: true, + keepOnTop: false, + showFPS: false, + foldsAtHinge: true, + chrome: unfolded.chromeIdentifier.flatMap { ChromeLocator.chrome(identifier: $0) }, + panelNativeRotation: unfolded.nativeRotation, + unfoldedPanel: unfolded, + cover: FoldableCover(panel: cover, session: try adapter.openDisplay(device.udid, panel: cover)), + retarget: { targeted.setTarget(screenID: $0) } + ) + let window = try XCTUnwrap(controller.window) + window.orderFront(nil) + let model = try XCTUnwrap(Self.find(DuoModelView.self, in: window)) + + let adapter = self.adapter! + let shared = try await IntegrationFoldable.shared.control(for: device.udid, adapter: adapter) + let foldables = FoldableController( + open: { _ in shared }, + hingeStream: { try adapter.openHingeStream($0) }, + displayReport: { try await adapter.displayReport($0) } + ) + let udid = device.udid + defer { + foldables.setAngle(DeviceControlBar.FoldMode.fullyOpen.angle, for: udid) + foldables.forget(udid) + manager.close(udid) + } + controller.onHingeAngle = { angle in foldables.setAngle(angle, for: udid) } + controller.onFoldPreset = { angle in foldables.setAngle(angle, for: udid, eased: true) } + foldables.onMove = { [weak controller, weak self] _, event in + switch event { + case .began(let target): controller?.beginFold(to: target) + case .angle(let angle): + self?.stepTimes.append(ContinuousClock.now) + controller?.showHingeAngle(angle) + case .ended: controller?.endFold() + } + } + foldables.follow( + udid, + onPanel: { [weak controller] panel in controller?.setActivePanel(screenID: panel.displayID) }, + onHinge: { [weak controller] degrees in controller?.showHingeAngle(degrees) } + ) + let open = DeviceControlBar.FoldMode.fullyOpen.angle + controller.showHingeAngle(open) + foldables.setAngle(open, for: udid) + _ = try run(["simctl", "launch", udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(Self.settleSeconds)) + let windowMemory = Self.footprintMB() - before + + let idle = try await measureIdle(controller) + let box = model.bounds + let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) + .filter { model.screenPoint(at: $0) != nil } + XCTAssertFalse(spots.isEmpty, "no click lands on the open screen") + let latency = try await measureClicks(controller, on: model, spots: spots) + + let control = try XCTUnwrap(Self.find(NSSegmentedControl.self, in: window), "the window's fold positions") + stepTimes.removeAll() + var gaps: [Double] = [] + let cpuStart = Self.cpuSeconds() + let wallStart = ContinuousClock.now + for mode in [DeviceControlBar.FoldMode.cover, .fullyOpen] { + stepTimes.removeAll() + control.selectedSegment = mode.rawValue + control.sendAction(control.action, to: control.target) + try await Task.sleep(for: .seconds(3)) + gaps += zip(stepTimes.dropFirst(), stepTimes).map { later, earlier in Self.milliseconds(later - earlier) } + } + let foldCPU = (Self.cpuSeconds() - cpuStart) / Self.seconds(ContinuousClock.now - wallStart) * 100 + XCTAssertGreaterThan(gaps.count, 10, "the window drew no run of fold steps") + + return WindowMeasurement( + name: "foldable", + device: "\(device.name) (\(device.runtimeName))", + idleCPUPercent: idle.cpu, + idleDrawsPerSecond: idle.draws, + idleFramesPerSecond: idle.frames, + windowMemoryMB: windowMemory, + processMemoryMB: Self.footprintMB(), + clickLatencyMs: latency, + foldStepGapMs: gaps.isEmpty ? nil : Spread(gaps), + foldCPUPercent: foldCPU + ) + } + + private struct Idle { + let cpu: Spread + let draws: Double + let frames: Double + } + + /// The whole process's CPU, which with one window open is that window's: the test runner itself + /// sits idle while it waits. + private func measureIdle(_ controller: DeviceWindowController) async throws -> Idle { + var cpu: [Double] = [] + let draws = controller.drawCount + let frames = controller.framesReceived + let started = ContinuousClock.now + for _ in 0.. Spread { + let window = try XCTUnwrap(view.window) + var samples: [Double] = [] + for index in -1..= 0, let reading = controller.latencyReading, reading.sampleCount > counted { + samples.append(reading.lastMilliseconds) + } + try await Task.sleep(for: .milliseconds(300)) + } + XCTAssertGreaterThanOrEqual(samples.count, Self.clicks * 2 / 3, "most clicks drew no frame") + guard !samples.isEmpty else { throw XCTSkip("no click drew a frame, so latency could not be measured") } + return Spread(samples) + } + + private static func spots(around centre: CGPoint, size: CGSize) -> [CGPoint] { + let step = CGSize(width: size.width * 0.15, height: size.height * 0.15) + return [(-1, -1), (1, -1), (0, 0), (-1, 1), (1, 1)].map { dx, dy in + CGPoint(x: centre.x + CGFloat(dx) * step.width, y: centre.y + CGFloat(dy) * step.height) + } + } + + static func environment() throws -> PerformanceEnvironment { + var size = 0 + sysctlbyname("hw.model", nil, &size, nil, 0) + var bytes = [CChar](repeating: 0, count: size) + sysctlbyname("hw.model", &bytes, &size, nil, 0) + let model = String(decoding: bytes.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self) + let system = ProcessInfo.processInfo.operatingSystemVersion + let xcode = try XcodeLocator.locate() + #if DEBUG + let configuration = "debug" + #else + let configuration = "release" + #endif + return PerformanceEnvironment( + hardwareModel: model, + macOS: "\(system.majorVersion).\(system.minorVersion).\(system.patchVersion)", + xcode: "Xcode \(xcode.version) (\(xcode.build))", + lowPowerMode: ProcessInfo.processInfo.isLowPowerModeEnabled, + buildConfiguration: configuration + ) + } + + private static func cpuSeconds() -> Double { + var usage = rusage() + getrusage(RUSAGE_SELF, &usage) + func seconds(_ time: timeval) -> Double { Double(time.tv_sec) + Double(time.tv_usec) / 1_000_000 } + return seconds(usage.ru_utime) + seconds(usage.ru_stime) + } + + private static func footprintMB() -> Double { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + return result == KERN_SUCCESS ? Double(info.phys_footprint) / 1_048_576 : 0 + } + + private static func seconds(_ duration: Duration) -> Double { + Double(duration.components.seconds) + Double(duration.components.attoseconds) / 1e18 + } + + private static func milliseconds(_ duration: Duration) -> Double { + seconds(duration) * 1000 + } + + private static func json(_ value: T) throws -> Data { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + return try encoder.encode(value) + } + + private static func find(_ type: T.Type, in window: NSWindow) -> T? { + func search(_ view: NSView) -> T? { + if let found = view as? T { return found } + for subview in view.subviews { + if let found = search(subview) { return found } + } + return nil + } + return window.contentView.flatMap(search) + } + + private static func mouse(_ type: NSEvent.EventType, at point: CGPoint, in window: NSWindow) throws -> NSEvent { + try XCTUnwrap(NSEvent.mouseEvent( + with: type, location: point, modifierFlags: [], + timestamp: ProcessInfo.processInfo.systemUptime, + windowNumber: window.windowNumber, context: nil, + eventNumber: 0, clickCount: 1, pressure: 1 + )) + } + + @discardableResult + private func run(_ arguments: [String]) throws -> String { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") + process.arguments = arguments + process.environment = ProcessInfo.processInfo.environment + let output = Pipe() + process.standardOutput = output + process.standardError = Pipe() + try process.run() + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + return String(decoding: data, as: UTF8.self) + } +} + +private final class DiscardingStorage: PreferenceStorage, @unchecked Sendable { + func text(forKey key: String) -> String? { nil } + func setText(_ text: String, forKey key: String) {} + func removeText(forKey key: String) {} + func keys(withPrefix prefix: String) -> [String] { [] } +} diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift new file mode 100644 index 0000000..c6ada31 --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift @@ -0,0 +1,117 @@ +import XCTest + +final class PerformanceComparisonTests: XCTestCase { + private let environment = PerformanceEnvironment( + hardwareModel: "Mac16,7", macOS: "27.0", xcode: "Xcode 27.0 (27A266a)", + lowPowerMode: false, buildConfiguration: "release" + ) + + private func window( + _ name: String = "foldable", + cpu: Double = 10, + draws: Double = 60, + memory: Double = 350, + latency: Double = 20, + gap: Double? = 34 + ) -> WindowMeasurement { + WindowMeasurement( + name: name, + device: "iPhone Duo (iOS 27.1)", + idleCPUPercent: Spread(median: cpu, p95: cpu * 2, samples: 10), + idleDrawsPerSecond: draws, + idleFramesPerSecond: 0.1, + windowMemoryMB: memory, + processMemoryMB: memory + 60, + clickLatencyMs: Spread(median: latency, p95: latency * 1.5, samples: 15), + foldStepGapMs: gap.map { Spread(median: $0 / 2, p95: $0, samples: 60) }, + foldCPUPercent: gap == nil ? nil : 15 + ) + } + + private func report(_ windows: [WindowMeasurement], environment: PerformanceEnvironment? = nil) -> PerformanceReport { + PerformanceReport(createdAt: "now", environment: environment ?? self.environment, windows: windows) + } + + func testTheLimitIsTheLargerOfAShareAndAFixedAmount() { + let tolerance = PerformanceComparison.Tolerance(relative: 0.5, absolute: 3) + XCTAssertEqual(tolerance.limit(for: 1), 4, "near zero the fixed amount decides") + XCTAssertEqual(tolerance.limit(for: 20), 30, "further up the share decides") + } + + func testTheSpreadTakesTheSampleAtEachPercentile() { + let spread = Spread([5, 1, 4, 2, 3, 9, 8, 7, 6, 10]) + XCTAssertEqual(spread.median, 5) + XCTAssertEqual(spread.p95, 10) + XCTAssertEqual(spread.samples, 10) + } + + func testARunWithinToleranceHasNoRegressions() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 12, memory: 380, latency: 24)]), + baseline: report([window()]) + ) + XCTAssertEqual(findings, []) + } + + func testAnImprovementIsNotARegression() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 0.5, draws: 0.2, memory: 120)]), + baseline: report([window()]) + ) + XCTAssertEqual(findings, []) + } + + func testEachMetricBeyondItsLimitIsReported() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 16, draws: 60, memory: 450, latency: 40, gap: 60)]), + baseline: report([window()]) + ) + XCTAssertEqual(Set(findings.map(\.metric)), [ + "idle CPU median (%)", "window memory (MB)", "click latency median (ms)", + "click latency p95 (ms)", "fold step gap p95 (ms)", + ]) + let cpu = try XCTUnwrap(findings.first { $0.metric == "idle CPU median (%)" }) + XCTAssertEqual(cpu.baseline, 10) + XCTAssertEqual(cpu.current, 16) + XCTAssertEqual(cpu.limit, 15) + } + + func testAWindowWithoutFoldMetricsIsComparedOnTheRest() throws { + let findings = try PerformanceComparison.regressions( + current: report([window("phone", cpu: 0, draws: 0, memory: 60, gap: nil)]), + baseline: report([window("phone", cpu: 0, draws: 0, memory: 60, gap: nil)]) + ) + XCTAssertEqual(findings, []) + } + + func testABaselineFromAnotherMachineIsRefused() { + var other = environment + other = PerformanceEnvironment( + hardwareModel: other.hardwareModel, macOS: other.macOS, xcode: "Xcode 26.5 (17F42)", + lowPowerMode: other.lowPowerMode, buildConfiguration: other.buildConfiguration + ) + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()]), + baseline: report([window()], environment: other) + )) { error in + guard case PerformanceComparison.Incompatible.environment = error else { + return XCTFail("wrong error: \(error)") + } + } + } + + func testAWindowTheBaselineMeasuredMustBeMeasuredAgain() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window("phone", gap: nil)]), + baseline: report([window("phone", gap: nil), window("foldable")]) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .missingWindow("foldable")) + } + } + + func testTheReportRoundTripsThroughJSON() throws { + let original = report([window(), window("phone", gap: nil)]) + let decoded = try JSONDecoder().decode(PerformanceReport.self, from: JSONEncoder().encode(original)) + XCTAssertEqual(decoded, original) + } +} diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift new file mode 100644 index 0000000..43bd759 --- /dev/null +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift @@ -0,0 +1,184 @@ +import Foundation + +/// What one run of the performance check measured, and the machine it measured it on. Numbers +/// only compare with a baseline taken on the same machine, OS, Xcode and build configuration. +struct PerformanceReport: Codable, Equatable { + static let schemaVersion = 1 + + var schemaVersion = Self.schemaVersion + let createdAt: String + let environment: PerformanceEnvironment + let windows: [WindowMeasurement] +} + +struct PerformanceEnvironment: Codable, Equatable { + let hardwareModel: String + let macOS: String + let xcode: String + let lowPowerMode: Bool + let buildConfiguration: String +} + +/// The median and the nearest rank 95th percentile: the sample at or above that share of the +/// sorted run. +struct Spread: Codable, Equatable { + let median: Double + let p95: Double + let samples: Int + + init(median: Double, p95: Double, samples: Int) { + self.median = median + self.p95 = p95 + self.samples = samples + } + + init(_ values: [Double]) { + precondition(!values.isEmpty, "a spread needs at least one sample") + median = Self.percentile(values, 0.5) + p95 = Self.percentile(values, 0.95) + samples = values.count + } + + static func percentile(_ values: [Double], _ fraction: Double) -> Double { + let sorted = values.sorted() + return sorted[max(0, Int((Double(sorted.count) * fraction).rounded(.up)) - 1)] + } +} + +struct WindowMeasurement: Codable, Equatable { + let name: String + let device: String + let idleCPUPercent: Spread + let idleDrawsPerSecond: Double + let idleFramesPerSecond: Double + let windowMemoryMB: Double + let processMemoryMB: Double + let clickLatencyMs: Spread + let foldStepGapMs: Spread? + let foldCPUPercent: Double? +} + +enum PerformanceComparison { + /// A limit of the baseline plus the larger of a share of it and a fixed amount, so a metric + /// near zero, idle CPU above all, is not failed for noise. + struct Tolerance: Equatable { + let relative: Double + let absolute: Double + + func limit(for baseline: Double) -> Double { + baseline + max(baseline * relative, absolute) + } + } + + struct Finding: Codable, Equatable { + let window: String + let metric: String + let baseline: Double + let current: Double + let limit: Double + } + + enum Incompatible: Error, Equatable, CustomStringConvertible { + case schema(baseline: Int, current: Int) + case environment(baseline: PerformanceEnvironment, current: PerformanceEnvironment) + case missingWindow(String) + + var description: String { + switch self { + case .schema(let baseline, let current): + "the baseline is schema \(baseline) and this run is schema \(current)" + case .environment(let baseline, let current): + "the baseline was measured on \(baseline) and this run on \(current)" + case .missingWindow(let name): + "the baseline measured a \(name) window and this run did not" + } + } + } + + /// Every metric is one where less is better. + static let tolerances: [String: Tolerance] = [ + "idle CPU median (%)": Tolerance(relative: 0.5, absolute: 3), + "idle draws per second": Tolerance(relative: 0.5, absolute: 5), + "window memory (MB)": Tolerance(relative: 0.2, absolute: 30), + "click latency median (ms)": Tolerance(relative: 0.3, absolute: 8), + "click latency p95 (ms)": Tolerance(relative: 0.3, absolute: 12), + "fold step gap p95 (ms)": Tolerance(relative: 0.3, absolute: 10), + "fold CPU (%)": Tolerance(relative: 0.5, absolute: 5), + ] + + static func metrics(of window: WindowMeasurement) -> [String: Double] { + var values = [ + "idle CPU median (%)": window.idleCPUPercent.median, + "idle draws per second": window.idleDrawsPerSecond, + "window memory (MB)": window.windowMemoryMB, + "click latency median (ms)": window.clickLatencyMs.median, + "click latency p95 (ms)": window.clickLatencyMs.p95, + ] + if let gaps = window.foldStepGapMs { values["fold step gap p95 (ms)"] = gaps.p95 } + if let cpu = window.foldCPUPercent { values["fold CPU (%)"] = cpu } + return values + } + + static func regressions( + current: PerformanceReport, + baseline: PerformanceReport + ) throws(Incompatible) -> [Finding] { + guard baseline.schemaVersion == current.schemaVersion else { + throw .schema(baseline: baseline.schemaVersion, current: current.schemaVersion) + } + guard baseline.environment == current.environment else { + throw .environment(baseline: baseline.environment, current: current.environment) + } + var findings: [Finding] = [] + for base in baseline.windows { + guard let now = current.windows.first(where: { $0.name == base.name }) else { + throw .missingWindow(base.name) + } + let before = metrics(of: base) + let after = metrics(of: now) + for (metric, value) in before.sorted(by: { $0.key < $1.key }) { + guard let measured = after[metric], let tolerance = tolerances[metric] else { continue } + let limit = tolerance.limit(for: value) + if measured > limit { + findings.append(Finding(window: base.name, metric: metric, baseline: value, current: measured, limit: limit)) + } + } + } + return findings + } + + /// The report as a person reads it, one window after another. + static func text(_ report: PerformanceReport, findings: [Finding]? = nil) -> String { + func number(_ value: Double) -> String { String(format: "%.1f", value) } + var lines = [ + "Performance check \(report.createdAt)", + "\(report.environment.hardwareModel), macOS \(report.environment.macOS), \(report.environment.xcode), \(report.environment.buildConfiguration)\(report.environment.lowPowerMode ? ", Low Power Mode" : "")", + "", + ] + for window in report.windows { + lines.append("\(window.name): \(window.device)") + lines.append(" idle CPU median \(number(window.idleCPUPercent.median))%, p95 \(number(window.idleCPUPercent.p95))% over \(window.idleCPUPercent.samples) samples") + lines.append(" idle drawing \(number(window.idleDrawsPerSecond)) pictures a second for \(number(window.idleFramesPerSecond)) frames a second from the device") + lines.append(" memory \(number(window.windowMemoryMB)) MB for the window, \(number(window.processMemoryMB)) MB for the process") + lines.append(" click to frame median \(number(window.clickLatencyMs.median)) ms, p95 \(number(window.clickLatencyMs.p95)) ms over \(window.clickLatencyMs.samples) clicks") + if let gaps = window.foldStepGapMs { + lines.append(" fold steps median gap \(number(gaps.median)) ms, p95 \(number(gaps.p95)) ms over \(gaps.samples) steps") + } + if let cpu = window.foldCPUPercent { + lines.append(" fold CPU \(number(cpu))%") + } + lines.append("") + } + if let findings { + if findings.isEmpty { + lines.append("No regressions against the baseline.") + } else { + lines.append("Regressions against the baseline:") + for finding in findings { + lines.append(" \(finding.window) \(finding.metric): \(number(finding.current)), baseline \(number(finding.baseline)), limit \(number(finding.limit))") + } + } + } + return lines.joined(separator: "\n") + "\n" + } +} From 0f47ec7dea5311c1342109ad9c24768eb10bad4a Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 07:50:52 +0100 Subject: [PATCH 03/17] build(scripts): add a performance check against a per machine baseline The recorded baseline is this Mac's, taken in Low Power Mode, and shows the foldable window drawing 60 pictures a second at rest while its device sends about two. --- RELEASING.md | 14 ++++++--- performance/baseline.json | 55 ++++++++++++++++++++++++++++++++ scripts/performance-check.sh | 61 ++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 performance/baseline.json create mode 100755 scripts/performance-check.sh diff --git a/RELEASING.md b/RELEASING.md index 3c3be9e..13fe0df 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -51,21 +51,25 @@ unsigned build that reaches a release page is worse than a run that stops. 1. Add the version's section to `CHANGELOG.md`. The release notes on GitHub and the notes inside the update feed both come from it, so they cannot drift apart. 2. Update `VERSION`. -3. Run the **Release** workflow from the Actions tab with `publish` left **false**. That is a dry +3. Run `scripts/performance-check.sh` on the Mac the baseline in `performance/baseline.json` was + recorded on, with an iPhone and an iPhone Duo booted. It exits 1 if a window got slower, heavier + or less responsive than the baseline allows. Record a new baseline with `--record-baseline` only + when a change is meant to move the numbers, and commit it with that change. +4. Run the **Release** workflow from the Actions tab with `publish` left **false**. That is a dry run: it builds, signs, notarizes, staples, packages, checksums and generates the feed, and creates no tag, no release and no commit anywhere. The artifacts are attached to the run under a short retention name so you can download and check them. -4. Install the dry run's DMG and use it. `stapler validate` on the app and on the DMG, and +5. Install the dry run's DMG and use it. `stapler validate` on the app and on the DMG, and `shasum -c SHA256SUMS` from a different directory. -5. Run the workflow again with `publish` **true**. It creates the tag and a release marked as the +6. Run the workflow again with `publish` **true**. It creates the tag and a release marked as the latest, carrying the DMG under its versioned name, the same DMG as `OpenDeviceHub.dmg`, `SHA256SUMS` and the signed `appcast.xml`. The README's download button and its Install link fetch `OpenDeviceHub.dmg` from the latest release, so they need no edit. -6. **Only once that release page exists**, commit the release's `appcast.xml` to the feed +7. **Only once that release page exists**, commit the release's `appcast.xml` to the feed repository. Installed copies read the feed from there, never from the release, which is why the feed URL can never move. A feed pointing at a download that is not there yet breaks updates for everyone who reads it in the meantime. -7. Confirm an already installed older copy finds the update, verifies it and installs it, ending on +8. Confirm an already installed older copy finds the update, verifies it and installs it, ending on the new version. Do this before announcing anything: a broken updater cannot be fixed by a later release. diff --git a/performance/baseline.json b/performance/baseline.json new file mode 100644 index 0000000..0b04903 --- /dev/null +++ b/performance/baseline.json @@ -0,0 +1,55 @@ +{ + "createdAt" : "2026-09-28T05:45:24Z", + "environment" : { + "buildConfiguration" : "release", + "hardwareModel" : "Mac15,6", + "lowPowerMode" : true, + "macOS" : "27.0.0", + "xcode" : "Xcode 27.1 (27A9269)" + }, + "schemaVersion" : 1, + "windows" : [ + { + "clickLatencyMs" : { + "median" : 12.259333333333334, + "p95" : 19.268083333333333, + "samples" : 20 + }, + "device" : "iPhone 17 (iOS 27.0)", + "idleCPUPercent" : { + "median" : 0.2005022855287288, + "p95" : 0.2525720731410715, + "samples" : 10 + }, + "idleDrawsPerSecond" : 0, + "idleFramesPerSecond" : 0, + "name" : "phone", + "processMemoryMB" : 152.43841552734375, + "windowMemoryMB" : 48.640785217285156 + }, + { + "clickLatencyMs" : { + "median" : 12.488833333333334, + "p95" : 16.651708333333335, + "samples" : 20 + }, + "device" : "iPhone Duo (iOS 27.1)", + "foldCPUPercent" : 23.19616284802404, + "foldStepGapMs" : { + "median" : 32.523292000000005, + "p95" : 34.215083, + "samples" : 73 + }, + "idleCPUPercent" : { + "median" : 17.592522514306282, + "p95" : 18.955358112325786, + "samples" : 10 + }, + "idleDrawsPerSecond" : 60.001300476919354, + "idleFramesPerSecond" : 2.2077338687337456, + "name" : "foldable", + "processMemoryMB" : 359.1573944091797, + "windowMemoryMB" : 295.26585388183594 + } + ] +} \ No newline at end of file diff --git a/scripts/performance-check.sh b/scripts/performance-check.sh new file mode 100755 index 0000000..6c110c6 --- /dev/null +++ b/scripts/performance-check.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Measures what a device window costs on this Mac and compares it with a baseline taken here +# before: CPU and drawing while nothing on screen changes, memory, click to frame latency and, on a +# foldable, how evenly a fold is drawn. +# +# Needs a booted iPhone and a booted iPhone Duo; a device that is not booted is not measured, and a +# baseline that measured it then fails the comparison. Keep the Mac otherwise quiet while it runs. +# +# A baseline only compares with a run on the same Mac model, macOS, Xcode, power mode and build +# configuration. +# +# Usage: scripts/performance-check.sh [--baseline FILE] [--record-baseline | --measure-only] +# (default) compare with the baseline and exit 1 on a regression +# --record-baseline measure and write the baseline, accepting what was measured +# --measure-only measure and report, comparing with nothing +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +baseline="${repo_root}/performance/baseline.json" +mode="compare" + +while [ $# -gt 0 ]; do + case "$1" in + --baseline) baseline="$2"; shift ;; + --record-baseline) mode="record" ;; + --measure-only) mode="measure" ;; + -h|--help) sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown option $1" >&2; exit 2 ;; + esac + shift +done + +if [ "${mode}" = "compare" ] && [ ! -f "${baseline}" ]; then + echo "No baseline at ${baseline}. Record one with --record-baseline first." >&2 + exit 2 +fi + +output="${repo_root}/build/performance/$(date +%Y%m%d-%H%M%S)" +mkdir -p "${output}" + +variables=(ODH_INTEGRATION=1 "ODH_PERFORMANCE_REPORT=${output}") +case "${mode}" in + compare) variables+=("ODH_PERFORMANCE_BASELINE=${baseline}") ;; + record) variables+=("ODH_PERFORMANCE_RECORD=${baseline}") ;; +esac + +status=0 +env "${variables[@]}" swift test --package-path "${repo_root}/engine" \ + -c release -Xswiftc -enable-testing --filter PerformanceCheckTests \ + > "${output}/test.log" 2>&1 || status=$? + +if [ -f "${output}/performance.txt" ]; then + cat "${output}/performance.txt" +else + echo "The check produced no report. The test log is at ${output}/test.log." >&2 + grep -E "error:|skipped" "${output}/test.log" | head -10 >&2 || true + exit 2 +fi +echo "Results are in ${output}" +[ "${mode}" = "record" ] && [ "${status}" -eq 0 ] && echo "Recorded the baseline at ${baseline}" +exit $(( status == 0 ? 0 : 1 )) From 2ac042e8283f86368b7f6f5c31542db804b8bf67 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 08:39:32 +0100 Subject: [PATCH 04/17] fix(viewer): draw the foldable only when something on it changes It redrew sixty times a second whatever the device sent, so a still screen cost a steady share of the CPU and GPU. SceneKit now draws when the scene changes: a new frame, a pose, a hover. A still screen draws nothing, and each change costs about a quarter of a second of drawing, which is SceneKit's own. --- engine/Sources/OpenDeviceHubViewer/DuoModelView.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift index 46da5aa..d9f68c2 100644 --- a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift +++ b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift @@ -189,7 +189,10 @@ public final class DuoModelView: SCNView { layer?.isOpaque = false antialiasingMode = .multisampling4X allowsCameraControl = false - rendersContinuously = true + // Drawn when the scene changes, so a still screen costs nothing. SceneKit keeps drawing for + // about a quarter of a second after each change, a new frame from the device included + // (Xcode 27.1, 27A9269). + rendersContinuously = false // Asked for 30, SceneKit draws at 24 on a 120 Hz display, and the rate cannot be changed // once the view exists; a fold needs 60 to look like one. preferredFramesPerSecond = 60 From 8e24c0cbc97a74b62c63011ae788d4b178c61e7c Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 08:39:32 +0100 Subject: [PATCH 05/17] build(scripts): record the baseline with the foldable drawn on demand --- performance/baseline.json | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/performance/baseline.json b/performance/baseline.json index 0b04903..365503f 100644 --- a/performance/baseline.json +++ b/performance/baseline.json @@ -1,5 +1,5 @@ { - "createdAt" : "2026-09-28T05:45:24Z", + "createdAt" : "2026-09-28T07:38:43Z", "environment" : { "buildConfiguration" : "release", "hardwareModel" : "Mac15,6", @@ -11,45 +11,45 @@ "windows" : [ { "clickLatencyMs" : { - "median" : 12.259333333333334, - "p95" : 19.268083333333333, + "median" : 13.091291666666667, + "p95" : 15.483583333333334, "samples" : 20 }, "device" : "iPhone 17 (iOS 27.0)", "idleCPUPercent" : { - "median" : 0.2005022855287288, - "p95" : 0.2525720731410715, + "median" : 0.23452933463056475, + "p95" : 0.3247799798450037, "samples" : 10 }, "idleDrawsPerSecond" : 0, "idleFramesPerSecond" : 0, "name" : "phone", - "processMemoryMB" : 152.43841552734375, - "windowMemoryMB" : 48.640785217285156 + "processMemoryMB" : 152.17279052734375, + "windowMemoryMB" : 47.82826232910156 }, { "clickLatencyMs" : { - "median" : 12.488833333333334, - "p95" : 16.651708333333335, + "median" : 13.330625, + "p95" : 16.869375, "samples" : 20 }, "device" : "iPhone Duo (iOS 27.1)", - "foldCPUPercent" : 23.19616284802404, + "foldCPUPercent" : 23.654378541557307, "foldStepGapMs" : { - "median" : 32.523292000000005, - "p95" : 34.215083, - "samples" : 73 + "median" : 32.90175, + "p95" : 34.264084, + "samples" : 70 }, "idleCPUPercent" : { - "median" : 17.592522514306282, - "p95" : 18.955358112325786, + "median" : 5.571381041923167, + "p95" : 8.18479697207644, "samples" : 10 }, - "idleDrawsPerSecond" : 60.001300476919354, - "idleFramesPerSecond" : 2.2077338687337456, + "idleDrawsPerSecond" : 18.98328772731046, + "idleFramesPerSecond" : 2.152743969076444, "name" : "foldable", - "processMemoryMB" : 359.1573944091797, - "windowMemoryMB" : 295.26585388183594 + "processMemoryMB" : 357.6105422973633, + "windowMemoryMB" : 295.1877746582031 } ] } \ No newline at end of file From 2d8bdbd8ef30f7b0d3670b330496bedf746db844 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 10:11:18 +0100 Subject: [PATCH 06/17] build(scripts): keep a baseline for each power mode Low Power Mode caps drawing at 60 a second, which hides half of what a window that draws at the display's rate costs on a 120 Hz Mac. The check now compares with the baseline for the mode the Mac is in; the existing one is kept as the Low Power Mode baseline and a normal power one is added. --- RELEASING.md | 10 ++-- ...{baseline.json => baseline-low-power.json} | 0 performance/baseline-normal-power.json | 55 +++++++++++++++++++ scripts/performance-check.sh | 12 +++- 4 files changed, 70 insertions(+), 7 deletions(-) rename performance/{baseline.json => baseline-low-power.json} (100%) create mode 100644 performance/baseline-normal-power.json diff --git a/RELEASING.md b/RELEASING.md index 13fe0df..0332b46 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -51,10 +51,12 @@ unsigned build that reaches a release page is worse than a run that stops. 1. Add the version's section to `CHANGELOG.md`. The release notes on GitHub and the notes inside the update feed both come from it, so they cannot drift apart. 2. Update `VERSION`. -3. Run `scripts/performance-check.sh` on the Mac the baseline in `performance/baseline.json` was - recorded on, with an iPhone and an iPhone Duo booted. It exits 1 if a window got slower, heavier - or less responsive than the baseline allows. Record a new baseline with `--record-baseline` only - when a change is meant to move the numbers, and commit it with that change. +3. Run `scripts/performance-check.sh` on the Mac the baselines in `performance/` were recorded on, + with an iPhone and an iPhone Duo booted and Low Power Mode off. It compares with + `baseline-normal-power.json`, or `baseline-low-power.json` when Low Power Mode is on, and exits 1 + if a window got slower, heavier or less responsive than the baseline allows. Record a new + baseline with `--record-baseline` only when a change is meant to move the numbers, and commit it + with that change. 4. Run the **Release** workflow from the Actions tab with `publish` left **false**. That is a dry run: it builds, signs, notarizes, staples, packages, checksums and generates the feed, and creates no tag, no release and no commit anywhere. The artifacts are attached to the run under a diff --git a/performance/baseline.json b/performance/baseline-low-power.json similarity index 100% rename from performance/baseline.json rename to performance/baseline-low-power.json diff --git a/performance/baseline-normal-power.json b/performance/baseline-normal-power.json new file mode 100644 index 0000000..590ebd2 --- /dev/null +++ b/performance/baseline-normal-power.json @@ -0,0 +1,55 @@ +{ + "createdAt" : "2026-09-28T09:09:23Z", + "environment" : { + "buildConfiguration" : "release", + "hardwareModel" : "Mac15,6", + "lowPowerMode" : false, + "macOS" : "27.0.0", + "xcode" : "Xcode 27.1 (27A9269)" + }, + "schemaVersion" : 1, + "windows" : [ + { + "clickLatencyMs" : { + "median" : 11.968583333333333, + "p95" : 13.459375, + "samples" : 20 + }, + "device" : "iPhone 17 (iOS 27.0)", + "idleCPUPercent" : { + "median" : 0.19903322942526908, + "p95" : 0.2747969613825266, + "samples" : 10 + }, + "idleDrawsPerSecond" : 0, + "idleFramesPerSecond" : 0, + "name" : "phone", + "processMemoryMB" : 152.65716552734375, + "windowMemoryMB" : 48.56263732910156 + }, + { + "clickLatencyMs" : { + "median" : 14.076875, + "p95" : 16.38925, + "samples" : 20 + }, + "device" : "iPhone Duo (iOS 27.1)", + "foldCPUPercent" : 36.23484187234482, + "foldStepGapMs" : { + "median" : 18.287666, + "p95" : 20.769958000000003, + "samples" : 108 + }, + "idleCPUPercent" : { + "median" : 9.649805164040677, + "p95" : 13.174145444260233, + "samples" : 10 + }, + "idleDrawsPerSecond" : 35.885166285889795, + "idleFramesPerSecond" : 2.116683333735612, + "name" : "foldable", + "processMemoryMB" : 357.2667922973633, + "windowMemoryMB" : 295.98462677001953 + } + ] +} \ No newline at end of file diff --git a/scripts/performance-check.sh b/scripts/performance-check.sh index 6c110c6..a184d62 100755 --- a/scripts/performance-check.sh +++ b/scripts/performance-check.sh @@ -7,7 +7,9 @@ # baseline that measured it then fails the comparison. Keep the Mac otherwise quiet while it runs. # # A baseline only compares with a run on the same Mac model, macOS, Xcode, power mode and build -# configuration. +# configuration. Low Power Mode caps drawing at 60 a second, so there is one baseline for each: +# performance/baseline-low-power.json and performance/baseline-normal-power.json, and the one that +# matches the Mac right now is used unless --baseline names another. # # Usage: scripts/performance-check.sh [--baseline FILE] [--record-baseline | --measure-only] # (default) compare with the baseline and exit 1 on a regression @@ -16,7 +18,11 @@ set -euo pipefail repo_root="$(cd "$(dirname "$0")/.." && pwd)" -baseline="${repo_root}/performance/baseline.json" +if [ "$(pmset -g | awk '/lowpowermode/ {print $2}')" = "1" ]; then + baseline="${repo_root}/performance/baseline-low-power.json" +else + baseline="${repo_root}/performance/baseline-normal-power.json" +fi mode="compare" while [ $# -gt 0 ]; do @@ -24,7 +30,7 @@ while [ $# -gt 0 ]; do --baseline) baseline="$2"; shift ;; --record-baseline) mode="record" ;; --measure-only) mode="measure" ;; - -h|--help) sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "Unknown option $1" >&2; exit 2 ;; esac shift From 72480e3e90f1987739de968b1bac1ad535daa691 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 11:51:24 +0100 Subject: [PATCH 07/17] test(viewer): measure open time, energy, gpu and memory growth Energy is read from the process's own CPU energy counter and GPU use from the GPU driver's per process time, since powermetrics needs admin rights. Every booted device's window is also opened at once and kept busy, and each window is reopened a few times, so a leak shows as memory that keeps growing. --- .../PerformanceCheckTests.swift | 441 ++++++++++++++---- .../PerformanceComparisonTests.swift | 177 ++++++- .../PerformanceReport.swift | 183 +++++++- 3 files changed, 703 insertions(+), 98 deletions(-) diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift index ce44a83..7103f76 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift @@ -1,11 +1,14 @@ import AppKit +import IOKit import XCTest import OpenDeviceHubEngine @testable import OpenDeviceHubViewer -/// Measures what a device window costs: CPU and drawing while nothing on screen changes, memory, -/// click to frame latency and, on a foldable, how evenly a fold is drawn. It runs only through -/// `scripts/performance-check.sh`, which says where the report goes and what to compare it with. +/// Measures what a device window costs: how long it takes to open, CPU, energy, GPU and drawing +/// while nothing on screen changes, memory, click to frame latency and, on a foldable, how evenly a +/// fold is drawn. Then every booted device's window is opened at once and kept busy to see whether +/// memory keeps growing. It runs only through `scripts/performance-check.sh`, which says where the +/// report goes and what to compare it with. @MainActor final class PerformanceCheckTests: XCTestCase { private var adapter: (any SimulatorAdapter)! @@ -16,6 +19,12 @@ final class PerformanceCheckTests: XCTestCase { static let idleRounds = 10 static let idleRoundSeconds = 2 static let clicks = 20 + static let hostSettleSeconds = 2 + static let closeSettleSeconds = 3 + static let reopens = 3 + static let soakSampleSeconds = 5.0 + static let soakFoldEverySeconds = 20.0 + static let soakClickSeconds = 0.5 func testMeasureEachKindOfWindow() async throws { try IntegrationGate.requireEnabled() @@ -26,6 +35,8 @@ final class PerformanceCheckTests: XCTestCase { adapter = try AdapterFactory.make(for: XcodeLocator.locate()) manager = DeviceWindowManager(frameStore: WindowFrameStore(storage: DiscardingStorage()), shutdown: { _ in }) + let soakSeconds = Double(environment["ODH_PERFORMANCE_BUSY_SECONDS"] ?? "") ?? 180 + let processStart = try Self.footprintMB() let booted = try adapter.devices().filter { $0.state == .booted } var windows: [WindowMeasurement] = [] if let phone = booted.first(where: { @@ -37,11 +48,13 @@ final class PerformanceCheckTests: XCTestCase { windows.append(try await measureFoldable(foldable)) } guard !windows.isEmpty else { throw XCTSkip("boot an iPhone and an iPhone Duo to measure them") } + let together = booted.count > 1 ? try await measureTogether(booted, soakSeconds: soakSeconds, processStart: processStart) : nil let report = PerformanceReport( createdAt: ISO8601DateFormatter().string(from: Date()), environment: try Self.environment(), - windows: windows + windows: windows, + together: together ) let directory = URL(fileURLWithPath: output, isDirectory: true) try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -75,24 +88,24 @@ final class PerformanceCheckTests: XCTestCase { private func measurePhone(_ device: DeviceInfo) async throws -> WindowMeasurement { try IntegrationHost.install(on: device.udid) _ = try run(["simctl", "launch", device.udid, IntegrationHost.bundleID]) - let before = Self.footprintMB() - let controller = try manager.open( - device: device, - session: try adapter.openDisplay(device.udid, panel: nil), - input: try adapter.openInput(device.udid), - scaleMode: .fit, - bezelEnabled: true, - keepOnTop: false, - showFPS: false - ) + try await Task.sleep(for: .seconds(Self.hostSettleSeconds)) + let before = try Self.footprintMB() + var measurement = try await measurePhoneWindow(device, before: before) + measurement.reopen = try await measureReopen(device, before: before) + return measurement + } + + /// Apart from the reopen, so nothing here still holds the window once it closes. + private func measurePhoneWindow(_ device: DeviceInfo, before: Double) async throws -> WindowMeasurement { + let opened = try await openWindow(device) + let controller = opened.controller defer { manager.close(device.udid) } let window = try XCTUnwrap(controller.window) - window.orderFront(nil) let screen = try XCTUnwrap(Self.find(DeviceScreenView.self, in: window)) try await Task.sleep(for: .seconds(Self.settleSeconds)) - let windowMemory = Self.footprintMB() - before + let windowMemory = try Self.footprintMB() - before - let idle = try await measureIdle(controller) + let idle = try await measureIdle([controller]) let box = screen.bounds let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) let latency = try await measureClicks(controller, on: screen, spots: spots) @@ -103,77 +116,43 @@ final class PerformanceCheckTests: XCTestCase { idleDrawsPerSecond: idle.draws, idleFramesPerSecond: idle.frames, windowMemoryMB: windowMemory, - processMemoryMB: Self.footprintMB(), + processMemoryMB: try Self.footprintMB(), clickLatencyMs: latency, foldStepGapMs: nil, - foldCPUPercent: nil + foldCPUPercent: nil, + idleCPUEnergyMilliwatts: idle.energy, + idleGPUPercent: idle.gpu, + open: opened.time ) } private func measureFoldable(_ device: DeviceInfo) async throws -> WindowMeasurement { try IntegrationHost.install(on: device.udid) - let panels = try adapter.panels(device.udid) - let unfolded = try XCTUnwrap(panels.first { $0.name == "Unfolded" }) - let cover = try XCTUnwrap(panels.first { $0.name == "Cover" }) - let input = try adapter.openInput(device.udid, screenID: unfolded.screenID) - let targeted = try XCTUnwrap(input as? PanelInputSession, "touches must be panel targeted") - let before = Self.footprintMB() - let controller = try manager.open( - device: device, - session: try adapter.openDisplay(device.udid, panel: unfolded), - input: input, - scaleMode: .fit, - bezelEnabled: true, - keepOnTop: false, - showFPS: false, - foldsAtHinge: true, - chrome: unfolded.chromeIdentifier.flatMap { ChromeLocator.chrome(identifier: $0) }, - panelNativeRotation: unfolded.nativeRotation, - unfoldedPanel: unfolded, - cover: FoldableCover(panel: cover, session: try adapter.openDisplay(device.udid, panel: cover)), - retarget: { targeted.setTarget(screenID: $0) } - ) - let window = try XCTUnwrap(controller.window) - window.orderFront(nil) - let model = try XCTUnwrap(Self.find(DuoModelView.self, in: window)) + _ = try run(["simctl", "launch", device.udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(Self.hostSettleSeconds)) + let before = try Self.footprintMB() + var measurement = try await measureFoldableWindow(device, before: before) + measurement.reopen = try await measureReopen(device, before: before) + return measurement + } - let adapter = self.adapter! - let shared = try await IntegrationFoldable.shared.control(for: device.udid, adapter: adapter) - let foldables = FoldableController( - open: { _ in shared }, - hingeStream: { try adapter.openHingeStream($0) }, - displayReport: { try await adapter.displayReport($0) } - ) + private func measureFoldableWindow(_ device: DeviceInfo, before: Double) async throws -> WindowMeasurement { + let opened = try await openWindow(device) + let controller = opened.controller let udid = device.udid + XCTAssertNotNil(opened.input as? PanelInputSession, "touches must be panel targeted") + let foldables = try await followFolds(of: controller, udid: udid) defer { foldables.setAngle(DeviceControlBar.FoldMode.fullyOpen.angle, for: udid) foldables.forget(udid) manager.close(udid) } - controller.onHingeAngle = { angle in foldables.setAngle(angle, for: udid) } - controller.onFoldPreset = { angle in foldables.setAngle(angle, for: udid, eased: true) } - foldables.onMove = { [weak controller, weak self] _, event in - switch event { - case .began(let target): controller?.beginFold(to: target) - case .angle(let angle): - self?.stepTimes.append(ContinuousClock.now) - controller?.showHingeAngle(angle) - case .ended: controller?.endFold() - } - } - foldables.follow( - udid, - onPanel: { [weak controller] panel in controller?.setActivePanel(screenID: panel.displayID) }, - onHinge: { [weak controller] degrees in controller?.showHingeAngle(degrees) } - ) - let open = DeviceControlBar.FoldMode.fullyOpen.angle - controller.showHingeAngle(open) - foldables.setAngle(open, for: udid) - _ = try run(["simctl", "launch", udid, IntegrationHost.bundleID]) + let window = try XCTUnwrap(controller.window) + let model = try XCTUnwrap(Self.find(DuoModelView.self, in: window)) try await Task.sleep(for: .seconds(Self.settleSeconds)) - let windowMemory = Self.footprintMB() - before + let windowMemory = try Self.footprintMB() - before - let idle = try await measureIdle(controller) + let idle = try await measureIdle([controller]) let box = model.bounds let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) .filter { model.screenPoint(at: $0) != nil } @@ -202,25 +181,203 @@ final class PerformanceCheckTests: XCTestCase { idleDrawsPerSecond: idle.draws, idleFramesPerSecond: idle.frames, windowMemoryMB: windowMemory, - processMemoryMB: Self.footprintMB(), + processMemoryMB: try Self.footprintMB(), clickLatencyMs: latency, foldStepGapMs: gaps.isEmpty ? nil : Spread(gaps), - foldCPUPercent: foldCPU + foldCPUPercent: foldCPU, + idleCPUEnergyMilliwatts: idle.energy, + idleGPUPercent: idle.gpu, + open: opened.time + ) + } + + /// Called once the first window has closed. The reopened windows are not wired for folding: the + /// point is what opening and closing costs. + private func measureReopen(_ device: DeviceInfo, before: Double) async throws -> Reopen { + try await Task.sleep(for: .seconds(Self.closeSettleSeconds)) + let afterFirst = try Self.footprintMB() + var opens: [OpenTime] = [] + var added: [Double] = [] + var freed = true + var previous = afterFirst + for _ in 0.. TogetherMeasurement { + var controllers: [DeviceWindowController] = [] + var foldables: [(controller: FoldableController, udid: String)] = [] + defer { + for foldable in foldables { + foldable.controller.setAngle(DeviceControlBar.FoldMode.fullyOpen.angle, for: foldable.udid) + foldable.controller.forget(foldable.udid) + } + for device in devices { manager.close(device.udid) } + } + for device in devices { + let opened = try await openWindow(device) + controllers.append(opened.controller) + if opened.controller.foldsAtHinge { + foldables.append((try await followFolds(of: opened.controller, udid: device.udid), device.udid)) + } + } + try await Task.sleep(for: .seconds(Self.settleSeconds)) + let windowsMemory = try Self.footprintMB() - processStart + let idle = try await measureIdle(controllers) + let soak = try await keepBusy(controllers, seconds: soakSeconds) + return TogetherMeasurement( + devices: devices.map { "\($0.name) (\($0.runtimeName))" }, + idleCPUPercent: idle.cpu, + idleCPUEnergyMilliwatts: idle.energy, + idleGPUPercent: idle.gpu, + idleDrawsPerSecond: idle.draws, + idleFramesPerSecond: idle.frames, + windowsMemoryMB: windowsMemory, + processMemoryMB: try Self.footprintMB(), + soak: soak + ) + } + + private struct OpenedWindow { + let controller: DeviceWindowController + let input: any InputSession + let time: OpenTime + } + + /// In the app's own order: panels, display and input sessions, the orientation from the guest, + /// then the window. The first picture is the first draw after the first frame from the device. + private func openWindow(_ device: DeviceInfo) async throws -> OpenedWindow { + let started = ContinuousClock.now + let panels = (try? adapter.panels(device.udid)) ?? [] + let foldsAtHinge = panels.count > 1 + let unfolded = foldsAtHinge ? panels.first { $0.name == "Unfolded" } : nil + let coverPanel = foldsAtHinge ? panels.first { $0.name == "Cover" } : nil + let session = try adapter.openDisplay(device.udid, panel: unfolded) + session.setBezelEnabled(true) + var cover: FoldableCover? + if let coverPanel { + let coverSession = try adapter.openDisplay(device.udid, panel: coverPanel) + coverSession.setBezelEnabled(true) + cover = FoldableCover(panel: coverPanel, session: coverSession) + } + let input = try adapter.openInput(device.udid, screenID: unfolded?.screenID ?? 0) + let retarget: ((Int) -> Void)? = (input as? PanelInputSession).map { targeted in + { targeted.setTarget(screenID: $0) } + } + let sessionsOpened = ContinuousClock.now + let orientation = DevicectlService().orientation(udid: device.udid) ?? .portrait + let oriented = ContinuousClock.now + let controller = try manager.open( + device: device, + session: session, + input: input, + scaleMode: .fit, + bezelEnabled: true, + keepOnTop: false, + showFPS: false, + foldsAtHinge: foldsAtHinge, + chrome: unfolded?.chromeIdentifier.flatMap { ChromeLocator.chrome(identifier: $0) }, + panelNativeRotation: unfolded?.nativeRotation ?? 0, + unfoldedPanel: unfolded, + cover: cover, + retarget: retarget, + orientation: orientation + ) + if let unfolded, !controller.foldsAtHinge { + controller.nativeRotation = unfolded.nativeRotation + } + _ = controller.applyScaleMode(.fit) + controller.window?.orderFront(nil) + let shown = ContinuousClock.now + var drawsBeforeFrame = controller.drawCount + while controller.framesReceived == 0 || controller.drawCount <= drawsBeforeFrame { + if controller.framesReceived == 0 { drawsBeforeFrame = controller.drawCount } + guard ContinuousClock.now - shown < .seconds(10) else { + manager.close(device.udid) + throw MeasurementUnavailable(what: "\(device.name)'s first picture after 10 seconds") + } + try await Task.sleep(for: .milliseconds(2)) + } + let drawn = ContinuousClock.now + return OpenedWindow( + controller: controller, + input: input, + time: OpenTime( + sessionsMs: Self.milliseconds(sessionsOpened - started), + orientationMs: Self.milliseconds(oriented - sessionsOpened), + windowMs: Self.milliseconds(shown - oriented), + firstPictureMs: Self.milliseconds(drawn - shown) + ) + ) + } + + /// Wired as the app wires a foldable's window, and left fully open. + private func followFolds(of controller: DeviceWindowController, udid: String) async throws -> FoldableController { + let adapter = self.adapter! + let shared = try await IntegrationFoldable.shared.control(for: udid, adapter: adapter) + let foldables = FoldableController( + open: { _ in shared }, + hingeStream: { try adapter.openHingeStream($0) }, + displayReport: { try await adapter.displayReport($0) } + ) + controller.onHingeAngle = { angle in foldables.setAngle(angle, for: udid) } + controller.onFoldPreset = { angle in foldables.setAngle(angle, for: udid, eased: true) } + foldables.onMove = { [weak controller, weak self] _, event in + switch event { + case .began(let target): controller?.beginFold(to: target) + case .angle(let angle): + self?.stepTimes.append(ContinuousClock.now) + controller?.showHingeAngle(angle) + case .ended: controller?.endFold() + } + } + foldables.follow( + udid, + onPanel: { [weak controller] panel in controller?.setActivePanel(screenID: panel.displayID) }, + onHinge: { [weak controller] degrees in controller?.showHingeAngle(degrees) } ) + let open = DeviceControlBar.FoldMode.fullyOpen.angle + controller.showHingeAngle(open) + foldables.setAngle(open, for: udid) + return foldables } private struct Idle { let cpu: Spread let draws: Double let frames: Double + let energy: Double + let gpu: Double } - /// The whole process's CPU, which with one window open is that window's: the test runner itself + /// The whole process's CPU, energy and GPU, which are the open windows': the test runner itself /// sits idle while it waits. - private func measureIdle(_ controller: DeviceWindowController) async throws -> Idle { + private func measureIdle(_ controllers: [DeviceWindowController]) async throws -> Idle { + func draws() -> Int { controllers.map(\.drawCount).reduce(0, +) } + func frames() -> Int { controllers.map(\.framesReceived).reduce(0, +) } var cpu: [Double] = [] - let draws = controller.drawCount - let frames = controller.framesReceived + let drawsStart = draws() + let framesStart = frames() + let energyStart = try Self.cpuEnergyJoules() + let gpuStart = try Self.gpuSeconds() let started = ContinuousClock.now for _ in 0.. MemorySoak { + var targets: [(controller: DeviceWindowController, view: NSView, spots: [CGPoint])] = [] + var folds: [NSSegmentedControl] = [] + for controller in controllers { + let window = try XCTUnwrap(controller.window) + if let model = Self.find(DuoModelView.self, in: window) { + let box = model.bounds + let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) + .filter { model.screenPoint(at: $0) != nil } + if !spots.isEmpty { targets.append((controller, model, spots)) } + folds.append(try XCTUnwrap(Self.find(NSSegmentedControl.self, in: window), "the window's fold positions")) + } else { + let screen = try XCTUnwrap(Self.find(DeviceScreenView.self, in: window)) + let box = screen.bounds + targets.append((controller, screen, Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size))) + } + } + XCTAssertFalse(targets.isEmpty, "no window has a spot to click") + + let started = ContinuousClock.now + func elapsed() -> Double { Self.seconds(ContinuousClock.now - started) } + var samples = [(seconds: 0.0, megabytes: try Self.footprintMB())] + var nextSample = Self.soakSampleSeconds + var nextFold = Self.soakFoldEverySeconds + var clicks = 0 + var foldCount = 0 + while elapsed() < seconds { + if !folds.isEmpty, elapsed() >= nextFold { + for mode in [DeviceControlBar.FoldMode.cover, .fullyOpen] { + for control in folds { + control.selectedSegment = mode.rawValue + control.sendAction(control.action, to: control.target) + } + try await Task.sleep(for: .seconds(3)) + } + foldCount += folds.count + nextFold += Self.soakFoldEverySeconds + } + if !targets.isEmpty { + let target = targets[clicks % targets.count] + let spot = target.spots[(clicks / targets.count) % target.spots.count] + let window = try XCTUnwrap(target.view.window) + let inWindow = target.view.convert(spot, to: nil) + target.view.mouseDown(with: try Self.mouse(.leftMouseDown, at: inWindow, in: window)) + try await Task.sleep(for: .milliseconds(60)) + target.view.mouseUp(with: try Self.mouse(.leftMouseUp, at: inWindow, in: window)) + clicks += 1 + } + try await Task.sleep(for: .seconds(Self.soakClickSeconds)) + while elapsed() >= nextSample { + samples.append((elapsed(), try Self.footprintMB())) + nextSample += Self.soakSampleSeconds + } + } + let warmUp = min(30, seconds / 4) + return MemorySoak( + seconds: seconds, + clicks: clicks, + folds: foldCount, + startMB: samples[0].megabytes, + endMB: samples[samples.count - 1].megabytes, + growthMBPerMinute: MemorySoak.slope(samples.filter { $0.seconds >= warmUp }) ) } @@ -300,7 +526,7 @@ final class PerformanceCheckTests: XCTestCase { return seconds(usage.ru_utime) + seconds(usage.ru_stime) } - private static func footprintMB() -> Double { + private static func footprintMB() throws -> Double { var info = task_vm_info_data_t() var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) let result = withUnsafeMutablePointer(to: &info) { pointer in @@ -308,7 +534,53 @@ final class PerformanceCheckTests: XCTestCase { task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) } } - return result == KERN_SUCCESS ? Double(info.phys_footprint) / 1_048_576 : 0 + guard result == KERN_SUCCESS else { throw MeasurementUnavailable(what: "the process's memory footprint") } + return Double(info.phys_footprint) / 1_048_576 + } + + /// The CPU's energy alone: a GPU bound Metal loop barely moves it (macOS 27.0, Mac15,6). + private static func cpuEnergyJoules() throws -> Double { + var info = task_power_info_v2() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_POWER_INFO_V2), $0, &count) + } + } + guard result == KERN_SUCCESS else { throw MeasurementUnavailable(what: "the process's energy") } + return Double(info.task_energy) / 1e9 + } + + /// The GPU driver's own undocumented count of each process's GPU time, kept on its user clients + /// in the I/O Registry. For frame sized work it read within a third of Metal's own timing, and + /// no closer (macOS 27.0, Mac15,6). + private static func gpuSeconds() throws -> Double { + func property(_ entry: io_registry_entry_t, _ key: String) -> Any? { + IORegistryEntryCreateCFProperty(entry, key as CFString, kCFAllocatorDefault, 0)?.takeRetainedValue() + } + let creator = "pid \(getpid())," + var accelerators: io_iterator_t = 0 + guard IOServiceGetMatchingServices(kIOMainPortDefault, IOServiceMatching("IOAccelerator"), &accelerators) == KERN_SUCCESS else { + throw MeasurementUnavailable(what: "the GPU") + } + defer { IOObjectRelease(accelerators) } + var nanoseconds: UInt64 = 0 + var found = false + while case let accelerator = IOIteratorNext(accelerators), accelerator != 0 { + defer { IOObjectRelease(accelerator) } + var clients: io_iterator_t = 0 + guard IORegistryEntryGetChildIterator(accelerator, kIOServicePlane, &clients) == KERN_SUCCESS else { continue } + defer { IOObjectRelease(clients) } + while case let client = IOIteratorNext(clients), client != 0 { + defer { IOObjectRelease(client) } + guard let owner = property(client, "IOUserClientCreator") as? String, owner.hasPrefix(creator), + let usage = property(client, "AppUsage") as? [[String: Any]] else { continue } + found = true + nanoseconds += usage.compactMap { ($0["accumulatedGPUTime"] as? NSNumber)?.uint64Value }.reduce(0, +) + } + } + guard found else { throw MeasurementUnavailable(what: "this process's GPU time") } + return Double(nanoseconds) / 1e9 } private static func seconds(_ duration: Duration) -> Double { @@ -361,6 +633,11 @@ final class PerformanceCheckTests: XCTestCase { } } +private struct MeasurementUnavailable: Error, CustomStringConvertible { + let what: String + var description: String { "could not read \(what)" } +} + private final class DiscardingStorage: PreferenceStorage, @unchecked Sendable { func text(forKey key: String) -> String? { nil } func setText(_ text: String, forKey key: String) {} diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift index c6ada31..13a1c44 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift @@ -12,11 +12,16 @@ final class PerformanceComparisonTests: XCTestCase { draws: Double = 60, memory: Double = 350, latency: Double = 20, - gap: Double? = 34 + gap: Double? = 34, + energy: Double? = nil, + gpu: Double? = nil, + open: Double? = nil, + kept: Double? = nil, + device: String = "iPhone Duo (iOS 27.1)" ) -> WindowMeasurement { WindowMeasurement( name: name, - device: "iPhone Duo (iOS 27.1)", + device: device, idleCPUPercent: Spread(median: cpu, p95: cpu * 2, samples: 10), idleDrawsPerSecond: draws, idleFramesPerSecond: 0.1, @@ -24,12 +29,50 @@ final class PerformanceComparisonTests: XCTestCase { processMemoryMB: memory + 60, clickLatencyMs: Spread(median: latency, p95: latency * 1.5, samples: 15), foldStepGapMs: gap.map { Spread(median: $0 / 2, p95: $0, samples: 60) }, - foldCPUPercent: gap == nil ? nil : 15 + foldCPUPercent: gap == nil ? nil : 15, + idleCPUEnergyMilliwatts: energy, + idleGPUPercent: gpu, + open: open.map(openTime), + reopen: kept.map { + Reopen( + opens: [openTime(420), openTime(380), openTime(400)], + keptAfterCloseMB: 250, + addedByEachReopenMB: [$0 * 3, $0, 0], + freedOnClose: true + ) + } + ) + } + + private func openTime(_ total: Double) -> OpenTime { + OpenTime(sessionsMs: total / 4, orientationMs: total / 2, windowMs: total / 8, firstPictureMs: total / 8) + } + + private func together( + cpu: Double = 12, + growth: Double = 1, + seconds: Double = 180, + devices: [String] = ["iPhone 17 (iOS 27.1)", "iPhone Duo (iOS 27.1)"] + ) -> TogetherMeasurement { + TogetherMeasurement( + devices: devices, + idleCPUPercent: Spread(median: cpu, p95: cpu * 2, samples: 10), + idleCPUEnergyMilliwatts: 150, + idleGPUPercent: 3, + idleDrawsPerSecond: 30, + idleFramesPerSecond: 0.2, + windowsMemoryMB: 400, + processMemoryMB: 520, + soak: MemorySoak(seconds: seconds, clicks: 300, folds: 8, startMB: 520, endMB: 530, growthMBPerMinute: growth) ) } - private func report(_ windows: [WindowMeasurement], environment: PerformanceEnvironment? = nil) -> PerformanceReport { - PerformanceReport(createdAt: "now", environment: environment ?? self.environment, windows: windows) + private func report( + _ windows: [WindowMeasurement], + together: TogetherMeasurement? = nil, + environment: PerformanceEnvironment? = nil + ) -> PerformanceReport { + PerformanceReport(createdAt: "now", environment: environment ?? self.environment, windows: windows, together: together) } func testTheLimitIsTheLargerOfAShareAndAFixedAmount() { @@ -38,6 +81,31 @@ final class PerformanceComparisonTests: XCTestCase { XCTAssertEqual(tolerance.limit(for: 20), 30, "further up the share decides") } + func testAReopenIsJudgedByItsMedians() { + let reopen = Reopen( + opens: [openTime(420), openTime(380), openTime(400)], + keptAfterCloseMB: 250, + addedByEachReopenMB: [24, 0.5, -1], + freedOnClose: true + ) + XCTAssertEqual(reopen.medianOpen, openTime(400)) + XCTAssertEqual(reopen.medianAddedMB, 0.5, "only the first reopen growing is a cache, not a leak") + } + + func testABaselineBelowZeroIsHeldToZeroPlusTheFixedAmount() { + let tolerance = PerformanceComparison.Tolerance(relative: 0.5, absolute: 5) + XCTAssertEqual(tolerance.limit(for: -6.8), 5, "memory that shrank is not a limit below zero") + } + + func testMemoryKeptByEachReopenBeyondItsLimitIsReported() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(kept: 40)]), + baseline: report([window(kept: 2)]) + ) + XCTAssertEqual(findings.map(\.metric), ["memory kept by each reopen (MB)"]) + XCTAssertEqual(findings.first?.limit, 12) + } + func testTheSpreadTakesTheSampleAtEachPercentile() { let spread = Spread([5, 1, 4, 2, 3, 9, 8, 7, 6, 10]) XCTAssertEqual(spread.median, 5) @@ -110,8 +178,105 @@ final class PerformanceComparisonTests: XCTestCase { } func testTheReportRoundTripsThroughJSON() throws { - let original = report([window(), window("phone", gap: nil)]) + let original = report( + [window(energy: 120, gpu: 4, open: 900, kept: 3), window("phone", gap: nil)], + together: together() + ) let decoded = try JSONDecoder().decode(PerformanceReport.self, from: JSONEncoder().encode(original)) XCTAssertEqual(decoded, original) } + + func testEnergyGPUAndOpenTimeBeyondTheirLimitsAreReported() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(energy: 200, gpu: 7, open: 1700)]), + baseline: report([window(energy: 100, gpu: 4, open: 800)]) + ) + XCTAssertEqual(Set(findings.map(\.metric)), [ + "idle CPU energy (mW)", "idle GPU (%)", "open to first picture (ms)", + ]) + let open = try XCTUnwrap(findings.first { $0.metric == "open to first picture (ms)" }) + XCTAssertEqual(open.limit, 1600, "the first open fails only once it doubles") + } + + func testABaselineFromBeforeTheNewerMetricsStillChecksTheRest() throws { + let json = String(decoding: try JSONEncoder().encode(report([window()])), as: UTF8.self) + XCTAssertFalse(json.contains("idleGPUPercent")) + XCTAssertFalse(json.contains("together")) + let baseline = try JSONDecoder().decode(PerformanceReport.self, from: Data(json.utf8)) + let findings = try PerformanceComparison.regressions( + current: report([window(cpu: 16, energy: 900, gpu: 50, open: 9000)], together: together()), + baseline: baseline + ) + XCTAssertEqual(findings.map(\.metric), ["idle CPU median (%)"]) + } + + func testTheWindowsOpenedTogetherAreComparedUnderTheirOwnName() throws { + let findings = try PerformanceComparison.regressions( + current: report([window()], together: together(cpu: 20, growth: 9)), + baseline: report([window()], together: together()) + ) + XCTAssertEqual(findings.map(\.window), ["together", "together"]) + XCTAssertEqual(Set(findings.map(\.metric)), ["idle CPU median (%)", "memory growth (MB per minute)"]) + } + + func testTheWindowsOpenedTogetherMustBeMeasuredAgain() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()]), + baseline: report([window()], together: together()) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .missingWindow("together")) + } + } + + func testOtherBootedDevicesAreRefused() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()], together: together(devices: ["iPhone 17 (iOS 27.1)"])), + baseline: report([window()], together: together()) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .differentDevices( + window: "together", + baseline: "iPhone 17 (iOS 27.1), iPhone Duo (iOS 27.1)", + current: "iPhone 17 (iOS 27.1)" + )) + } + } + + func testAWindowOnAnotherDeviceIsRefused() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window(device: "iPhone Duo (iOS 27.0)")]), + baseline: report([window()]) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .differentDevices( + window: "foldable", baseline: "iPhone Duo (iOS 27.1)", current: "iPhone Duo (iOS 27.0)" + )) + } + } + + func testAnotherBusyTimeIsRefused() { + XCTAssertThrowsError(try PerformanceComparison.regressions( + current: report([window()], together: together(seconds: 60)), + baseline: report([window()], together: together()) + )) { error in + XCTAssertEqual(error as? PerformanceComparison.Incompatible, .differentSoak(baseline: 180, current: 60)) + } + } + + func testTheGrowthIsTheSlopeInMegabytesAMinute() { + XCTAssertEqual(MemorySoak.slope([(0, 100), (30, 101), (60, 102)]), 2, accuracy: 1e-9) + XCTAssertEqual(MemorySoak.slope([(0, 100), (5, 104), (10, 100), (15, 104)]), 9.6, accuracy: 1e-9) + XCTAssertEqual(MemorySoak.slope([(10, 100), (20, 100)]), 0) + XCTAssertEqual(MemorySoak.slope([(10, 100)]), 0, "one sample has no slope") + } + + func testTheOpenTimeIsTheSumOfItsParts() { + let open = OpenTime(sessionsMs: 40, orientationMs: 700, windowMs: 60, firstPictureMs: 25) + XCTAssertEqual(open.totalMs, 825) + } + + func testTheReportSetsTheWindowsTogetherBesideTheirSumAlone() { + let phone = window("phone", cpu: 0.5, gap: nil, device: "iPhone 17 (iOS 27.1)") + let text = PerformanceComparison.text(report([phone, window(cpu: 6)], together: together(cpu: 7))) + XCTAssertTrue(text.contains("All 2 windows together: iPhone 17 (iOS 27.1), iPhone Duo (iOS 27.1)"), text) + XCTAssertTrue(text.contains("median 7.0%, p95 14.0%, the windows alone add up to 6.5%"), text) + } } diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift index 43bd759..180499b 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift @@ -9,6 +9,8 @@ struct PerformanceReport: Codable, Equatable { let createdAt: String let environment: PerformanceEnvironment let windows: [WindowMeasurement] + /// Absent from reports taken before every window was also measured open at once. + var together: TogetherMeasurement? = nil } struct PerformanceEnvironment: Codable, Equatable { @@ -56,17 +58,97 @@ struct WindowMeasurement: Codable, Equatable { let clickLatencyMs: Spread let foldStepGapMs: Spread? let foldCPUPercent: Double? + /// These are absent from reports taken before they were measured. + var idleCPUEnergyMilliwatts: Double? = nil + var idleGPUPercent: Double? = nil + var open: OpenTime? = nil + var reopen: Reopen? = nil +} + +/// The window opened and closed a few more times after the first closed. What the first left behind +/// can be a cache the next reuses; what grows again with every reopen is kept by every window. +struct Reopen: Codable, Equatable { + let opens: [OpenTime] + let keptAfterCloseMB: Double + /// For each reopen, memory once it closed less memory once the window before it closed. + let addedByEachReopenMB: [Double] + /// Whether every reopened window's controller was freed once it closed. + let freedOnClose: Bool + + var medianOpen: OpenTime { + let median = Spread(opens.map(\.totalMs)).median + return opens.first { $0.totalMs == median } ?? opens[0] + } + + var medianAddedMB: Double { Spread(addedByEachReopenMB).median } +} + +/// From starting to open a window, as the app does, to the device's first picture drawn in it. +struct OpenTime: Codable, Equatable { + /// The panels, the display sessions and the input session. + let sessionsMs: Double + let orientationMs: Double + let windowMs: Double + let firstPictureMs: Double + let totalMs: Double + + init(sessionsMs: Double, orientationMs: Double, windowMs: Double, firstPictureMs: Double) { + self.sessionsMs = sessionsMs + self.orientationMs = orientationMs + self.windowMs = windowMs + self.firstPictureMs = firstPictureMs + totalMs = sessionsMs + orientationMs + windowMs + firstPictureMs + } +} + +/// Every booted device's window open at once, then kept busy for a while to see whether memory +/// keeps growing. +struct TogetherMeasurement: Codable, Equatable { + let devices: [String] + let idleCPUPercent: Spread + let idleCPUEnergyMilliwatts: Double + let idleGPUPercent: Double + let idleDrawsPerSecond: Double + let idleFramesPerSecond: Double + /// Over the process before any window opened, since what the windows measured alone kept after + /// closing is reused by these and would otherwise be left out. + let windowsMemoryMB: Double + let processMemoryMB: Double + let soak: MemorySoak +} + +struct MemorySoak: Codable, Equatable { + let seconds: Double + let clicks: Int + let folds: Int + let startMB: Double + let endMB: Double + /// The least squares slope of the samples taken after the first quarter, at most 30 seconds, + /// which is when caches are still filling. + let growthMBPerMinute: Double + + static func slope(_ samples: [(seconds: Double, megabytes: Double)]) -> Double { + guard samples.count > 1 else { return 0 } + let count = Double(samples.count) + let meanTime = samples.map(\.seconds).reduce(0, +) / count + let meanSize = samples.map(\.megabytes).reduce(0, +) / count + let covariance = samples.map { ($0.seconds - meanTime) * ($0.megabytes - meanSize) }.reduce(0, +) + let variance = samples.map { ($0.seconds - meanTime) * ($0.seconds - meanTime) }.reduce(0, +) + return variance > 0 ? covariance / variance * 60 : 0 + } } enum PerformanceComparison { /// A limit of the baseline plus the larger of a share of it and a fixed amount, so a metric - /// near zero, idle CPU above all, is not failed for noise. + /// near zero, idle CPU above all, is not failed for noise. A baseline below zero, memory that + /// shrank, is held to zero plus the fixed amount for the same reason. struct Tolerance: Equatable { let relative: Double let absolute: Double func limit(for baseline: Double) -> Double { - baseline + max(baseline * relative, absolute) + let base = max(baseline, 0) + return base + max(base * relative, absolute) } } @@ -82,6 +164,8 @@ enum PerformanceComparison { case schema(baseline: Int, current: Int) case environment(baseline: PerformanceEnvironment, current: PerformanceEnvironment) case missingWindow(String) + case differentDevices(window: String, baseline: String, current: String) + case differentSoak(baseline: Double, current: Double) var description: String { switch self { @@ -91,6 +175,10 @@ enum PerformanceComparison { "the baseline was measured on \(baseline) and this run on \(current)" case .missingWindow(let name): "the baseline measured a \(name) window and this run did not" + case .differentDevices(let window, let baseline, let current): + "the baseline measured the \(window) window on \(baseline) and this run on \(current)" + case .differentSoak(let baseline, let current): + "the baseline kept the windows busy for \(Int(baseline)) seconds and this run for \(Int(current))" } } } @@ -104,6 +192,14 @@ enum PerformanceComparison { "click latency p95 (ms)": Tolerance(relative: 0.3, absolute: 12), "fold step gap p95 (ms)": Tolerance(relative: 0.3, absolute: 10), "fold CPU (%)": Tolerance(relative: 0.5, absolute: 5), + "idle CPU energy (mW)": Tolerance(relative: 0.5, absolute: 10), + "idle GPU (%)": Tolerance(relative: 0.5, absolute: 2), + // The first open in a process pays for loading what the rest reuse and moved by more than + // half between runs, so only a doubling fails it. + "open to first picture (ms)": Tolerance(relative: 1, absolute: 300), + "open again to first picture (ms)": Tolerance(relative: 0.3, absolute: 150), + "memory kept by each reopen (MB)": Tolerance(relative: 0.5, absolute: 10), + "memory growth (MB per minute)": Tolerance(relative: 0.5, absolute: 5), ] static func metrics(of window: WindowMeasurement) -> [String: Double] { @@ -116,9 +212,27 @@ enum PerformanceComparison { ] if let gaps = window.foldStepGapMs { values["fold step gap p95 (ms)"] = gaps.p95 } if let cpu = window.foldCPUPercent { values["fold CPU (%)"] = cpu } + if let energy = window.idleCPUEnergyMilliwatts { values["idle CPU energy (mW)"] = energy } + if let gpu = window.idleGPUPercent { values["idle GPU (%)"] = gpu } + if let open = window.open { values["open to first picture (ms)"] = open.totalMs } + if let reopen = window.reopen { + values["open again to first picture (ms)"] = reopen.medianOpen.totalMs + values["memory kept by each reopen (MB)"] = reopen.medianAddedMB + } return values } + static func metrics(of together: TogetherMeasurement) -> [String: Double] { + [ + "idle CPU median (%)": together.idleCPUPercent.median, + "idle draws per second": together.idleDrawsPerSecond, + "window memory (MB)": together.windowsMemoryMB, + "idle CPU energy (mW)": together.idleCPUEnergyMilliwatts, + "idle GPU (%)": together.idleGPUPercent, + "memory growth (MB per minute)": together.soak.growthMBPerMinute, + ] + } + static func regressions( current: PerformanceReport, baseline: PerformanceReport @@ -134,19 +248,41 @@ enum PerformanceComparison { guard let now = current.windows.first(where: { $0.name == base.name }) else { throw .missingWindow(base.name) } - let before = metrics(of: base) - let after = metrics(of: now) - for (metric, value) in before.sorted(by: { $0.key < $1.key }) { - guard let measured = after[metric], let tolerance = tolerances[metric] else { continue } - let limit = tolerance.limit(for: value) - if measured > limit { - findings.append(Finding(window: base.name, metric: metric, baseline: value, current: measured, limit: limit)) - } + guard now.device == base.device else { + throw .differentDevices(window: base.name, baseline: base.device, current: now.device) } + findings += compare(base.name, metrics(of: base), metrics(of: now)) + } + if let base = baseline.together { + guard let now = current.together else { throw .missingWindow(togetherName) } + guard now.devices == base.devices else { + throw .differentDevices( + window: togetherName, + baseline: base.devices.joined(separator: ", "), + current: now.devices.joined(separator: ", ") + ) + } + guard now.soak.seconds == base.soak.seconds else { + throw .differentSoak(baseline: base.soak.seconds, current: now.soak.seconds) + } + findings += compare(togetherName, metrics(of: base), metrics(of: now)) } return findings } + static let togetherName = "together" + + /// A metric only the baseline or only this run has is skipped, so a baseline taken before a + /// metric existed still checks the rest. + private static func compare(_ window: String, _ before: [String: Double], _ after: [String: Double]) -> [Finding] { + before.sorted(by: { $0.key < $1.key }).compactMap { metric, value in + guard let measured = after[metric], let tolerance = tolerances[metric] else { return nil } + let limit = tolerance.limit(for: value) + guard measured > limit else { return nil } + return Finding(window: window, metric: metric, baseline: value, current: measured, limit: limit) + } + } + /// The report as a person reads it, one window after another. static func text(_ report: PerformanceReport, findings: [Finding]? = nil) -> String { func number(_ value: Double) -> String { String(format: "%.1f", value) } @@ -167,6 +303,33 @@ enum PerformanceComparison { if let cpu = window.foldCPUPercent { lines.append(" fold CPU \(number(cpu))%") } + if let energy = window.idleCPUEnergyMilliwatts, let gpu = window.idleGPUPercent { + lines.append(" idle energy \(number(energy)) mW of CPU, GPU busy \(number(gpu))% of the time") + } + func parts(_ open: OpenTime) -> String { + "\(number(open.totalMs)) ms to the first picture: \(number(open.sessionsMs)) ms sessions, \(number(open.orientationMs)) ms orientation, \(number(open.windowMs)) ms window, \(number(open.firstPictureMs)) ms first picture" + } + if let open = window.open { + lines.append(" open \(parts(open))") + } + if let reopen = window.reopen { + lines.append(" open again median of \(reopen.opens.count), \(parts(reopen.medianOpen))") + lines.append(" after closing \(number(reopen.keptAfterCloseMB)) MB kept by the first window, then \(reopen.addedByEachReopenMB.map(number).joined(separator: ", ")) MB by each reopen; \(reopen.freedOnClose ? "every reopened window was freed" : "a reopened window outlived its window")") + } + lines.append("") + } + if let together = report.together { + let alone = together.devices.compactMap { device in report.windows.first { $0.device == device } } + let sum = alone.count == together.devices.count + ? ", the windows alone add up to \(number(alone.map(\.idleCPUPercent.median).reduce(0, +)))%" + : "" + lines.append("All \(together.devices.count) windows together: \(together.devices.joined(separator: ", "))") + lines.append(" idle CPU median \(number(together.idleCPUPercent.median))%, p95 \(number(together.idleCPUPercent.p95))%\(sum)") + lines.append(" idle drawing \(number(together.idleDrawsPerSecond)) pictures a second for \(number(together.idleFramesPerSecond)) frames a second from the devices") + lines.append(" idle energy \(number(together.idleCPUEnergyMilliwatts)) mW of CPU, GPU busy \(number(together.idleGPUPercent))% of the time") + lines.append(" memory \(number(together.windowsMemoryMB)) MB over the process before any window opened, \(number(together.processMemoryMB)) MB for the process") + let soak = together.soak + lines.append(" kept busy \(Int(soak.seconds)) s, \(soak.clicks) clicks, \(soak.folds) folds: \(number(soak.startMB)) MB to \(number(soak.endMB)) MB, growing \(number(soak.growthMBPerMinute)) MB a minute") lines.append("") } if let findings { From 66e73ce3d266d87b9f8b7e93ace7d31a00327d4a Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 11:51:24 +0100 Subject: [PATCH 08/17] build(scripts): let the performance check keep windows busy for longer --- RELEASING.md | 11 ++++++----- scripts/performance-check.sh | 34 ++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 0332b46..83ae822 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -52,11 +52,12 @@ unsigned build that reaches a release page is worse than a run that stops. update feed both come from it, so they cannot drift apart. 2. Update `VERSION`. 3. Run `scripts/performance-check.sh` on the Mac the baselines in `performance/` were recorded on, - with an iPhone and an iPhone Duo booted and Low Power Mode off. It compares with - `baseline-normal-power.json`, or `baseline-low-power.json` when Low Power Mode is on, and exits 1 - if a window got slower, heavier or less responsive than the baseline allows. Record a new - baseline with `--record-baseline` only when a change is meant to move the numbers, and commit it - with that change. + with only an iPhone and an iPhone Duo booted and Low Power Mode off. Every booted device joins + the windows it opens at once, and the baseline names them. It takes about eight minutes, compares + with `baseline-normal-power.json`, or `baseline-low-power.json` when Low Power Mode is on, and + exits 1 if a window got slower to open, heavier, less responsive or used more energy or GPU than + the baseline allows, or if memory kept growing. Record a new baseline with `--record-baseline` + only when a change is meant to move the numbers, and commit it with that change. 4. Run the **Release** workflow from the Actions tab with `publish` left **false**. That is a dry run: it builds, signs, notarizes, staples, packages, checksums and generates the feed, and creates no tag, no release and no commit anywhere. The artifacts are attached to the run under a diff --git a/scripts/performance-check.sh b/scripts/performance-check.sh index a184d62..4093158 100755 --- a/scripts/performance-check.sh +++ b/scripts/performance-check.sh @@ -1,18 +1,24 @@ #!/bin/bash # Measures what a device window costs on this Mac and compares it with a baseline taken here -# before: CPU and drawing while nothing on screen changes, memory, click to frame latency and, on a -# foldable, how evenly a fold is drawn. +# before: how long it takes to open, CPU, energy, GPU and drawing while nothing on screen changes, +# memory, click to frame latency and, on a foldable, how evenly a fold is drawn. Then every booted +# device's window is opened at once, measured the same way and kept busy with clicks and folds to +# see whether memory keeps growing. # # Needs a booted iPhone and a booted iPhone Duo; a device that is not booted is not measured, and a -# baseline that measured it then fails the comparison. Keep the Mac otherwise quiet while it runs. +# baseline that measured it then fails the comparison. Every other booted device joins the windows +# opened at once. Keep the Mac otherwise quiet while it runs. # -# A baseline only compares with a run on the same Mac model, macOS, Xcode, power mode and build -# configuration. Low Power Mode caps drawing at 60 a second, so there is one baseline for each: -# performance/baseline-low-power.json and performance/baseline-normal-power.json, and the one that -# matches the Mac right now is used unless --baseline names another. +# A baseline only compares with a run on the same Mac model, macOS, Xcode, power mode, build +# configuration, booted devices and busy time. Low Power Mode caps drawing at 60 a second, so there +# is one baseline for each: performance/baseline-low-power.json and +# performance/baseline-normal-power.json, and the one that matches the Mac right now is used unless +# --baseline names another. # -# Usage: scripts/performance-check.sh [--baseline FILE] [--record-baseline | --measure-only] +# Usage: scripts/performance-check.sh [--baseline FILE] [--busy-minutes N] +# [--record-baseline | --measure-only] # (default) compare with the baseline and exit 1 on a regression +# --busy-minutes N how long to keep the windows busy, 3 unless given # --record-baseline measure and write the baseline, accepting what was measured # --measure-only measure and report, comparing with nothing set -euo pipefail @@ -24,18 +30,26 @@ else baseline="${repo_root}/performance/baseline-normal-power.json" fi mode="compare" +busy_minutes=3 while [ $# -gt 0 ]; do case "$1" in --baseline) baseline="$2"; shift ;; + --busy-minutes) busy_minutes="$2"; shift ;; --record-baseline) mode="record" ;; --measure-only) mode="measure" ;; - -h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -h|--help) sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "Unknown option $1" >&2; exit 2 ;; esac shift done +if ! [[ "${busy_minutes}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + echo "--busy-minutes takes a number of minutes, not ${busy_minutes}" >&2 + exit 2 +fi +busy_seconds="$(awk -v minutes="${busy_minutes}" 'BEGIN { printf "%d", minutes * 60 }')" + if [ "${mode}" = "compare" ] && [ ! -f "${baseline}" ]; then echo "No baseline at ${baseline}. Record one with --record-baseline first." >&2 exit 2 @@ -44,7 +58,7 @@ fi output="${repo_root}/build/performance/$(date +%Y%m%d-%H%M%S)" mkdir -p "${output}" -variables=(ODH_INTEGRATION=1 "ODH_PERFORMANCE_REPORT=${output}") +variables=(ODH_INTEGRATION=1 "ODH_PERFORMANCE_REPORT=${output}" "ODH_PERFORMANCE_BUSY_SECONDS=${busy_seconds}") case "${mode}" in compare) variables+=("ODH_PERFORMANCE_BASELINE=${baseline}") ;; record) variables+=("ODH_PERFORMANCE_RECORD=${baseline}") ;; From b6236f9cd5fe46e5e5ef8ab19d473fc3701ff325 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 13:02:55 +0100 Subject: [PATCH 09/17] test(viewer): read window memory as the median while idle A single reading eight seconds after a window opened sometimes caught a passing rise of about 100 MB in the process, which recorded the iPhone window at 142 MB instead of about 45. --- .../PerformanceCheckTests.swift | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift index 7103f76..047db47 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift @@ -103,9 +103,9 @@ final class PerformanceCheckTests: XCTestCase { let window = try XCTUnwrap(controller.window) let screen = try XCTUnwrap(Self.find(DeviceScreenView.self, in: window)) try await Task.sleep(for: .seconds(Self.settleSeconds)) - let windowMemory = try Self.footprintMB() - before let idle = try await measureIdle([controller]) + let windowMemory = idle.memory - before let box = screen.bounds let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) let latency = try await measureClicks(controller, on: screen, spots: spots) @@ -150,9 +150,9 @@ final class PerformanceCheckTests: XCTestCase { let window = try XCTUnwrap(controller.window) let model = try XCTUnwrap(Self.find(DuoModelView.self, in: window)) try await Task.sleep(for: .seconds(Self.settleSeconds)) - let windowMemory = try Self.footprintMB() - before let idle = try await measureIdle([controller]) + let windowMemory = idle.memory - before let box = model.bounds let spots = Self.spots(around: CGPoint(x: box.midX, y: box.midY), size: box.size) .filter { model.screenPoint(at: $0) != nil } @@ -240,8 +240,8 @@ final class PerformanceCheckTests: XCTestCase { } } try await Task.sleep(for: .seconds(Self.settleSeconds)) - let windowsMemory = try Self.footprintMB() - processStart let idle = try await measureIdle(controllers) + let windowsMemory = idle.memory - processStart let soak = try await keepBusy(controllers, seconds: soakSeconds) return TogetherMeasurement( devices: devices.map { "\($0.name) (\($0.runtimeName))" }, @@ -366,14 +366,17 @@ final class PerformanceCheckTests: XCTestCase { let frames: Double let energy: Double let gpu: Double + let memory: Double } - /// The whole process's CPU, energy and GPU, which are the open windows': the test runner itself - /// sits idle while it waits. + /// The whole process's CPU, energy, GPU and memory, which are the open windows': the test runner + /// itself sits idle while it waits. Memory is the median of a reading each round, so one passing + /// spike does not decide it. private func measureIdle(_ controllers: [DeviceWindowController]) async throws -> Idle { func draws() -> Int { controllers.map(\.drawCount).reduce(0, +) } func frames() -> Int { controllers.map(\.framesReceived).reduce(0, +) } var cpu: [Double] = [] + var memory: [Double] = [] let drawsStart = draws() let framesStart = frames() let energyStart = try Self.cpuEnergyJoules() @@ -384,6 +387,7 @@ final class PerformanceCheckTests: XCTestCase { let wallStart = ContinuousClock.now try await Task.sleep(for: .seconds(Self.idleRoundSeconds)) cpu.append((Self.cpuSeconds() - cpuStart) / Self.seconds(ContinuousClock.now - wallStart) * 100) + memory.append(try Self.footprintMB()) } let elapsed = Self.seconds(ContinuousClock.now - started) return Idle( @@ -391,7 +395,8 @@ final class PerformanceCheckTests: XCTestCase { draws: Double(draws() - drawsStart) / elapsed, frames: Double(frames() - framesStart) / elapsed, energy: (try Self.cpuEnergyJoules() - energyStart) / elapsed * 1000, - gpu: (try Self.gpuSeconds() - gpuStart) / elapsed * 100 + gpu: (try Self.gpuSeconds() - gpuStart) / elapsed * 100, + memory: Spread(memory).median ) } From 65ae275130e1574aa87c02add1978abcb9c5a329 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 13:02:55 +0100 Subject: [PATCH 10/17] build(scripts): record the normal power baseline with the new measurements --- performance/baseline-normal-power.json | 149 +++++++++++++++++++++---- 1 file changed, 128 insertions(+), 21 deletions(-) diff --git a/performance/baseline-normal-power.json b/performance/baseline-normal-power.json index 590ebd2..69fb184 100644 --- a/performance/baseline-normal-power.json +++ b/performance/baseline-normal-power.json @@ -1,5 +1,5 @@ { - "createdAt" : "2026-09-28T09:09:23Z", + "createdAt" : "2026-09-28T11:53:21Z", "environment" : { "buildConfiguration" : "release", "hardwareModel" : "Mac15,6", @@ -8,48 +8,155 @@ "xcode" : "Xcode 27.1 (27A9269)" }, "schemaVersion" : 1, + "together" : { + "devices" : [ + "iPhone 17 (iOS 27.0)", + "iPhone Duo (iOS 27.1)" + ], + "idleCPUEnergyMilliwatts" : 12.972761723841723, + "idleCPUPercent" : { + "median" : 9.572144208538406, + "p95" : 12.162444850751928, + "samples" : 10 + }, + "idleDrawsPerSecond" : 33.691864374922474, + "idleFramesPerSecond" : 2.308334731226467, + "idleGPUPercent" : 3.638691069104495, + "processMemoryMB" : 413.5949401855469, + "soak" : { + "clicks" : 227, + "endMB" : 413.5949401855469, + "folds" : 8, + "growthMBPerMinute" : 2.351137537762155, + "seconds" : 180, + "startMB" : 401.1105651855469 + }, + "windowsMemoryMB" : 363.6254348754883 + }, "windows" : [ { "clickLatencyMs" : { - "median" : 11.968583333333333, - "p95" : 13.459375, + "median" : 8.787416666666665, + "p95" : 13.040416666666665, "samples" : 20 }, "device" : "iPhone 17 (iOS 27.0)", + "idleCPUEnergyMilliwatts" : 0.708254376266516, "idleCPUPercent" : { - "median" : 0.19903322942526908, - "p95" : 0.2747969613825266, + "median" : 0.2054317745774193, + "p95" : 0.7792023230559177, "samples" : 10 }, - "idleDrawsPerSecond" : 0, - "idleFramesPerSecond" : 0, + "idleDrawsPerSecond" : 0.09746648917987798, + "idleFramesPerSecond" : 0.09746648917987798, + "idleGPUPercent" : 0.005427460436577177, "name" : "phone", - "processMemoryMB" : 152.65716552734375, - "windowMemoryMB" : 48.56263732910156 + "open" : { + "firstPictureMs" : 47.018209, + "orientationMs" : 410.716292, + "sessionsMs" : 26.218917, + "totalMs" : 670.460834, + "windowMs" : 186.507416 + }, + "processMemoryMB" : 152.50091552734375, + "reopen" : { + "addedByEachReopenMB" : [ + 5.671875, + 4.4375, + 4.578125 + ], + "freedOnClose" : true, + "keptAfterCloseMB" : 26.875091552734375, + "opens" : [ + { + "firstPictureMs" : 18.98425, + "orientationMs" : 338.607375, + "sessionsMs" : 28.813417, + "totalMs" : 402.951042, + "windowMs" : 16.546000000000003 + }, + { + "firstPictureMs" : 16.394459, + "orientationMs" : 321.07404099999997, + "sessionsMs" : 31.128209000000002, + "totalMs" : 380.868834, + "windowMs" : 12.272125 + }, + { + "firstPictureMs" : 16.304917, + "orientationMs" : 319.44795799999997, + "sessionsMs" : 24.152292, + "totalMs" : 375.04879199999993, + "windowMs" : 15.143625 + } + ] + }, + "windowMemoryMB" : 47.82823944091797 }, { "clickLatencyMs" : { - "median" : 14.076875, - "p95" : 16.38925, + "median" : 11.423833333333334, + "p95" : 14.965791666666666, "samples" : 20 }, "device" : "iPhone Duo (iOS 27.1)", - "foldCPUPercent" : 36.23484187234482, + "foldCPUPercent" : 33.88276299687477, "foldStepGapMs" : { - "median" : 18.287666, - "p95" : 20.769958000000003, - "samples" : 108 + "median" : 18.637625, + "p95" : 24.799459, + "samples" : 104 }, + "idleCPUEnergyMilliwatts" : 20.23728821071902, "idleCPUPercent" : { - "median" : 9.649805164040677, - "p95" : 13.174145444260233, + "median" : 9.5980171669382, + "p95" : 12.037014510216425, "samples" : 10 }, - "idleDrawsPerSecond" : 35.885166285889795, - "idleFramesPerSecond" : 2.116683333735612, + "idleDrawsPerSecond" : 35.61601065391605, + "idleFramesPerSecond" : 2.213702319649478, + "idleGPUPercent" : 4.268911748829975, "name" : "foldable", - "processMemoryMB" : 357.2667922973633, - "windowMemoryMB" : 295.98462677001953 + "open" : { + "firstPictureMs" : 46.669416, + "orientationMs" : 334.866417, + "sessionsMs" : 48.03337500000001, + "totalMs" : 1341.278208, + "windowMs" : 911.709 + }, + "processMemoryMB" : 371.9854965209961, + "reopen" : { + "addedByEachReopenMB" : [ + -4.4530792236328125, + -25.375, + 9.078147888183594 + ], + "freedOnClose" : true, + "keptAfterCloseMB" : 89.57835388183594, + "opens" : [ + { + "firstPictureMs" : 76.78133299999999, + "orientationMs" : 355.050792, + "sessionsMs" : 63.769625, + "totalMs" : 975.6773750000001, + "windowMs" : 480.075625 + }, + { + "firstPictureMs" : 87.002, + "orientationMs" : 355.940292, + "sessionsMs" : 36.425833, + "totalMs" : 954.561875, + "windowMs" : 475.19375 + }, + { + "firstPictureMs" : 81.960792, + "orientationMs" : 340.14175, + "sessionsMs" : 52.940417, + "totalMs" : 902.361084, + "windowMs" : 427.318125 + } + ] + }, + "windowMemoryMB" : 315.10960388183594 } ] } \ No newline at end of file From 5ad58e10932cdc194391d62e03ae694ff4ee5b36 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 14:18:55 +0100 Subject: [PATCH 11/17] fix(simctl): ask an ordinary device for its orientation once --- .../Sources/ODHubViewerApp/ViewerMain.swift | 2 +- .../Simctl/DevicectlService.swift | 8 +++--- .../DevicectlServiceTests.swift | 12 +++++++-- .../OpenSidewaysDriveTests.swift | 27 +++++++++++++++++-- .../PerformanceCheckTests.swift | 2 +- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/engine/Sources/ODHubViewerApp/ViewerMain.swift b/engine/Sources/ODHubViewerApp/ViewerMain.swift index 2efb26f..3df6d4a 100644 --- a/engine/Sources/ODHubViewerApp/ViewerMain.swift +++ b/engine/Sources/ODHubViewerApp/ViewerMain.swift @@ -637,7 +637,7 @@ struct ODHubViewer: ParsableCommand { } let panel = opened.unfolded let session = opened.session - let orientation = DevicectlService().orientation(udid: device.udid) ?? .portrait + let orientation = DevicectlService().orientation(udid: device.udid, foldable: opened.foldsAtHinge) ?? .portrait let controller = try manager.open( device: device, diff --git a/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift b/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift index 3b5fa89..c15a61e 100644 --- a/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift +++ b/engine/Sources/OpenDeviceHubEngine/Simctl/DevicectlService.swift @@ -24,10 +24,12 @@ public struct DevicectlService: Sendable { } /// On a foldable the first answer after a turn repeats the answer given before it, however long - /// ago the turn was (27A266a), so the question is asked twice and the second answer kept. - public func orientation(udid: String) -> DeviceOrientation? { + /// ago the turn was (27A266a, 27A9269), so a foldable is asked twice and the second answer kept. + /// An ordinary iPhone answers right the first time (27A9269), and each ask costs 150 to 400 ms. + public func orientation(udid: String, foldable: Bool) -> DeviceOrientation? { let arguments = Self.orientationArguments(udid: udid) - guard run(arguments) != nil, let output = run(arguments) else { return nil } + if foldable { guard run(arguments) != nil else { return nil } } + guard let output = run(arguments) else { return nil } return Self.parseOrientation(Data(output.utf8)) } diff --git a/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift b/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift index fbcb70c..22dd709 100644 --- a/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift +++ b/engine/Tests/OpenDeviceHubEngineTests/DevicectlServiceTests.swift @@ -37,13 +37,21 @@ final class DevicectlServiceTests: XCTestCase { func testKeepsTheSecondAnswerSinceAFoldableRepeatsItsLastOne() { let answers = Answers([report("portrait"), report("landscapeRight")]) let service = DevicectlService { _ in answers.next() } - XCTAssertEqual(service.orientation(udid: "ABC"), .landscapeLeft) + XCTAssertEqual(service.orientation(udid: "ABC", foldable: true), .landscapeLeft) XCTAssertEqual(answers.asked, 2) } + func testAsksAnOrdinaryDeviceOnce() { + let answers = Answers([report("landscapeRight"), report("portrait")]) + let service = DevicectlService { _ in answers.next() } + XCTAssertEqual(service.orientation(udid: "ABC", foldable: false), .landscapeLeft) + XCTAssertEqual(answers.asked, 1) + } + func testNoAnswerWhenDevicectlCannotSeeTheDevice() { let service = DevicectlService { _ in nil } - XCTAssertNil(service.orientation(udid: "ABC")) + XCTAssertNil(service.orientation(udid: "ABC", foldable: true)) + XCTAssertNil(service.orientation(udid: "ABC", foldable: false)) } func testAnythingElseIsNoAnswer() { diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift index cd4e762..3112844 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/OpenSidewaysDriveTests.swift @@ -25,7 +25,7 @@ final class OpenSidewaysDriveTests: XCTestCase { defer { try? adapter.setOrientation(.portrait, udid: udid) } try await Task.sleep(for: .seconds(3)) - let orientation = try XCTUnwrap(DevicectlService().orientation(udid: udid), "devicectl did not say how the device is turned") + let orientation = try XCTUnwrap(DevicectlService().orientation(udid: udid, foldable: false), "devicectl did not say how the device is turned") XCTAssertEqual(orientation, .landscapeLeft) let manager = DeviceWindowManager(shutdown: { _ in }) @@ -64,6 +64,29 @@ final class OpenSidewaysDriveTests: XCTestCase { try await checkQuadrants(screen, in: window, log: log, shown: "upright") } + /// One ask, straight after each turn, twice round, so a first answer that repeats the last one + /// would show. + func testAnOrdinaryIPhoneSaysHowItIsTurnedAtTheFirstAsk() async throws { + try IntegrationGate.requireEnabled() + let adapter = try AdapterFactory.make(for: XcodeLocator.locate()) + guard let device = try adapter.devices().first(where: { + $0.state == .booted && $0.name.hasPrefix("iPhone") && ((try? adapter.panels($0.udid).count) ?? 0) < 2 + }) else { throw XCTSkip("boot an ordinary iPhone to run this test") } + let udid = device.udid + try IntegrationHost.install(on: udid) + _ = try run(["simctl", "launch", udid, IntegrationHost.bundleID]) + try await Task.sleep(for: .seconds(3)) + defer { try? adapter.setOrientation(.portrait, udid: udid) } + + for orientation in [DeviceOrientation.landscapeLeft, .landscapeRight, .portrait, .landscapeRight, .portrait, .landscapeLeft, .portrait] { + try adapter.setOrientation(orientation, udid: udid) + try await Task.sleep(for: .seconds(2)) + let read = DevicectlService().orientation(udid: udid, foldable: false) + print("RESULT iPhone turned \(orientation.rawValue), first answer \(read?.rawValue ?? "nothing")") + XCTAssertEqual(read, orientation) + } + } + func testAFoldableSaysHowItIsTurned() async throws { try IntegrationGate.requireEnabled() let adapter = try AdapterFactory.make(for: XcodeLocator.locate()) @@ -76,7 +99,7 @@ final class OpenSidewaysDriveTests: XCTestCase { for orientation in [DeviceOrientation.landscapeRight, .landscapeLeft, .portrait] { try control.setOrientation(orientation) try await Task.sleep(for: .seconds(3)) - let read = DevicectlService().orientation(udid: device.udid) + let read = DevicectlService().orientation(udid: device.udid, foldable: true) print("RESULT foldable turned \(orientation.rawValue), read back \(read?.rawValue ?? "nothing")") XCTAssertEqual(read, orientation) } diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift index 047db47..7f07dde 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceCheckTests.swift @@ -283,7 +283,7 @@ final class PerformanceCheckTests: XCTestCase { { targeted.setTarget(screenID: $0) } } let sessionsOpened = ContinuousClock.now - let orientation = DevicectlService().orientation(udid: device.udid) ?? .portrait + let orientation = DevicectlService().orientation(udid: device.udid, foldable: foldsAtHinge) ?? .portrait let oriented = ContinuousClock.now let controller = try manager.open( device: device, From 782c2ff0f45c7b8941ee15604164301490be2a93 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 14:36:09 +0100 Subject: [PATCH 12/17] fix(viewer): free a closed window's screen view The click and gesture handlers stored on the screen view captured the view itself, so every closed window left its view, drawables and input session behind, about 4 MB each time a window opened. --- .../DeviceWindowController.swift | 5 +- .../WindowReleaseTests.swift | 80 +++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 engine/Tests/OpenDeviceHubViewerTests/WindowReleaseTests.swift diff --git a/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift b/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift index ee6875f..de7d520 100644 --- a/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift +++ b/engine/Sources/OpenDeviceHubViewer/DeviceWindowController.swift @@ -798,7 +798,6 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate private func installClickToTap() { let pixelSize = session.pixelSize - let screenView = screenView // The model's hit test already yields the guest's own coordinates; nothing is turned here. modelView?.onTouch = { [weak self] point, phase in @@ -817,7 +816,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate let orientation = self.orientation guard let primary = CoordinateMapper.normalize( viewPoint: point, - viewSize: screenView.bounds.size, + viewSize: self.screenView.bounds.size, pixelSize: orientation.displayedSize(portraitNative: pixelSize) ) else { return } @@ -865,7 +864,7 @@ public final class DeviceWindowController: NSWindowController, NSWindowDelegate screenView.onGesture = { [weak self] phase, spread, angle in guard let self else { return } - let size = screenView.bounds.size + let size = self.screenView.bounds.size guard size.width > 0, size.height > 0 else { return } let normalizedSpread = spread / min(size.width, size.height) diff --git a/engine/Tests/OpenDeviceHubViewerTests/WindowReleaseTests.swift b/engine/Tests/OpenDeviceHubViewerTests/WindowReleaseTests.swift new file mode 100644 index 0000000..4117483 --- /dev/null +++ b/engine/Tests/OpenDeviceHubViewerTests/WindowReleaseTests.swift @@ -0,0 +1,80 @@ +import AppKit +import XCTest +@testable import OpenDeviceHubEngine +@testable import OpenDeviceHubViewer + +/// A view that outlives its window keeps its drawables and its device's input session with it, +/// about 4 MB for every window opened. +@MainActor +final class WindowReleaseTests: XCTestCase { + func testAClosedWindowFreesItsControllerWindowAndScreen() async throws { + let manager = DeviceWindowManager( + frameStore: WindowFrameStore(storage: InMemoryPreferences(), prefix: "test."), + settings: ViewerSettings(storage: InMemoryPreferences(), prefix: "test."), + shutdown: { _ in } + ) + weak var controller: DeviceWindowController? + weak var window: NSWindow? + weak var screen: DeviceScreenView? + do { + let opened = try manager.open( + device: DeviceInfo( + udid: "released", + name: "iPhone 17", + deviceTypeIdentifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-17", + runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-27-0", + runtimeName: "iOS 27.0", + state: .booted, + isAvailable: true + ), + session: EmptySession(), + input: NoInput(), + scaleMode: .fit, + bezelEnabled: true, + keepOnTop: false, + showFPS: false + ) + controller = opened + window = opened.window + screen = opened.window.flatMap { Self.screen(in: $0) } + XCTAssertNotNil(screen) + manager.close("released") + } + for _ in 0..<20 where controller != nil || window != nil || screen != nil { + try await Task.sleep(for: .milliseconds(50)) + } + XCTAssertNil(controller, "the controller outlived its window") + XCTAssertNil(window, "the window outlived being closed") + XCTAssertNil(screen, "the screen view outlived its window") + } + + private static func screen(in window: NSWindow) -> DeviceScreenView? { + func search(_ view: NSView) -> DeviceScreenView? { + if let found = view as? DeviceScreenView { return found } + for subview in view.subviews { + if let found = search(subview) { return found } + } + return nil + } + return window.contentView.flatMap(search) + } +} + +/// Clicks are only wired up for a window that can send them. +private final class NoInput: InputSession, @unchecked Sendable { + func touch(_ event: TouchEvent) async throws {} + func key(_ event: KeyEvent) async throws {} + func button(_ button: HardwareButton, phase: ButtonPhase) async throws {} + func close() {} +} + +private final class EmptySession: DisplaySession, @unchecked Sendable { + let frames = AsyncStream { $0.finish() } + let screenChanges = AsyncStream { $0.finish() } + let pixelSize = CGSize(width: 1206, height: 2622) + let pointScale: CGFloat = 3 + let pixelsPerInch: CGFloat? = 460 + let supportsBezel = false + func setBezelEnabled(_ enabled: Bool) {} + func close() {} +} From ea1fc0c318851a6a5e75ab67ad5465cb9bb0b9d7 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 14:52:20 +0100 Subject: [PATCH 13/17] fix(viewer): draw the foldable once for each change An SCNView drawing on demand draws about 16 more times after every change, so two frames a second from the device kept the model drawing 35 times a second. The model now draws itself through SceneKit's renderer into the same pixel format, colour space and 4x multisampling, once per change, and on a timer only while a button lift animates. --- .../OpenDeviceHubViewer/DuoModelView.swift | 119 ++++++++++++------ 1 file changed, 84 insertions(+), 35 deletions(-) diff --git a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift index d9f68c2..e6262c2 100644 --- a/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift +++ b/engine/Sources/OpenDeviceHubViewer/DuoModelView.swift @@ -1,14 +1,19 @@ import AppKit import IOSurface import Metal +import MetalKit import OpenDeviceHubEngine -import os +import QuartzCore import SceneKit import simd /// A foldable drawn as the physical device, bent by the hinge, with the guest's screen on it. +/// +/// Drawn by its own renderer once for each change. An `SCNView` drawing on demand draws about 16 +/// more times after every change, a new frame from the device included, which at 2 frames a second +/// from the device was 35 draws and 9% of the CPU at rest (Xcode 27.1, 27A9269). @MainActor -public final class DuoModelView: SCNView { +public final class DuoModelView: MTKView { /// Times in the asset's closing clip. Its other flat-to-shut stretches also turn the hardware. enum Pose { static let open: TimeInterval = 260.0 / 24 @@ -84,9 +89,13 @@ public final class DuoModelView: SCNView { /// How many probe renders have been made, for a test that expects a move to make none. var probeRenders = 0 - /// SceneKit calls back on its render thread, so the count is kept behind a lock. - private let renders = OSAllocatedUnfairLock(initialState: 0) - public var renderCount: Int { renders.withLock { $0 } } + public private(set) var renderCount = 0 + private let sceneRenderer: SCNRenderer + private let commandQueue: MTLCommandQueue + /// Drawing runs on a timer until then, for a lift that SceneKit animates. + private var animatingUntil: CFTimeInterval = 0 + public var scene: SCNScene? { sceneRenderer.scene } + public var pointOfView: SCNNode? { sceneRenderer.pointOfView } private let cameraNode = SCNNode() private let innerScreen: SCNNode private let coverScreen: SCNNode @@ -133,6 +142,9 @@ public final class DuoModelView: SCNView { let cover = Self.closest(to: CGSize(width: 11.230, height: 7.739), among: faces), inner !== cover else { return nil } + guard let queue = metalDevice.makeCommandQueue() else { return nil } + commandQueue = queue + sceneRenderer = SCNRenderer(device: metalDevice, options: nil) self.metalDevice = metalDevice self.nativeQuarterTurns = ((-nativeRotation / 90) % 4 + 4) % 4 innerScreen = inner @@ -141,8 +153,8 @@ public final class DuoModelView: SCNView { lifts = Self.lifts(for: hardwareButtons, in: scene.rootNode) activeScreen = showingCover ? cover : inner content = scene.rootNode - // No animation players: a player churn per pose blocks behind the render thread for most - // of a frame, while setting thirty transforms by hand is free. + // No animation players: under an SCNView a player churn per pose blocked behind its render + // thread for most of a frame, while setting thirty transforms by hand is free. tracks = Self.freeze(scene.rootNode).flatMap { PoseTrack.tracks(of: $0.animation, on: $0.node) } // A source hands out its clips once, so the twin comes from a source of its own. guard let twinSource = SCNSceneSource(url: asset, options: nil), @@ -168,9 +180,7 @@ public final class DuoModelView: SCNView { tracks: twinTracks ) - super.init(frame: .zero, options: [ - SCNView.Option.preferredRenderingAPI.rawValue: SCNRenderingAPI.metal.rawValue, - ]) + super.init(frame: .zero, device: metalDevice) let camera = SCNCamera() camera.fieldOfView = 31 @@ -182,29 +192,24 @@ public final class DuoModelView: SCNView { scene.rootNode.addChildNode(cameraNode) installLighting(in: scene) - self.scene = scene - pointOfView = cameraNode - backgroundColor = .clear - wantsLayer = true + sceneRenderer.scene = scene + sceneRenderer.pointOfView = cameraNode + // What the SCNView this replaced drew into (27A9269), so the device looks the same. + colorPixelFormat = .bgra8Unorm_srgb + depthStencilPixelFormat = .depth32Float + sampleCount = 4 + clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) layer?.isOpaque = false - antialiasingMode = .multisampling4X - allowsCameraControl = false - // Drawn when the scene changes, so a still screen costs nothing. SceneKit keeps drawing for - // about a quarter of a second after each change, a new frame from the device included - // (Xcode 27.1, 27A9269). - rendersContinuously = false - // Asked for 30, SceneKit draws at 24 on a 120 Hz display, and the rate cannot be changed - // once the view exists; a fold needs 60 to look like one. + (layer as? CAMetalLayer)?.colorspace = CGColorSpace(name: CGColorSpace.sRGB) + isPaused = true + enableSetNeedsDisplay = true preferredFramesPerSecond = 60 - isPlaying = false - loops = false - delegate = self setHingeAngle(180) } @available(*, unavailable) - required init?(coder: NSCoder) { + required init(coder: NSCoder) { fatalError("not supported") } @@ -215,7 +220,7 @@ public final class DuoModelView: SCNView { settleLifts() if let move, let fraction = move.fraction(of: hingeAngle) { let correction = move.start + (move.end - move.start) * Float(fraction) - // The twin's bones, not this scene's: a read here waits behind the render thread. + // The twin's bones, not this scene's, which under an SCNView waited behind its render thread. place( cameraNode, angle: hingeAngle, centre: twinCentre(at: hingeAngle), correction: correction, distanceScale: move.scale(at: hingeAngle) @@ -223,6 +228,7 @@ public final class DuoModelView: SCNView { } else { frameCamera() } + needsDisplay = true } /// Framed from both ends, measured on the twin now; a probe render costs more than a frame, so @@ -291,6 +297,50 @@ public final class DuoModelView: SCNView { needsLayout = true } + public override func viewDidMoveToWindow() { + super.viewDidMoveToWindow() + needsDisplay = true + } + + public override func viewDidChangeBackingProperties() { + super.viewDidChangeBackingProperties() + needsDisplay = true + } + + public override func draw(_ dirtyRect: NSRect) { + let now = CACurrentMediaTime() + defer { + if !isPaused, now >= animatingUntil { + isPaused = true + enableSetNeedsDisplay = true + } + } + guard let pass = currentRenderPassDescriptor, let drawable = currentDrawable, + let buffer = commandQueue.makeCommandBuffer() else { return } + sceneRenderer.render( + atTime: now, + viewport: CGRect(origin: .zero, size: drawableSize), + commandBuffer: buffer, + passDescriptor: pass + ) + buffer.present(drawable) + buffer.commit() + renderCount += 1 + } + + /// Drawn on a timer for as long as SceneKit animates, then on demand again. + private func animate(for duration: TimeInterval) { + animatingUntil = max(animatingUntil, CACurrentMediaTime() + duration) + enableSetNeedsDisplay = false + isPaused = false + } + + /// The picture as it is drawn now, at the drawable's size, and empty before the view has one. + public func snapshot() -> NSImage { + guard drawableSize.width > 0, drawableSize.height > 0 else { return NSImage(size: .zero) } + return sceneRenderer.snapshot(atTime: CACurrentMediaTime(), with: drawableSize, antialiasingMode: .multisampling4X) + } + /// The hardware turns too: the guest has already turned its picture inside the panel. public func setOrientation(_ orientation: DeviceOrientation) { let turns = ((orientation.degrees / 90) % 4 + 4) % 4 @@ -355,6 +405,7 @@ public final class DuoModelView: SCNView { material.diffuse.wrapT = .clamp material.lightingModel = .constant } + needsDisplay = true } /// Frozen players rather than the view's scene time, which the view advances itself. @@ -456,6 +507,7 @@ public final class DuoModelView: SCNView { ) silhouette = framing.silhouette projectButtons() + needsDisplay = true } /// Measurements belong to a window size and a way up; either changing throws them away. @@ -881,17 +933,19 @@ public final class DuoModelView: SCNView { let amount = restLift(of: button) + rise let position = lift.outward * amount let scale = SIMD3(repeating: 1) + abs(lift.outward) * (amount / lift.depth) - // An explicit transaction commits behind the render thread, which costs most of a frame - // in a live window; a plain set costs nothing and the run loop commits it. + // Under an SCNView an explicit transaction committed behind its render thread, which cost + // most of a frame in a live window; a plain set costs nothing. if liftDuration > 0 { SCNTransaction.begin() SCNTransaction.animationDuration = liftDuration lift.node.simdPosition = position lift.node.simdScale = scale SCNTransaction.commit() + animate(for: liftDuration) } else { lift.node.simdPosition = position lift.node.simdScale = scale + needsDisplay = true } liftedReach = rise > 0 ? reach(of: button, lifted: amount) : nil } @@ -924,6 +978,7 @@ public final class DuoModelView: SCNView { for material in node.geometry?.materials ?? [] { material.emission.contents = lit ? NSColor(white: 0.35, alpha: 1) : NSColor.black } + needsDisplay = true } func liftOffset(of button: HardwareButton) -> Float { @@ -1277,9 +1332,3 @@ extension DuoModelView { return (SCNVector3(origin), SCNVector3(far)) } } - -extension DuoModelView: SCNSceneRendererDelegate { - public nonisolated func renderer(_ renderer: any SCNSceneRenderer, didRenderScene scene: SCNScene, atTime time: TimeInterval) { - renders.withLock { $0 += 1 } - } -} From ef127fa8ef1a34be73fd9ed20a4dba9fab4d5c4c Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 15:01:48 +0100 Subject: [PATCH 14/17] fix(viewer): keep fold steps on a fixed tick Each step slept a fixed 16 ms after its work, so the model now drawing on the main thread stretched the gap between steps to about 23 ms. Sleeping until the next tick keeps them 16 ms apart. --- engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift | 7 +++++++ .../Sources/OpenDeviceHubViewer/FoldableController.swift | 5 ++++- .../Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift b/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift index 1f472c2..f5155ff 100644 --- a/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift +++ b/engine/Sources/OpenDeviceHubEngine/Input/HingeMove.swift @@ -27,4 +27,11 @@ public struct HingeMove: Sendable, Hashable { } public static let stepInterval: Duration = .milliseconds(16) + + /// When the next step is due, counted from the start of the move: the first tick after + /// `elapsed`, so time taken between steps shortens the wait instead of adding to it, and a late + /// step is followed by the next tick rather than a burst. + public static func nextStep(after elapsed: Duration) -> Duration { + stepInterval * (Int(max(elapsed, .zero) / stepInterval) + 1) + } } diff --git a/engine/Sources/OpenDeviceHubViewer/FoldableController.swift b/engine/Sources/OpenDeviceHubViewer/FoldableController.swift index eb42399..bb999e0 100644 --- a/engine/Sources/OpenDeviceHubViewer/FoldableController.swift +++ b/engine/Sources/OpenDeviceHubViewer/FoldableController.swift @@ -102,7 +102,10 @@ public final class FoldableController { send(angle, to: control, for: udid, poke: false) onMove?(udid, .angle(angle)) if progress >= 1 { break } - try? await Task.sleep(for: HingeMove.stepInterval) + try? await Task.sleep( + until: started + HingeMove.nextStep(after: ContinuousClock.now - started), + clock: .continuous + ) } guard let self, !Task.isCancelled else { return } moves[udid] = nil diff --git a/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift b/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift index fee22ef..5286d2a 100644 --- a/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift +++ b/engine/Tests/OpenDeviceHubEngineTests/HingeMoveTests.swift @@ -33,4 +33,13 @@ import Testing #expect(HingeMove(start: 0, target: 90).duration == 0.5) #expect(HingeMove(start: 180, target: 0).duration == 1) } + + @Test func stepsOnTheNextTickWhateverTheWorkBetweenTook() { + #expect(HingeMove.nextStep(after: .zero) == .milliseconds(16)) + #expect(HingeMove.nextStep(after: .milliseconds(6)) == .milliseconds(16), "work between steps shortens the wait") + #expect(HingeMove.nextStep(after: .milliseconds(16)) == .milliseconds(32)) + #expect(HingeMove.nextStep(after: .milliseconds(22)) == .milliseconds(32)) + #expect(HingeMove.nextStep(after: .milliseconds(55)) == .milliseconds(64), "a late step skips to the next tick, not a burst") + #expect(HingeMove.nextStep(after: .milliseconds(-3)) == .milliseconds(16)) + } } From dcb47b3b7bfc2f666ea34a6c731bbfdcbf8cdfee Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 15:15:58 +0100 Subject: [PATCH 15/17] test(viewer): allow for the measured noise in memory growth --- .../PerformanceComparisonTests.swift | 2 +- .../OpenDeviceHubIntegrationTests/PerformanceReport.swift | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift index 13a1c44..d6cb292 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift @@ -212,7 +212,7 @@ final class PerformanceComparisonTests: XCTestCase { func testTheWindowsOpenedTogetherAreComparedUnderTheirOwnName() throws { let findings = try PerformanceComparison.regressions( - current: report([window()], together: together(cpu: 20, growth: 9)), + current: report([window()], together: together(cpu: 20, growth: 17)), baseline: report([window()], together: together()) ) XCTAssertEqual(findings.map(\.window), ["together", "together"]) diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift index 180499b..be2e2f1 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift @@ -199,7 +199,9 @@ enum PerformanceComparison { "open to first picture (ms)": Tolerance(relative: 1, absolute: 300), "open again to first picture (ms)": Tolerance(relative: 0.3, absolute: 150), "memory kept by each reopen (MB)": Tolerance(relative: 0.5, absolute: 10), - "memory growth (MB per minute)": Tolerance(relative: 0.5, absolute: 5), + // Three busy minutes with nothing leaking read from -9 to 13 MB a minute over nine runs, as + // the model's memory dips and comes back by tens of MB. + "memory growth (MB per minute)": Tolerance(relative: 0.5, absolute: 15), ] static func metrics(of window: WindowMeasurement) -> [String: Double] { From da9d077875eaf637f95564c3dcbfaffd7d024394 Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 15:23:47 +0100 Subject: [PATCH 16/17] test(viewer): give the foldable's reopen memory its own limit --- .../PerformanceComparisonTests.swift | 13 +++++++++++-- .../PerformanceReport.swift | 12 +++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift index d6cb292..fc4e5a2 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceComparisonTests.swift @@ -81,6 +81,15 @@ final class PerformanceComparisonTests: XCTestCase { XCTAssertEqual(tolerance.limit(for: 20), 30, "further up the share decides") } + func testTheFoldablesReopensHaveTheirOwnWiderLimit() throws { + let findings = try PerformanceComparison.regressions( + current: report([window(kept: 40), window("phone", gap: nil, kept: 40)]), + baseline: report([window(kept: 0), window("phone", gap: nil, kept: 0)]) + ) + XCTAssertEqual(findings.map(\.window), ["phone"], "the model's memory swings; the phone's does not") + XCTAssertEqual(findings.first?.limit, 10) + } + func testAReopenIsJudgedByItsMedians() { let reopen = Reopen( opens: [openTime(420), openTime(380), openTime(400)], @@ -99,8 +108,8 @@ final class PerformanceComparisonTests: XCTestCase { func testMemoryKeptByEachReopenBeyondItsLimitIsReported() throws { let findings = try PerformanceComparison.regressions( - current: report([window(kept: 40)]), - baseline: report([window(kept: 2)]) + current: report([window("phone", gap: nil, kept: 40)]), + baseline: report([window("phone", gap: nil, kept: 2)]) ) XCTAssertEqual(findings.map(\.metric), ["memory kept by each reopen (MB)"]) XCTAssertEqual(findings.first?.limit, 12) diff --git a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift index be2e2f1..8103477 100644 --- a/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift +++ b/engine/Tests/OpenDeviceHubIntegrationTests/PerformanceReport.swift @@ -204,6 +204,15 @@ enum PerformanceComparison { "memory growth (MB per minute)": Tolerance(relative: 0.5, absolute: 15), ] + /// Where one kind of window is noisier than the rest, keyed by window and then metric. + static let windowTolerances: [String: [String: Tolerance]] = [ + "foldable": [ + // The model's memory moves by up to 50 MB from one reopen to the next with nothing + // leaking (27A9269), while a whole window is about 300 MB. + "memory kept by each reopen (MB)": Tolerance(relative: 0.5, absolute: 60), + ], + ] + static func metrics(of window: WindowMeasurement) -> [String: Double] { var values = [ "idle CPU median (%)": window.idleCPUPercent.median, @@ -278,7 +287,8 @@ enum PerformanceComparison { /// metric existed still checks the rest. private static func compare(_ window: String, _ before: [String: Double], _ after: [String: Double]) -> [Finding] { before.sorted(by: { $0.key < $1.key }).compactMap { metric, value in - guard let measured = after[metric], let tolerance = tolerances[metric] else { return nil } + guard let measured = after[metric], + let tolerance = windowTolerances[window]?[metric] ?? tolerances[metric] else { return nil } let limit = tolerance.limit(for: value) guard measured > limit else { return nil } return Finding(window: window, metric: metric, baseline: value, current: measured, limit: limit) From e86ab1095ba525f644f0421f5b38f313873ab57d Mon Sep 17 00:00:00 2001 From: Codefarmer Date: Mon, 28 Sep 2026 15:23:47 +0100 Subject: [PATCH 17/17] build(scripts): record the normal power baseline with the fixes in --- performance/baseline-normal-power.json | 170 ++++++++++++------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/performance/baseline-normal-power.json b/performance/baseline-normal-power.json index 69fb184..180c063 100644 --- a/performance/baseline-normal-power.json +++ b/performance/baseline-normal-power.json @@ -1,5 +1,5 @@ { - "createdAt" : "2026-09-28T11:53:21Z", + "createdAt" : "2026-09-28T14:22:18Z", "environment" : { "buildConfiguration" : "release", "hardwareModel" : "Mac15,6", @@ -13,150 +13,150 @@ "iPhone 17 (iOS 27.0)", "iPhone Duo (iOS 27.1)" ], - "idleCPUEnergyMilliwatts" : 12.972761723841723, + "idleCPUEnergyMilliwatts" : 1.653480535325553, "idleCPUPercent" : { - "median" : 9.572144208538406, - "p95" : 12.162444850751928, + "median" : 1.2256303924930867, + "p95" : 1.6418494785014763, "samples" : 10 }, - "idleDrawsPerSecond" : 33.691864374922474, - "idleFramesPerSecond" : 2.308334731226467, - "idleGPUPercent" : 3.638691069104495, - "processMemoryMB" : 413.5949401855469, + "idleDrawsPerSecond" : 2.356308363298762, + "idleFramesPerSecond" : 2.356308363298762, + "idleGPUPercent" : 0.30492614919984484, + "processMemoryMB" : 402.1261215209961, "soak" : { - "clicks" : 227, - "endMB" : 413.5949401855469, + "clicks" : 228, + "endMB" : 402.1261215209961, "folds" : 8, - "growthMBPerMinute" : 2.351137537762155, + "growthMBPerMinute" : 2.3454191976695675, "seconds" : 180, - "startMB" : 401.1105651855469 + "startMB" : 391.4698715209961 }, - "windowsMemoryMB" : 363.6254348754883 + "windowsMemoryMB" : 380.3284912109375 }, "windows" : [ { "clickLatencyMs" : { - "median" : 8.787416666666665, - "p95" : 13.040416666666665, + "median" : 12.212416666666666, + "p95" : 20.951125, "samples" : 20 }, "device" : "iPhone 17 (iOS 27.0)", - "idleCPUEnergyMilliwatts" : 0.708254376266516, + "idleCPUEnergyMilliwatts" : 1.7159928379051423, "idleCPUPercent" : { - "median" : 0.2054317745774193, - "p95" : 0.7792023230559177, + "median" : 0.2203408604497964, + "p95" : 0.874295276459982, "samples" : 10 }, - "idleDrawsPerSecond" : 0.09746648917987798, - "idleFramesPerSecond" : 0.09746648917987798, - "idleGPUPercent" : 0.005427460436577177, + "idleDrawsPerSecond" : 0.09745445393163028, + "idleFramesPerSecond" : 0.09745445393163028, + "idleGPUPercent" : 0.008464528414299162, "name" : "phone", "open" : { - "firstPictureMs" : 47.018209, - "orientationMs" : 410.716292, - "sessionsMs" : 26.218917, - "totalMs" : 670.460834, - "windowMs" : 186.507416 + "firstPictureMs" : 75.851209, + "orientationMs" : 303.730458, + "sessionsMs" : 21.275917, + "totalMs" : 688.528167, + "windowMs" : 287.67058299999997 }, - "processMemoryMB" : 152.50091552734375, + "processMemoryMB" : 149.95404052734375, "reopen" : { "addedByEachReopenMB" : [ - 5.671875, - 4.4375, - 4.578125 + 3.5, + 0.171875, + 0 ], "freedOnClose" : true, - "keptAfterCloseMB" : 26.875091552734375, + "keptAfterCloseMB" : 11.562591552734375, "opens" : [ { - "firstPictureMs" : 18.98425, - "orientationMs" : 338.607375, - "sessionsMs" : 28.813417, - "totalMs" : 402.951042, - "windowMs" : 16.546000000000003 + "firstPictureMs" : 17.954167, + "orientationMs" : 158.828875, + "sessionsMs" : 18.226499999999998, + "totalMs" : 213.850917, + "windowMs" : 18.841375 }, { - "firstPictureMs" : 16.394459, - "orientationMs" : 321.07404099999997, - "sessionsMs" : 31.128209000000002, - "totalMs" : 380.868834, - "windowMs" : 12.272125 + "firstPictureMs" : 17.68375, + "orientationMs" : 166.909625, + "sessionsMs" : 31.184917, + "totalMs" : 230.490542, + "windowMs" : 14.71225 }, { - "firstPictureMs" : 16.304917, - "orientationMs" : 319.44795799999997, - "sessionsMs" : 24.152292, - "totalMs" : 375.04879199999993, - "windowMs" : 15.143625 + "firstPictureMs" : 17.194417, + "orientationMs" : 177.183541, + "sessionsMs" : 32.812792, + "totalMs" : 239.63904200000002, + "windowMs" : 12.448292 } ] }, - "windowMemoryMB" : 47.82823944091797 + "windowMemoryMB" : 36.23448944091797 }, { "clickLatencyMs" : { - "median" : 11.423833333333334, - "p95" : 14.965791666666666, + "median" : 9.792833333333334, + "p95" : 17.169291666666666, "samples" : 20 }, "device" : "iPhone Duo (iOS 27.1)", - "foldCPUPercent" : 33.88276299687477, + "foldCPUPercent" : 21.496221395481253, "foldStepGapMs" : { - "median" : 18.637625, - "p95" : 24.799459, - "samples" : 104 + "median" : 16.396542, + "p95" : 19.593458000000002, + "samples" : 126 }, - "idleCPUEnergyMilliwatts" : 20.23728821071902, + "idleCPUEnergyMilliwatts" : 4.596634097579818, "idleCPUPercent" : { - "median" : 9.5980171669382, - "p95" : 12.037014510216425, + "median" : 0.6751292239530251, + "p95" : 1.126026700718238, "samples" : 10 }, - "idleDrawsPerSecond" : 35.61601065391605, - "idleFramesPerSecond" : 2.213702319649478, - "idleGPUPercent" : 4.268911748829975, + "idleDrawsPerSecond" : 2.210415008503586, + "idleFramesPerSecond" : 2.161294674981284, + "idleGPUPercent" : 0.2093037894564363, "name" : "foldable", "open" : { - "firstPictureMs" : 46.669416, - "orientationMs" : 334.866417, - "sessionsMs" : 48.03337500000001, - "totalMs" : 1341.278208, - "windowMs" : 911.709 + "firstPictureMs" : 122.354041, + "orientationMs" : 693.304166, + "sessionsMs" : 174.235959, + "totalMs" : 2182.395, + "windowMs" : 1192.5008340000002 }, - "processMemoryMB" : 371.9854965209961, + "processMemoryMB" : 348.6104736328125, "reopen" : { "addedByEachReopenMB" : [ - -4.4530792236328125, - -25.375, - 9.078147888183594 + 32.390647888183594, + -49.140625, + 48.21875 ], "freedOnClose" : true, - "keptAfterCloseMB" : 89.57835388183594, + "keptAfterCloseMB" : 70.48458099365234, "opens" : [ { - "firstPictureMs" : 76.78133299999999, - "orientationMs" : 355.050792, - "sessionsMs" : 63.769625, - "totalMs" : 975.6773750000001, - "windowMs" : 480.075625 + "firstPictureMs" : 94.341583, + "orientationMs" : 386.609916, + "sessionsMs" : 76.84329199999999, + "totalMs" : 1182.4217079999999, + "windowMs" : 624.6269169999999 }, { - "firstPictureMs" : 87.002, - "orientationMs" : 355.940292, - "sessionsMs" : 36.425833, - "totalMs" : 954.561875, - "windowMs" : 475.19375 + "firstPictureMs" : 77.06837499999999, + "orientationMs" : 333.732709, + "sessionsMs" : 59.454791, + "totalMs" : 875.5543749999999, + "windowMs" : 405.2985 }, { - "firstPictureMs" : 81.960792, - "orientationMs" : 340.14175, - "sessionsMs" : 52.940417, - "totalMs" : 902.361084, - "windowMs" : 427.318125 + "firstPictureMs" : 79.98979200000001, + "orientationMs" : 382.287416, + "sessionsMs" : 39.989584, + "totalMs" : 948.8692089999998, + "windowMs" : 446.60241699999995 } ] }, - "windowMemoryMB" : 315.10960388183594 + "windowMemoryMB" : 318.92205810546875 } ] } \ No newline at end of file