From 744c395c31e1ab19dd7c76886ced8be442c5de06 Mon Sep 17 00:00:00 2001 From: Matjaz Domen Pecan Date: Mon, 13 Jul 2026 19:32:45 +0200 Subject: [PATCH 1/2] fix(tests): stop snapshot tests failing on newer macOS, and hitting the Keychain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two unrelated defects, both visible when running the suite locally. Snapshot tests compared images pixel-exact. SwiftUI renders gradients and antialiased edges differently across macOS versions, so all 8 failed on any OS other than the one that recorded the references — CI passes, a modern machine does not. Compare perceptually instead. The tolerance is measured, not guessed: the same icon drifts by at most ~10 deltaE across OS versions, while a different icon differs by ~128, so allowing deltaE <= 10 keeps a 12x margin. Reference images are unchanged, so CI keeps comparing against exactly what it did before. A guard test asserts a changed status colour is still rejected, so the tolerance cannot be loosened into asserting nothing. The test bundle is hosted by the app, so running tests launched the real menu bar: it read the session key from the Keychain (prompting the user, repeatedly, since the test binary is signed differently) and fetched live usage over the network. Skip bootstrap when running under XCTest. Co-Authored-By: Claude Opus 4.8 (1M context) --- ClaudeMeter/App/AppDelegate.swift | 9 +++++ .../MenuBarIconSnapshotTests.swift | 40 +++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/ClaudeMeter/App/AppDelegate.swift b/ClaudeMeter/App/AppDelegate.swift index 69ffda6..cd88a6e 100644 --- a/ClaudeMeter/App/AppDelegate.swift +++ b/ClaudeMeter/App/AppDelegate.swift @@ -20,7 +20,16 @@ final class AppDelegate: NSObject, NSApplicationDelegate { } #endif + /// The test bundle is hosted by the app, so launching it for tests would otherwise + /// bootstrap the real menu bar — reading the session key from the Keychain and + /// hitting the network. + private var isRunningTests: Bool { + ProcessInfo.processInfo.environment["XCTestConfigurationFilePath"] != nil + } + func applicationDidFinishLaunching(_ notification: Notification) { + guard !isRunningTests else { return } + SessionKeyImportPromptCoordinator.install() guard let appModel else { diff --git a/ClaudeMeterTests/MenuBarIconSnapshotTests.swift b/ClaudeMeterTests/MenuBarIconSnapshotTests.swift index a6578ff..3cbae26 100644 --- a/ClaudeMeterTests/MenuBarIconSnapshotTests.swift +++ b/ClaudeMeterTests/MenuBarIconSnapshotTests.swift @@ -12,52 +12,76 @@ import XCTest @MainActor final class MenuBarIconSnapshotTests: XCTestCase { + /// SwiftUI renders gradients and antialiased edges differently across macOS versions, + /// so a pixel-exact comparison fails on any OS other than the one that recorded the + /// references. Compare perceptually instead. + /// + /// The tolerance is sized from measurement, not taste: the same icon rendered on a + /// newer macOS drifts by at most ~10 ΔE, while a genuinely different icon differs by + /// ~128 ΔE. Allowing ΔE ≤ 10 absorbs the drift and still leaves a 12x margin. + private let strategy: Snapshotting = .image( + precision: 0.99, + perceptualPrecision: 0.90 + ) + func test_menuBarIcon_showsBatteryStyleWhenWarning() { let image = renderIcon(style: .battery) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) } func test_menuBarIcon_showsCircularStyleWhenWarning() { let image = renderIcon(style: .circular) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) } func test_menuBarIcon_showsMinimalStyleWhenWarning() { let image = renderIcon(style: .minimal) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) } func test_menuBarIcon_showsSegmentsStyleWhenWarning() { let image = renderIcon(style: .segments) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) } func test_menuBarIcon_showsDualBarStyleWhenWarning() { let image = renderIcon(style: .dualBar) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) } func test_menuBarIcon_showsGaugeStyleWhenWarning() { let image = renderIcon(style: .gauge) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) } func test_menuBarIcon_showsLoadingIndicatorInBatteryStyle() { let image = renderIcon(style: .battery, status: .safe, isLoading: true) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) } func test_menuBarIcon_showsStaleIndicatorInBatteryStyle() { let image = renderIcon(style: .battery, status: .safe, isStale: true) - assertSnapshot(of: image, as: .image, record: isRecording) + assertSnapshot(of: image, as: strategy, record: isRecording) + } + + /// Keeps the tolerance above honest: it must still reject an icon whose status colour + /// changed, otherwise the snapshots assert nothing. + func test_snapshotTolerance_rejectsChangedStatusColour() { + let warning = renderIcon(style: .battery, status: .warning) + let critical = renderIcon(style: .battery, status: .critical) + + XCTAssertNotNil( + strategy.diffing.diff(warning, critical), + "Tolerance is too loose: a changed status colour compares as unchanged" + ) } private func renderIcon( From 2195491826bfe4fcc920c6f29506bec3764049c8 Mon Sep 17 00:00:00 2001 From: Matjaz Domen Pecan Date: Mon, 13 Jul 2026 19:47:40 +0200 Subject: [PATCH 2/2] refactor(tests): keep the Keychain workaround out of production code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit skipped menu bar bootstrap by having AppDelegate check for XCTestConfigurationFilePath — production code that knows it is under test. The app already has the hook needed: `--demo ` routes to startWithoutBootstrap(), which sets up the menu bar without reading the Keychain or fetching usage. The scheme's test action now passes it, so AppDelegate is untouched. The cost is that the behaviour lives in the scheme, where it can be deleted without anyone noticing. TestHostLaunchTests asserts the test host was launched with a valid --demo argument, so removing it fails the suite instead of silently restoring the Keychain prompts. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../xcschemes/ClaudeMeter.xcscheme | 12 +++++- ClaudeMeter/App/AppDelegate.swift | 9 ----- ClaudeMeterTests/TestHostLaunchTests.swift | 38 +++++++++++++++++++ 3 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 ClaudeMeterTests/TestHostLaunchTests.swift diff --git a/ClaudeMeter.xcodeproj/xcshareddata/xcschemes/ClaudeMeter.xcscheme b/ClaudeMeter.xcodeproj/xcshareddata/xcschemes/ClaudeMeter.xcscheme index b90c50c..c27f3be 100644 --- a/ClaudeMeter.xcodeproj/xcshareddata/xcschemes/ClaudeMeter.xcscheme +++ b/ClaudeMeter.xcodeproj/xcshareddata/xcschemes/ClaudeMeter.xcscheme @@ -27,8 +27,18 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - shouldUseLaunchSchemeArgsEnv = "YES" + shouldUseLaunchSchemeArgsEnv = "NO" shouldAutocreateTestPlan = "YES"> + + + + + +