diff --git a/Resources/analytics-events.psv b/Resources/analytics-events.psv index 1398c3078..c1413d4c3 100644 --- a/Resources/analytics-events.psv +++ b/Resources/analytics-events.psv @@ -80,6 +80,7 @@ onboarding_reporting_toggle_changed|available,enabled,reporting_kind,step_id onboarding_shown|analytics_available,crash_reporting_available,entrypoint,has_target,meeting_recording_ready,mic_status,model_state,pasteback_status onboarding_step_viewed|flow_elapsed_bucket,model_state,step_id,step_index product_friction_observed|elapsed_bucket,failure_kind,model_state,result,route_shape,stage,surface +reliability_failure_observed|accessibility_permission_granted,app_version,build_revision,correlation_id,failure_kind,failure_stage,input_device_class,mic_permission_granted,os_major,output_device_class,screen_permission_granted,selection_reason,session_id,trigger settings_action_clicked|action_id,page_id settings_capture_library_changed|location_type,page_id settings_feature_discovered|feature_area,page_id,source @@ -96,6 +97,7 @@ update_installed|previous_version,version update_ready_to_install|automatic_downloads_enabled,state,version update_relaunching|version update_setting_changed|enabled,setting_id +usage_digest|app_version,dictation_median_duration_bucket,dictations_completed,digest_day,digest_is_partial,install_uuid,meeting_minutes_bucket,meetings_completed,meetings_started,os_major workflow_abandoned|elapsed_bucket,prior_ready_state,reason_kind,stage,surface,workflow_kind workflow_recovery_attempted|artifact_retained,failure_kind,recovery_attempt_bucket,retry_source,surface,workflow_kind workflow_recovery_failed|artifact_retained,elapsed_bucket,failure_kind,recovery_attempt_bucket,result,retry_source,surface,workflow_kind diff --git a/Resources/analytics-reviewed-properties.psv b/Resources/analytics-reviewed-properties.psv index 00b3ab442..f84b66dc6 100644 --- a/Resources/analytics-reviewed-properties.psv +++ b/Resources/analytics-reviewed-properties.psv @@ -7,6 +7,7 @@ # against the compiled allowlist. Marked `merge=union` in .gitattributes; run # python3 scripts/ops/normalize-analytics-taxonomy.py # after a union merge. Lines beginning with # are ignored. +accessibility_permission_granted action action_id action_kind @@ -53,6 +54,7 @@ completion_flow completion_kind cooldown_reason copy_reason +correlation_id crash_reporting_available crash_reporting_enabled cross_app_capture_status @@ -77,11 +79,15 @@ default_system_output_volume_dropped default_system_output_volume_during delivery dictation_ready +dictations_completed +digest_day +digest_is_partial elapsed_bucket enabled entrypoint failure_code failure_kind +failure_stage feature_area first_artifact_kind first_dictation_saved @@ -96,11 +102,15 @@ input_channels input_device_class input_rate_hz input_volume_scalar_available +install_uuid last_event location_type meeting_dry_run_completed meeting_recording_ready +meetings_completed +meetings_started mic_boost_prompt +mic_permission_granted mic_processed_peak mic_processing mic_raw_peak @@ -158,11 +168,13 @@ route_stability_warning runtime sample_flow_started save_outcome +screen_permission_granted second_artifact_kind selected_input_class selection_overrode_default selection_reason session_active +session_id session_kind session_stage setting_id diff --git a/Sources/Meeting/MeetingCaptureHealthTelemetry.swift b/Sources/Meeting/MeetingCaptureHealthTelemetry.swift index 89ea70abf..7b1768fa5 100644 --- a/Sources/Meeting/MeetingCaptureHealthTelemetry.swift +++ b/Sources/Meeting/MeetingCaptureHealthTelemetry.swift @@ -19,6 +19,7 @@ enum MeetingCaptureHealthTelemetry { let durationSeconds: Double let systemStreamPresent: Bool let stopTimedOut: Bool + var captureOutcome: String = "unknown" } struct DegradedReportInput { @@ -35,7 +36,7 @@ enum MeetingCaptureHealthTelemetry { } static func snapshotProperties(_ input: SnapshotInput) -> [String: String] { - input.captureDiagnostics.merging( + var properties = input.captureDiagnostics.merging( sharedProperties( health: input.health, trigger: input.trigger, @@ -46,6 +47,8 @@ enum MeetingCaptureHealthTelemetry { ), uniquingKeysWith: { _, new in new } ) + properties["capture_outcome"] = input.stopTimedOut ? "stop_timed_out" : input.captureOutcome + return properties } static func shouldReportDegraded(_ input: DegradedReportInput) -> Bool { @@ -88,7 +91,7 @@ enum MeetingCaptureHealthTelemetry { ) -> [String: String] { [ "capture_quality": health.captureQuality, - "quality_reason": health.qualityReason, + "quality_reason": health.qualityReason.isEmpty ? "unknown" : health.qualityReason, "duration_bucket": AnalyticsReporter.durationBucket(seconds: durationSeconds), "gap_count_bucket": AnalyticsReporter.countBucket(health.audioGaps), "reason": reason, diff --git a/Sources/Meeting/MeetingSessionController.swift b/Sources/Meeting/MeetingSessionController.swift index ec116d7e5..fbf4da725 100644 --- a/Sources/Meeting/MeetingSessionController.swift +++ b/Sources/Meeting/MeetingSessionController.swift @@ -100,6 +100,7 @@ final class MeetingSessionController: ObservableObject { } private struct RecordingStopSnapshot { + let telemetryIdentity: UUID? let trigger: StartTrigger let systemAudioStatus: SystemAudioStatus let durationSeconds: TimeInterval @@ -841,6 +842,7 @@ final class MeetingSessionController: ObservableObject { : TranscriptedConstants.meetingStartTimeout let started = await capture.startRecording(timeout: startTimeout) guard started else { + let failedStartIdentity = activeRecordingIdentity await capture.flushSharedDictationMicHandler() clearSharedDictationMicRelay() activeRecordingTrigger = .unknown @@ -856,7 +858,7 @@ final class MeetingSessionController: ObservableObject { let pipelineSnapshot = capture.pipelineDiagnosticsSnapshot( overrideSystemAudioStatus: capture.startFailureStage == .systemAudio ? .failed : nil ) - let failureProperties = meetingCaptureAnalyticsProperties(snapshot: pipelineSnapshot).merging( + let failureProperties = TelemetryContext.enrich(event: "meeting_recording_start_failed", properties: meetingCaptureAnalyticsProperties(snapshot: pipelineSnapshot, telemetryIdentity: failedStartIdentity).merging( [ "failure_kind": meetingStartFailureKind( from: failureMessage, @@ -866,7 +868,7 @@ final class MeetingSessionController: ObservableObject { "trigger": trigger.rawValue, ], uniquingKeysWith: { _, new in new } - ) + )) DiagnosticsTrail.record( level: .error, engine: "meeting", @@ -883,7 +885,8 @@ final class MeetingSessionController: ObservableObject { stage: "meeting_start", result: .failed, failureKind: failureProperties["failure_kind"], - modelState: state.diagnosticName + modelState: state.diagnosticName, + context: failureProperties ) transition(to: .error(failureMessage), reason: "capture_start_failed") Self.runtimeDiagnosticsRecorder?.clearSession(kind: "meeting", outcome: "start_failed") @@ -1148,7 +1151,7 @@ final class MeetingSessionController: ObservableObject { let afterStopVolumeContext = capture.routeVolumeDiagnosticsContext(currentPhase: "after") var stopCaptureDiagnostics = MeetingCaptureVolumeDiagnostics.annotatedStopContext( liveAttenuationCueObserved: capture.micAttenuationCueObserved, - baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot), + baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot, telemetryIdentity: recordingSnapshot.telemetryIdentity), afterStopContext: afterStopVolumeContext ) // Read the prompt outcome before any state mutations below; it is only @@ -1213,7 +1216,8 @@ final class MeetingSessionController: ObservableObject { "trigger": recordingSnapshot.trigger.rawValue, ], uniquingKeysWith: { _, new in new } - ) + ), + usageDurationSeconds: recordingSnapshot.durationSeconds ) var healthSnapshotProperties = MeetingCaptureHealthTelemetry.snapshotProperties( .init( @@ -1641,9 +1645,8 @@ final class MeetingSessionController: ObservableObject { _ = audioInactivityDetector.stopRecording() audioInactivityWarning = nil isMicBoostPromptVisible = false - clearActiveRecordingIdentity() - let recordingSnapshot = makeRecordingStopSnapshot() + clearActiveRecordingIdentity() DiagnosticsTrail.record( engine: "meeting", @@ -1666,7 +1669,7 @@ final class MeetingSessionController: ObservableObject { let afterStopVolumeContext = capture.routeVolumeDiagnosticsContext(currentPhase: "after") var cancelCaptureDiagnostics = MeetingCaptureVolumeDiagnostics.annotatedStopContext( liveAttenuationCueObserved: capture.micAttenuationCueObserved, - baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot), + baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot, telemetryIdentity: recordingSnapshot.telemetryIdentity), afterStopContext: afterStopVolumeContext ) // Mirror stopRecording(): cancelled meetings carry the prompt outcome @@ -1727,7 +1730,8 @@ final class MeetingSessionController: ObservableObject { reason: reason.rawValue, durationSeconds: recordingSnapshot.durationSeconds, systemStreamPresent: files.systemURL != nil, - stopTimedOut: stopResult.didTimeOut + stopTimedOut: stopResult.didTimeOut, + captureOutcome: "cancelled" ) ) ) @@ -2196,13 +2200,20 @@ final class MeetingSessionController: ObservableObject { splitLocalSpeakers: LocalSpeakerPreferences.isEnabled() ) + let failureOutcome = CaptureOutcome(micURL: files.micURL, systemURL: files.systemURL, didTimeOut: stopResult.didTimeOut) + let failureContext = TelemetryContext.enrich(event: "meeting_capture_stopped_under_controller", properties: + meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot, telemetryIdentity: recordingSnapshot.telemetryIdentity).merging([ + "failure_kind": files.micURL == nil && files.systemURL == nil ? "no_audio" : "unexpected_capture_stop", + "failure_stage": "capture_stop", "capture_outcome": failureOutcome.rawValue, + "trigger": recordingSnapshot.trigger.rawValue, + ], uniquingKeysWith: { _, new in new }), isFailure: true) DiagnosticsTrail.record( level: .error, engine: "meeting", event: "meeting_capture_stopped_under_controller", message: "Meeting capture stopped before the app stop path ran", context: baseDiagnosticsContext( - extra: [ + extra: failureContext.merging([ "mic_file_present": boolString(files.micURL != nil), "system_file_present": boolString(files.systemURL != nil), "preserved_for_retry": boolString(preserved), @@ -2210,24 +2221,24 @@ final class MeetingSessionController: ObservableObject { "quality_reason": recordingSnapshot.healthInfo.qualityReason.rawValue, "audio_gaps": "\(recordingSnapshot.healthInfo.audioGaps)", "device_switches": "\(recordingSnapshot.healthInfo.deviceSwitches)" - ] + ], uniquingKeysWith: { _, new in new }) ) ) - AnalyticsReporter.track( - "meeting_capture_stopped_under_controller", - properties: MeetingCaptureHealthTelemetry.snapshotProperties( + let healthProperties = MeetingCaptureHealthTelemetry.snapshotProperties( .init( - captureDiagnostics: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot), + captureDiagnostics: failureContext, health: captureHealthFacts(from: recordingSnapshot.healthInfo), trigger: recordingSnapshot.trigger.rawValue, reason: "internal_stop", durationSeconds: recordingSnapshot.durationSeconds, systemStreamPresent: files.systemURL != nil, - stopTimedOut: stopResult.didTimeOut + stopTimedOut: stopResult.didTimeOut, + captureOutcome: failureOutcome.rawValue ) ) - ) + AnalyticsReporter.track("meeting_capture_stopped_under_controller", properties: healthProperties) + AnalyticsReporter.track("meeting_capture_health_snapshot", properties: healthProperties) transition( to: preserved @@ -3003,29 +3014,25 @@ final class MeetingSessionController: ObservableObject { if failureKind == .speakerFinalizationFailed || failureKind == .speakerNameFinalizationFailed { activeQueuedTranscriptionJobID = nil let queueDepthBucket = AnalyticsReporter.queueDepthBucket(transcriptionQueue.queuedTranscriptionJobs.count) + let failureTelemetryContext = meetingFailureTelemetryContext(failureKind: failureKind, transcriptionTrigger: transcriptionTrigger) DiagnosticsTrail.record( level: .error, engine: "meeting", event: "speaker_finalization_failed", message: "Meeting speaker naming finalization failed", context: baseDiagnosticsContext( - extra: [ + extra: failureTelemetryContext.merging([ "failure_kind": failureKind.rawValue, "session_stage": "save", "queue_depth": "\(transcriptionQueue.queuedTranscriptionJobs.count)", "queue_depth_bucket": queueDepthBucket, "trigger": transcriptionTrigger.rawValue - ] + ], uniquingKeysWith: { current, _ in current }) ) ) AnalyticsReporter.track( "meeting_speaker_finalization_failed", - properties: [ - "session_stage": "save", - "failure_kind": failureKind.rawValue, - "queue_depth_bucket": queueDepthBucket, - "trigger": transcriptionTrigger.rawValue, - ] + properties: failureTelemetryContext ) trackDetectedPromptOutcome( .speakerFinalizationFailed, @@ -3038,7 +3045,8 @@ final class MeetingSessionController: ObservableObject { stage: "speaker_finalization", result: .failed, failureKind: failureKind.rawValue, - modelState: state.diagnosticName + modelState: state.diagnosticName, + context: failureTelemetryContext ) activeTranscriptionCaptureDiagnostics = nil Self.runtimeDiagnosticsRecorder?.clearSession(kind: "meeting", outcome: "speaker_finalization_failed") @@ -3081,7 +3089,8 @@ final class MeetingSessionController: ObservableObject { stage: "meeting_transcription", result: .failed, failureKind: failureKind.rawValue, - modelState: state.diagnosticName + modelState: state.diagnosticName, + context: failureTelemetryContext ) activeTranscriptionCaptureDiagnostics = nil Self.runtimeDiagnosticsRecorder?.clearSession(kind: "meeting", outcome: "transcript_failed") @@ -3157,11 +3166,15 @@ final class MeetingSessionController: ObservableObject { MeetingStartFailureClassifier.kind(from: message, stage: stage) } - private func meetingCaptureAnalyticsProperties(snapshot: AudioPipelineDiagnosticsSnapshot) -> [String: String] { + private func meetingCaptureAnalyticsProperties(snapshot: AudioPipelineDiagnosticsSnapshot, telemetryIdentity: UUID? = nil) -> [String: String] { var properties = snapshot.privacySafeContext.merging( MeetingCaptureVolumeDiagnostics.measurementScope, uniquingKeysWith: { _, scope in scope } ) + if let id = (telemetryIdentity ?? activeRecordingIdentity)?.uuidString { + properties["session_id"] = id + properties["correlation_id"] = id + } properties["gap_count_bucket"] = AnalyticsReporter.countBucket(snapshot.gapCount) properties["route_change_count_bucket"] = AnalyticsReporter.countBucket(snapshot.routeChangeCount) properties["recovery_attempt_bucket"] = AnalyticsReporter.countBucket(snapshot.recoveryAttemptCount) @@ -3172,14 +3185,15 @@ final class MeetingSessionController: ObservableObject { failureKind: MeetingFailureKind, transcriptionTrigger: StartTrigger ) -> [String: String] { - (activeTranscriptionCaptureDiagnostics ?? [:]).merging( + TelemetryContext.enrich(event: "meeting_transcript_failed", properties: (activeTranscriptionCaptureDiagnostics ?? [:]).merging( [ + "failure_stage": failureKind == .speakerFinalizationFailed || failureKind == .speakerNameFinalizationFailed ? "speaker_finalization" : "transcription", "failure_kind": failureKind.rawValue, "queue_depth_bucket": AnalyticsReporter.queueDepthBucket(transcriptionQueue.queuedTranscriptionJobs.count), "trigger": transcriptionTrigger.rawValue, ], uniquingKeysWith: { _, new in new } - ) + )) } private func trackDetectedPromptOutcome( @@ -3249,7 +3263,7 @@ final class MeetingSessionController: ObservableObject { ) else { return } DiagnosticsTrail.record( - level: .error, + level: .warning, engine: "meeting", event: "recording_capture_degraded", message: "Meeting capture health degraded", @@ -3421,6 +3435,7 @@ final class MeetingSessionController: ObservableObject { healthInfo = baseHealthInfo } return RecordingStopSnapshot( + telemetryIdentity: activeRecordingIdentity, trigger: activeRecordingTrigger, systemAudioStatus: systemAudioStatus, durationSeconds: durationSeconds, diff --git a/Sources/Observability/ActivationTelemetry.swift b/Sources/Observability/ActivationTelemetry.swift index eac0eb5d1..2e3c9463c 100644 --- a/Sources/Observability/ActivationTelemetry.swift +++ b/Sources/Observability/ActivationTelemetry.swift @@ -524,7 +524,8 @@ enum ProductFrictionTelemetry { failureKind: String? = nil, elapsedBucket: String? = nil, routeShape: String? = nil, - modelState: String? = nil + modelState: String? = nil, + context: [String: String] = [:] ) { var properties = [ "result": result.rawValue, @@ -545,6 +546,7 @@ enum ProductFrictionTelemetry { properties["model_state"] = modelState } + properties.merge(context) { current, _ in current } AnalyticsReporter.track("product_friction_observed", properties: properties) } diff --git a/Sources/Observability/AnalyticsPayloadSanitizer.swift b/Sources/Observability/AnalyticsPayloadSanitizer.swift index 1d19d982f..67a4107c9 100644 --- a/Sources/Observability/AnalyticsPayloadSanitizer.swift +++ b/Sources/Observability/AnalyticsPayloadSanitizer.swift @@ -15,6 +15,9 @@ enum AnalyticsPayloadSanitizer { for (key, value) in properties { guard allowedKeys.contains(key) else { continue } guard !shouldDrop(key: key) else { continue } + if ["session_id", "correlation_id", "install_uuid"].contains(key), PayloadSanitizationCore.uuid(value) == nil { continue } + if ["failure_kind", "failure_stage", "start_failure_stage", "selection_reason", "trigger", "quality_reason", "capture_outcome"].contains(key), + PayloadSanitizationCore.category(value) == nil { continue } let cleaned = sanitizeText(value) guard !cleaned.isEmpty else { continue } diff --git a/Sources/Observability/AnalyticsReporter.swift b/Sources/Observability/AnalyticsReporter.swift index ba4e170b9..4eb2e0561 100644 --- a/Sources/Observability/AnalyticsReporter.swift +++ b/Sources/Observability/AnalyticsReporter.swift @@ -1,17 +1,58 @@ import Foundation -private struct AnalyticsCaptureRequest: Encodable { +struct AnalyticsCaptureRequest: Encodable { let apiKey: String let event: String let distinctID: String let timestamp: String let properties: [String: String] + var uuid: String? = nil + var personProperties: [String: String]? = nil + var aggregateProperties: [String: [String: String]]? = nil + + private struct PropertyKey: CodingKey { + let stringValue: String + var intValue: Int? { nil } + init(_ value: String) { stringValue = value } + init?(stringValue: String) { self.stringValue = stringValue } + init?(intValue: Int) { return nil } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(apiKey, forKey: .apiKey) + try container.encode(event, forKey: .event) + try container.encode(distinctID, forKey: .distinctID) + try container.encode(timestamp, forKey: .timestamp) + try container.encodeIfPresent(uuid, forKey: .uuid) + var values = container.nestedContainer(keyedBy: PropertyKey.self, forKey: .properties) + for (key, value) in properties { + try values.encode(value, forKey: PropertyKey(key)) + } + // PostHog creates/updates an anonymous install profile via $set on capture. + // Never accept caller-provided person properties or collect an email. + if let personProperties { + try values.encode(personProperties, forKey: PropertyKey("$set")) + } + if event == "usage_digest", let aggregateProperties { + for key in ["failures_by_kind", "capture_quality_counts"] { + if let counts = aggregateProperties[key] { + let safeCounts = counts.filter { + PayloadSanitizationCore.category($0.key) != nil && ["0", "1", "2_3", "4_9", "10_plus"].contains($0.value) + } + try values.encode(safeCounts, forKey: PropertyKey(key)) + } + } + } + try values.encode(true, forKey: PropertyKey("$geoip_disable")) + } enum CodingKeys: String, CodingKey { case apiKey = "api_key" case event case distinctID = "distinct_id" case timestamp + case uuid case properties } } @@ -25,6 +66,8 @@ struct PendingAnalyticsCapture: Codable, Equatable { var attemptCount: Int var nextRetryAt: TimeInterval? let properties: [String: String] + var personProperties: [String: String]? = nil + var aggregateProperties: [String: [String: String]]? = nil } struct AnalyticsDeliveryBufferStore { @@ -74,11 +117,12 @@ struct AnalyticsDeliveryBufferStore { } } - func save(_ records: [PendingAnalyticsCapture], now: Date = Date()) { + @discardableResult + func save(_ records: [PendingAnalyticsCapture], now: Date = Date()) -> Bool { let capped = cappedRecords(records, now: now) guard !capped.isEmpty else { remove() - return + return !fileManager.fileExists(atPath: fileURL.path) } do { @@ -86,8 +130,9 @@ struct AnalyticsDeliveryBufferStore { let data = try JSONEncoder().encode(BufferFile(version: 1, records: capped)) try data.write(to: fileURL, options: [.atomic]) fileManager.restrictFileToOwnerOnly(at: fileURL) + return true } catch { - return + return false } } @@ -97,8 +142,9 @@ struct AnalyticsDeliveryBufferStore { func cappedRecords(_ records: [PendingAnalyticsCapture], now: Date = Date()) -> [PendingAnalyticsCapture] { let cutoff = now.timeIntervalSince1970 - ttl + let digestCutoff = now.timeIntervalSince1970 - 14 * 24 * 60 * 60 var capped = records - .filter { $0.enqueuedAt >= cutoff } + .filter { $0.enqueuedAt >= ($0.event == "usage_digest" ? digestCutoff : cutoff) } .sorted { lhs, rhs in if lhs.enqueuedAt == rhs.enqueuedAt { return lhs.id < rhs.id @@ -106,8 +152,14 @@ struct AnalyticsDeliveryBufferStore { return lhs.enqueuedAt < rhs.enqueuedAt } - if capped.count > maxRecordCount { - capped = Array(capped.suffix(maxRecordCount)) + // Reserve capacity for at most 14 daily rollups. Lifecycle bursts must + // never evict their only durable copy. The whole file keeps its original + // count/byte bounds; ordinary events yield space first. + let retainedDigestIDs = Set(capped.filter { $0.event == "usage_digest" }.suffix(14).map(\.id)) + capped.removeAll { $0.event == "usage_digest" && !retainedDigestIDs.contains($0.id) } + while capped.count > maxRecordCount { + let index = capped.firstIndex { $0.event != "usage_digest" } ?? capped.startIndex + capped.remove(at: index) } // Encode once up front, then trim by subtracting each removed record's own @@ -116,7 +168,8 @@ struct AnalyticsDeliveryBufferStore { // encoded bytes plus one array separator. var totalBytes = encodedByteCount(capped) while !capped.isEmpty && totalBytes > maxFileBytes { - let removed = capped.removeFirst() + let index = capped.firstIndex { $0.event != "usage_digest" } ?? capped.startIndex + let removed = capped.remove(at: index) let removedBytes = (try? JSONEncoder().encode(removed).count) ?? 0 let separatorBytes = capped.isEmpty ? 0 : 1 totalBytes = max(0, totalBytes - removedBytes - separatorBytes) @@ -274,8 +327,8 @@ final class AnalyticsReporter { shared.apiKey != nil && shared.captureHost != nil } - static func track(_ event: String, properties: [String: String] = [:]) { - shared.trackEvent(event, properties: properties) + static func track(_ event: String, properties: [String: String] = [:], usageDurationSeconds: Double? = nil) { + shared.trackEvent(event, properties: properties, usageDurationSeconds: usageDurationSeconds) } /// Shared bucketing for `duration_ms` string context values (Sentry policy @@ -422,7 +475,8 @@ final class AnalyticsReporter { fileURL: AnalyticsDeliveryBufferStore.defaultFileURL() ), userDefaults: .standard, - observePreferenceChanges: true + observePreferenceChanges: true, + usageStore: .shared ) } @@ -436,9 +490,11 @@ final class AnalyticsReporter { retryDelay: @escaping (Int) -> TimeInterval = AnalyticsDeliveryPolicy.retryDelay(afterAttempt:), persistDebounceInterval: TimeInterval = AnalyticsReporter.defaultPersistDebounceInterval, analyticsEnabled: (() -> Bool)? = nil, - observePreferenceChanges: Bool = false + observePreferenceChanges: Bool = false, + usageStore: UsageHealthStore? = nil ) { self.apiKey = apiKey + self.usageStore = usageStore self.captureHost = captureHost self.session = session self.bufferStore = bufferStore @@ -456,7 +512,10 @@ final class AnalyticsReporter { queue: nil ) { [weak self] _ in guard let self else { return } - if !self.analyticsEnabled() { + // Defaults mutations made by the ledger also notify synchronously. + // Serialize clearing after the writer releases its ledger lock. + self.deliveryQueue.async { [weak self] in + guard let self, !self.analyticsEnabled() else { return } self.clearPendingCaptures() } } @@ -468,10 +527,22 @@ final class AnalyticsReporter { queue: nil ) { [weak self] _ in // Final synchronous persist so debounced buffer writes are not lost on quit. + self?.enqueueUsageDigests(includeCurrentDay: true) self?.persistPendingCapturesNow() } + if usageStore != nil { + let timer = DispatchSource.makeTimerSource(queue: deliveryQueue) + timer.schedule(deadline: .now() + 60, repeating: 60) + timer.setEventHandler { [weak self] in + self?.enqueueUsageDigests(includeCurrentDay: false) + self?.flushPendingCapturesLocked() + } + digestTimer = timer + timer.resume() + } if self.analyticsEnabled() { + enqueueUsageDigests(includeCurrentDay: false) flushPendingCaptures() } else { clearPendingCaptures() @@ -479,6 +550,7 @@ final class AnalyticsReporter { } deinit { + digestTimer?.cancel() if let preferenceObserver { NotificationCenter.default.removeObserver(preferenceObserver) } @@ -489,10 +561,10 @@ final class AnalyticsReporter { // Config is read once from env/plist/overrides file and cached for the app lifetime. private let apiKey: String? + private let usageStore: UsageHealthStore? private let captureHost: String? private static let isoDateFormatter = ISO8601DateFormatter() - private let storageKey = "observability-anonymous-analytics-id" - private let sessionID = UUID().uuidString + private let sessionID = TelemetryContext.launchSessionID private let session: URLSession private let bufferStore: AnalyticsDeliveryBufferStore private let userDefaults: UserDefaults @@ -505,6 +577,7 @@ final class AnalyticsReporter { private var inFlightCaptureIDs: Set = [] private var preferenceObserver: NSObjectProtocol? private var terminationObserver: NSObjectProtocol? + private var digestTimer: DispatchSourceTimer? // The in-memory buffer is the source of truth after the first load; disk writes // are debounced so a burst of tracked events costs one file write, not one per @@ -518,41 +591,32 @@ final class AnalyticsReporter { private var needsPersist = false private var pendingPersistWorkItem: DispatchWorkItem? - private lazy var distinctID: String = { - if let existing = userDefaults.string(forKey: storageKey) { - return existing - } - - let newValue = UUID().uuidString - userDefaults.set(newValue, forKey: storageKey) - return newValue - }() + private var distinctID: String { InstallIdentity.id(userDefaults: userDefaults) } - func trackEvent(_ event: String, properties: [String: String] = [:]) { + func trackEvent(_ event: String, properties: [String: String] = [:], usageDurationSeconds: Double? = nil) { guard analyticsEnabled() else { clearPendingCaptures() return } - guard apiKey != nil, - let captureHost, - normalizedCaptureURL(from: captureHost) != nil, - let policy = AnalyticsEventPolicy.policy(forEvent: event) else { - return - } - + guard let policy = AnalyticsEventPolicy.policy(forEvent: event) else { return } + let enrichedProperties = TelemetryContext.enrich(event: event, properties: properties) let sanitizedProperties = AnalyticsPayloadSanitizer.sanitizeProperties( - properties, - allowedKeys: policy.allowedProperties + enrichedProperties, + allowedKeys: policy.allowedProperties.union(TelemetryContext.keys) ) + usageStore?.record(event: event, properties: sanitizedProperties, durationSeconds: usageDurationSeconds, now: currentDate()) + guard apiKey != nil, let captureHost, normalizedCaptureURL(from: captureHost) != nil else { return } - let eventProperties = Self.captureProperties( + var eventProperties = Self.captureProperties( sanitizedProperties: sanitizedProperties, distinctID: distinctID, sessionID: sessionID ) let now = currentDate() + let traits = InstallIdentity.traits(userDefaults: userDefaults, now: now) + eventProperties.merge(traits) { current, _ in current } let capture = PendingAnalyticsCapture( id: UUID().uuidString, event: policy.name, @@ -561,12 +625,45 @@ final class AnalyticsReporter { enqueuedAt: now.timeIntervalSince1970, attemptCount: 0, nextRetryAt: nil, - properties: eventProperties + properties: eventProperties, + personProperties: traits ) enqueue(capture) } + func enqueueUsageDigests(includeCurrentDay: Bool) { + syncOnDeliveryQueue { + guard self.analyticsEnabled(), let store = self.usageStore, + self.apiKey != nil, let host = self.captureHost, + self.normalizedCaptureURL(from: host) != nil, + let policy = AnalyticsEventPolicy.policy(forEvent: "usage_digest") else { return } + let now = self.currentDate() + self.loadPendingCapturesIfNeededLocked(now: now) + for digest in store.pendingDigests(includeCurrentDay: includeCurrentDay, now: now) { + if !self.pendingCaptures.contains(where: { $0.id == digest.id }) { + var properties = digest.properties + properties["install_uuid"] = self.distinctID + let safe = AnalyticsPayloadSanitizer.sanitizeProperties(properties, allowedKeys: policy.allowedProperties) + self.pendingCaptures.append(PendingAnalyticsCapture( + id: digest.id, event: "usage_digest", distinctID: self.distinctID, + timestamp: Self.isoDateFormatter.string(from: now), enqueuedAt: now.timeIntervalSince1970, + attemptCount: 0, nextRetryAt: nil, + properties: Self.captureProperties(sanitizedProperties: safe, distinctID: self.distinctID, sessionID: self.sessionID), + personProperties: InstallIdentity.traits(userDefaults: self.userDefaults, now: now), + aggregateProperties: digest.aggregates + )) + self.pendingCaptures = self.bufferStore.cappedRecords(self.pendingCaptures, now: now) + self.needsPersist = true + } + guard self.pendingCaptures.contains(where: { $0.id == digest.id }), + self.persistPendingCapturesLocked() else { continue } + store.markDigestEnqueued(id: digest.id) + } + self.flushPendingCapturesLocked() + } + } + func flushPendingCapturesForTesting() { flushPendingCaptures() } @@ -619,6 +716,7 @@ final class AnalyticsReporter { } private func clearPendingCaptures() { + usageStore?.clear() syncOnDeliveryQueue { self.inFlightCaptureIDs.removeAll() self.clearBufferedCapturesLocked() @@ -651,14 +749,17 @@ final class AnalyticsReporter { deliveryQueue.asyncAfter(deadline: .now() + persistDebounceInterval, execute: workItem) } - private func persistPendingCapturesLocked() { + @discardableResult + private func persistPendingCapturesLocked() -> Bool { pendingPersistWorkItem?.cancel() pendingPersistWorkItem = nil - guard needsPersist else { return } + guard needsPersist else { return true } needsPersist = false // `save` re-applies TTL/count/byte caps and removes the file when empty, so // the on-disk format, owner-only permissions, and cap semantics are unchanged. - bufferStore.save(pendingCaptures, now: currentDate()) + let saved = bufferStore.save(pendingCaptures, now: currentDate()) + needsPersist = !saved + return saved } private func syncOnDeliveryQueue(_ work: () -> Void) { @@ -702,7 +803,10 @@ final class AnalyticsReporter { event: capture.event, distinctID: capture.distinctID, timestamp: capture.timestamp, - properties: capture.properties + properties: capture.properties, + uuid: capture.id, + personProperties: capture.personProperties, + aggregateProperties: capture.aggregateProperties ) guard let data = try? JSONEncoder().encode(payload) else { diff --git a/Sources/Observability/CrashReporter.swift b/Sources/Observability/CrashReporter.swift index c4e6044f5..b33675d8c 100644 --- a/Sources/Observability/CrashReporter.swift +++ b/Sources/Observability/CrashReporter.swift @@ -54,6 +54,7 @@ final class CrashReporter { } SentrySDK.start(options: options) + SentrySDK.setUser(User(userId: InstallIdentity.id())) shared.hasStarted = true @@ -271,7 +272,8 @@ final class CrashReporter { } event.request = nil - event.user = nil + // Replace the entire SDK user object; preserve only the app-generated UUID. + event.user = User(userId: InstallIdentity.id()) event.breadcrumbs = nil event.serverName = nil diff --git a/Sources/Observability/EventReporter.swift b/Sources/Observability/EventReporter.swift index e206ccdb7..e4e41a610 100644 --- a/Sources/Observability/EventReporter.swift +++ b/Sources/Observability/EventReporter.swift @@ -214,6 +214,7 @@ final class EventReporter { mergedContext["build_revision"] = AnalyticsRuntimeConfiguration.buildRevision(infoDictionary: infoDictionary) } + mergedContext = TelemetryContext.enrich(event: event, properties: mergedContext, isFailure: level == .error) let entry = ObservabilityEvent( timestamp: isoFormatter.string(from: Date()), level: level.rawValue, @@ -246,6 +247,10 @@ final class EventReporter { if level == .error, let sentryPolicy = SentryEventPolicy.policy(forEngine: engine, event: event) { + // One canonical analytics counterpart for every allowlisted hard failure, + // including low-level engine failures without a product lifecycle event. + // Both sinks receive the exact same UUIDs and failure taxonomy. + AnalyticsReporter.track("reliability_failure_observed", properties: mergedContext) CrashReporter.shared.captureObservabilityEvent( level: level, engine: sentryPolicy.engine, diff --git a/Sources/Observability/InstallIdentity.swift b/Sources/Observability/InstallIdentity.swift new file mode 100644 index 000000000..9e9646e98 --- /dev/null +++ b/Sources/Observability/InstallIdentity.swift @@ -0,0 +1,48 @@ +import Foundation + +/// App-generated identity only. Never derived from hardware, account, email, or content. +enum InstallIdentity { + static let storageKey = "observability-anonymous-analytics-id" + private static let firstLaunchKey = "observability-first-launch-day" + private static let lock = NSLock() + + static func id(userDefaults: UserDefaults = .standard) -> String { + lock.lock() + defer { lock.unlock() } + // Preserve the existing PostHog UUID byte-for-byte so upgrades do not split people. + if let existing = userDefaults.string(forKey: storageKey), UUID(uuidString: existing) != nil { + return existing + } + let id = UUID().uuidString + userDefaults.set(id, forKey: storageKey) + return id + } + + static func firstLaunchDay(userDefaults: UserDefaults = .standard, now: Date = Date()) -> String { + lock.lock() + defer { lock.unlock() } + if let existing = userDefaults.string(forKey: firstLaunchKey), + existing.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil { + return existing + } + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(secondsFromGMT: 0) + formatter.dateFormat = "yyyy-MM-dd" + let day = formatter.string(from: now) + userDefaults.set(day, forKey: firstLaunchKey) + return day + } + + static func traits(userDefaults: UserDefaults = .standard, now: Date = Date()) -> [String: String] { + [ + "analytics_opt_in": "true", + "app_version": Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown", + "build_revision": AnalyticsRuntimeConfiguration.buildRevision(), + "os_major": "\(ProcessInfo.processInfo.operatingSystemVersion.majorVersion)", + "install_channel": AnalyticsRuntimeConfiguration.buildChannel(), + // For upgrades this is the first observed day with this instrumentation. + "first_launch_at": firstLaunchDay(userDefaults: userDefaults, now: now), + ] + } +} diff --git a/Sources/Observability/PayloadSanitizationCore.swift b/Sources/Observability/PayloadSanitizationCore.swift index 0274b1392..bd9d385ff 100644 --- a/Sources/Observability/PayloadSanitizationCore.swift +++ b/Sources/Observability/PayloadSanitizationCore.swift @@ -11,6 +11,24 @@ import Foundation /// Generic free-text patterns live in `PrivacyTextRedactor`; the app-specific /// path profile stays behind `ObservabilityTextRedactor`. enum PayloadSanitizationCore { + static let commonTelemetryKeys: Set = [ + "session_id", "correlation_id", "failure_kind", "failure_stage", "start_failure_stage", + "app_version", "build_revision", "os_major", "input_device_class", "output_device_class", + "selection_reason", "mic_permission_granted", "screen_permission_granted", + "accessibility_permission_granted", "trigger", "quality_reason", "capture_outcome", + ] + static func uuid(_ value: String?) -> String? { + guard let value, UUID(uuidString: value) != nil else { return nil } + return value + } + + static func category(_ value: String?) -> String? { + guard let value, !value.isEmpty, value.count <= 80, + value.range(of: #"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$"#, options: .regularExpression) != nil, + redactAndCap(value, maxValueLength: 80) == value else { return nil } + return value + } + /// Sensitive-key fragments shared by every off-device destination. A value /// is dropped when its lowercased key contains any of these as a substring. /// The Sentry, Analytics, and local sanitizers start from this list and diff --git a/Sources/Observability/SentryEventPolicy.swift b/Sources/Observability/SentryEventPolicy.swift index 4df3a77b6..cbec13427 100644 --- a/Sources/Observability/SentryEventPolicy.swift +++ b/Sources/Observability/SentryEventPolicy.swift @@ -16,31 +16,19 @@ struct SentryEventPolicy: Equatable { ) -> [String: String] { guard policy(forEngine: engine, event: event) != nil else { return [:] } - var tags = context.filter { allowedDiagnosticTagKeys.contains($0.key) } + var tags = context.filter { allowedDiagnosticTagKeys.union(TelemetryContext.keys).contains($0.key) } if let waitBucket = AnalyticsReporter.durationBucket(fromMilliseconds: context["wait_ms"]) { tags["wait_bucket"] = waitBucket } - // `reason` is allowlisted because callers usually set it to a short - // enum-style value (e.g. "preferred_built_in_for_bluetooth_headset"), - // but some producers pass free-text error strings. The redactor in - // `sanitizeTags` strips obvious paths/emails/secrets, yet a hard length - // cap is the cheaper backstop against an accidental free-text blob - // riding off-device under a "safe" key. We keep `reason` (it carries - // diagnostic signal `failure_kind` does not always duplicate) but bound - // it tightly; enum-style values fit well under the cap. - if let reason = tags["reason"], reason.count > maxReasonTagLength { - tags["reason"] = String(reason.prefix(maxReasonTagLength)) + "..." + // Reasons must be codes, never a shortened excerpt of a raw error. + if let reason = tags["reason"], PayloadSanitizationCore.category(reason) == nil { + tags["reason"] = "unknown" } return SentryPayloadSanitizer.sanitizeTags(tags) } - /// Hard cap for the free-text-capable `reason` diagnostic tag. Enum-style - /// reason values stay well below this; anything longer is truncated before - /// redaction so a stray error message cannot ship a large blob to Sentry. - static let maxReasonTagLength = 80 - private static let allowedDiagnosticTagKeys: Set = [ "attenuation_kind", "buffer_success_bucket", @@ -116,6 +104,11 @@ struct SentryEventPolicy: Equatable { ] private static let allowedPolicies: [String: SentryEventPolicy] = [ + "meeting.meeting_capture_stopped_under_controller": .init( + engine: "meeting", + event: "meeting_capture_stopped_under_controller", + summary: "Meeting recording stopped unexpectedly before completion." + ), "app.session_stall_detected": .init( engine: "app", event: "session_stall_detected", @@ -216,11 +209,6 @@ struct SentryEventPolicy: Equatable { event: "meeting_start_failed", summary: "Meeting recording could not start." ), - "meeting.recording_capture_degraded": .init( - engine: "meeting", - event: "recording_capture_degraded", - summary: "Meeting capture health degraded." - ), "meeting.recording_stop_timeout": .init( engine: "meeting", event: "recording_stop_timeout", diff --git a/Sources/Observability/SentryPayloadSanitizer.swift b/Sources/Observability/SentryPayloadSanitizer.swift index ca637512a..59a22c665 100644 --- a/Sources/Observability/SentryPayloadSanitizer.swift +++ b/Sources/Observability/SentryPayloadSanitizer.swift @@ -21,6 +21,7 @@ enum SentryPayloadSanitizer { for (key, value) in tags { guard !shouldDrop(key: key) else { continue } + if ["session_id", "correlation_id", "install_uuid"].contains(key), PayloadSanitizationCore.uuid(value) == nil { continue } let cleaned = sanitizeText(value) guard !cleaned.isEmpty else { continue } sanitized[key] = cleaned @@ -34,6 +35,7 @@ enum SentryPayloadSanitizer { for (key, value) in context { guard !shouldDrop(key: key) else { continue } + if ["session_id", "correlation_id", "install_uuid"].contains(key), PayloadSanitizationCore.uuid(value) == nil { continue } let cleaned = sanitizeText(value) guard !cleaned.isEmpty else { continue } sanitized[key] = cleaned diff --git a/Sources/Observability/TelemetryContext.swift b/Sources/Observability/TelemetryContext.swift new file mode 100644 index 000000000..a16aa0eef --- /dev/null +++ b/Sources/Observability/TelemetryContext.swift @@ -0,0 +1,74 @@ +import Foundation + +/// Shared metadata contract. Only UUIDs and categorical state cross the reporting boundary. +enum TelemetryContext { + static let launchSessionID = UUID().uuidString + static let keys = PayloadSanitizationCore.commonTelemetryKeys + static let deviceClasses: Set = [ + "built_in", "bluetooth", "usb", "aggregate", "virtual", "continuity", "wired", "external", + "hdmi", "displayport", "airplay", "thunderbolt", "firewire", "pci", "unknown", "none", + ] + + static func permissions() -> [String: String] { + [ + "mic_permission_granted": String(TranscriptedPermissionAccess.isGranted(.microphone)), + "screen_permission_granted": String(TranscriptedPermissionAccess.isGranted(.systemAudioRecording)), + "accessibility_permission_granted": String(TranscriptedPermissionAccess.isGranted(.accessibility)), + ] + } + + static func enrich( + event: String, + properties: [String: String], + isFailure: Bool = false, + environment: [String: String] = permissions() + ) -> [String: String] { + var result = properties + for (key, value) in environment where result[key] == nil { result[key] = value } + result["app_version"] = result["app_version"] ?? (Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown") + result["build_revision"] = result["build_revision"] ?? AnalyticsRuntimeConfiguration.buildRevision() + result["os_major"] = result["os_major"] ?? String(ProcessInfo.processInfo.operatingSystemVersion.majorVersion) + let session = uuid(result["session_id"]) ?? uuid(result["dictation_session_id"]) ?? launchSessionID + result["session_id"] = session + result["correlation_id"] = uuid(result["correlation_id"]) ?? UUID().uuidString + for key in ["input_device_class", "output_device_class"] { + if !deviceClasses.contains(result[key] ?? "") { result[key] = "unknown" } + } + result["selection_reason"] = category(result["selection_reason"]) ?? "unknown" + result["trigger"] = category(result["trigger"]) ?? "unknown" + for key in ["mic_permission_granted", "screen_permission_granted", "accessibility_permission_granted"] { + result[key] = result[key] == "true" ? "true" : "false" + } + let outcome = category(result["capture_outcome"]) ?? "unknown" + let outcomeFailure = ["no_audio", "timed_out", "stop_timed_out", "failed"].contains(outcome) + let failed = isFailure || event.hasSuffix("_failed") || event == "reliability_failure_observed" || outcomeFailure + if failed || event == "product_friction_observed" || event == "meeting_capture_health_snapshot" { + let fallbackKind = outcomeFailure ? outcome + : failed ? (category(event) ?? "unknown") + : event == "meeting_capture_health_snapshot" && outcome == "unknown" ? "unknown" : "none" + result["failure_kind"] = category(result["failure_kind"]) ?? fallbackKind + result["failure_stage"] = category(result["failure_stage"]) + ?? category(result["start_failure_stage"]) ?? category(result["stage"]) + ?? category(result["session_stage"]) ?? stage(for: event) + } + if event == "meeting_capture_health_snapshot" { + result["quality_reason"] = category(result["quality_reason"]) ?? "unknown" + result["capture_outcome"] = category(result["capture_outcome"]) ?? "unknown" + } + return result + } + + static func uuid(_ value: String?) -> String? { PayloadSanitizationCore.uuid(value) } + static func category(_ value: String?) -> String? { PayloadSanitizationCore.category(value) } + + static func stage(for event: String) -> String { + if event.contains("speaker_finalization") { return "speaker_finalization" } + if event.contains("start") || event.contains("mic_not_authorized") { return "start" } + if event.contains("transcript") { return "transcription" } + if event.contains("stop") || event.contains("health") { return "capture_stop" } + if event.contains("delivery") || event.contains("paste") { return "delivery" } + if event.contains("model") || event.contains("prewarm") { return "model_loading" } + if event.contains("recovery") || event.contains("engine") { return "recovery" } + return "unknown" + } +} diff --git a/Sources/Observability/UsageHealthModels.swift b/Sources/Observability/UsageHealthModels.swift new file mode 100644 index 000000000..8c830ad26 --- /dev/null +++ b/Sources/Observability/UsageHealthModels.swift @@ -0,0 +1,40 @@ +import Foundation + +struct UsageFailure: Codable, Equatable, Identifiable { + var id: String + var kind: String + var stage: String + var time: Date + var version: String +} + +struct UsageDay: Codable, Equatable { + var day: String + var startedAt: Date + var meetingsStarted = 0 + var meetingsCompleted = 0 + // Aggregate rounded minutes only; no per-recording duration or content is retained. + var meetingMinutes = 0 + var dictationsCompleted = 0 + var dictationDurationCounts: [String: Int] = [:] + var failuresByKind: [String: Int] = [:] + var captureQualityCounts: [String: Int] = [:] + var seenOutcomes: [String] = [] + var digestID = UUID().uuidString + var digestEnqueued = false +} + +struct UsageHealthSnapshot: Equatable { + var meetings = 0 + var dictations = 0 + var meetingMinutesBucket = "0" + var qualityCounts: [String: Int] = [:] + var failures: [UsageFailure] = [] +} + +struct UsageDigest: Equatable { + var id: String + var day: String + var properties: [String: String] + var aggregates: [String: [String: String]] +} diff --git a/Sources/Observability/UsageHealthStore.swift b/Sources/Observability/UsageHealthStore.swift new file mode 100644 index 000000000..b16844821 --- /dev/null +++ b/Sources/Observability/UsageHealthStore.swift @@ -0,0 +1,193 @@ +import Foundation + +/// Bounded metadata ledger, populated from event enums only. Never scans a capture or a log. +final class UsageHealthStore { + static let shared = UsageHealthStore() + static let didChange = Notification.Name("TranscriptedUsageHealthDidChange") + static let storageKey = "observability-usage-health-v1" + static let digestReceiptKey = "observability-usage-digest-days-v1" + static let durationBuckets = ["lt_10s", "10_29s", "30_119s", "2_9m", "10_29m", "30m_plus"] + private struct State: Codable { + var days: [UsageDay] = [] + var failures: [UsageFailure] = [] + } + private let lock = NSLock() + private let defaults: UserDefaults + private var state: State + + init(userDefaults: UserDefaults = .standard) { + defaults = userDefaults + state = userDefaults.data(forKey: Self.storageKey) + .flatMap { try? JSONDecoder().decode(State.self, from: $0) } ?? State() + } + + func clear() { + lock.lock() + let hadData = !state.days.isEmpty || !state.failures.isEmpty || defaults.data(forKey: Self.storageKey) != nil + state = State() + lock.unlock() + guard hadData else { return } + // UserDefaults notifications can re-enter observers synchronously. Never + // mutate defaults under this lock when clearing; repeated clears are no-ops. + defaults.removeObject(forKey: Self.storageKey) + NotificationCenter.default.post(name: Self.didChange, object: nil) + } + + func record(event: String, properties: [String: String], durationSeconds: Double? = nil, + now: Date = Date(), calendar: Calendar = .current) { + let relevant = ["meeting_recording_started", "meeting_transcript_saved", "meeting_recording_stopped", + "dictation_completed", "meeting_capture_health_snapshot", "reliability_failure_observed", + "meeting_capture_stopped_under_controller"].contains(event) + || event.hasSuffix("_failed") + guard relevant else { return } + lock.lock() + defer { + lock.unlock() + NotificationCenter.default.post(name: Self.didChange, object: nil) + } + guard AnalyticsPreferences.isEnabled(userDefaults: defaults) else { return } + prune(now: now, calendar: calendar) + let key = Self.dayKey(now, calendar: calendar) + if !state.days.contains(where: { $0.day == key }) { + state.days.append(UsageDay(day: key, startedAt: calendar.startOfDay(for: now))) + } + let index = state.days.firstIndex(where: { $0.day == key })! + var day = state.days[index] + let isFailure = event.hasSuffix("_failed") || event == "reliability_failure_observed" + || event == "meeting_capture_stopped_under_controller" + let kind = TelemetryContext.category(properties["failure_kind"]) ?? "unknown" + let correlation = TelemetryContext.uuid(properties["correlation_id"]) ?? UUID().uuidString + let outcomeKey = correlation + ":" + (isFailure ? "failure:" + kind : event) + guard !day.seenOutcomes.contains(outcomeKey) else { return } + day.seenOutcomes.append(outcomeKey) + day.seenOutcomes = Array(day.seenOutcomes.suffix(2_000)) + switch event { + case "meeting_recording_started": day.meetingsStarted += 1 + case "meeting_transcript_saved": day.meetingsCompleted += 1 + case "meeting_recording_stopped": + if let seconds = durationSeconds, seconds.isFinite, seconds > 0 { + day.meetingMinutes += Int(min(seconds / 60, 24 * 60).rounded()) + } + case "dictation_completed": + day.dictationsCompleted += 1 + if let bucket = properties["duration_bucket"], Self.durationBuckets.contains(bucket) { + day.dictationDurationCounts[bucket, default: 0] += 1 + } + case "meeting_capture_health_snapshot": + let outcome = properties["capture_outcome"] ?? "unknown" + if outcome != "cancelled" { + let quality: String + if ["no_audio", "timed_out", "stop_timed_out", "failed"].contains(outcome) { quality = "failed" } + else if ["degraded", "fair"].contains(properties["capture_quality"] ?? "") { quality = "degraded" } + else if ["excellent", "good"].contains(properties["capture_quality"] ?? "") { quality = "good" } + else { quality = "unknown" } + day.captureQualityCounts[quality, default: 0] += 1 + } + default: break + } + if isFailure { + day.failuresByKind[kind, default: 0] += 1 + state.failures.append(UsageFailure(id: outcomeKey, kind: kind, + stage: TelemetryContext.category(properties["failure_stage"]) ?? "unknown", time: now, + version: TelemetryContext.category(properties["app_version"]) ?? "unknown")) + state.failures = Array(state.failures.suffix(3)) + } + state.days[index] = day + persist() + } + + func snapshot(now: Date = Date(), calendar: Calendar = .current) -> UsageHealthSnapshot { + lock.lock() + defer { lock.unlock() } + let week = calendar.dateInterval(of: .weekOfYear, for: now) + let days = state.days.filter { week?.contains($0.startedAt) == true } + var snapshot = UsageHealthSnapshot() + snapshot.meetings = days.reduce(0) { $0 + $1.meetingsCompleted } + snapshot.dictations = days.reduce(0) { $0 + $1.dictationsCompleted } + snapshot.meetingMinutesBucket = Self.minutesBucket(days.reduce(0) { $0 + $1.meetingMinutes }) + for day in days { + for (key, count) in day.captureQualityCounts { snapshot.qualityCounts[key, default: 0] += count } + } + snapshot.failures = Array(state.failures.reversed()) + return snapshot + } + + func pendingDigests(includeCurrentDay: Bool, now: Date = Date(), calendar: Calendar = .current) -> [UsageDigest] { + lock.lock() + defer { lock.unlock() } + guard AnalyticsPreferences.isEnabled(userDefaults: defaults) else { return [] } + prune(now: now, calendar: calendar) + let today = Self.dayKey(now, calendar: calendar) + let receipts = Set(defaults.stringArray(forKey: Self.digestReceiptKey) ?? []) + return state.days.filter { !$0.digestEnqueued && !receipts.contains($0.day) && ($0.day < today || (includeCurrentDay && $0.day == today)) }.map { day in + UsageDigest(id: day.digestID, day: day.day, properties: [ + "digest_day": day.day, + "digest_is_partial": String(day.day == today), + "meetings_started": AnalyticsReporter.countBucket(day.meetingsStarted), + "meetings_completed": AnalyticsReporter.countBucket(day.meetingsCompleted), + "meeting_minutes_bucket": Self.minutesBucket(day.meetingMinutes), + "dictations_completed": AnalyticsReporter.countBucket(day.dictationsCompleted), + "dictation_median_duration_bucket": Self.medianDurationBucket(day.dictationDurationCounts), + ], aggregates: [ + "failures_by_kind": day.failuresByKind.mapValues(AnalyticsReporter.countBucket), + "capture_quality_counts": Dictionary(uniqueKeysWithValues: ["good", "degraded", "failed", "unknown"].map { + ($0, AnalyticsReporter.countBucket(day.captureQualityCounts[$0, default: 0])) + }), + ]) + } + } + + /// Called only after the matching capture is durably enqueued. Its stable insert ID + /// covers the small crash window between buffer persistence and this marker. + func markDigestEnqueued(id: String) { + lock.lock() + defer { lock.unlock() } + guard AnalyticsPreferences.isEnabled(userDefaults: defaults), + let index = state.days.firstIndex(where: { $0.digestID == id }) else { return } + state.days[index].digestEnqueued = true + // Retain only day receipts across opt-out. No counts, identity or activity + // remain; these prevent a second partial digest after re-enabling that day. + var receipts = Set(defaults.stringArray(forKey: Self.digestReceiptKey) ?? []) + receipts.insert(state.days[index].day) + defaults.set(Array(receipts.sorted().suffix(14)), forKey: Self.digestReceiptKey) + persist() + } + + static func medianDurationBucket(_ counts: [String: Int]) -> String { + let total = durationBuckets.reduce(0) { $0 + counts[$1, default: 0] } + guard total > 0 else { return "none" } + let rank = (total + 1) / 2 + var cumulative = 0 + for bucket in durationBuckets { + cumulative += counts[bucket, default: 0] + if cumulative >= rank { return bucket } + } + return "none" + } + + static func minutesBucket(_ minutes: Int) -> String { + switch minutes { + case ..<1: return "0" + case ..<15: return "1_14m" + case ..<60: return "15_59m" + case ..<180: return "1_2h" + case ..<600: return "3_9h" + default: return "10h_plus" + } + } + + static func dayKey(_ date: Date, calendar: Calendar) -> String { + let components = calendar.dateComponents([.year, .month, .day], from: date) + return String(format: "%04d-%02d-%02d", components.year!, components.month!, components.day!) + } + + private func prune(now: Date, calendar: Calendar) { + let oldest = calendar.date(byAdding: .day, value: -14, to: now) ?? now + state.days.removeAll { $0.startedAt < oldest } + state.days = Array(state.days.sorted { $0.startedAt < $1.startedAt }.suffix(14)) + } + + private func persist() { + if let data = try? JSONEncoder().encode(state) { defaults.set(data, forKey: Self.storageKey) } + } +} diff --git a/Sources/Speech/DictationSession.swift b/Sources/Speech/DictationSession.swift index 8dedbb646..f4f26a253 100644 --- a/Sources/Speech/DictationSession.swift +++ b/Sources/Speech/DictationSession.swift @@ -504,6 +504,7 @@ extension DictationSession { for (key, value) in extra { context[key] = value } + context.merge(telemetryContext) { current, _ in current } return context } } diff --git a/Sources/Speech/DictationSessionTypes.swift b/Sources/Speech/DictationSessionTypes.swift index f54721ce1..d8e609984 100644 --- a/Sources/Speech/DictationSessionTypes.swift +++ b/Sources/Speech/DictationSessionTypes.swift @@ -16,6 +16,9 @@ import Foundation @MainActor final class DictationSession: ObservableObject { + /// Correlates the engine-start failure with its owning overlay attempt. + var telemetryContext: [String: String] = [:] + // NOTE: this type deliberately does NOT publish its own lifecycle/state // enum. `DictationSessionController.isDictating` plus the overlay's own // state remain the single source of truth for "is a dictation session diff --git a/Sources/UI/Overlay/DictationSessionController.swift b/Sources/UI/Overlay/DictationSessionController.swift index 302c6027b..b0de3bca7 100644 --- a/Sources/UI/Overlay/DictationSessionController.swift +++ b/Sources/UI/Overlay/DictationSessionController.swift @@ -163,6 +163,11 @@ class DictationSessionController: ObservableObject { } isDictating = true currentDictationSessionID = UUID() + dictationSession.telemetryContext = [ + "session_id": currentDictationSessionID.uuidString, + "correlation_id": currentDictationSessionID.uuidString, + "trigger": trigger.rawValue, + ] stoppedAudioRecovery = nil stoppedAudioRecoveryPreservationSessionID = nil stoppedAudioCheckpointSignal = nil @@ -263,7 +268,8 @@ class DictationSessionController: ObservableObject { result: .blocked, failureKind: failureKind, routeShape: analyticsProperties["route_shape"], - modelState: ProductFrictionTelemetry.modelState(isReady: appState?.sttRouter.isModelLoaded) + modelState: ProductFrictionTelemetry.modelState(isReady: appState?.sttRouter.isModelLoaded), + context: analyticsProperties ) } @@ -2057,6 +2063,9 @@ class DictationSessionController: ObservableObject { private func dictationContext(extra: [String: String] = [:]) -> [String: String] { var context: [String: String] = [ + "session_id": currentDictationSessionID.uuidString, + "correlation_id": currentDictationSessionID.uuidString, + "trigger": currentDictationTrigger.rawValue, "audio_device": appState?.sttRouter.inputDeviceName ?? "" ] if let routeContext = appState?.sttRouter.dictationAudioRouteAnalyticsContext { @@ -2074,6 +2083,9 @@ class DictationSessionController: ObservableObject { private func dictationAnalyticsProperties(extra: [String: String] = [:]) -> [String: String] { var properties = appState?.sttRouter.dictationAudioRouteAnalyticsContext ?? [:] + properties["session_id"] = currentDictationSessionID.uuidString + properties["correlation_id"] = currentDictationSessionID.uuidString + properties["trigger"] = currentDictationTrigger.rawValue for (key, value) in extra { properties[key] = value } diff --git a/Sources/UI/Settings/TranscriptedSettingsView.swift b/Sources/UI/Settings/TranscriptedSettingsView.swift index 51e18461e..7c888ebb6 100644 --- a/Sources/UI/Settings/TranscriptedSettingsView.swift +++ b/Sources/UI/Settings/TranscriptedSettingsView.swift @@ -2342,7 +2342,7 @@ struct TranscriptedSettingsView: View { help: analyticsFootnote, info: GeneralInfo( title: "Usage stats", - message: "Anonymous feature usage from a strict allowlist. No content, ever." + message: "Shares feature use, duration and count ranges, error codes, permissions, and an anonymous install ID. Never shares recordings, words, titles, names, or email." ), automationIdentifier: "transcripted.settings.general.usage-stats", showsDivider: false diff --git a/Sources/UI/Shared/SupportDiagnosticsBundle.swift b/Sources/UI/Shared/SupportDiagnosticsBundle.swift index 19640abd0..39480c1f3 100644 --- a/Sources/UI/Shared/SupportDiagnosticsBundle.swift +++ b/Sources/UI/Shared/SupportDiagnosticsBundle.swift @@ -24,6 +24,9 @@ struct SupportDiagnosticsSnapshot: Equatable { var meetingShortcut: String = "unknown" var reliabilityPackets: [String] var recentLogLines: [String] + var installUUID: String = "unknown" + var buildRevision: String = "unknown" + var recentFailures: [UsageFailure] = [] } enum SupportDiagnosticsBundle { @@ -31,14 +34,9 @@ enum SupportDiagnosticsBundle { static let maxReliabilityPackets = 8 static func text(snapshot: SupportDiagnosticsSnapshot, now: Date = Date()) -> String { - let reliabilityPackets = snapshot.reliabilityPackets - .suffix(maxReliabilityPackets) - .map(AnalyticsPayloadSanitizer.redact) - .filter { !$0.isEmpty } - let recentLogs = snapshot.recentLogLines - .suffix(maxRecentLogLines) - .map(AnalyticsPayloadSanitizer.redact) - .filter { !$0.isEmpty } + let failures = snapshot.recentFailures.prefix(3).map { + "\($0.time.formatted(date: .abbreviated, time: .shortened)) | \(PayloadSanitizationCore.category($0.kind) ?? "unknown") | \(PayloadSanitizationCore.category($0.stage) ?? "unknown") | version \(PayloadSanitizationCore.category($0.version) ?? "unknown")" + } return """ Transcripted diagnostics @@ -47,6 +45,8 @@ enum SupportDiagnosticsBundle { App Version: \(snapshot.appVersion) Build: \(snapshot.buildVersion) + Revision: \(PayloadSanitizationCore.category(snapshot.buildRevision) ?? "unknown") + Install UUID: \(PayloadSanitizationCore.uuid(snapshot.installUUID) ?? "unknown") macOS: \(snapshot.osVersion) Reporting @@ -77,11 +77,8 @@ enum SupportDiagnosticsBundle { Queued meetings: \(snapshot.queuedMeetingCount) Meeting shortcut: \(snapshot.meetingShortcut) - Reliability Packets - \(reliabilityPackets.isEmpty ? "No recent reliability packets." : reliabilityPackets.joined(separator: "\n")) - - Recent Events - \(recentLogs.isEmpty ? "No recent in-app events." : recentLogs.joined(separator: "\n")) + Recent failures + \(failures.isEmpty ? "No recent failures recorded." : failures.joined(separator: "\n")) Privacy This diagnostic summary is designed to exclude transcript text, raw audio, file paths, device names, meeting titles, speaker names, emails, tokens, and raw URLs. @@ -90,6 +87,11 @@ enum SupportDiagnosticsBundle { static func sentryContext(snapshot: SupportDiagnosticsSnapshot) -> [String: String] { var context: [String: String] = [ + "install_uuid": PayloadSanitizationCore.uuid(snapshot.installUUID) ?? "unknown", + "build_revision": PayloadSanitizationCore.category(snapshot.buildRevision) ?? "unknown", + "last_failure_kind": PayloadSanitizationCore.category(snapshot.recentFailures.first?.kind) ?? "none", + "last_failure_stage": PayloadSanitizationCore.category(snapshot.recentFailures.first?.stage) ?? "none", + "last_failure_version": PayloadSanitizationCore.category(snapshot.recentFailures.first?.version) ?? "unknown", "analytics_available": bool(snapshot.analyticsAvailable), "analytics_enabled": bool(snapshot.analyticsEnabled), "app_version": snapshot.appVersion, @@ -115,15 +117,15 @@ enum SupportDiagnosticsBundle { // Sentry. (The human-readable diagnostics text, built separately, still // summarizes recent reliability packets for support diagnostic payloads.) - for (key, value) in snapshot.audioRoute { + for (key, value) in safeMetadata(snapshot.audioRoute) { context["route_\(key)"] = value } - for (key, value) in snapshot.runtime { + for (key, value) in safeMetadata(snapshot.runtime) { context["runtime_\(key)"] = value } - for (key, value) in snapshot.storage { + for (key, value) in safeMetadata(snapshot.storage) { context["storage_\(key)"] = value } @@ -143,6 +145,11 @@ enum SupportDiagnosticsBundle { /// suffix is sensitive (e.g. `runtime_file_path`, `route_raw_url`) is also /// dropped downstream. Keep in sync with `sentryContext` above. static let sentryContextAllowedKeys: Set = [ + "install_uuid", + "build_revision", + "last_failure_kind", + "last_failure_stage", + "last_failure_version", "analytics_available", "analytics_enabled", "app_version", @@ -175,12 +182,31 @@ enum SupportDiagnosticsBundle { static func allowlistedSentryContext(_ context: [String: String]) -> [String: String] { context.filter { key, _ in sentryContextAllowedKeys.contains(key) - || sentryContextAllowedKeyPrefixes.contains(where: { key.hasPrefix($0) }) + || sentryContextAllowedKeyPrefixes.contains(where: { prefix in + key.hasPrefix(prefix) && safeMetadata([String(key.dropFirst(prefix.count)): context[key]!]).count == 1 + }) + } + } + + private static func safeMetadata(_ values: [String: String]) -> [String: String] { + let keys = PayloadSanitizationCore.commonTelemetryKeys.union([ + "session_stage", "session_kind", "session_active", "previous_clean_shutdown", "heartbeat_age_bucket", + "last_event", "session_duration_bucket", "route_shape", "default_input_class", "default_output_class", + "selected_input_class", "selection_overrode_default", "input_channels", "output_channels", + "input_rate_hz", "output_rate_hz", "recovering", "format_ready", "sample_flow_started", + "known_stale_model_count", "model_cache_total", "known_stale_model_size", + ]) + return values.filter { key, value in + guard keys.contains(key) else { return false } + if key == "model_cache_total" || key == "known_stale_model_size" { + return value.range(of: #"^[0-9]+(?:[.,][0-9]+)? (?:bytes|KB|MB|GB|TB)$"#, options: .regularExpression) != nil + } + return PayloadSanitizationCore.category(value) != nil } } private static func render(_ values: [String: String]) -> String { - let sanitized = AnalyticsPayloadSanitizer.sanitizeDiagnosticContextForDisplay(values) + let sanitized = AnalyticsPayloadSanitizer.sanitizeDiagnosticContextForDisplay(safeMetadata(values)) guard !sanitized.isEmpty else { return "Unavailable" } return sanitized .sorted { $0.key < $1.key } diff --git a/Sources/UI/Shared/TranscriptedSupportActions.swift b/Sources/UI/Shared/TranscriptedSupportActions.swift index 2edada555..0c41b66c8 100644 --- a/Sources/UI/Shared/TranscriptedSupportActions.swift +++ b/Sources/UI/Shared/TranscriptedSupportActions.swift @@ -20,7 +20,7 @@ enum TranscriptedSupportActions { static func feedbackEmailURL(appState: TranscriptedAppState) -> URL? { FeedbackIssueBuilder.emailURL( - rawLogLines: appState.logger.entries, + rawLogLines: [], diagnostics: diagnosticsText(appState: appState) ) } @@ -94,7 +94,10 @@ enum TranscriptedSupportActions { queuedMeetingCount: queuedMeetingCount, meetingShortcut: meetingShortcut, reliabilityPackets: ReliabilityPacketRecorder.recentPacketSummaries(), - recentLogLines: appState.logger.entries + recentLogLines: [], + installUUID: InstallIdentity.id(), + buildRevision: AnalyticsRuntimeConfiguration.buildRevision(), + recentFailures: UsageHealthStore.shared.snapshot().failures ) } diff --git a/Tests/AnalyticsPayloadSanitizerTests.swift b/Tests/AnalyticsPayloadSanitizerTests.swift index b7cc508cd..e685c37ce 100644 --- a/Tests/AnalyticsPayloadSanitizerTests.swift +++ b/Tests/AnalyticsPayloadSanitizerTests.swift @@ -11,6 +11,13 @@ func testAnalyticsPayloadSanitizer() { } } + runSuite("Analytics taxonomy drops free text instead of forwarding redacted excerpts") { + for key in ["failure_kind", "failure_stage", "start_failure_stage", "trigger", "quality_reason", "capture_outcome", "selection_reason"] { + let safe = AnalyticsPayloadSanitizer.sanitizeProperties([key: "Confidential words /Users/example/private.txt"], allowedKeys: [key]) + assertNil(safe[key], "taxonomy values must be codes") + } + } + let corpus = loadJSONFixture("Tests/Fixtures/ObservabilitySanitizerCorpus.json", as: ObservabilitySanitizerCorpus.self) runSuite("AnalyticsPayloadSanitizer keeps only allowlisted coarse properties") { @@ -33,12 +40,12 @@ func testAnalyticsPayloadSanitizer() { runSuite("AnalyticsPayloadSanitizer redacts file paths and emails from values") { let sanitized = AnalyticsPayloadSanitizer.sanitizeProperties( [ - "failure_kind": "Saved to /Users/redbars/Library/Application Support/Transcripted/logs/app.jsonl by person@example.com on Redbarss-MacBook-Pro.local", + "diagnostic_value": "Saved to /Users/redbars/Library/Application Support/Transcripted/logs/app.jsonl by person@example.com on Redbarss-MacBook-Pro.local", ], - allowedKeys: ["failure_kind"] + allowedKeys: ["diagnostic_value"] ) - let value = sanitized["failure_kind"] ?? "" + let value = sanitized["diagnostic_value"] ?? "" assertFalse(value.contains("/Users/redbars/"), "user paths should be redacted") assertFalse(value.contains("Application Support/Transcripted/logs/app.jsonl"), "app support paths should be fully redacted") assertFalse(value.contains("person@example.com"), "emails should be redacted") @@ -51,12 +58,12 @@ func testAnalyticsPayloadSanitizer() { runSuite("AnalyticsPayloadSanitizer redacts synthetic-root macOS paths from values") { let sanitized = AnalyticsPayloadSanitizer.sanitizeProperties( [ - "failure_kind": "Saved to /System/Volumes/Data/Users/redbars/Library/Application Support/Transcripted/logs/app.jsonl before retry", + "diagnostic_value": "Saved to /System/Volumes/Data/Users/redbars/Library/Application Support/Transcripted/logs/app.jsonl before retry", ], - allowedKeys: ["failure_kind"] + allowedKeys: ["diagnostic_value"] ) - let value = sanitized["failure_kind"] ?? "" + let value = sanitized["diagnostic_value"] ?? "" assertFalse(value.contains("/System/Volumes/Data/Users/redbars/"), "synthetic-root home paths should be redacted") assertFalse(value.contains("Application Support/Transcripted/logs/app.jsonl"), "synthetic-root app support path should be fully redacted") assertTrue(value.contains("[redacted-path]"), "path marker should remain") @@ -100,12 +107,12 @@ func testAnalyticsPayloadSanitizer() { runSuite("AnalyticsPayloadSanitizer redacts raw URLs and common secret values") { let sanitized = AnalyticsPayloadSanitizer.sanitizeProperties( [ - "failure_kind": "Upload failed at https://example.com/path?token=abc123 with github_pat_abcdefghijklmnopqrstuvwxyz_1234567890 AKIAIOSFODNN7EXAMPLE AIzaSyA-BCDEFGHIJKLMNOPQRSTUVWXYZ123456 and api_key=secret-value password=hunter2 client_secret:supersecret credential=temp-pass", + "diagnostic_value": "Upload failed at https://example.com/path?token=abc123 with github_pat_abcdefghijklmnopqrstuvwxyz_1234567890 AKIAIOSFODNN7EXAMPLE AIzaSyA-BCDEFGHIJKLMNOPQRSTUVWXYZ123456 and api_key=secret-value password=hunter2 client_secret:supersecret credential=temp-pass", ], - allowedKeys: ["failure_kind"] + allowedKeys: ["diagnostic_value"] ) - let value = sanitized["failure_kind"] ?? "" + let value = sanitized["diagnostic_value"] ?? "" assertFalse(value.contains("https://example.com/path?token=abc123"), "raw URLs should be redacted") assertFalse(value.contains("github_pat_abcdefghijklmnopqrstuvwxyz_1234567890"), "GitHub fine-grained tokens should be redacted") assertFalse(value.contains("AKIAIOSFODNN7EXAMPLE"), "AWS access key IDs should be redacted") @@ -121,12 +128,12 @@ func testAnalyticsPayloadSanitizer() { runSuite("AnalyticsPayloadSanitizer redacts bearer headers and sk-style keys") { let sanitized = AnalyticsPayloadSanitizer.sanitizeProperties( [ - "failure_kind": "Request used Bearer abc123 and sk-proj-secret-value while retrying", + "diagnostic_value": "Request used Bearer abc123 and sk-proj-secret-value while retrying", ], - allowedKeys: ["failure_kind"] + allowedKeys: ["diagnostic_value"] ) - let value = sanitized["failure_kind"] ?? "" + let value = sanitized["diagnostic_value"] ?? "" assertFalse(value.contains("Bearer abc123"), "bearer headers should be redacted") assertFalse(value.contains("sk-proj-secret-value"), "sk-style API keys should be redacted") assertTrue(value.contains("Bearer ****"), "bearer marker should remain") @@ -145,12 +152,12 @@ func testAnalyticsPayloadSanitizer() { runSuite("AnalyticsPayloadSanitizer redacts basic auth headers and authorization assignments") { let sanitized = AnalyticsPayloadSanitizer.sanitizeProperties( [ - "failure_kind": "Authorization: Basic dXNlcjpwYXNz Basic ZGVtbzpwYXNz", + "diagnostic_value": "Authorization: Basic dXNlcjpwYXNz Basic ZGVtbzpwYXNz", ], - allowedKeys: ["failure_kind"] + allowedKeys: ["diagnostic_value"] ) - let value = sanitized["failure_kind"] ?? "" + let value = sanitized["diagnostic_value"] ?? "" assertFalse(value.contains("dXNlcjpwYXNz"), "basic auth payloads should be redacted") assertFalse(value.contains("ZGVtbzpwYXNz"), "standalone basic auth values should be redacted") assertTrue(value.contains("Authorization=[redacted-secret]"), "authorization assignments should collapse to a redacted marker") @@ -160,17 +167,17 @@ func testAnalyticsPayloadSanitizer() { runSuite("AnalyticsPayloadSanitizer redacts PEM private key material") { let sanitized = AnalyticsPayloadSanitizer.sanitizeProperties( [ - "failure_kind": """ + "diagnostic_value": """ Failure included: -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAz7i9W5tQ3k3FdemoKeyMaterial -----END RSA PRIVATE KEY----- """, ], - allowedKeys: ["failure_kind"] + allowedKeys: ["diagnostic_value"] ) - let value = sanitized["failure_kind"] ?? "" + let value = sanitized["diagnostic_value"] ?? "" assertFalse(value.contains("BEGIN RSA PRIVATE KEY"), "private key header should be redacted") assertFalse(value.contains("MIIEpAIBAAKCAQEA"), "private key body should be redacted") assertFalse(value.contains("END RSA PRIVATE KEY"), "private key footer should be redacted") diff --git a/Tests/AnalyticsReporterTests.swift b/Tests/AnalyticsReporterTests.swift index bcee78698..20868166d 100644 --- a/Tests/AnalyticsReporterTests.swift +++ b/Tests/AnalyticsReporterTests.swift @@ -1,6 +1,69 @@ import Foundation func testAnalyticsReporter() { + runSuite("Install identity survives upgrades and person traits are content-free") { + let fixture = makeAnalyticsReporterFixture(responses: [.networkFailure]) + defer { fixture.cleanup() } + let existing = UUID().uuidString.lowercased() + fixture.userDefaults.set(existing, forKey: InstallIdentity.storageKey) + assertEqual(InstallIdentity.id(userDefaults: fixture.userDefaults), existing, "preserve the existing UUID exactly") + let first = InstallIdentity.firstLaunchDay(userDefaults: fixture.userDefaults, now: Date(timeIntervalSince1970: 0)) + assertEqual(first, "1970-01-01", "persist only day precision") + assertEqual(InstallIdentity.firstLaunchDay(userDefaults: fixture.userDefaults), first, "first observed launch stays stable") + fixture.reporter.trackEvent("app_launched", properties: ["email": "private@example.com", "$set": "private words"]) + assertTrue(waitUntil { loadBufferedAnalyticsCaptures(from: fixture.bufferURL).count == 1 }, "capture is buffered") + let capture = loadBufferedAnalyticsCaptures(from: fixture.bufferURL).first! + assertEqual(capture.distinctID, existing, "PostHog uses the install UUID") + assertEqual(capture.personProperties?["analytics_opt_in"], "true", "opted-in trait is present") + assertNil(capture.personProperties?["email"], "person never has an email") + let payload = AnalyticsCaptureRequest(apiKey: "test", event: capture.event, distinctID: capture.distinctID, + timestamp: capture.timestamp, properties: capture.properties, personProperties: capture.personProperties) + let data = try! JSONEncoder().encode(payload) + let json = try! JSONSerialization.jsonObject(with: data) as! [String: Any] + let properties = json["properties"] as! [String: Any] + assertEqual((properties["$set"] as? [String: String])?["first_launch_at"], first, "person traits use a nested JSON object") + assertEqual(properties["$geoip_disable"] as? Bool, true, "disable server-derived location enrichment") + assertFalse(String(decoding: data, as: UTF8.self).contains("private"), "caller cannot inject personal properties") + } + + runSuite("Install identity rejects non-UUID legacy values") { + let name = "IdentityTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: name)! + defer { defaults.removePersistentDomain(forName: name) } + defaults.set("private@example.com", forKey: InstallIdentity.storageKey) + let id = InstallIdentity.id(userDefaults: defaults) + assertNotNil(UUID(uuidString: id), "replace invalid identity with a random UUID") + assertEqual(InstallIdentity.id(userDefaults: defaults), id, "replacement is stable") + } + + runSuite("Usage digest is durably queued once and carries only bucketed metadata") { + let fixture = makeAnalyticsReporterFixture(responses: [.networkFailure, .networkFailure], includeUsageStore: true) + defer { fixture.cleanup() } + fixture.reporter.trackEvent("dictation_completed", properties: ["duration_bucket": "10_29s", "transcript_text": "Private words"]) + fixture.reporter.enqueueUsageDigests(includeCurrentDay: true) + fixture.reporter.enqueueUsageDigests(includeCurrentDay: true) + fixture.reporter.persistPendingCapturesNow() + let digests = loadBufferedAnalyticsCaptures(from: fixture.bufferURL).filter { $0.event == "usage_digest" } + assertEqual(digests.count, 1, "repeated quit/timer callbacks enqueue one digest") + let digest = digests.first! + assertEqual(digest.properties["dictations_completed"], "1", "count uses the count bucket") + assertEqual(digest.properties["dictation_median_duration_bucket"], "10_29s", "median uses only histogram buckets") + assertEqual(digest.properties["install_uuid"], digest.distinctID, "digest joins the same install") + let payload = AnalyticsCaptureRequest(apiKey: "test", event: digest.event, distinctID: digest.distinctID, + timestamp: digest.timestamp, properties: digest.properties, uuid: digest.id, personProperties: digest.personProperties, + aggregateProperties: digest.aggregateProperties) + let json = String(decoding: try! JSONEncoder().encode(payload), as: UTF8.self) + assertTrue(json.contains("capture_quality_counts"), "quality is a JSON aggregate object") + assertFalse(json.contains("Private words"), "no capture content enters the digest") + let reloaded = UsageHealthStore(userDefaults: fixture.userDefaults) + assertEqual(reloaded.pendingDigests(includeCurrentDay: true, now: Date(timeIntervalSince1970: 2_000)).count, 0, "restart retains sent-day marker") + AnalyticsPreferences.setEnabled(false, userDefaults: fixture.userDefaults) + fixture.reporter.trackEvent("dictation_completed") + fixture.reporter.enqueueUsageDigests(includeCurrentDay: true) + assertNil(fixture.userDefaults.data(forKey: UsageHealthStore.storageKey), "opt-out clears rollups") + assertFalse(FileManager.default.fileExists(atPath: fixture.bufferURL.path), "opt-out clears unsent digests and person traits") + } + runSuite("AnalyticsRuntimeConfiguration prefers Transcripted overrides before legacy Draft") { let appSupport = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) .appendingPathComponent("AnalyticsReporterTests-\(UUID().uuidString)", isDirectory: true) @@ -436,6 +499,11 @@ func testAnalyticsReporter() { waitUntil { AnalyticsReporterTestURLProtocol.requestCount() == 2 && !FileManager.default.fileExists(atPath: fixture.bufferURL.path) }, "successful retry should delete the persisted capture" ) + let payloads = AnalyticsReporterTestURLProtocol.wirePayloads() + assertEqual(payloads.count, 2, "both transport attempts were recorded") + assertNotNil(payloads.first?["uuid"] as? String, "capture provides PostHog's top-level UUID") + assertEqual(payloads.first?["uuid"] as? String, payloads.last?["uuid"] as? String, "retries preserve the dedup UUID") + assertEqual(payloads.first?["timestamp"] as? String, payloads.last?["timestamp"] as? String, "retries preserve occurrence time") } runSuite("AnalyticsReporter drops non-429 4xx responses and retains 429 responses") { @@ -489,16 +557,17 @@ func testAnalyticsReporter() { } runSuite("AnalyticsReporter opt-out wipes the retry buffer and prevents delivery") { - let fixture = makeAnalyticsReporterFixture(responses: [.networkFailure], observePreferenceChanges: true) + let fixture = makeAnalyticsReporterFixture(responses: [.networkFailure], observePreferenceChanges: true, includeUsageStore: true) defer { fixture.cleanup() } - fixture.reporter.trackEvent("app_launched") + fixture.reporter.trackEvent("dictation_completed") assertTrue( waitUntil { AnalyticsReporterTestURLProtocol.requestCount() == 1 && loadBufferedAnalyticsCaptures(from: fixture.bufferURL).count == 1 }, "failed delivery should create a retry file before opt-out" ) + assertNotNil(fixture.userDefaults.data(forKey: UsageHealthStore.storageKey), "observed reporter populated the ledger") AnalyticsPreferences.setEnabled(false, userDefaults: fixture.userDefaults) assertTrue( @@ -506,6 +575,7 @@ func testAnalyticsReporter() { "analytics opt-out notification should delete the retry file" ) + assertTrue(waitUntil { fixture.userDefaults.data(forKey: UsageHealthStore.storageKey) == nil }, "opt-out clears ledger without recursive defaults deadlock") fixture.reporter.trackEvent("app_launched") assertTrue( @@ -591,6 +661,18 @@ func testAnalyticsReporter() { let ttlCaptures = ttlStore.load(now: Date(timeIntervalSince1970: 1_020)) assertEqual(ttlCaptures.map(\.id), ["fresh"], "TTL should drop records older than one day in production") + let day: TimeInterval = 24 * 60 * 60 + let digest = makePendingAnalyticsCapture(id: "daily-rollup", event: "usage_digest", enqueuedAt: 1_000) + let burst = (0..<150).map { makePendingAnalyticsCapture(id: "burst-\($0)", enqueuedAt: 1_000 + 2 * day) } + defaultStore.save([digest] + burst, now: Date(timeIntervalSince1970: 1_000 + 2 * day)) + let offline = defaultStore.load(now: Date(timeIntervalSince1970: 1_000 + 2 * day)) + assertTrue(offline.contains { $0.id == digest.id }, "daily digest survives 48 hours offline and lifecycle count pressure") + assertTrue(offline.count <= 100, "reserved digest still respects the total count bound") + tinyStore.save([digest] + burst, now: Date(timeIntervalSince1970: 1_000 + 2 * day)) + assertTrue(tinyStore.load(now: Date(timeIntervalSince1970: 1_000 + 2 * day)).contains { $0.id == digest.id }, "byte pressure discards ordinary events before a digest") + assertTrue((try? Data(contentsOf: fixture.bufferURL).count) ?? Int.max <= 1_200, "reserved digest respects total bytes") + assertFalse(defaultStore.cappedRecords([digest], now: Date(timeIntervalSince1970: 1_000 + 15 * day)).contains { $0.id == digest.id }, "digest retention remains bounded at 14 days") + try? Data("not-json".utf8).write(to: fixture.bufferURL, options: [.atomic]) assertEqual(defaultStore.load().count, 0, "corrupt retry files should recover as an empty buffer") assertFalse(FileManager.default.fileExists(atPath: fixture.bufferURL.path), "corrupt retry files should be deleted") @@ -656,7 +738,8 @@ private func makeAnalyticsReporterFixture( persistDebounceInterval: TimeInterval = 0.05, analyticsEnabled: (() -> Bool)? = nil, observePreferenceChanges: Bool = false, - autostart: Bool = true + autostart: Bool = true, + includeUsageStore: Bool = false ) -> AnalyticsReporterFixture { AnalyticsReporterTestURLProtocol.reset(responses: responses) @@ -690,7 +773,8 @@ private func makeAnalyticsReporterFixture( retryDelay: retryDelay, persistDebounceInterval: persistDebounceInterval, analyticsEnabled: analyticsEnabled, - observePreferenceChanges: observePreferenceChanges + observePreferenceChanges: observePreferenceChanges, + usageStore: includeUsageStore ? UsageHealthStore(userDefaults: defaults) : nil ) } ) @@ -761,11 +845,19 @@ private final class AnalyticsReporterTestURLProtocol: URLProtocol { private static let lock = NSLock() private static var responses: [AnalyticsReporterTestResponse] = [] private static var requests: [URLRequest] = [] + private static var payloads: [[String: Any]] = [] + + static func wirePayloads() -> [[String: Any]] { + lock.lock() + defer { lock.unlock() } + return payloads + } static func reset(responses: [AnalyticsReporterTestResponse]) { lock.lock() self.responses = responses self.requests = [] + self.payloads = [] lock.unlock() } @@ -805,8 +897,21 @@ private final class AnalyticsReporterTestURLProtocol: URLProtocol { override func stopLoading() {} private static func nextResponse(recording request: URLRequest) -> AnalyticsReporterTestResponse { + var body = request.httpBody ?? Data() + if body.isEmpty, let stream = request.httpBodyStream { + stream.open() + var bytes = [UInt8](repeating: 0, count: 4096) + while stream.hasBytesAvailable { + let count = stream.read(&bytes, maxLength: bytes.count) + guard count > 0 else { break } + body.append(contentsOf: bytes.prefix(count)) + } + stream.close() + } + let payload = (try? JSONSerialization.jsonObject(with: body)) as? [String: Any] ?? [:] lock.lock() requests.append(request) + payloads.append(payload) let response = responses.isEmpty ? .status(200) : responses.removeFirst() lock.unlock() return response diff --git a/Tests/E2E/TranscriptedE2ESmoke.swift b/Tests/E2E/TranscriptedE2ESmoke.swift index 46d22d7a0..14402b436 100644 --- a/Tests/E2E/TranscriptedE2ESmoke.swift +++ b/Tests/E2E/TranscriptedE2ESmoke.swift @@ -511,7 +511,10 @@ private final class TranscriptedE2ESmokeHarness { recentLogLines: [ "DIAG source_app_name=Codex transcript_text=\(secretTranscript)", "DIAG meeting_title=\(secretMeetingTitle) speaker_name=\(secretSpeaker)", - ] + ], + installUUID: "F31DB235-6730-4620-9646-55F7CBE6FA0C", + buildRevision: "fixture-revision", + recentFailures: [.init(id: "fixture", kind: "mic_unavailable", stage: "start", time: try fixedDate("2026-05-18T15:00:00Z"), version: "9.9.9")] ), now: try fixedDate("2026-05-18T15:10:00Z") ) @@ -533,27 +536,20 @@ private final class TranscriptedE2ESmokeHarness { try expect(!diagnostics.contains(forbidden), "Support diagnostics should not contain sensitive value: \(forbidden)") } - // Absence of forbidden values alone also passes if a whole section was silently dropped or - // the body came back empty. Assert the redacted structure actually survived: known-safe - // fields must still be present, and the explicit redaction markers must appear, proving the - // sensitive sections were emitted and redacted rather than omitted. + // Known-safe structure proves the summary is useful even though raw + // logs and reliability packet strings are intentionally omitted. for expected in [ "Version: 9.9.9", "input_device_class: built_in", "session_stage: recording", + "Install UUID: F31DB235-6730-4620-9646-55F7CBE6FA0C", + "mic_unavailable | start | version 9.9.9", + "Revision: fixture-revision", ] { try expect(diagnostics.contains(expected), "Support diagnostics should keep known-safe field: \(expected)") } - // Note: the injected sk- token only appears inside `token=...` assignments, so the - // apiKeyRegex's "sk-****" is superseded by a later secret-assignment pass; the raw token's - // absence is already asserted above. These markers prove sections were emitted and redacted. - for marker in [ - "[redacted-path]", - "[redacted-email]", - "[redacted-sensitive-value]", - ] { - try expect(diagnostics.contains(marker), "Support diagnostics should surface redaction marker: \(marker)") - } + try expect(!diagnostics.contains("DIAG "), "Raw log lines must be omitted from copied diagnostics") + try expect(!diagnostics.contains("event=retry"), "Raw reliability packets must be omitted from copied diagnostics") let eventLog = try String(contentsOf: fixtures.eventLogURL, encoding: .utf8) try expect(!eventLog.contains(fixtures.meetingURL.path), "Sanitized observability event should not contain file paths") diff --git a/Tests/MeetingCaptureVolumeDiagnosticsTests.swift b/Tests/MeetingCaptureVolumeDiagnosticsTests.swift index 111cfcce8..f0fc2834e 100644 --- a/Tests/MeetingCaptureVolumeDiagnosticsTests.swift +++ b/Tests/MeetingCaptureVolumeDiagnosticsTests.swift @@ -479,6 +479,17 @@ func testMeetingCaptureVolumeDiagnostics() { ) } + runSuite("MeetingCaptureHealthTelemetry never leaves outcome or quality reason blank") { + let input = MeetingCaptureHealthTelemetry.SnapshotInput( + captureDiagnostics: [:], health: .init(captureQuality: "good", audioGaps: 0, deviceSwitches: 0), + trigger: "menu", reason: "stop_button", durationSeconds: 30, systemStreamPresent: true, stopTimedOut: false) + assertEqual(MeetingCaptureHealthTelemetry.snapshotProperties(input)["capture_outcome"], "unknown", "unmeasured outcome cannot claim completion") + assertEqual(MeetingCaptureHealthTelemetry.snapshotProperties(input)["quality_reason"], "none", "good health has an explicit reason") + var cancelled = input + cancelled.captureOutcome = "cancelled" + assertEqual(MeetingCaptureHealthTelemetry.snapshotProperties(cancelled)["capture_outcome"], "cancelled", "discard is not a successful completion") + } + runSuite("MeetingCaptureHealthTelemetry builds shared capture health payloads") { let properties = MeetingCaptureHealthTelemetry.snapshotProperties( .init( diff --git a/Tests/SentryEventPolicyTests.swift b/Tests/SentryEventPolicyTests.swift index 94d60c200..4c2ebcfad 100644 --- a/Tests/SentryEventPolicyTests.swift +++ b/Tests/SentryEventPolicyTests.swift @@ -139,7 +139,7 @@ func testSentryEventPolicy() { assertEqual(deviceRecoveryTimeout?.summary, "Speech engine device-change recovery timed out.", "device recovery timeouts should be visible in Sentry with privacy-safe route context") assertEqual(recordingInterrupted?.summary, "Dictation recording was interrupted by audio device recovery.", "recording interruptions should be visible in Sentry with privacy-safe route context") assertEqual(meetingStartFailed?.summary, "Meeting recording could not start.", "meeting start failures should be visible without raw device names") - assertEqual(meetingCaptureDegraded?.summary, "Meeting capture health degraded.", "degraded meeting capture should be visible without raw device names") + assertNil(meetingCaptureDegraded, "completed degradation must never open a Sentry error even if a caller logs at error") assertEqual(meetingStopTimeout?.summary, "Meeting recording stop timed out.", "stop timeouts should be visible without raw device names") assertEqual(meetingTranscriptFailed?.summary, "Meeting transcription failed.", "meeting transcript failures should be visible with sanitized context") assertNil(meetingTranscriptSkipped, "expected empty/no-speech meeting outcomes should stay out of Sentry") @@ -329,7 +329,7 @@ func testSentryEventPolicy() { runSuite("SentryEventPolicy diagnosticTags keeps issue 500 volume-drop flags searchable") { let tags = SentryEventPolicy.diagnosticTags( forEngine: "meeting", - event: "recording_capture_degraded", + event: "meeting_transcript_failed", context: [ "default_output_volume_dropped": "true", "default_system_output_volume_dropped": "true", @@ -348,8 +348,8 @@ func testSentryEventPolicy() { ] ) - assertEqual(tags["default_output_volume_dropped"], "true", "output volume drops should be queryable in APPLE-MACOS-1B") - assertEqual(tags["default_system_output_volume_dropped"], "true", "system output drops should be queryable in APPLE-MACOS-1B") + assertEqual(tags["default_output_volume_dropped"], "true", "output volume drops should be queryable on hard failures") + assertEqual(tags["default_system_output_volume_dropped"], "true", "system output drops should be queryable on hard failures") assertEqual(tags["default_input_volume_dropped"], "false", "input volume state should stay available as a control") assertEqual(tags["buffer_success_bucket"], "98_100", "coarse buffer success should distinguish expected silence from write loss") assertEqual(tags["output_ducking_detected"], "true", "ducking classification should stay queryable") @@ -418,11 +418,11 @@ func testSentryEventPolicy() { assertNil(tags["transcript_text"], "transcript text must stay out of Sentry") } - runSuite("SentryEventPolicy diagnosticTags hard-caps the free-text reason tag") { + runSuite("SentryEventPolicy diagnosticTags rejects free-text reason tags") { let enumReason = "preferred_built_in_for_bluetooth_headset" let shortTags = SentryEventPolicy.diagnosticTags( forEngine: "meeting", - event: "recording_capture_degraded", + event: "meeting_transcript_failed", context: ["reason": enumReason] ) assertEqual(shortTags["reason"], enumReason, "short enum-style reasons should pass through untruncated") @@ -430,19 +430,12 @@ func testSentryEventPolicy() { let freeText = String(repeating: "a", count: 400) let cappedTags = SentryEventPolicy.diagnosticTags( forEngine: "meeting", - event: "recording_capture_degraded", + event: "meeting_transcript_failed", context: ["reason": freeText] ) - let cappedReason = cappedTags["reason"] - assertTrue(cappedReason != nil, "reason should still be forwarded after capping") - assertTrue( - (cappedReason?.count ?? 0) <= SentryEventPolicy.maxReasonTagLength + 3, - "an oversized free-text reason should be truncated to the hard cap plus ellipsis" - ) - assertTrue( - cappedReason?.hasSuffix("...") ?? false, - "a truncated reason should be marked with an ellipsis" - ) + assertEqual(cappedTags["reason"], "unknown", "free-text reasons never leave as a truncated excerpt") + let privateReason = SentryEventPolicy.diagnosticTags(forEngine: "meeting", event: "meeting_transcript_failed", context: ["reason": "Confidential meeting words"]) + assertEqual(privateReason["reason"], "unknown", "short free text is also excluded") } runSuite("Meeting stop emits one canonical Sentry terminal before generic degraded capture") { diff --git a/Tests/SupportDiagnosticsBundleTests.swift b/Tests/SupportDiagnosticsBundleTests.swift index c41ae9a9f..2365fa491 100644 --- a/Tests/SupportDiagnosticsBundleTests.swift +++ b/Tests/SupportDiagnosticsBundleTests.swift @@ -43,6 +43,7 @@ func testSupportDiagnosticsBundle() { "2026-05-03T01:15:11Z meeting.stop recovered event=meeting_recording_stopped route_change_count_bucket=2_3 path=/Users/redbars/private.txt" ], recentLogLines: [ + "Private content without an obvious sensitive prefix", "Opened /Users/redbars/Library/Application Support/Transcripted/logs/app.jsonl", "DIAG | capture.dictation_toggle_requested | source_app_bundle_id=com.openai.codex source_app_name=Codex trigger=physical_key", "DIAG | dictation.dictation_started | audio_device=MacBook Pro Microphone route_shape=built_in_input_to_built_in_output", @@ -56,6 +57,7 @@ func testSupportDiagnosticsBundle() { now: Date(timeIntervalSince1970: 100) ) + assertFalse(text.contains("Private content"), "arbitrary log content is never copied") assertTrue(text.contains("Version: 1.2.3"), "diagnostics should include app version") assertTrue(text.contains("input_device_class: bluetooth"), "diagnostics should include coarse route facts") assertTrue(text.contains("session_stage: recording"), "diagnostics should include runtime session stage") @@ -64,7 +66,7 @@ func testSupportDiagnosticsBundle() { assertTrue(text.contains("Speaker review pending: true"), "diagnostics should include pending speaker review state") assertTrue(text.contains("Queued meetings: 1"), "diagnostics should include queued meeting count") assertTrue(text.contains("Meeting shortcut: ⌥C"), "diagnostics should include the active meeting shortcut") - assertTrue(text.contains("meeting.stop recovered"), "diagnostics should include recent reliability packet summaries") + assertFalse(text.contains("meeting.stop recovered"), "diagnostics must not copy raw reliability packet blobs") assertFalse(text.contains("/Users/redbars"), "diagnostics should redact home paths") assertFalse(text.contains("person@example.com"), "diagnostics should redact emails") assertFalse(text.contains("Application Support/Transcripted"), "diagnostics should redact app support paths") diff --git a/Tests/TelemetryContextTests.swift b/Tests/TelemetryContextTests.swift new file mode 100644 index 000000000..20220d2ab --- /dev/null +++ b/Tests/TelemetryContextTests.swift @@ -0,0 +1,49 @@ +import Foundation + +func testTelemetryContext() { + let permissions = ["mic_permission_granted": "true", "screen_permission_granted": "false", "accessibility_permission_granted": "true"] + runSuite("Failures preserve identical safe metadata across both sinks") { + let correlation = UUID().uuidString + let properties = TelemetryContext.enrich(event: "meeting_recording_start_failed", properties: [ + "failure_kind": "mic_unavailable", "start_failure_stage": "microphone", "correlation_id": correlation, + "session_id": UUID().uuidString, "input_device_class": "bluetooth", "output_device_class": "built_in", + "selection_reason": "preferredBuiltInForBluetoothHeadset", "trigger": "hotkey", + "transcript_text": "Private content", "speaker_name": "Private Person", "audio_path": "/private/audio.wav", + ], environment: permissions) + let analytics = AnalyticsPayloadSanitizer.sanitizeProperties(properties, allowedKeys: TelemetryContext.keys) + let sentry = SentryEventPolicy.diagnosticTags(forEngine: "meeting", event: "meeting_start_failed", context: properties) + for key in TelemetryContext.keys.subtracting(["quality_reason", "capture_outcome"]) { + assertNotNil(analytics[key], "failure has required field \(key)") + assertEqual(sentry[key], analytics[key], "same \(key) reaches both systems") + } + assertEqual(sentry["correlation_id"], correlation, "join survives filtering") + assertEqual(sentry["failure_stage"], "microphone", "preserve precise start stage") + assertFalse(analytics.values.contains("Private content"), "content is excluded") + assertNil(sentry["audio_path"], "audio paths are excluded") + } + runSuite("Health and friction distinguish unknown metadata from success") { + let health = TelemetryContext.enrich(event: "meeting_capture_health_snapshot", properties: [:], environment: permissions) + assertEqual(health["quality_reason"], "unknown", "missing measurements never imply good health") + assertEqual(health["capture_outcome"], "unknown", "missing outcome never implies completion") + assertEqual(health["failure_kind"], "unknown", "unmeasured health does not claim no failure") + for outcome in ["no_audio", "timed_out", "stop_timed_out"] { + let failed = TelemetryContext.enrich(event: "meeting_capture_health_snapshot", properties: ["capture_outcome": outcome], environment: permissions) + assertEqual(failed["failure_kind"], outcome, "failed outcome has a stable failure code") + assertEqual(failed["failure_stage"], "capture_stop", "failed outcome has a stage") + } + let friction = TelemetryContext.enrich(event: "product_friction_observed", properties: ["stage": "dictation_start", "result": "started"], environment: permissions) + assertEqual(friction["failure_kind"], "none", "normal friction observations are not failures") + assertEqual(friction["failure_stage"], "dictation_start", "stage is explicit") + } + runSuite("Free text cannot impersonate shared taxonomy or identifiers") { + let properties = TelemetryContext.enrich(event: "dictation_start_failed", properties: [ + "session_id": "private@example.com", "correlation_id": "private words", "failure_kind": "Private transcript words", + "input_device_class": "Jane's AirPods", "trigger": "Private meeting title", + ], environment: permissions) + assertNotNil(UUID(uuidString: properties["session_id"]!), "session fallback is a UUID") + assertNotNil(UUID(uuidString: properties["correlation_id"]!), "correlation fallback is a UUID") + assertEqual(properties["input_device_class"], "unknown", "device names never become device classes") + assertEqual(properties["trigger"], "unknown", "free text never becomes a trigger") + assertEqual(properties["failure_kind"], "dictation_start_failed", "failure fallback is a code") + } +} diff --git a/Tests/UsageHealthStoreTests.swift b/Tests/UsageHealthStoreTests.swift new file mode 100644 index 000000000..b93631a16 --- /dev/null +++ b/Tests/UsageHealthStoreTests.swift @@ -0,0 +1,91 @@ +import Foundation + +func testUsageHealthStore() { + runSuite("Daily rollups handle local midnight, DST, and partial quit snapshots") { + let name = "UsageDaysTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: name)! + defer { defaults.removePersistentDomain(forName: name) } + let store = UsageHealthStore(userDefaults: defaults) + var calendar = Calendar(identifier: .gregorian) + calendar.timeZone = TimeZone(identifier: "America/Chicago")! + let now = ISO8601DateFormatter().date(from: "2026-03-08T07:30:00Z")! + let nextDay = calendar.date(byAdding: .day, value: 1, to: now)! + store.record(event: "dictation_completed", properties: ["duration_bucket": "10_29s"], now: now, calendar: calendar) + assertEqual(store.pendingDigests(includeCurrentDay: false, now: now, calendar: calendar).count, 0, "do not send an empty launch-day rollup") + let complete = store.pendingDigests(includeCurrentDay: false, now: nextDay, calendar: calendar) + assertEqual(complete.count, 1, "a closed local day rolls over across DST") + assertEqual(complete.first?.day, "2026-03-08", "report the activity's local day") + assertEqual(complete.first?.properties["digest_is_partial"], "false", "closed day is complete") + let partial = store.pendingDigests(includeCurrentDay: true, now: now, calendar: calendar).first! + assertEqual(partial.properties["digest_is_partial"], "true", "quit snapshots disclose partial coverage") + assertEqual(partial.id, complete.first?.id, "quit and rollover share a stable insert ID") + store.markDigestEnqueued(id: partial.id) + assertEqual(store.pendingDigests(includeCurrentDay: true, now: nextDay, calendar: calendar).count, 0, "at most one digest per reported local day") + AnalyticsPreferences.setEnabled(false, userDefaults: defaults) + store.clear() + AnalyticsPreferences.setEnabled(true, userDefaults: defaults) + store.record(event: "dictation_completed", properties: [:], now: now, calendar: calendar) + assertEqual(store.pendingDigests(includeCurrentDay: true, now: now, calendar: calendar).count, 0, "date-only receipt prevents duplicate digest after same-day re-opt-in") + assertEqual(UsageHealthStore.medianDurationBucket(["lt_10s": 1, "10_29s": 3, "30m_plus": 1]), "10_29s", "median uses ordered duration bins") + } + runSuite("Usage health counts lifecycle outcomes and deduplicates paired failures") { + let name = "UsageHealthTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: name)! + defer { defaults.removePersistentDomain(forName: name) } + let store = UsageHealthStore(userDefaults: defaults) + let now = Date() + let id = UUID().uuidString + var properties = ["correlation_id": id, "failure_kind": "mic_unavailable", "failure_stage": "microphone", "app_version": "1.1.59"] + store.record(event: "meeting_recording_started", properties: properties, now: now) + store.record(event: "meeting_recording_stopped", properties: properties, durationSeconds: 20 * 60, now: now) + store.record(event: "meeting_transcript_saved", properties: properties, now: now) + properties["duration_bucket"] = "10_29s" + store.record(event: "dictation_completed", properties: properties, now: now) + store.record(event: "reliability_failure_observed", properties: properties, now: now) + store.record(event: "meeting_recording_start_failed", properties: properties, now: now) + store.record(event: "product_friction_observed", properties: properties, now: now) + properties["capture_quality"] = "fair" + properties["capture_outcome"] = "complete" + store.record(event: "meeting_capture_health_snapshot", properties: properties, now: now) + let summary = store.snapshot(now: now) + assertEqual(summary.meetings, 1, "saved meetings count once") + assertEqual(summary.dictations, 1, "completed dictations count once") + assertEqual(summary.meetingMinutesBucket, "15_59m", "meeting minutes stay bucketed in summary") + assertEqual(summary.failures.count, 1, "paired analytics and Sentry failure count once") + assertEqual(summary.qualityCounts["degraded"], 1, "fair capture quality is degraded, not an unknown or hard failure") + assertEqual(UsageHealthStore(userDefaults: defaults).snapshot(now: now), summary, "metadata survives restart") + } + runSuite("Unexpected no-audio stop records one failure and a failed quality outcome") { + let name = "UsageUnexpectedStopTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: name)! + defer { defaults.removePersistentDomain(forName: name) } + let store = UsageHealthStore(userDefaults: defaults) + let properties = ["correlation_id": UUID().uuidString, "failure_kind": "no_audio", "failure_stage": "capture_stop", "capture_outcome": "no_audio"] + store.record(event: "meeting_capture_stopped_under_controller", properties: properties) + store.record(event: "reliability_failure_observed", properties: properties) + store.record(event: "meeting_capture_health_snapshot", properties: properties) + assertEqual(store.snapshot().failures.map(\.kind), ["no_audio"], "terminal and canonical failure are deduplicated") + assertEqual(store.snapshot().qualityCounts["failed"], 1, "health records the failed capture once") + let digest = store.pendingDigests(includeCurrentDay: true).first! + assertEqual(digest.aggregates["failures_by_kind"]?["no_audio"], "1", "digest includes the terminal failure") + assertNotNil(SentryEventPolicy.policy(forEngine: "meeting", event: "meeting_capture_stopped_under_controller"), "unexpected stop remains a hard Sentry failure") + } + runSuite("Usage health excludes cancellations and ignores content fields") { + let name = "UsageHealthTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: name)! + defer { defaults.removePersistentDomain(forName: name) } + let store = UsageHealthStore(userDefaults: defaults) + store.record(event: "meeting_capture_health_snapshot", properties: ["capture_outcome": "cancelled", "capture_quality": "excellent", "transcript": "Private content"]) + for i in 0..<5 { + store.record(event: "dictation_start_failed", properties: ["failure_kind": "failure_\(i)", "transcript": "Private content"]) + } + assertEqual(store.snapshot().failures.count, 3, "only the last three failures are retained") + assertEqual(store.snapshot().qualityCounts.count, 0, "discard is not a quality completion") + let persisted = String(decoding: defaults.data(forKey: UsageHealthStore.storageKey)!, as: UTF8.self) + assertFalse(persisted.contains("Private content"), "content fields cannot enter the local ledger") + AnalyticsPreferences.setEnabled(false, userDefaults: defaults) + store.clear() + store.record(event: "dictation_completed", properties: [:]) + assertNil(defaults.data(forKey: UsageHealthStore.storageKey), "opt-out erases the ledger and blocks new collection") + } +} diff --git a/docs/privacy-first-observability.md b/docs/privacy-first-observability.md index 5dcafa0f8..f53489208 100644 --- a/docs/privacy-first-observability.md +++ b/docs/privacy-first-observability.md @@ -81,6 +81,8 @@ This list should match `Resources/analytics-events.psv`, which `Sources/Observability/AnalyticsEventPolicy.swift` compiles into the runtime allowlist. +- `usage_digest` +- `reliability_failure_observed` - `app_launched` - `app_unclean_shutdown_detected` - `app_session_stall_detected` @@ -300,3 +302,102 @@ packets and the already-allowlisted meeting analytics/failure events. They add n process identity, audio content, hardware reads, or newly forwarded events. Live audio compatibility requires the receiving-participant checks in [Meeting Audio QA](qa-issue-500-meeting-audio.md). + +## Shared install and failure metadata + +The app reuses its persisted anonymous install UUID as PostHog `distinct_id` and +Sentry `user.id`. The Sentry sanitizer replaces the full user object with this ID +only. Analytics-enabled captures update a PostHog person through a fixed `$set` +object: `analytics_opt_in`, `app_version`, `build_revision`, `os_major`, +`install_channel`, and `first_launch_at` (UTC day). For existing installs the first +launch day means first observed by this version, not the original installation. +No email is collected. GeoIP enrichment is disabled on new PostHog requests. + +Every app analytics event can carry the common `TelemetryContext.keys` allowlist: +`session_id`, `correlation_id`, `app_version`, `build_revision`, `os_major`, +`input_device_class`, `output_device_class`, `selection_reason`, `trigger`, and +microphone/screen/accessibility permission booleans. Session and correlation IDs +must be app-generated UUIDs. Missing route observations are explicitly `unknown`; +a missing observation is not evidence of a healthy route or a denied permission. +Screen permission reflects the app's cached System Audio Recording grant. + +Failure, friction, and health events also carry `failure_kind` and `failure_stage`. +Health snapshots always carry `quality_reason` and `capture_outcome`; cancelled +captures have their own outcome. `none` means no failure; `unknown` means missing +measurement. Every allowlisted Sentry hard failure has a matching +`reliability_failure_observed` PostHog record using the exact same correlation ID +and taxonomy, even when the low-level failure has no product lifecycle event. +Product lifecycle failures remain available for funnel analysis; do not sum them +with their canonical reliability counterparts. Meeting start, transcription, +speaker finalization, and dictation microphone timeout preserve the same operation +ID across their existing lifecycle reports too. + +Capture degradation is a local warning and a PostHog health observation. It is +never a Sentry error, even if an old producer accidentally requests error level. +Hard start, transcript, audio-loss, stop-timeout, and engine-loop failures keep +their existing Sentry path. No audio-quality or routing policy changes here. + +## Support diagnostics and daily digest + +The existing Usage stats toggle controls analytics collection. A local metadata +ledger supplies the daily digest and recent failure details for support diagnostics. +`fair` capture grades join degraded; discarded captures do not count as successful +quality outcomes. Missing quality measurements remain explicitly unknown. + +The local ledger retains at most 14 local days and three failure summaries in +preferences. It consumes reviewed lifecycle metadata, not capture files or logs. +It stores aggregate rounded minutes and a histogram of dictation duration buckets, +not individual durations. Matching failure kind + correlation ID is counted once +across the canonical and lifecycle reports. Copy diagnostics and support drafts +include install UUID, release, OS, permissions, coarse runtime/route state, and +last failure taxonomy; they no longer append raw event or reliability-log text. + +`usage_digest` is emitted for closed local days on launch or the minute timer, +and for an unsent current day on normal quit. A quit snapshot has +`digest_is_partial=true`: later activity after a same-day relaunch is still visible +in lifecycle events and the local ledger but does not generate a second digest. +`digest_day` identifies the activity's local date; timestamps identify delivery. +Calendar arithmetic handles local midnight and DST rather than adding 24 hours. + +Digest fields `meetings_started`, `meetings_completed`, `dictations_completed`, +and values inside `failures_by_kind` / `capture_quality_counts` use the shared +count buckets `0`, `1`, `2_3`, `4_9`, `10_plus`. `meeting_minutes_bucket` uses +`0`, `1_14m`, `15_59m`, `1_2h`, `3_9h`, `10h_plus`. +`dictation_median_duration_bucket` is the lower median bin of the duration +histogram, or `none` when no dictations completed. No exact count or duration is +sent by the digest. This deliberately follows the brief's bucket-only hard rule. +The two aggregate maps are encoded as JSON objects, never free-form JSON strings. + +Each digest is persisted in the existing bounded retry buffer before its day is +marked enqueued. Retries keep the same top-level `uuid`, event, timestamp, and distinct ID, so a retry after an uncertain +HTTP response does not intentionally count the rollup twice. Transport remains +best effort: digests expire after 14 days, ordinary events after 24 hours. +Up to 14 digests receive priority over lifecycle traffic within the same +100-record / 64 KiB file bound. Off disables new events/person properties/digests, +purges unsent captures, and clears the local usage ledger. Up to 14 date-only +enqueue receipts remain to prevent duplicate same-day digests after re-enabling; +they contain no IDs or activity counts. Crash reporting retains its separate +preference. No vendor, recording behavior, or release configuration changes. + +## Verify the candidate + +- Exercise a failure with a controlled test install. Match Sentry `user.id` to + PostHog `distinct_id`, and `correlation_id` + `failure_kind` on the Sentry event, + `reliability_failure_observed`, and the corresponding lifecycle failure. +- Break down new-build health snapshots by `quality_reason` and `capture_outcome`; + both must be present. Separate older releases when assessing null rates. +- Complete a degraded capture: the PostHog snapshot and local warning remain, + but `meeting.recording_capture_degraded` is absent from Sentry errors. +- Compare unique users across the whole selected window; never sum daily DAU. + Do not add canonical failure counts to the same lifecycle failures. +- Check Settings, copy diagnostics, then turn Usage stats off. Reopen and quit: + there must be no additional analytics/person/digest requests after opt-out. +- Check local-day rollover, repeated quit/relaunch, retry after a network failure, + and nested count maps. Synthetic local tests validate plumbing, not production + ingestion or physical audio behavior. Fleet changes require deployment. + +PostHog's [capture API](https://posthog.com/docs/api/capture) and +[person properties](https://posthog.com/docs/product-analytics/person-properties) +provide the wire contract for `$set` and event-based anonymous install profiles. + +Every capture disables GeoIP enrichment with `$geoip_disable=true`. Request transport IP handling is a server setting: PostHog's [Discard IP data setting](https://posthog.com/tutorials/web-redact-properties#hiding-customer-ip-address) should be verified separately; a client-side `ip: false` option does not provide that guarantee. diff --git a/scripts/entrypoints/lib/shared-smoke-sources.sh b/scripts/entrypoints/lib/shared-smoke-sources.sh index 0e58c7f05..b5ab14197 100644 --- a/scripts/entrypoints/lib/shared-smoke-sources.sh +++ b/scripts/entrypoints/lib/shared-smoke-sources.sh @@ -56,6 +56,7 @@ SHARED_TEST_STORAGE_SOURCES=( "Sources/Meeting/MeetingArtifactRenamer.swift" "Sources/Observability/ObservabilityTextRedactor.swift" "Sources/Observability/PayloadSanitizationCore.swift" + "Sources/Observability/UsageHealthModels.swift" "Sources/Observability/AnalyticsPayloadSanitizer.swift" "Sources/TranscriptedCore/Audio/MicRecordingSegment.swift" "Sources/TranscriptedCore/Logging/PrivacyTextRedactor.swift" diff --git a/scripts/entrypoints/run-tests.sh b/scripts/entrypoints/run-tests.sh index 9ffc49861..908032080 100755 --- a/scripts/entrypoints/run-tests.sh +++ b/scripts/entrypoints/run-tests.sh @@ -438,6 +438,9 @@ APP_SOURCES=( "Sources/UI/MenuBar/MenuBarHeaderLayoutPolicy.swift" "Sources/UI/MenuBar/MenuBarHeaderStatusPresentation.swift" "Sources/UI/MenuBar/PasteLastDictationFeedback.swift" + "Sources/Observability/UsageHealthStore.swift" + "Sources/Observability/TelemetryContext.swift" + "Sources/Observability/InstallIdentity.swift" "Sources/Observability/AnalyticsReporter.swift" "Sources/Observability/DictationPasteRetryTelemetry.swift" "Sources/Observability/SpeakerRecognitionTelemetry.swift" diff --git a/scripts/ops/transcripted-qa-bench.sh b/scripts/ops/transcripted-qa-bench.sh index ea7ab1992..649569310 100755 --- a/scripts/ops/transcripted-qa-bench.sh +++ b/scripts/ops/transcripted-qa-bench.sh @@ -721,7 +721,7 @@ run_deep_tail() { run_full_tail() { run_step_when_present "60-release-health" "Deterministic release health gate" "yes" \ "scripts/ops/nightly-security-check.py" \ - "python3 scripts/ops/nightly-security-check.py --strict --automation-toml Tests/Fixtures/nightly-security-automation.toml --github-release-json Tests/Fixtures/release-health-github-release-1.1.58.json --write-report $(shell_quote "${RAW_DIR}/release-health.json")" + "python3 scripts/ops/nightly-security-check.py --strict --automation-toml Tests/Fixtures/nightly-security-automation.toml --github-release-json Tests/Fixtures/release-health-github-release-1.1.59.json --write-report $(shell_quote "${RAW_DIR}/release-health.json")" run_step_when_present "62-posthog-product-tasks" "PostHog product task fixture gate" "yes" \ "scripts/ops/posthog-product-dashboard-summary.py" \