Skip to content

Improve analytics identity and failure diagnostics #1733

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
r3dbars wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

Improve analytics identity and failure diagnostics #1733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d286802
Unify anonymous install identity across Sentry and PostHog
r3dbars Sep 10, 2026
cdbe8c1
Keep capture degradation in health analytics instead of Sentry errors
r3dbars Sep 10, 2026
3751e43
Complete failure taxonomy and correlate reliability reports across sinks
r3dbars Sep 10, 2026
610b260
Add local Usage and health settings with content-free support diagnos…
r3dbars Sep 10, 2026
e1312cf
Add daily bucketed usage digests with durable consent-aware delivery
r3dbars Sep 10, 2026
9783455
Remove customer-facing usage and health panel
r3dbars Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Resources/analytics-events.psv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ onboarding_reporting_toggle_changed|available,enabled,reporting_kind,step_id
onboarding_shown|analytics_available,crash_reporting_available,entrypoint,has_target,meeting_recording_ready,mic_status,model_state,pasteback_status
onboarding_step_viewed|flow_elapsed_bucket,model_state,step_id,step_index
product_friction_observed|elapsed_bucket,failure_kind,model_state,result,route_shape,stage,surface
reliability_failure_observed|accessibility_permission_granted,app_version,build_revision,correlation_id,failure_kind,failure_stage,input_device_class,mic_permission_granted,os_major,output_device_class,screen_permission_granted,selection_reason,session_id,trigger
settings_action_clicked|action_id,page_id
settings_capture_library_changed|location_type,page_id
settings_feature_discovered|feature_area,page_id,source
Expand All @@ -96,6 +97,7 @@ update_installed|previous_version,version
update_ready_to_install|automatic_downloads_enabled,state,version
update_relaunching|version
update_setting_changed|enabled,setting_id
usage_digest|app_version,dictation_median_duration_bucket,dictations_completed,digest_day,digest_is_partial,install_uuid,meeting_minutes_bucket,meetings_completed,meetings_started,os_major
workflow_abandoned|elapsed_bucket,prior_ready_state,reason_kind,stage,surface,workflow_kind
workflow_recovery_attempted|artifact_retained,failure_kind,recovery_attempt_bucket,retry_source,surface,workflow_kind
workflow_recovery_failed|artifact_retained,elapsed_bucket,failure_kind,recovery_attempt_bucket,result,retry_source,surface,workflow_kind
Expand Down
12 changes: 12 additions & 0 deletions Resources/analytics-reviewed-properties.psv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
# against the compiled allowlist. Marked `merge=union` in .gitattributes; run
# python3 scripts/ops/normalize-analytics-taxonomy.py
# after a union merge. Lines beginning with # are ignored.
accessibility_permission_granted
action
action_id
action_kind
Expand Down Expand Up @@ -53,6 +54,7 @@ completion_flow
completion_kind
cooldown_reason
copy_reason
correlation_id
crash_reporting_available
crash_reporting_enabled
cross_app_capture_status
Expand All @@ -77,11 +79,15 @@ default_system_output_volume_dropped
default_system_output_volume_during
delivery
dictation_ready
dictations_completed
digest_day
digest_is_partial
elapsed_bucket
enabled
entrypoint
failure_code
failure_kind
failure_stage
feature_area
first_artifact_kind
first_dictation_saved
Expand All @@ -96,11 +102,15 @@ input_channels
input_device_class
input_rate_hz
input_volume_scalar_available
install_uuid
last_event
location_type
meeting_dry_run_completed
meeting_recording_ready
meetings_completed
meetings_started
mic_boost_prompt
mic_permission_granted
mic_processed_peak
mic_processing
mic_raw_peak
Expand Down Expand Up @@ -158,11 +168,13 @@ route_stability_warning
runtime
sample_flow_started
save_outcome
screen_permission_granted
second_artifact_kind
selected_input_class
selection_overrode_default
selection_reason
session_active
session_id
session_kind
session_stage
setting_id
Expand Down
7 changes: 5 additions & 2 deletions Sources/Meeting/MeetingCaptureHealthTelemetry.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ enum MeetingCaptureHealthTelemetry {
let durationSeconds: Double
let systemStreamPresent: Bool
let stopTimedOut: Bool
var captureOutcome: String = "unknown"
}

struct DegradedReportInput {
Expand All @@ -35,7 +36,7 @@ enum MeetingCaptureHealthTelemetry {
}

static func snapshotProperties(_ input: SnapshotInput) -> [String: String] {
input.captureDiagnostics.merging(
var properties = input.captureDiagnostics.merging(
sharedProperties(
health: input.health,
trigger: input.trigger,
Expand All @@ -46,6 +47,8 @@ enum MeetingCaptureHealthTelemetry {
),
uniquingKeysWith: { _, new in new }
)
properties["capture_outcome"] = input.stopTimedOut ? "stop_timed_out" : input.captureOutcome
return properties
}

static func shouldReportDegraded(_ input: DegradedReportInput) -> Bool {
Expand Down Expand Up @@ -88,7 +91,7 @@ enum MeetingCaptureHealthTelemetry {
) -> [String: String] {
[
"capture_quality": health.captureQuality,
"quality_reason": health.qualityReason,
"quality_reason": health.qualityReason.isEmpty ? "unknown" : health.qualityReason,
"duration_bucket": AnalyticsReporter.durationBucket(seconds: durationSeconds),
"gap_count_bucket": AnalyticsReporter.countBucket(health.audioGaps),
"reason": reason,
Expand Down
77 changes: 46 additions & 31 deletions Sources/Meeting/MeetingSessionController.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ final class MeetingSessionController: ObservableObject {
}

private struct RecordingStopSnapshot {
let telemetryIdentity: UUID?
let trigger: StartTrigger
let systemAudioStatus: SystemAudioStatus
let durationSeconds: TimeInterval
Expand Down Expand Up @@ -841,6 +842,7 @@ final class MeetingSessionController: ObservableObject {
: TranscriptedConstants.meetingStartTimeout
let started = await capture.startRecording(timeout: startTimeout)
guard started else {
let failedStartIdentity = activeRecordingIdentity
await capture.flushSharedDictationMicHandler()
clearSharedDictationMicRelay()
activeRecordingTrigger = .unknown
Expand All @@ -856,7 +858,7 @@ final class MeetingSessionController: ObservableObject {
let pipelineSnapshot = capture.pipelineDiagnosticsSnapshot(
overrideSystemAudioStatus: capture.startFailureStage == .systemAudio ? .failed : nil
)
let failureProperties = meetingCaptureAnalyticsProperties(snapshot: pipelineSnapshot).merging(
let failureProperties = TelemetryContext.enrich(event: "meeting_recording_start_failed", properties: meetingCaptureAnalyticsProperties(snapshot: pipelineSnapshot, telemetryIdentity: failedStartIdentity).merging(
[
"failure_kind": meetingStartFailureKind(
from: failureMessage,
Expand All @@ -866,7 +868,7 @@ final class MeetingSessionController: ObservableObject {
"trigger": trigger.rawValue,
],
uniquingKeysWith: { _, new in new }
)
))
DiagnosticsTrail.record(
level: .error,
engine: "meeting",
Expand All @@ -883,7 +885,8 @@ final class MeetingSessionController: ObservableObject {
stage: "meeting_start",
result: .failed,
failureKind: failureProperties["failure_kind"],
modelState: state.diagnosticName
modelState: state.diagnosticName,
context: failureProperties
)
transition(to: .error(failureMessage), reason: "capture_start_failed")
Self.runtimeDiagnosticsRecorder?.clearSession(kind: "meeting", outcome: "start_failed")
Expand Down Expand Up @@ -1148,7 +1151,7 @@ final class MeetingSessionController: ObservableObject {
let afterStopVolumeContext = capture.routeVolumeDiagnosticsContext(currentPhase: "after")
var stopCaptureDiagnostics = MeetingCaptureVolumeDiagnostics.annotatedStopContext(
liveAttenuationCueObserved: capture.micAttenuationCueObserved,
baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot),
baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot, telemetryIdentity: recordingSnapshot.telemetryIdentity),
afterStopContext: afterStopVolumeContext
)
// Read the prompt outcome before any state mutations below; it is only
Expand Down Expand Up @@ -1213,7 +1216,8 @@ final class MeetingSessionController: ObservableObject {
"trigger": recordingSnapshot.trigger.rawValue,
],
uniquingKeysWith: { _, new in new }
)
),
usageDurationSeconds: recordingSnapshot.durationSeconds
)
var healthSnapshotProperties = MeetingCaptureHealthTelemetry.snapshotProperties(
.init(
Expand Down Expand Up @@ -1641,9 +1645,8 @@ final class MeetingSessionController: ObservableObject {
_ = audioInactivityDetector.stopRecording()
audioInactivityWarning = nil
isMicBoostPromptVisible = false
clearActiveRecordingIdentity()

let recordingSnapshot = makeRecordingStopSnapshot()
clearActiveRecordingIdentity()

DiagnosticsTrail.record(
engine: "meeting",
Expand All @@ -1666,7 +1669,7 @@ final class MeetingSessionController: ObservableObject {
let afterStopVolumeContext = capture.routeVolumeDiagnosticsContext(currentPhase: "after")
var cancelCaptureDiagnostics = MeetingCaptureVolumeDiagnostics.annotatedStopContext(
liveAttenuationCueObserved: capture.micAttenuationCueObserved,
baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot),
baseContext: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot, telemetryIdentity: recordingSnapshot.telemetryIdentity),
afterStopContext: afterStopVolumeContext
)
// Mirror stopRecording(): cancelled meetings carry the prompt outcome
Expand Down Expand Up @@ -1727,7 +1730,8 @@ final class MeetingSessionController: ObservableObject {
reason: reason.rawValue,
durationSeconds: recordingSnapshot.durationSeconds,
systemStreamPresent: files.systemURL != nil,
stopTimedOut: stopResult.didTimeOut
stopTimedOut: stopResult.didTimeOut,
captureOutcome: "cancelled"
)
)
)
Expand Down Expand Up @@ -2196,38 +2200,45 @@ final class MeetingSessionController: ObservableObject {
splitLocalSpeakers: LocalSpeakerPreferences.isEnabled()
)

let failureOutcome = CaptureOutcome(micURL: files.micURL, systemURL: files.systemURL, didTimeOut: stopResult.didTimeOut)
let failureContext = TelemetryContext.enrich(event: "meeting_capture_stopped_under_controller", properties:
meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot, telemetryIdentity: recordingSnapshot.telemetryIdentity).merging([
"failure_kind": files.micURL == nil && files.systemURL == nil ? "no_audio" : "unexpected_capture_stop",
"failure_stage": "capture_stop", "capture_outcome": failureOutcome.rawValue,
"trigger": recordingSnapshot.trigger.rawValue,
], uniquingKeysWith: { _, new in new }), isFailure: true)
DiagnosticsTrail.record(
level: .error,
engine: "meeting",
event: "meeting_capture_stopped_under_controller",
message: "Meeting capture stopped before the app stop path ran",
context: baseDiagnosticsContext(
extra: [
extra: failureContext.merging([
"mic_file_present": boolString(files.micURL != nil),
"system_file_present": boolString(files.systemURL != nil),
"preserved_for_retry": boolString(preserved),
"capture_quality": recordingSnapshot.healthInfo.captureQuality.rawValue,
"quality_reason": recordingSnapshot.healthInfo.qualityReason.rawValue,
"audio_gaps": "\(recordingSnapshot.healthInfo.audioGaps)",
"device_switches": "\(recordingSnapshot.healthInfo.deviceSwitches)"
]
], uniquingKeysWith: { _, new in new })
)
)

AnalyticsReporter.track(
"meeting_capture_stopped_under_controller",
properties: MeetingCaptureHealthTelemetry.snapshotProperties(
let healthProperties = MeetingCaptureHealthTelemetry.snapshotProperties(
.init(
captureDiagnostics: meetingCaptureAnalyticsProperties(snapshot: recordingSnapshot.pipelineSnapshot),
captureDiagnostics: failureContext,
health: captureHealthFacts(from: recordingSnapshot.healthInfo),
trigger: recordingSnapshot.trigger.rawValue,
reason: "internal_stop",
durationSeconds: recordingSnapshot.durationSeconds,
systemStreamPresent: files.systemURL != nil,
stopTimedOut: stopResult.didTimeOut
stopTimedOut: stopResult.didTimeOut,
captureOutcome: failureOutcome.rawValue
)
)
)
AnalyticsReporter.track("meeting_capture_stopped_under_controller", properties: healthProperties)
AnalyticsReporter.track("meeting_capture_health_snapshot", properties: healthProperties)

transition(
to: preserved
Expand Down Expand Up @@ -3003,29 +3014,25 @@ final class MeetingSessionController: ObservableObject {
if failureKind == .speakerFinalizationFailed || failureKind == .speakerNameFinalizationFailed {
activeQueuedTranscriptionJobID = nil
let queueDepthBucket = AnalyticsReporter.queueDepthBucket(transcriptionQueue.queuedTranscriptionJobs.count)
let failureTelemetryContext = meetingFailureTelemetryContext(failureKind: failureKind, transcriptionTrigger: transcriptionTrigger)
DiagnosticsTrail.record(
level: .error,
engine: "meeting",
event: "speaker_finalization_failed",
message: "Meeting speaker naming finalization failed",
context: baseDiagnosticsContext(
extra: [
extra: failureTelemetryContext.merging([
"failure_kind": failureKind.rawValue,
"session_stage": "save",
"queue_depth": "\(transcriptionQueue.queuedTranscriptionJobs.count)",
"queue_depth_bucket": queueDepthBucket,
"trigger": transcriptionTrigger.rawValue
]
], uniquingKeysWith: { current, _ in current })
)
)
AnalyticsReporter.track(
"meeting_speaker_finalization_failed",
properties: [
"session_stage": "save",
"failure_kind": failureKind.rawValue,
"queue_depth_bucket": queueDepthBucket,
"trigger": transcriptionTrigger.rawValue,
]
properties: failureTelemetryContext
)
trackDetectedPromptOutcome(
.speakerFinalizationFailed,
Expand All @@ -3038,7 +3045,8 @@ final class MeetingSessionController: ObservableObject {
stage: "speaker_finalization",
result: .failed,
failureKind: failureKind.rawValue,
modelState: state.diagnosticName
modelState: state.diagnosticName,
context: failureTelemetryContext
)
activeTranscriptionCaptureDiagnostics = nil
Self.runtimeDiagnosticsRecorder?.clearSession(kind: "meeting", outcome: "speaker_finalization_failed")
Expand Down Expand Up @@ -3081,7 +3089,8 @@ final class MeetingSessionController: ObservableObject {
stage: "meeting_transcription",
result: .failed,
failureKind: failureKind.rawValue,
modelState: state.diagnosticName
modelState: state.diagnosticName,
context: failureTelemetryContext
)
activeTranscriptionCaptureDiagnostics = nil
Self.runtimeDiagnosticsRecorder?.clearSession(kind: "meeting", outcome: "transcript_failed")
Expand Down Expand Up @@ -3157,11 +3166,15 @@ final class MeetingSessionController: ObservableObject {
MeetingStartFailureClassifier.kind(from: message, stage: stage)
}

private func meetingCaptureAnalyticsProperties(snapshot: AudioPipelineDiagnosticsSnapshot) -> [String: String] {
private func meetingCaptureAnalyticsProperties(snapshot: AudioPipelineDiagnosticsSnapshot, telemetryIdentity: UUID? = nil) -> [String: String] {
var properties = snapshot.privacySafeContext.merging(
MeetingCaptureVolumeDiagnostics.measurementScope,
uniquingKeysWith: { _, scope in scope }
)
if let id = (telemetryIdentity ?? activeRecordingIdentity)?.uuidString {
properties["session_id"] = id
properties["correlation_id"] = id
}
properties["gap_count_bucket"] = AnalyticsReporter.countBucket(snapshot.gapCount)
properties["route_change_count_bucket"] = AnalyticsReporter.countBucket(snapshot.routeChangeCount)
properties["recovery_attempt_bucket"] = AnalyticsReporter.countBucket(snapshot.recoveryAttemptCount)
Expand All @@ -3172,14 +3185,15 @@ final class MeetingSessionController: ObservableObject {
failureKind: MeetingFailureKind,
transcriptionTrigger: StartTrigger
) -> [String: String] {
(activeTranscriptionCaptureDiagnostics ?? [:]).merging(
TelemetryContext.enrich(event: "meeting_transcript_failed", properties: (activeTranscriptionCaptureDiagnostics ?? [:]).merging(
[
"failure_stage": failureKind == .speakerFinalizationFailed || failureKind == .speakerNameFinalizationFailed ? "speaker_finalization" : "transcription",
"failure_kind": failureKind.rawValue,
"queue_depth_bucket": AnalyticsReporter.queueDepthBucket(transcriptionQueue.queuedTranscriptionJobs.count),
"trigger": transcriptionTrigger.rawValue,
],
uniquingKeysWith: { _, new in new }
)
))
}

private func trackDetectedPromptOutcome(
Expand Down Expand Up @@ -3249,7 +3263,7 @@ final class MeetingSessionController: ObservableObject {
) else { return }

DiagnosticsTrail.record(
level: .error,
level: .warning,
engine: "meeting",
event: "recording_capture_degraded",
message: "Meeting capture health degraded",
Expand Down Expand Up @@ -3421,6 +3435,7 @@ final class MeetingSessionController: ObservableObject {
healthInfo = baseHealthInfo
}
return RecordingStopSnapshot(
telemetryIdentity: activeRecordingIdentity,
trigger: activeRecordingTrigger,
systemAudioStatus: systemAudioStatus,
durationSeconds: durationSeconds,
Expand Down
4 changes: 3 additions & 1 deletion Sources/Observability/ActivationTelemetry.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,7 +524,8 @@ enum ProductFrictionTelemetry {
failureKind: String? = nil,
elapsedBucket: String? = nil,
routeShape: String? = nil,
modelState: String? = nil
modelState: String? = nil,
context: [String: String] = [:]
) {
var properties = [
"result": result.rawValue,
Expand All @@ -545,6 +546,7 @@ enum ProductFrictionTelemetry {
properties["model_state"] = modelState
}

properties.merge(context) { current, _ in current }
AnalyticsReporter.track("product_friction_observed", properties: properties)
}

Expand Down
3 changes: 3 additions & 0 deletions Sources/Observability/AnalyticsPayloadSanitizer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ enum AnalyticsPayloadSanitizer {
for (key, value) in properties {
guard allowedKeys.contains(key) else { continue }
guard !shouldDrop(key: key) else { continue }
if ["session_id", "correlation_id", "install_uuid"].contains(key), PayloadSanitizationCore.uuid(value) == nil { continue }
if ["failure_kind", "failure_stage", "start_failure_stage", "selection_reason", "trigger", "quality_reason", "capture_outcome"].contains(key),
PayloadSanitizationCore.category(value) == nil { continue }

let cleaned = sanitizeText(value)
guard !cleaned.isEmpty else { continue }
Expand Down
Loading
Loading