diff --git a/App/iOSDeveloperToolkit/Model/LiveLogsModel.swift b/App/iOSDeveloperToolkit/Model/LiveLogsModel.swift index 8de7123..a390012 100644 --- a/App/iOSDeveloperToolkit/Model/LiveLogsModel.swift +++ b/App/iOSDeveloperToolkit/Model/LiveLogsModel.swift @@ -40,6 +40,8 @@ final class LogSession: Identifiable { var findings: [LiveLogFinding] = [] var investigationReference = "" var hasUnsavedData = true + /// The .logarchive or .trace a collected source keeps, once it exists. + var artifactURL: URL? fileprivate var task: Task? private var pending: [LogLine] = [] private var flushScheduled = false @@ -105,6 +107,8 @@ final class LogSession: Identifiable { final class LiveLogsModel { var sessions: [LogSession] = [] var selectedSessionID: UUID? + /// The time window for OSLog archives and DVT recordings, in seconds. + var collectionSeconds = 60 var selectedSession: LogSession? { sessions.first { $0.id == selectedSessionID } ?? sessions.last @@ -123,15 +127,18 @@ final class LiveLogsModel { selectedSessionID = session.id let runner = app.runner let started = Date() + let seconds = collectionSeconds session.task = Task { [weak session] in do { - let stream = try await LiveLogSource.open(kind, target: target, runner: runner) + let artifact = kind.artifactExtension.map { capture.spoolURL.deletingPathExtension().appendingPathExtension($0) } + let stream = try await LiveLogSource.open(kind, target: target, runner: runner, windowSeconds: seconds, artifact: artifact) session?.state = .running for try await chunk in stream { try await capture.append(chunk) session?.receive(chunk) } - session?.stop(reason: "the device ended the stream") + if let artifact, FileManager.default.fileExists(atPath: artifact.path) { session?.artifactURL = artifact } + session?.stop(reason: kind.isCollected ? "collection finished" : "the device ended the stream") app.record(title: kind.title, workspace: .liveLogs, target: target, transport: kind.serviceDescription, argv: [], started: started, finished: Date(), outcome: .succeeded, error: nil, outputPaths: [capture.spoolURL.path]) } catch is CancellationError { app.record(title: kind.title, workspace: .liveLogs, target: target, transport: kind.serviceDescription, argv: [], started: started, finished: Date(), outcome: .cancelled, error: nil, outputPaths: [capture.spoolURL.path]) diff --git a/App/iOSDeveloperToolkit/Views/DataViews.swift b/App/iOSDeveloperToolkit/Views/DataViews.swift index 73d1b3d..c53128b 100644 --- a/App/iOSDeveloperToolkit/Views/DataViews.swift +++ b/App/iOSDeveloperToolkit/Views/DataViews.swift @@ -141,6 +141,8 @@ struct EvidenceView: View { Toggle("Network packet capture (PCAP)", isOn: $evidence.options.includePacketCapture) Toggle("Screenshot", isOn: $evidence.options.includeScreenshot) Toggle("Copy crash reports", isOn: $evidence.options.includeCrashReports) + Toggle("OSLog archive: the device's saved logs from the last hour", isOn: $evidence.options.includeOSLogArchive) + Toggle("DVT logging through Instruments (\(evidence.options.dvtSeconds) s; needs Developer Mode and the developer image)", isOn: $evidence.options.includeDVTLogging) Text("Logs, packet captures, screenshots, and crash reports can contain private information.").font(.caption).foregroundStyle(.secondary) ReadinessStatusView(requirements: evidence.options.requirements, device: device, subject: "this collection") } diff --git a/App/iOSDeveloperToolkit/Views/LogViews.swift b/App/iOSDeveloperToolkit/Views/LogViews.swift index e40a8c6..0a5a616 100644 --- a/App/iOSDeveloperToolkit/Views/LogViews.swift +++ b/App/iOSDeveloperToolkit/Views/LogViews.swift @@ -13,30 +13,51 @@ struct LiveLogsView: View { TargetHeader(allowedKinds: [.physical, .simulator]) if let device = model.selectedDevice { let kinds = LogStreamKind.available(for: device.kind) - HStack(spacing: 10) { - ForEach(kinds) { kind in - Button { - model.logs.start(kind, target: device.target, app: model) - } label: { - Label("Start \(kind.title)", systemImage: "play.fill") - } - .help(kind.summary) - .accessibilityIdentifier("start-\(kind.rawValue)") + let live = kinds.filter { !$0.isCollected } + let collected = kinds.filter(\.isCollected) + if kinds.isEmpty { + Text("Live logs are not available for the demo device.").foregroundStyle(.secondary) + } + if !live.isEmpty { + HStack(spacing: 10) { + Text("Stream").font(.callout.weight(.semibold)).frame(width: 64, alignment: .leading) + ForEach(live) { kind in startButton(kind, device: device, label: "Start \(kind.title)", symbol: "play.fill") } + Spacer() } - if kinds.isEmpty { - Text("Live logs are not available for the demo device.").foregroundStyle(.secondary) + } + if !collected.isEmpty { + @Bindable var logs = model.logs + HStack(spacing: 10) { + Text("Collect").font(.callout.weight(.semibold)).frame(width: 64, alignment: .leading) + Picker("Window", selection: $logs.collectionSeconds) { + ForEach(CollectedLogs.windows, id: \.self) { seconds in + Text(seconds < 60 ? "\(seconds) s" : "\(seconds / 60) min").tag(seconds) + } + } + .fixedSize() + .help("OSLog Archive collects this much saved history; DVT Logging records for this long.") + .accessibilityIdentifier("collection-window") + ForEach(collected) { kind in + startButton(kind, device: device, label: kind == .osLogArchive ? "Collect OSLog Archive" : "Record DVT Logging", symbol: kind == .osLogArchive ? "tray.and.arrow.down" : "record.circle") + } + Spacer() } - Spacer() } if !kinds.isEmpty { - Text(kinds.map(\.summary).joined(separator: " ")) - .font(.callout) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) + DisclosureGroup("About these sources") { + VStack(alignment: .leading, spacing: 4) { + ForEach(kinds) { kind in + Text("**\(kind.title):** \(kind.summary)").fixedSize(horizontal: false, vertical: true) + } + } + .padding(.top, 4) + } + .font(.callout) + .foregroundStyle(.secondary) } } if model.logs.sessions.isEmpty { - ContentUnavailableView("No Log Streams", systemImage: "text.alignleft", description: Text("Start a stream above. Every byte is saved to a private spool on this Mac, even while the view is paused or filtered.")) + ContentUnavailableView("No Logs Yet", systemImage: "text.alignleft", description: Text("Start a stream or a collection above. Every byte is saved to a private spool on this Mac, even while the view is paused or filtered.")) .frame(maxWidth: .infinity, maxHeight: .infinity) } else { Picker("Stream", selection: Binding(get: { model.logs.selectedSession?.id }, set: { model.logs.selectedSessionID = $0 })) { @@ -65,6 +86,16 @@ struct LiveLogsView: View { .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) .padding(20) } + + private func startButton(_ kind: LogStreamKind, device: Device, label: String, symbol: String) -> some View { + Button { + model.logs.start(kind, target: device.target, app: model) + } label: { + Label(label, systemImage: symbol) + } + .help(kind.summary) + .accessibilityIdentifier("start-\(kind.rawValue)") + } } /// The working view for one log stream. @@ -131,7 +162,7 @@ struct LogSessionView: View { .accessibilityLabel("Log lines") .overlay { if lines.isEmpty { - Text(session.state == .starting ? "Connecting…" : (session.filter.isEmpty ? "Waiting for log messages…" : "No lines match the filter.")) + Text(session.state == .starting ? "Connecting…" : (session.filter.isEmpty ? (session.kind.isCollected ? "Collecting… the lines appear when it finishes." : "Waiting for log messages…") : "No lines match the filter.")) .foregroundStyle(.secondary) } } @@ -169,6 +200,11 @@ struct LogSessionView: View { Button("Show Spool in Finder") { FilePanels.reveal(session.capture.spoolURL) } } .fixedSize() + if let artifact = session.artifactURL { + Button(session.kind == .dvt ? "Open in Instruments" : "Open in Console") { NSWorkspace.shared.open(artifact) } + .help(artifact.path) + Button("Show in Finder") { FilePanels.reveal(artifact) } + } } } .sheet(isPresented: $isMarkingFinding) { diff --git a/MIGRATION.md b/MIGRATION.md index 4addde2..4e363e9 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -69,7 +69,7 @@ Swift device discovery (usbmuxd + CoreDevice + simctl), so nothing is lost. | 10 | Location Lab (coordinate, nudge, saved places, offline map, map-link parsing, route generator, GPX inspection/replay, evidence log, clear) | `pmd3 developer dvt simulate-location` | Physical: CoreDevice `simulate location coordinate/route/clear`; Simulator: `simctl location`; GPX replay driven by the app; offline MapKit-free world map | devicectl, simctl | No | ✅ simulators · 🟡 physical | | 11 | Live Logs — Unified | `pmd3 syslog live --format json` (os_trace_relay) | Native `com.apple.os_trace_relay` client; Simulator: `simctl spawn log stream --style ndjson` | Lockdown service / simctl | No | ✅ simulators · 🟡 physical | | 12 | Live Logs — Classic syslog | `pmd3 syslog live-old` | Native `com.apple.syslog_relay` client | Lockdown service | No | 🟡 | -| 13 | Live Logs — DVT OSLog | `pmd3 developer dvt oslog` | 🔁 Covered by #11 (os_trace_relay needs no DDI); DVT/DTX is not an Apple-public interface | — | No | 🔁 🟡 | +| 13 | Live Logs — DVT OSLog | `pmd3 developer dvt oslog` | 🔁 Live Logs › **DVT Logging**: a timed Instruments Logging recording (`xctrace record`, then `xctrace export` of the os-log table), shown line by line and kept as a `.trace`; also an Evidence Capture option. Live streaming as in 0.3.x would need a DTX client over Xcode's private tunnel. The live Unified stream (#11) needs no developer image. Live Logs › **OSLog Archive** adds the device's saved history (`log collect`). | Xcode (xctrace); macOS `log` | No | 🔁 ✅ simulator · 🟡 physical | | 14 | Live log spool, pause, filter (literal/regex/case), findings, review, raw/filtered save, evidence bundle, metadata sidecar | `live_logs.py` | Ported (`LogCapture`, `FindingsStore`, `InvestigationReport`) | Foundation | No | ✅ | | 15 | Command Center — 49 `pmd3` presets + Advanced Mode + risk classes + typed confirmation | `command_catalog.py`, `action_safety.py` | 🔁 Guided **Actions** catalog backed by native services / devicectl / simctl / xctrace, same risk classes and device-bound `RUN XXXXXX` / `IRREVERSIBLE XXXXXX` phrases; Advanced Mode for `devicectl` with safety classification | Yes | No | 🔁 ✅ simulators · 🟡 physical | | 16 | Guided Command Drift | `pmd3 --help` probes | 🔁 **Toolchain Check**: verifies every devicectl/simctl/xctrace route the app uses is present in the installed Xcode | Yes | No | 🔁 ✅ | @@ -297,8 +297,9 @@ that changes the device was run: no mounting, location, installation, or backup. macOS is 14 (was 13). - Guided actions replace the 49 raw `pymobiledevice3` presets; Advanced Mode runs `devicectl` instead of arbitrary `pymobiledevice3` subcommands. -- DVT telemetry streams are replaced by Instruments recordings (`xctrace`); the DVT OSLog stream by - the Unified Logging stream, which needs no developer image. +- DVT telemetry streams are replaced by Instruments recordings (`xctrace`). The DVT OSLog stream is + replaced by DVT Logging, a timed Instruments Logging recording shown line by line; the live + Unified Logging stream needs no developer image, and the OSLog Archive adds the saved history. - Features that need a developer tunnel (iOS 17+) now require Xcode, which owns the tunnel. ## 7. Migration log @@ -407,7 +408,7 @@ and 0.3.x's shortcuts for the tenth and later pages and focus (⌘0, ⇧⌘E/M/S | dvt-list (`developer dvt ls`) | — | — | §6.1: DTX over RemoteXPC on iOS 17+ | | crash-list, crash-pull | Actions (native AFC) | = | `BackupAndAFCTests` | | syslog | Live Logs · Classic syslog | = | `ServiceTests` | -| oslog (DVT) | Live Logs · Unified (os_trace_relay, no DDI) | 🔁 | `ServiceTests`, real-simulator test | +| oslog (DVT) | Live Logs · DVT Logging (Instruments Logging recording, exported) and Unified (os_trace_relay, no DDI); OSLog Archive (`log collect`) | 🔁 | `CollectedLogTests`, real-simulator test (DVT: 9,608 lines in 3 s) | | pcap | Action `packet-capture`, Evidence stream | = | `nativeActionsRunAgainstTheCapturedTarget` | | btlogger (`--format pcapng`) | Action `bluetooth-capture` (native, `.pklg`) | 🔁 (**G4 resolved**; PacketLogger format instead of pcapng) | `bluetoothRecordsBecomeAPacketLoggerFile`, `nativeActionsRunAgainstTheCapturedTarget` | | dvt-device, dvt-proclist, dvt-applist | device details / processes / apps | 🔁 | — | @@ -448,7 +449,7 @@ and 0.3.x's shortcuts for the tenth and later pages and focus (⌘0, ⇧⌘E/M/S | Item | Swift | Class | |---|---|---| -| `ios-developer-collect` (all options) | `idt collect` — `--include-oslog` renamed `--include-unified-logs`; the old name is still accepted (hidden from help) | = (**G7 resolved**) | +| `ios-developer-collect` (all options) | `idt collect` — `--include-oslog` (0.3.x DVT OSLog) is accepted and selects `--include-dvt-logs`; `--include-oslog-archive` adds `log collect` | = (**G7 resolved**) | | `ios-ipa-inspect`, `ios-local-ddi` | `idt inspect-ipa`, `idt ddi` | = / 🔁 | | Evidence snapshots (17) | lockdown, images, diagnostics ×4, apps, provisioning, crashes, AFC root, CoreDevice details; processes and configuration profiles (native over USB since G1/G2); cryptex list and DVT ×3 excluded (RemoteXPC/DTX) | = (**G1**, **G2** resolved) | | `tests/` behaviours | ported to Swift tests (see §5.1); packaging/runtime tests replaced by `scripts/build-release.sh` checks | = | @@ -463,7 +464,7 @@ and 0.3.x's shortcuts for the tenth and later pages and focus (⌘0, ⇧⌘E/M/S | G4 | Bluetooth HCI capture (`com.apple.bluetooth.BTPacketLogger`) to `.pklg` | P1 | ✅ resolved — action “Bluetooth capture” | | G5 | Guided reconnect | P2 | ✅ resolved — Device › Reconnect a Device…, also on the Connection diagnostics and No-device cards | | G6 | Import 0.3.x workspace profiles; profile fields for the developer-image mechanism and selected action | P2 | ✅ resolved — tested with a profile written by 0.3.4's own exporter | -| G7 | `idt collect --include-oslog` accepted as an alias | P2 | ✅ resolved — hidden alias of `--include-unified-logs` | +| G7 | `idt collect --include-oslog` accepted as an alias | P2 | ✅ resolved — hidden alias of `--include-dvt-logs` (DVT logging through Instruments) | | G8 | Instruments readiness row (replaces the DVT row) | P2 | ✅ resolved — Readiness row “Instruments (xctrace)” from `xctrace list devices` (available / offline / not listed); the Instruments recording action waits for it | | G9 | Add current coordinate as a route waypoint | P3 | ✅ resolved — Location Lab › Route › Add Current Coordinate | | G10 | Copy the findings register | P3 | ✅ resolved — Live Logs › Findings › Copy Register (Markdown, same as the evidence bundle's report) | diff --git a/README.md b/README.md index 99b4a41..f4eb24d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Instruments. It needs no Python, no Homebrew packages, and no administrator righ | **Devices** | Automatic discovery of USB and Wi-Fi–synced devices (event-driven, no polling), Xcode-paired network devices, and simulators — shown in separate *Physical Devices* and *Simulators* sections. | | **Plain-language device details** | Name, model, hardware identifier, UDID, iOS version and build, architecture, connection, trust, Developer Mode, and developer-service status, each with an explanation. Identifying values stay hidden until you choose to show them. | | **Readiness Check** | A read-only check of every prerequisite (Xcode, the macOS device service, connection, trust, Developer Mode, Xcode's device service, developer services, Instruments, lock state, logging and backup services, Safari Web Inspector) with a next step for anything not ready. | -| **Live Logs** | Unified Logging and classic syslog from physical devices, and the simulator's unified log. Every byte is spooled and hashed; the view can be paused and filtered (literal or regex) without affecting capture. Mark findings, then export the raw capture, filtered lines, or an evidence bundle. | +| **Live Logs** | Unified Logging and classic syslog streamed from physical devices, and the simulator's unified log; plus two collected sources: an **OSLog archive** (the device's saved log history for a time window, kept as a `.logarchive` for Console) and **DVT logging** (os_log recorded through Instruments, kept as a `.trace`). Every byte is spooled and hashed; the view can be paused and filtered (literal or regex) without affecting capture. Mark findings, then export the raw capture, filtered lines, or an evidence bundle. | | **Location Lab** | Set a coordinate (offline world map, map-link parsing, nudges, saved places), move along a route at constant speed, or replay a GPX track. Always clearable; every change is logged. | | **Apps** | Search and sort installed apps (with sizes over USB), launch, and remove with confirmation. | | **Install App** | Inspect an `.ipa` on the Mac first — contents, provisioning profile, and code signature verified with Security.framework — then install it on a device, or install an `.app` on a simulator. | @@ -253,7 +253,7 @@ idt ddi unmount --udid --confirm "RUN ABC123" idt toolchain # check the installed Xcode ``` -`--include-oslog`, the 0.3.x name of `--include-unified-logs`, is still accepted. +`--include-oslog-archive` adds the device's saved Unified Log history for the last hour (`log collect`); `--include-dvt-logs` records os_log through Instruments (DVT) for the stream duration. `--include-oslog`, the 0.3.x flag for the DVT OSLog stream, is still accepted and selects DVT logging. `idt collect` exits with `0` when complete, `2` when finished with coverage gaps, and `1` when the device could not be identified. diff --git a/Sources/ToolkitCore/AppleTools.swift b/Sources/ToolkitCore/AppleTools.swift index 824cd7b..37d1e2d 100644 --- a/Sources/ToolkitCore/AppleTools.swift +++ b/Sources/ToolkitCore/AppleTools.swift @@ -12,6 +12,7 @@ public enum AppleTool: String, CaseIterable, Sendable { case ditto case rvictl case swVers = "sw_vers" + case log public var candidates: [URL] { switch self { @@ -24,6 +25,7 @@ public enum AppleTool: String, CaseIterable, Sendable { case .ditto: return [URL(fileURLWithPath: "/usr/bin/ditto")] case .rvictl: return [URL(fileURLWithPath: "/Library/Apple/usr/bin/rvictl"), URL(fileURLWithPath: "/usr/bin/rvictl")] case .swVers: return [URL(fileURLWithPath: "/usr/bin/sw_vers")] + case .log: return [URL(fileURLWithPath: "/usr/bin/log")] } } diff --git a/Sources/ToolkitFeatures/Evidence/EvidenceCollector.swift b/Sources/ToolkitFeatures/Evidence/EvidenceCollector.swift index cb0bbd6..b4a172d 100644 --- a/Sources/ToolkitFeatures/Evidence/EvidenceCollector.swift +++ b/Sources/ToolkitFeatures/Evidence/EvidenceCollector.swift @@ -11,16 +11,44 @@ public struct CollectionOptions: Codable, Sendable, Hashable { public var includePacketCapture: Bool public var includeScreenshot: Bool public var includeCrashReports: Bool + /// The device's saved Unified Log history for the last hour (`log collect`). + public var includeOSLogArchive: Bool + /// os_log recorded through Instruments (DVT) for the stream duration. + public var includeDVTLogging: Bool - public init(durationSeconds: Int = 60, includeClassicSyslog: Bool = false, includeUnifiedLogs: Bool = true, includePacketCapture: Bool = false, includeScreenshot: Bool = false, includeCrashReports: Bool = false) { + public init(durationSeconds: Int = 60, includeClassicSyslog: Bool = false, includeUnifiedLogs: Bool = true, includePacketCapture: Bool = false, includeScreenshot: Bool = false, includeCrashReports: Bool = false, includeOSLogArchive: Bool = false, includeDVTLogging: Bool = false) { self.durationSeconds = durationSeconds self.includeClassicSyslog = includeClassicSyslog self.includeUnifiedLogs = includeUnifiedLogs self.includePacketCapture = includePacketCapture self.includeScreenshot = includeScreenshot self.includeCrashReports = includeCrashReports + self.includeOSLogArchive = includeOSLogArchive + self.includeDVTLogging = includeDVTLogging } + enum CodingKeys: String, CodingKey { + case durationSeconds, includeClassicSyslog, includeUnifiedLogs, includePacketCapture, includeScreenshot, includeCrashReports, includeOSLogArchive, includeDVTLogging + } + + /// Profiles and manifests written before the OSLog archive and DVT options existed still load. + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.init( + durationSeconds: try c.decode(Int.self, forKey: .durationSeconds), + includeClassicSyslog: try c.decode(Bool.self, forKey: .includeClassicSyslog), + includeUnifiedLogs: try c.decode(Bool.self, forKey: .includeUnifiedLogs), + includePacketCapture: try c.decode(Bool.self, forKey: .includePacketCapture), + includeScreenshot: try c.decode(Bool.self, forKey: .includeScreenshot), + includeCrashReports: try c.decode(Bool.self, forKey: .includeCrashReports), + includeOSLogArchive: try c.decodeIfPresent(Bool.self, forKey: .includeOSLogArchive) ?? false, + includeDVTLogging: try c.decodeIfPresent(Bool.self, forKey: .includeDVTLogging) ?? false + ) + } + + /// How long a DVT recording runs: the stream duration, within Instruments' practical limits. + public var dvtSeconds: Int { min(max(durationSeconds, 10), CollectedLogs.maximumDVTSeconds) } + public func validated() throws -> CollectionOptions { guard (0...3600).contains(durationSeconds) else { throw ToolkitError.invalidInput("The stream duration must be between 0 and 3,600 seconds.") } return self @@ -29,7 +57,7 @@ public struct CollectionOptions: Codable, Sendable, Hashable { /// What a collection with these options needs: a trusted connection, and Xcode's device /// service for the screenshot. public var requirements: [ActionRequirement] { - [.trustedDevice] + (includeScreenshot ? [.coreDevice] : []) + [.trustedDevice] + (includeScreenshot ? [.coreDevice] : []) + (includeDVTLogging ? [.developerMode, .instruments] : []) } var hasStreams: Bool { durationSeconds > 0 && (includeClassicSyslog || includeUnifiedLogs || includePacketCapture) } @@ -394,6 +422,30 @@ public actor EvidenceCollector { _ = try await coreDevice.screenshot(target, to: folder.appendingPathComponent("artifacts/screen.png")) } } + let runner = self.runner + if options.includeOSLogArchive { + await snapshotFile("oslog-archive", "OSLog archive (last hour)", "log collect --device-udid", file: "artifacts/device.logarchive", events: events) { + let archive = folder.appendingPathComponent("artifacts/device.logarchive") + try SecureFileIO.createPrivateDirectory(at: archive.deletingLastPathComponent()) + let result = try await runner.run(try CollectedLogs.osLogArchiveRequest(udid: target.udid, windowSeconds: CollectedLogs.evidenceArchiveSeconds, output: archive)) + guard result.succeeded, FileManager.default.fileExists(atPath: archive.path) else { throw CollectedLogs.archiveError(result) } + } + } + if options.includeDVTLogging { + let seconds = options.dvtSeconds + await snapshotFile("dvt-logging", "DVT logging (Instruments, \(seconds) s)", "xctrace record --template Logging, xctrace export", file: "streams/dvt-logging.jsonl", timeout: TimeInterval(seconds + 900), events: events) { + let trace = folder.appendingPathComponent("artifacts/dvt-logging.trace") + let lines = folder.appendingPathComponent("streams/dvt-logging.jsonl") + try SecureFileIO.createPrivateDirectory(at: trace.deletingLastPathComponent()) + try SecureFileIO.createPrivateDirectory(at: lines.deletingLastPathComponent()) + try SecureFileIO.writeNewFile(Data(), to: lines) + let output = try FileHandle(forWritingTo: lines) + defer { try? output.close() } + for try await chunk in CollectedLogs.stream(.dvt, target: target, seconds: seconds, artifact: trace, runner: runner) { + try output.write(contentsOf: chunk.spoolBytes) + } + } + } if options.includeCrashReports { await snapshotFile("crash-reports", "Crash reports", "crashreportcopymobile (AFC)", file: "artifacts/crashes", events: events) { let destination = folder.appendingPathComponent("artifacts/crashes") @@ -411,7 +463,7 @@ public actor EvidenceCollector { } } - private func snapshotFile(_ id: String, _ title: String, _ mechanism: String, file: String, unavailableUnless available: Bool = true, events: @escaping @Sendable (CollectionEvent) -> Void, _ body: @escaping @Sendable () async throws -> Void) async { + private func snapshotFile(_ id: String, _ title: String, _ mechanism: String, file: String, unavailableUnless available: Bool = true, timeout: TimeInterval = 900, events: @escaping @Sendable (CollectionEvent) -> Void, _ body: @escaping @Sendable () async throws -> Void) async { events(.stepStarted(title)) let started = Date() guard available else { @@ -419,7 +471,7 @@ public actor EvidenceCollector { return } do { - try await withTimeout(900, operation: title) { try await body() } + try await withTimeout(timeout, operation: title) { try await body() } record(CollectionStep(id: id, title: title, mechanism: mechanism, required: false, status: .succeeded, attempts: 1, startedAt: started, finishedAt: Date(), outputPath: file, detail: ""), events: events) } catch { record(CollectionStep(id: id, title: title, mechanism: mechanism, required: false, status: .failed, attempts: 1, startedAt: started, finishedAt: Date(), outputPath: nil, detail: (error as? ToolkitError)?.message ?? error.localizedDescription), events: events) diff --git a/Sources/ToolkitFeatures/LiveLogs/CollectedLogSources.swift b/Sources/ToolkitFeatures/LiveLogs/CollectedLogSources.swift new file mode 100644 index 0000000..4a87b4f --- /dev/null +++ b/Sources/ToolkitFeatures/LiveLogs/CollectedLogSources.swift @@ -0,0 +1,310 @@ +import DeviceKit +import Foundation +import ToolkitCore + +/// Log sources that are gathered over a time window instead of streamed, both through Apple's own +/// tools: +/// - **OSLog archive:** macOS's `log collect` copies the device's saved Unified Log history into a +/// `.logarchive` (Console.app opens it); `log show` reads it back for the viewer. +/// - **DVT logging:** Instruments' Logging template (`xctrace record`) records os_log through the +/// device's developer services (DVT); `xctrace export` reads the recording's os-log table. This +/// is Apple's supported route to DVT logging; a direct DVT connection would need Xcode's private +/// tunnel. +public enum CollectedLogs { + /// Time windows offered in the app, in seconds. + public static let windows = [30, 60, 300, 900] + /// DVT recordings keep every event in memory while exporting, so they are kept short. + public static let maximumDVTSeconds = 900 + /// How much saved history Evidence Capture's OSLog archive covers. + public static let evidenceArchiveSeconds = 3600 + + public static func osLogArchiveRequest(udid: String, windowSeconds: Int, output: URL) throws -> CommandRequest { + guard (1...86_400).contains(windowSeconds) else { throw ToolkitError.invalidInput("Choose a window between 1 second and 24 hours.") } + guard output.pathExtension == "logarchive" else { throw ToolkitError.invalidInput("Log archives are saved as .logarchive.") } + return CommandRequest( + executable: try AppleTool.log.locate(), + arguments: ["collect", "--device-udid", udid, "--last", "\(windowSeconds)s", "--output", output.path], + timeout: 900, + displayName: "log collect (device)" + ) + } + + public static func logShowRequest(archive: URL) throws -> CommandRequest { + CommandRequest( + executable: try AppleTool.log.locate(), + arguments: ["show", "--archive", archive.path, "--style", "ndjson", "--info", "--debug"], + timeout: 1800, + displayName: "log show (archive)" + ) + } + + public static func dvtRecordRequest(target: DeviceTarget, seconds: Int, output: URL) throws -> CommandRequest { + guard (1...maximumDVTSeconds).contains(seconds) else { + throw ToolkitError.invalidInput("DVT recordings can be up to \(maximumDVTSeconds / 60) minutes long.") + } + return try InstrumentsRecorder.request(template: "Logging", target: target, durationSeconds: seconds, output: output) + } + + public static func dvtTableOfContentsRequest(trace: URL, output: URL) throws -> CommandRequest { + try XcodeTool.xctrace.request(["export", "--input", trace.path, "--toc", "--output", output.path], timeout: 300, displayName: "xctrace export (contents)") + } + + public static func dvtExportRequest(trace: URL, output: URL) throws -> CommandRequest { + try XcodeTool.xctrace.request( + ["export", "--input", trace.path, "--xpath", #"/trace-toc/run[@number="1"]/data/table[@schema="os-log"]"#, "--output", output.path], + timeout: 1800, + displayName: "xctrace export (os-log)" + ) + } + + /// The recording's start time from `xctrace export --toc` (``). + public static func startDate(tableOfContents: Data) -> Date? { + let text = String(decoding: tableOfContents, as: UTF8.self) + guard let open = text.range(of: ""), let close = text.range(of: "", range: open.upperBound.. Data { + var object: [String: Any] = ["eventMessage": line.message] + if let timestamp = line.timestamp { object["timestamp"] = logShowTimestamp.string(from: timestamp) } + if let process = line.process { object["processImagePath"] = process } + if let pid = line.pid { object["processID"] = pid } + if let level = line.level { object["messageType"] = level } + if let subsystem = line.subsystem { object["subsystem"] = subsystem } + if let category = line.category { object["category"] = category } + var data = (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .withoutEscapingSlashes])) ?? Data(line.message.utf8) + data.append(0x0A) + return data + } + + /// The timestamp format `log show --style ndjson` uses (and `SimulatorLogParser` reads). + static var logShowTimestamp: DateFormatter { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSSZ" + return formatter + } + + /// Gathers `kind` over `seconds` and yields its lines. The archive or recording is kept at + /// `artifact`. + public static func stream(_ kind: LogStreamKind, target: DeviceTarget, seconds: Int, artifact: URL, runner: CommandRunning) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + let task = Task { + do { + switch kind { + case .osLogArchive: + try await collectArchive(target: target, seconds: seconds, artifact: artifact, runner: runner, continuation: continuation) + case .dvt: + try await recordDVT(target: target, seconds: seconds, artifact: artifact, runner: runner, continuation: continuation) + default: + throw ToolkitError(.unsupported, message: "\(kind.title) is streamed, not collected.") + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + } + + static func note(_ message: String) -> LogChunk { LogChunk(spoolBytes: Data(), lines: [LogLine(level: "note", message: message)]) } + + private static func collectArchive(target: DeviceTarget, seconds: Int, artifact: URL, runner: CommandRunning, continuation: AsyncThrowingStream.Continuation) async throws { + guard target.kind == .physical else { throw ToolkitError(.unsupported, message: "Log archives are collected from iPhones and iPads.") } + continuation.yield(note("Collecting the device's saved logs from the last \(Self.describe(seconds)). This can take a few minutes.")) + let collected = try await runner.run(try osLogArchiveRequest(udid: target.udid, windowSeconds: seconds, output: artifact)) + guard collected.succeeded, FileManager.default.fileExists(atPath: artifact.path) else { throw archiveError(collected) } + continuation.yield(note("Saved the log archive to \(artifact.path). Console.app can open it.")) + var splitter = LineSplitter() + for try await event in runner.stream(try logShowRequest(archive: artifact)) { + switch event { + case .standardOutput(let data): + let lines = splitter.consume(data).compactMap(SimulatorLogParser.parse(line:)) + continuation.yield(LogChunk(spoolBytes: data, lines: lines)) + case .standardError: + break + case .finished(let result): + if !result.succeeded, !Task.isCancelled { + throw ToolkitError(.commandFailed, message: "The log archive was collected but could not be read.", recovery: "Open it in Console.app instead.", technicalDetail: result.technicalSummary) + } + } + } + let remainder = splitter.flush().compactMap(SimulatorLogParser.parse(line:)) + if !remainder.isEmpty { continuation.yield(LogChunk(spoolBytes: Data(), lines: remainder)) } + } + + static func archiveError(_ result: CommandResult) -> ToolkitError { + let text = (result.standardErrorText + result.standardOutputText).lowercased() + if text.contains("root") || text.contains("permission") || text.contains("not permitted") { + return ToolkitError(.permissionDenied, message: "macOS did not allow collecting the device's log archive.", recovery: "The toolkit never uses administrator rights. Use Unified Logs (live) instead, or run `log collect --device-udid` yourself.", technicalDetail: result.technicalSummary) + } + return ToolkitError(.commandFailed, message: "The device's log archive could not be collected.", recovery: "Unlock the device, keep it connected by USB, tap Trust if asked, and try again.", technicalDetail: result.technicalSummary) + } + + private static func recordDVT(target: DeviceTarget, seconds: Int, artifact: URL, runner: CommandRunning, continuation: AsyncThrowingStream.Continuation) async throws { + let seconds = min(seconds, maximumDVTSeconds) + continuation.yield(note("Recording os_log through Instruments (DVT) for \(Self.describe(seconds)). The lines appear when the recording ends.")) + let recorded = try await runner.run(try dvtRecordRequest(target: target, seconds: seconds, output: artifact)) + guard recorded.succeeded, FileManager.default.fileExists(atPath: artifact.path) else { + throw ToolkitError(.commandFailed, message: "Instruments could not record from \(target.name).", recovery: target.kind == .simulator ? "Make sure the simulator is running, then try again." : "Turn on Developer Mode, mount the developer image (Developer Image page), keep the device unlocked, and try again.", technicalDetail: recorded.technicalSummary) + } + continuation.yield(note("Saved the recording to \(artifact.path). Instruments can open it.")) + let work = try SecureFileIO.makeTemporaryDirectory(prefix: "dvt-export") + defer { try? FileManager.default.removeItem(at: work) } + let toc = work.appendingPathComponent("toc.xml") + var start: Date? + if (try? await runner.run(try dvtTableOfContentsRequest(trace: artifact, output: toc)))?.succeeded == true { + start = startDate(tableOfContents: (try? Data(contentsOf: toc)) ?? Data()) + } + let xml = work.appendingPathComponent("os-log.xml") + let exported = try await runner.run(try dvtExportRequest(trace: artifact, output: xml)) + guard exported.succeeded, FileManager.default.fileExists(atPath: xml.path) else { + throw ToolkitError(.commandFailed, message: "The recording was saved but its log could not be read.", recovery: "Open the .trace in Instruments instead.", technicalDetail: exported.technicalSummary) + } + try Task.checkCancellation() + try XctraceOSLogReader.read(xml, start: start, batchSize: 2_000) { lines in + continuation.yield(LogChunk(spoolBytes: lines.reduce(into: Data()) { $0.append(ndjson($1)) }, lines: lines)) + } + } + + static func describe(_ seconds: Int) -> String { + seconds % 3600 == 0 ? "\(seconds / 3600) h" : seconds % 60 == 0 ? "\(seconds / 60) min" : "\(seconds) s" + } +} + +/// Reads the os-log table that `xctrace export` writes, without loading it into memory. +/// +/// Each `` lists one value per schema column, in column order. A value either carries +/// `id`/`fmt` attributes (its display text) or a `ref` to an earlier `id`; `` means no +/// value. Nested elements (a thread's process, for example) define ids too. +public final class XctraceOSLogReader: NSObject, XMLParserDelegate { + private var columns: [String] = [] + private var inMnemonic = false + private var mnemonic = "" + private var texts: [String: String] = [:] + private var rowDepth: Int? + private var depth = 0 + private var row: [String?] = [] + private var batch: [LogLine] = [] + private let start: Date? + private let batchSize: Int + private let deliver: ([LogLine]) -> Void + + init(start: Date?, batchSize: Int, deliver: @escaping ([LogLine]) -> Void) { + self.start = start + self.batchSize = batchSize + self.deliver = deliver + } + + /// Parses `url` and delivers lines in batches of up to `batchSize`. + public static func read(_ url: URL, start: Date?, batchSize: Int = 2_000, deliver: @escaping ([LogLine]) -> Void) throws { + guard let parser = XMLParser(contentsOf: url) else { throw ToolkitError.fileSystem("The exported log could not be opened.", path: url.path) } + let reader = XctraceOSLogReader(start: start, batchSize: batchSize, deliver: deliver) + parser.delegate = reader + guard parser.parse() else { + throw ToolkitError(.commandFailed, message: "The exported log could not be read.", technicalDetail: parser.parserError.map { String(describing: $0) }) + } + if !reader.batch.isEmpty { deliver(reader.batch) } + } + + /// Parses exported XML held in memory (tests). + public static func lines(from data: Data, start: Date? = nil) throws -> [LogLine] { + let parser = XMLParser(data: data) + var all: [LogLine] = [] + let reader = XctraceOSLogReader(start: start, batchSize: .max) { all.append(contentsOf: $0) } + parser.delegate = reader + guard parser.parse() else { throw ToolkitError(.commandFailed, message: "The exported log could not be read.") } + all.append(contentsOf: reader.batch) + return all + } + + public func parser(_ parser: XMLParser, didStartElement name: String, namespaceURI: String?, qualifiedName: String?, attributes: [String: String] = [:]) { + depth += 1 + switch name { + case "mnemonic": + inMnemonic = true; mnemonic = "" + return + case "row": + rowDepth = depth; row = [] + return + default: + break + } + guard let rowDepth else { return } + var value: String? + if let id = attributes["id"], let fmt = attributes["fmt"] { + texts[id] = fmt + value = fmt + } else if let ref = attributes["ref"] { + value = texts[ref] + } + if depth == rowDepth + 1 { row.append(name == "sentinel" ? nil : value) } + } + + public func parser(_ parser: XMLParser, foundCharacters string: String) { + if inMnemonic { mnemonic += string } + } + + public func parser(_ parser: XMLParser, didEndElement name: String, namespaceURI: String?, qualifiedName: String?) { + defer { depth -= 1 } + if name == "mnemonic" { + inMnemonic = false + if rowDepth == nil { columns.append(mnemonic) } + } else if name == "row" { + rowDepth = nil + if let line = makeLine() { batch.append(line) } + if batch.count >= batchSize { deliver(batch); batch.removeAll(keepingCapacity: true) } + } + } + + private func value(_ column: String) -> String? { + guard let index = columns.firstIndex(of: column), index < row.count else { return nil } + return row[index].flatMap { $0.isEmpty ? nil : $0 } + } + + private func makeLine() -> LogLine? { + guard let message = value("message") ?? value("format-string") else { return nil } + var process: String?, pid: Int? + if let text = value("process") { + // "Name (1234)" + if text.hasSuffix(")"), let open = text.lastIndex(of: "(") { + process = String(text[.. Double? { + let parts = text.split(separator: ":") + guard let last = parts.last else { return nil } + let fraction = last.split(separator: ".") + guard let wholeSeconds = Double(fraction.first ?? "") else { return nil } + var total = wholeSeconds + if fraction.count > 1, let milliseconds = Double(fraction[1]) { total += milliseconds / 1_000 } + if fraction.count > 2, let microseconds = Double(fraction[2]) { total += microseconds / 1_000_000 } + for (index, part) in parts.dropLast().reversed().enumerated() { + guard let number = Double(part) else { return nil } + total += number * pow(60, Double(index + 1)) + } + return total + } +} diff --git a/Sources/ToolkitFeatures/LiveLogs/LiveLogSources.swift b/Sources/ToolkitFeatures/LiveLogs/LiveLogSources.swift index 195bde5..8d2a095 100644 --- a/Sources/ToolkitFeatures/LiveLogs/LiveLogSources.swift +++ b/Sources/ToolkitFeatures/LiveLogs/LiveLogSources.swift @@ -7,6 +7,10 @@ public enum LogStreamKind: String, CaseIterable, Sendable, Codable, Identifiable case unified case classic case simulator + /// The device's saved Unified Log history, collected with `log collect`. + case osLogArchive + /// os_log recorded through Instruments' developer services (DVT). + case dvt public var id: String { rawValue } @@ -15,6 +19,8 @@ public enum LogStreamKind: String, CaseIterable, Sendable, Codable, Identifiable case .unified: return "Unified Logs" case .classic: return "Classic Syslog" case .simulator: return "Simulator Logs" + case .osLogArchive: return "OSLog Archive" + case .dvt: return "DVT Logging" } } @@ -23,10 +29,24 @@ public enum LogStreamKind: String, CaseIterable, Sendable, Codable, Identifiable case .unified: return "Structured Unified Logging with process, level, subsystem, and category. Works on any trusted device; no developer image needed." case .classic: return "The older plain-text syslog relay. Useful for tools that expect traditional syslog lines." case .simulator: return "The simulator's Unified Log, streamed with `log stream`." + case .osLogArchive: return "The device's saved Unified Log history for a time window, collected with macOS's `log collect` and kept as a .logarchive that Console.app opens. No Xcode needed." + case .dvt: return "os_log and signposts recorded through Instruments' developer services (DVT) for a set time, then shown here and kept as a .trace. Needs Xcode, Developer Mode, and the developer image." } } public var isStructured: Bool { self != .classic } + + /// Collected once over a time window, rather than streamed until stopped. + public var isCollected: Bool { self == .osLogArchive || self == .dvt } + + /// The file a collected source keeps (Console or Instruments opens it). + public var artifactExtension: String? { + switch self { + case .osLogArchive: return "logarchive" + case .dvt: return "trace" + default: return nil + } + } public var spoolExtension: String { isStructured ? "jsonl" : "log" } public var serviceDescription: String { @@ -34,13 +54,15 @@ public enum LogStreamKind: String, CaseIterable, Sendable, Codable, Identifiable case .unified: return "com.apple.os_trace_relay (lockdown)" case .classic: return "com.apple.syslog_relay (lockdown)" case .simulator: return "xcrun simctl spawn log stream --style ndjson" + case .osLogArchive: return "/usr/bin/log collect --device-udid , then log show --archive" + case .dvt: return "xcrun xctrace record --template Logging, then xctrace export" } } public static func available(for kind: DeviceKind) -> [LogStreamKind] { switch kind { - case .physical: return [.unified, .classic] - case .simulator: return [.simulator] + case .physical: return [.unified, .classic, .osLogArchive, .dvt] + case .simulator: return [.simulator, .dvt] case .demo: return [] } } @@ -48,8 +70,13 @@ public enum LogStreamKind: String, CaseIterable, Sendable, Codable, Identifiable /// Opens a live log stream for a target. public enum LiveLogSource { - public static func open(_ kind: LogStreamKind, target: DeviceTarget, runner: CommandRunning = ProcessCommandRunner(), usbmux: USBMuxClient = USBMuxClient()) async throws -> AsyncThrowingStream { + /// Opens `kind`. Collected sources gather `windowSeconds` of logs and keep their archive or + /// recording at `artifact`. + public static func open(_ kind: LogStreamKind, target: DeviceTarget, runner: CommandRunning = ProcessCommandRunner(), usbmux: USBMuxClient = USBMuxClient(), windowSeconds: Int = 300, artifact: URL? = nil) async throws -> AsyncThrowingStream { switch kind { + case .osLogArchive, .dvt: + let file = try artifact ?? SecureFileIO.makeTemporaryDirectory(prefix: "collected-logs").appendingPathComponent("\(kind.rawValue).\(kind.artifactExtension ?? "out")") + return CollectedLogs.stream(kind, target: target, seconds: windowSeconds, artifact: file, runner: runner) case .unified, .classic: guard target.kind == .physical else { throw ToolkitError(.unsupported, message: "\(kind.title) are only available for physical devices.") } let session = try await DeviceSession.open(target: target, usbmux: usbmux) diff --git a/Sources/ToolkitFeatures/LiveLogs/LogCapture.swift b/Sources/ToolkitFeatures/LiveLogs/LogCapture.swift index a7cacd6..ee4abe9 100644 --- a/Sources/ToolkitFeatures/LiveLogs/LogCapture.swift +++ b/Sources/ToolkitFeatures/LiveLogs/LogCapture.swift @@ -277,11 +277,11 @@ public actor LogCapture { case .classic: var parser = SyslogRecordParser() lines = (parser.consume(data) + parser.flush()).map(\.message) - case .unified, .simulator: + case .unified, .simulator, .osLogArchive, .dvt: var splitter = LineSplitter() let raw = splitter.consume(data) + splitter.flush() lines = raw.compactMap { line -> String? in - if kind == .simulator { return SimulatorLogParser.parse(line: line)?.rendered } + if kind != .unified { return SimulatorLogParser.parse(line: line)?.rendered } guard let json = try? JSONValue.parse(Data(line.utf8)) else { return String(line) } return LogLine(timestamp: json["timestamp"]?.string.flatMap(ISO8601.parse), process: json["process"]?.string, pid: json["pid"]?.int, level: json["level"]?.string, subsystem: json["subsystem"]?.string, category: json["category"]?.string, message: json["message"]?.string ?? "").rendered } diff --git a/Sources/ToolkitFeatures/Shared/LegacyWorkspaceProfile.swift b/Sources/ToolkitFeatures/Shared/LegacyWorkspaceProfile.swift index 2881ed0..e299368 100644 --- a/Sources/ToolkitFeatures/Shared/LegacyWorkspaceProfile.swift +++ b/Sources/ToolkitFeatures/Shared/LegacyWorkspaceProfile.swift @@ -175,7 +175,7 @@ enum LegacyWorkspaceProfile { } notes.append("Developer-image source “\(settings.ddiSource)” → \(DeveloperImageMechanism.native.label). Images come from Xcode or a folder you add; nothing is downloaded.") if evidence.includeOSLog { - notes.append("DVT OSLog → Unified Logging (no developer image needed).") + notes.append("DVT OSLog → DVT logging through Instruments (needs Xcode, Developer Mode, and the developer image).") } var profile = WorkspaceProfile( @@ -187,7 +187,7 @@ enum LegacyWorkspaceProfile { developerImageMechanism: .native, apps: .init(calculateSizes: settings.appWorkflow.calculateAppSizes, includeSystemApps: false, installAsDeveloperPackage: settings.appWorkflow.installAsDeveloperPackage), backup: .init(forceFullBackup: settings.backupWorkflow.forceFullBackup, requireEncryption: settings.backupWorkflow.requireEncryption), - evidence: CollectionOptions(durationSeconds: evidence.captureDurationSeconds, includeClassicSyslog: evidence.includeSyslog, includeUnifiedLogs: evidence.includeOSLog, includePacketCapture: evidence.includePcap, includeScreenshot: evidence.includeScreenshot, includeCrashReports: evidence.includeCrashPull), + evidence: CollectionOptions(durationSeconds: evidence.captureDurationSeconds, includeClassicSyslog: evidence.includeSyslog, includeUnifiedLogs: false, includePacketCapture: evidence.includePcap, includeScreenshot: evidence.includeScreenshot, includeCrashReports: evidence.includeCrashPull, includeDVTLogging: evidence.includeOSLog), location: .init(timingJitterMilliseconds: settings.locationWorkflow.timingRandomnessMilliseconds, ignoreRecordedTiming: settings.locationWorkflow.ignoreTimingDelays, routeSpeedKmh: settings.locationWorkflow.routeSpeedKmh, routeIntervalSeconds: settings.locationWorkflow.routeIntervalSeconds, routeTraversals: settings.locationWorkflow.routeTraversals) ) profile.createdWithVersion = file.createdWithVersion diff --git a/Sources/ToolkitFeatures/Shared/WorkspaceProfile.swift b/Sources/ToolkitFeatures/Shared/WorkspaceProfile.swift index 51f1648..ee3de60 100644 --- a/Sources/ToolkitFeatures/Shared/WorkspaceProfile.swift +++ b/Sources/ToolkitFeatures/Shared/WorkspaceProfile.swift @@ -171,7 +171,7 @@ public struct WorkspaceProfile: Codable, Sendable, Hashable { developerImageMechanism.map { "Developer image: mount with \($0.label)" }, "Apps: sizes \(apps.calculateSizes ? "on" : "off"), system apps \(apps.includeSystemApps ? "shown" : "hidden"), developer package installs \(apps.installAsDeveloperPackage ? "on" : "off")", "Backup: \(backup.forceFullBackup ? "always full" : "incremental when possible"), encryption \(backup.requireEncryption ? "required" : "optional")", - "Evidence: \(evidence.durationSeconds)s streams; syslog \(evidence.includeClassicSyslog ? "on" : "off"), unified logs \(evidence.includeUnifiedLogs ? "on" : "off"), packet capture \(evidence.includePacketCapture ? "on" : "off"), screenshot \(evidence.includeScreenshot ? "on" : "off"), crash reports \(evidence.includeCrashReports ? "on" : "off")", + "Evidence: \(evidence.durationSeconds)s streams; syslog \(evidence.includeClassicSyslog ? "on" : "off"), unified logs \(evidence.includeUnifiedLogs ? "on" : "off"), packet capture \(evidence.includePacketCapture ? "on" : "off"), screenshot \(evidence.includeScreenshot ? "on" : "off"), crash reports \(evidence.includeCrashReports ? "on" : "off"), OSLog archive \(evidence.includeOSLogArchive ? "on" : "off"), DVT logging \(evidence.includeDVTLogging ? "on" : "off")", "Location: route \(location.routeSpeedKmh) km/h every \(location.routeIntervalSeconds)s × \(location.routeTraversals); GPX timing \(location.ignoreRecordedTiming ? "ignored" : "kept") with ±\(location.timingJitterMilliseconds) ms", ].compactMap { $0 }.joined(separator: "\n") } diff --git a/Sources/idt/IDT.swift b/Sources/idt/IDT.swift index 66bf326..102b51a 100644 --- a/Sources/idt/IDT.swift +++ b/Sources/idt/IDT.swift @@ -81,7 +81,9 @@ struct Collect: AsyncParsableCommand { @Option(help: "Stream duration in seconds (0–3600).") var duration = 60 @Flag(help: "Capture the classic syslog stream.") var includeSyslog = false @Flag(help: "Capture the Unified Logging stream.") var includeUnifiedLogs = false - /// The 0.3.x name of --include-unified-logs, still accepted so existing scripts keep working. + @Flag(help: "Collect the device's saved Unified Log history for the last hour (log collect).") var includeOSLogArchive = false + @Flag(help: "Record os_log through Instruments (DVT) for the stream duration (needs Xcode, Developer Mode, and the developer image).") var includeDVTLogs = false + /// The 0.3.x flag for the DVT OSLog stream, still accepted so existing scripts keep working. @Flag(name: .customLong("include-oslog"), help: .hidden) var includeOSLog = false @Flag(help: "Capture network packets (PCAP).") var includePcap = false @Flag(help: "Save a screenshot (needs Xcode).") var includeScreenshot = false @@ -103,7 +105,7 @@ struct Collect: AsyncParsableCommand { } else { folder = try CaseWorkflow.createCaseFolder(in: URL(fileURLWithPath: ((outputRoot ?? "") as NSString).expandingTildeInPath), target: device.target) } - let options = CollectionOptions(durationSeconds: duration, includeClassicSyslog: includeSyslog, includeUnifiedLogs: includeUnifiedLogs || includeOSLog, includePacketCapture: includePcap, includeScreenshot: includeScreenshot, includeCrashReports: includeCrashPull) + let options = CollectionOptions(durationSeconds: duration, includeClassicSyslog: includeSyslog, includeUnifiedLogs: includeUnifiedLogs, includePacketCapture: includePcap, includeScreenshot: includeScreenshot, includeCrashReports: includeCrashPull, includeOSLogArchive: includeOSLogArchive, includeDVTLogging: includeDVTLogs || includeOSLog) let collector = try EvidenceCollector(device: device, caseFolder: folder, options: options) print("Collecting from \(device.name) into \(folder.path)") let manifest = await collector.run { event in diff --git a/Tests/ToolkitFeaturesTests/CollectedLogTests.swift b/Tests/ToolkitFeaturesTests/CollectedLogTests.swift new file mode 100644 index 0000000..206cd2a --- /dev/null +++ b/Tests/ToolkitFeaturesTests/CollectedLogTests.swift @@ -0,0 +1,157 @@ +import Foundation +import Testing +@testable import DeviceKit +@testable import ToolkitFeatures +import ToolkitCore + +@Suite("Collected logs: OSLog archive and DVT") +struct CollectedLogTests { + /// The structure `xctrace export` writes for the os-log table (abridged from a real recording): + /// values carry id/fmt or ref an earlier id, nested elements define ids too, and + /// marks an empty column. + static let exportedXML = #""" + + + timethreadprocessmessage-typeformat-stringbacktracesubsystemcategorymessageemit-location + 54784312582649949207DefaultLaunched %{public}scom.apple.extensionkitdefaultLaunched + 62000500000048775Infocom.apple.diagnosticd + 62100000000 + + """# + + @Test func readsTheOSLogTableFromXctraceExport() throws { + let start = Date(timeIntervalSince1970: 1_800_000_000) + let lines = try XctraceOSLogReader.lines(from: Data(Self.exportedXML.utf8), start: start) + // The second row has neither a message nor a format string, so it is skipped. + #expect(lines.count == 2) + #expect(lines[0].message == "Launched Weather") + #expect(lines[0].process == "SpringBoard" && lines[0].pid == 49207) + #expect(lines[0].level == "Default") + #expect(lines[0].subsystem == "com.apple.extensionkit" && lines[0].category == "default") + #expect(lines[0].timestamp == start.addingTimeInterval(0.547843)) + // Values given by reference resolve to the earlier definitions, including nested ones. + #expect(lines[1].message == "Launched Maps" && lines[1].process == "SpringBoard" && lines[1].level == "Info") + #expect(lines[1].timestamp == start.addingTimeInterval(62.1)) + #expect(try XctraceOSLogReader.lines(from: Data(Self.exportedXML.utf8)).allSatisfy { $0.timestamp == nil }) + } + + @Test func parsesXctraceTimes() { + func close(_ text: String, _ expected: Double) -> Bool { XctraceOSLogReader.seconds(fromFormattedTime: text).map { abs($0 - expected) < 0.000001 } ?? false } + #expect(close("00:00.547.843", 0.547843)) + #expect(close("01:02.000.500", 62.0005)) + #expect(close("1:02:03.004.005", 3723.004005)) + #expect(XctraceOSLogReader.seconds(fromFormattedTime: "later") == nil) + } + + @Test func spooledLinesReadBackLikeLogShowOutput() { + let line = LogLine(timestamp: Date(timeIntervalSince1970: 1_800_000_000.25), process: "SpringBoard", pid: 42, level: "Error", subsystem: "com.example", category: "net", message: "boom \"quoted\"") + let back = SimulatorLogParser.parse(line: Substring(String(decoding: CollectedLogs.ndjson(line).dropLast(), as: UTF8.self))) + #expect(back?.message == line.message && back?.process == "SpringBoard" && back?.pid == 42) + #expect(back?.level == "Error" && back?.subsystem == "com.example" && back?.category == "net") + #expect(back?.timestamp.map { abs($0.timeIntervalSince(line.timestamp!)) < 0.001 } == true) + } + + @Test func buildsAppleToolCommands() throws { + let archive = URL(fileURLWithPath: "/tmp/x/device.logarchive") + let collect = try CollectedLogs.osLogArchiveRequest(udid: "00008150-000B33334444002E", windowSeconds: 300, output: archive) + #expect(collect.executable.path == "/usr/bin/log") + #expect(collect.arguments == ["collect", "--device-udid", "00008150-000B33334444002E", "--last", "300s", "--output", archive.path]) + #expect(try CollectedLogs.logShowRequest(archive: archive).arguments == ["show", "--archive", archive.path, "--style", "ndjson", "--info", "--debug"]) + #expect(throws: ToolkitError.self) { try CollectedLogs.osLogArchiveRequest(udid: "u", windowSeconds: 300, output: URL(fileURLWithPath: "/tmp/x/out.zip")) } + #expect(throws: ToolkitError.self) { try CollectedLogs.osLogArchiveRequest(udid: "u", windowSeconds: 0, output: archive) } + if xcodeAvailable { + let target = DeviceTarget(kind: .simulator, udid: "SIM", name: "iPhone", osVersion: "26.0", usbmuxDeviceID: nil, coreDeviceIdentifier: nil, transport: nil) + let trace = FileManager.default.temporaryDirectory.appendingPathComponent("never-\(UUID().uuidString).trace") + let record = try CollectedLogs.dvtRecordRequest(target: target, seconds: 30, output: trace) + #expect(record.arguments.contains("Logging") && record.arguments.contains("30s") && record.arguments.contains("SIM")) + #expect(throws: ToolkitError.self) { try CollectedLogs.dvtRecordRequest(target: target, seconds: 3600, output: trace) } + let export = try CollectedLogs.dvtExportRequest(trace: trace, output: URL(fileURLWithPath: "/tmp/x/os-log.xml")) + #expect(export.arguments.contains(#"/trace-toc/run[@number="1"]/data/table[@schema="os-log"]"#)) + } + } + + @Test func explainsARefusedArchive() { + let request = CommandRequest(executable: URL(fileURLWithPath: "/usr/bin/log"), arguments: [], displayName: "log collect") + let refused = CommandResult(request: request, termination: .exited(1), standardOutput: Data(), standardError: Data("log: Must be run as root to collect logs".utf8), startedAt: Date(), finishedAt: Date()) + #expect(CollectedLogs.archiveError(refused).kind == .permissionDenied) + #expect(CollectedLogs.archiveError(refused).recovery?.contains("never uses administrator rights") == true) + let other = CommandResult(request: request, termination: .exited(1), standardOutput: Data(), standardError: Data("device not found".utf8), startedAt: Date(), finishedAt: Date()) + #expect(CollectedLogs.archiveError(other).message == "The device's log archive could not be collected.") + } + + @Test func sourcesAreOfferedPerDeviceKind() { + #expect(LogStreamKind.available(for: .physical) == [.unified, .classic, .osLogArchive, .dvt]) + #expect(LogStreamKind.available(for: .simulator) == [.simulator, .dvt]) + #expect(LogStreamKind.osLogArchive.isCollected && LogStreamKind.dvt.isCollected && !LogStreamKind.unified.isCollected) + #expect(LogStreamKind.osLogArchive.artifactExtension == "logarchive" && LogStreamKind.dvt.artifactExtension == "trace") + } + + /// Plays both collected sources end to end with a runner that stands in for the Apple tools. + @Test func collectedSourcesYieldLinesAndKeepTheirFiles() async throws { + let folder = try SecureFileIO.makeTemporaryDirectory(prefix: "collected") + defer { try? FileManager.default.removeItem(at: folder) } + let runner = FakeToolRunner(exportedXML: Data(Self.exportedXML.utf8)) + let phone = DeviceTarget(kind: .physical, udid: "00008150-000B33334444002E", name: "Phone", osVersion: "26.3", usbmuxDeviceID: 1, coreDeviceIdentifier: nil, transport: .usb) + + let archive = folder.appendingPathComponent("a.logarchive") + var archiveLines: [LogLine] = [] + var archiveBytes = 0 + for try await chunk in CollectedLogs.stream(.osLogArchive, target: phone, seconds: 60, artifact: archive, runner: runner) { + archiveLines += chunk.lines; archiveBytes += chunk.spoolBytes.count + } + #expect(FileManager.default.fileExists(atPath: archive.path)) + #expect(archiveLines.contains { $0.message == "from the archive" && $0.process == "backboardd" }) + #expect(archiveLines.first?.level == "note" && archiveBytes > 0) + + if xcodeAvailable { + let trace = folder.appendingPathComponent("d.trace") + var dvtLines: [LogLine] = [] + for try await chunk in CollectedLogs.stream(.dvt, target: phone, seconds: 30, artifact: trace, runner: runner) { dvtLines += chunk.lines } + #expect(FileManager.default.fileExists(atPath: trace.path)) + #expect(dvtLines.contains { $0.message == "Launched Maps" && $0.timestamp != nil }) + } + + // A refusal from `log collect` surfaces as a plain-language error. + runner.refuseCollect = true + await #expect(throws: ToolkitError.self) { + for try await _ in CollectedLogs.stream(.osLogArchive, target: phone, seconds: 60, artifact: folder.appendingPathComponent("b.logarchive"), runner: runner) {} + } + } +} + +/// Stands in for `log` and `xctrace`: creates the files they would write and streams output. +final class FakeToolRunner: CommandRunning, @unchecked Sendable { + let exportedXML: Data + var refuseCollect = false + let requests = LockedValue<[CommandRequest]>([]) + init(exportedXML: Data) { self.exportedXML = exportedXML } + + private func output(_ request: CommandRequest) -> URL? { + request.arguments.firstIndex(of: "--output").map { URL(fileURLWithPath: request.arguments[$0 + 1]) } + } + + func run(_ request: CommandRequest) async throws -> CommandResult { + requests.withLock { $0.append(request) } + let args = request.arguments + var code: Int32 = 0, stderr = "" + if args.first == "collect" { + if refuseCollect { code = 1; stderr = "log: Must be run as root" } else if let out = output(request) { try FileManager.default.createDirectory(at: out, withIntermediateDirectories: true) } + } else if args.contains("record"), let out = output(request) { + try FileManager.default.createDirectory(at: out, withIntermediateDirectories: true) + } else if args.contains("--toc"), let out = output(request) { + try Data("2027-01-15T08:00:00.000Z".utf8).write(to: out) + } else if args.contains("--xpath"), let out = output(request) { + try exportedXML.write(to: out) + } + return CommandResult(request: request, termination: .exited(code), standardOutput: Data(), standardError: Data(stderr.utf8), startedAt: Date(), finishedAt: Date()) + } + + func stream(_ request: CommandRequest) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + // `log show --style ndjson` + continuation.yield(.standardOutput(Data(#"{"timestamp":"2027-01-15 08:00:01.000000+0000","processImagePath":"/usr/libexec/backboardd","processID":61,"messageType":"Default","eventMessage":"from the archive"}"#.utf8 + [0x0A]))) + continuation.yield(.finished(CommandResult(request: request, termination: .exited(0), standardOutput: Data(), standardError: Data(), startedAt: Date(), finishedAt: Date()))) + continuation.finish() + } + } +} diff --git a/Tests/ToolkitFeaturesTests/EvidenceAndReadinessTests.swift b/Tests/ToolkitFeaturesTests/EvidenceAndReadinessTests.swift index 8ead766..fafe069 100644 --- a/Tests/ToolkitFeaturesTests/EvidenceAndReadinessTests.swift +++ b/Tests/ToolkitFeaturesTests/EvidenceAndReadinessTests.swift @@ -130,6 +130,39 @@ struct EvidenceTests { #expect(manifest.outcome.exitCode == 2) } + @Test(.enabled(if: xcodeAvailable)) func collectsOSLogArchiveAndDVTLogging() async throws { + let server = try FakeDeviceServer() + registerStandardServices(server, afc: FakeAFCFileSystem(files: [:]), crashes: FakeAFCFileSystem(files: [:])) + try await server.start() + defer { Task { await server.stop() } } + let root = try SecureFileIO.makeTemporaryDirectory(prefix: "collect-logs") + defer { try? FileManager.default.removeItem(at: root) } + let device = physicalDevice(server) + let runner = FakeToolRunner(exportedXML: Data(CollectedLogTests.exportedXML.utf8)) + let options = CollectionOptions(durationSeconds: 0, includeUnifiedLogs: false, includeOSLogArchive: true, includeDVTLogging: true) + + let folder = try CaseWorkflow.createCaseFolder(in: root, target: device.target) + let manifest = try await EvidenceCollector(device: device, caseFolder: folder, options: options, runner: runner, usbmux: server.client).run { _ in } + let status = Dictionary(manifest.steps.map { ($0.id, $0) }, uniquingKeysWith: { $1 }) + #expect(status["oslog-archive"]?.status == .succeeded, "\(status["oslog-archive"]?.detail ?? "missing")") + #expect(status["dvt-logging"]?.status == .succeeded, "\(status["dvt-logging"]?.detail ?? "missing")") + #expect(FileManager.default.fileExists(atPath: folder.appendingPathComponent("artifacts/device.logarchive").path)) + #expect(FileManager.default.fileExists(atPath: folder.appendingPathComponent("artifacts/dvt-logging.trace").path)) + #expect(try String(contentsOf: folder.appendingPathComponent("streams/dvt-logging.jsonl"), encoding: .utf8).contains("Launched Maps")) + #expect(try HashManifest.verify(folder: folder, fileName: "SHA256SUMS").isEmpty) + // The archive request asks for the last hour. + #expect(runner.requests.current.contains { $0.arguments.starts(with: ["collect"]) && $0.arguments.contains("3600s") }) + + // A refused archive is a coverage gap with a plain-language reason, not a silent skip. + runner.refuseCollect = true + let second = try CaseWorkflow.createCaseFolder(in: root.appendingPathComponent("second"), target: device.target) + let refused = try await EvidenceCollector(device: device, caseFolder: second, options: options, runner: runner, usbmux: server.client).run { _ in } + let archive = refused.steps.first { $0.id == "oslog-archive" } + #expect(archive?.status == .failed) + #expect(archive?.detail == "macOS did not allow collecting the device's log archive.") + #expect(refused.outcome == .partial) + } + @Test func failsWhenTheDeviceCannotBeIdentified() async throws { let server = try FakeDeviceServer() server.pairRecordAvailable = false diff --git a/Tests/ToolkitFeaturesTests/LiveLogTests.swift b/Tests/ToolkitFeaturesTests/LiveLogTests.swift index abf93e0..517d323 100644 --- a/Tests/ToolkitFeaturesTests/LiveLogTests.swift +++ b/Tests/ToolkitFeaturesTests/LiveLogTests.swift @@ -105,8 +105,8 @@ struct LiveLogTests { } @Test func streamKindsMatchDeviceKinds() { - #expect(LogStreamKind.available(for: .physical) == [.unified, .classic]) - #expect(LogStreamKind.available(for: .simulator) == [.simulator]) + #expect(LogStreamKind.available(for: .physical) == [.unified, .classic, .osLogArchive, .dvt]) + #expect(LogStreamKind.available(for: .simulator) == [.simulator, .dvt]) #expect(LogStreamKind.available(for: .demo).isEmpty) } } diff --git a/Tests/ToolkitFeaturesTests/RealSimulatorTests.swift b/Tests/ToolkitFeaturesTests/RealSimulatorTests.swift index c31934c..f5b84d9 100644 --- a/Tests/ToolkitFeaturesTests/RealSimulatorTests.swift +++ b/Tests/ToolkitFeaturesTests/RealSimulatorTests.swift @@ -69,6 +69,26 @@ struct RealSimulatorTests { #expect(metadata.rawBytes > 0, "no log bytes captured (\(lines) lines)") #expect(metadata.rawSHA256?.count == 64) + // DVT logging: a short Instruments Logging recording, exported and parsed. + step("dvt logging") + let dvtFolder = try SecureFileIO.makeTemporaryDirectory(prefix: "dvt-e2e") + defer { try? FileManager.default.removeItem(at: dvtFolder) } + let trace = dvtFolder.appendingPathComponent("logging.trace") + var dvtLines: [LogLine] = [] + do { + for try await chunk in CollectedLogs.stream(.dvt, target: target, seconds: 3, artifact: trace, runner: ProcessCommandRunner()) { dvtLines += chunk.lines } + let recorded = dvtLines.filter { $0.level != "note" } + print("[simulator e2e] DVT logging: \(recorded.count) lines") + #expect(FileManager.default.fileExists(atPath: trace.path)) + #expect(recorded.count > 10, "no os_log lines were read from the recording") + #expect(recorded.contains { $0.timestamp != nil && $0.process != nil && $0.subsystem != nil }) + } catch let error as ToolkitError where error.kind == .timedOut { + // On a heavily loaded runner (a simulator that took minutes to boot) Instruments can + // take longer than the app allows; the app then reports a timeout, which is what it + // should do. Any other failure still fails the test. + print("[simulator e2e] DVT logging: xctrace did not finish in time on this machine (\(error.message)); not verified in this run") + } + let executor = ActionExecutor() let openURL = try await executor.execute(try #require(ActionCatalog.descriptor("open-url")), target: target, values: ["url": "https://example.com"]) #expect(openURL.summary.contains("example.com")) diff --git a/Tests/ToolkitFeaturesTests/SharedFeatureTests.swift b/Tests/ToolkitFeaturesTests/SharedFeatureTests.swift index 22526bb..be24849 100644 --- a/Tests/ToolkitFeaturesTests/SharedFeatureTests.swift +++ b/Tests/ToolkitFeaturesTests/SharedFeatureTests.swift @@ -168,11 +168,11 @@ struct SharedFeatureTests { #expect(profile.developerImageMechanism == .native) #expect(profile.apps == .init(calculateSizes: false, includeSystemApps: false, installAsDeveloperPackage: true)) #expect(profile.backup == .init(forceFullBackup: true, requireEncryption: true)) - #expect(profile.evidence == CollectionOptions(durationSeconds: 120, includeClassicSyslog: true, includeUnifiedLogs: true, includePacketCapture: false, includeScreenshot: false, includeCrashReports: true)) + #expect(profile.evidence == CollectionOptions(durationSeconds: 120, includeClassicSyslog: true, includeUnifiedLogs: false, includePacketCapture: false, includeScreenshot: false, includeCrashReports: true, includeDVTLogging: true)) #expect(profile.location == .init(timingJitterMilliseconds: 250, ignoreRecordedTiming: true, routeSpeedKmh: 35, routeIntervalSeconds: 3, routeTraversals: 4)) #expect(imported.notes.contains { $0.contains("“btlogger” → action “Bluetooth capture”") }) #expect(imported.notes.contains { $0.contains("nothing is downloaded") }) - #expect(imported.notes.contains { $0.contains("DVT OSLog → Unified Logging") }) + #expect(imported.notes.contains { $0.contains("DVT OSLog → DVT logging through Instruments") }) // Once imported it is an ordinary profile. #expect(try WorkspaceProfile.decode(try profile.encoded()) == profile) @@ -384,3 +384,27 @@ struct DeveloperImagePageTests { #expect(DeveloperImageMechanism.native.explanation.contains("online")) } } + +@Suite("Evidence options with OSLog archive and DVT") +struct EvidenceLogOptionTests { + @Test func olderSavedOptionsStillLoad() throws { + let old = Data(#"{"durationSeconds":120,"includeClassicSyslog":true,"includeUnifiedLogs":true,"includePacketCapture":false,"includeScreenshot":false,"includeCrashReports":true}"#.utf8) + let options = try JSONOutput.decoder().decode(CollectionOptions.self, from: old) + #expect(options.durationSeconds == 120 && options.includeCrashReports) + #expect(!options.includeOSLogArchive && !options.includeDVTLogging) + let round = try JSONOutput.decoder().decode(CollectionOptions.self, from: JSONOutput.encode(CollectionOptions(includeOSLogArchive: true, includeDVTLogging: true))) + #expect(round.includeOSLogArchive && round.includeDVTLogging) + } + + @Test func dvtNeedsDeveloperServicesAndStaysBounded() { + var options = CollectionOptions(durationSeconds: 0) + #expect(options.dvtSeconds == 10) + options.durationSeconds = 3600 + #expect(options.dvtSeconds == CollectedLogs.maximumDVTSeconds) + options.includeDVTLogging = true + #expect(options.requirements.contains(.developerMode) && options.requirements.contains(.instruments)) + options.includeDVTLogging = false + options.includeOSLogArchive = true + #expect(options.requirements == [.trustedDevice]) + } +}