Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 177 additions & 16 deletions apps/ios/Zeron/Composer/Attachments.swift
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
// back out to render thumbnails. RunRequest.attachments additionally carries
// the paths so a harness can inline the bytes.

import ImageIO
import Photos
import PhotosUI
import SwiftUI
import UniformTypeIdentifiers

// MARK: - Text transport (message-attachments.ts)

Expand Down Expand Up @@ -264,8 +266,9 @@ func uploadAttachmentChunked(relay: DeviceRelayClient, name: String, data: Data,

/// Decoded transcript images keyed by `(deviceId, path)`, loaded over the
/// owning device's relay in 45KB base64 chunks, seeded locally after a send
/// so own bubbles never round-trip. Bounded by an encoded-byte LRU budget;
/// failed loads retry on the 2s→15s ladder.
/// so own bubbles never round-trip. Generated images count decoded memory
/// toward the LRU budget; ordinary attachments retain their existing accounting.
/// Failed loads retry on the 2s→15s ladder.
@MainActor
@Observable
final class AttachmentImageCache {
Expand All @@ -280,6 +283,9 @@ final class AttachmentImageCache {
private struct Key: Hashable {
let deviceId: String
let path: String
// A generated image must pass its declared raster policy even if a
// generic attachment for the same path is already cached.
var expectedMimeType: String? = nil
}

private enum Entry {
Expand All @@ -304,24 +310,26 @@ final class AttachmentImageCache {
}
}

func snapshot(deviceId: String, path: String) -> Snapshot {
switch entries[Key(deviceId: deviceId, path: path)] {
func snapshot(deviceId: String, path: String, expectedMimeType: String? = nil) -> Snapshot {
switch entries[Key(deviceId: deviceId, path: path, expectedMimeType: expectedMimeType)] {
case .loaded(let name, let image, _, _):
return .loaded(name: name, image: image)
case .error(let attempts, let at)
where Date().timeIntervalSince(at) < Self.retryDelay(attempts):
return .error
case .error:
return .loading // ladder elapsed; the next load() attempt owns it
// Generated-image fallback selection must still see a failed
// owner after backoff expires; load() owns the retry transition.
return expectedMimeType == nil ? .loading : .error
case .loading, .none:
return .loading
}
}

/// Kick a load if this source isn't already loaded/loading (errored
/// sources retry only after their backoff).
func load(deviceId: String, path: String) {
let key = Key(deviceId: deviceId, path: path)
func load(deviceId: String, path: String, expectedMimeType: String? = nil) {
let key = Key(deviceId: deviceId, path: path, expectedMimeType: expectedMimeType)
let attempts: Int
switch entries[key] {
case .loaded, .loading:
Expand All @@ -340,8 +348,8 @@ final class AttachmentImageCache {
return client
}()
Task { @MainActor [weak self] in
let loaded = await Self.readImage(relay: relay, path: path)
guard let self else { return }
let loaded = await Self.readImage(relay: relay, path: path, expectedMimeType: expectedMimeType)
guard let self, case .loading? = self.entries[key] else { return }
if let loaded {
self.store(key: key, name: loaded.name, image: loaded.image, bytes: loaded.bytes)
} else {
Expand All @@ -352,9 +360,15 @@ final class AttachmentImageCache {

/// Seed after a successful upload (composer send path) so the just-sent
/// bubble renders from local bytes instead of a round-trip.
func seed(deviceId: String, path: String, name: String, data: Data) {
guard let image = UIImage(data: data) else { return }
store(key: Key(deviceId: deviceId, path: path), name: name, image: image, bytes: data.count)
func seed(deviceId: String, path: String, name: String, data: Data, expectedMimeType: String? = nil) {
if let expectedMimeType {
guard let loaded = Self.decodeGeneratedImage(data, mimeType: expectedMimeType) else { return }
store(key: Key(deviceId: deviceId, path: path, expectedMimeType: expectedMimeType),
name: name, image: loaded.image, bytes: loaded.bytes)
} else {
guard let image = UIImage(data: data) else { return }
store(key: Key(deviceId: deviceId, path: path), name: name, image: image, bytes: data.count)
}
}

private func store(key: Key, name: String, image: UIImage, bytes: Int) {
Expand Down Expand Up @@ -382,7 +396,7 @@ final class AttachmentImageCache {

/// `ReadAttachmentChunk` loop: 45KB base64 chunks until `done` (bounded,
/// with a stuck-offset guard).
private static func readImage(relay: DeviceRelayClient, path: String)
private static func readImage(relay: DeviceRelayClient, path: String, expectedMimeType: String?)
async -> (name: String, image: UIImage, bytes: Int)? {
struct Chunk: Decodable {
var name: String
Expand All @@ -400,17 +414,53 @@ final class AttachmentImageCache {
method: "ReadAttachmentChunk",
params: ["path": path, "offset": offset],
timeoutSeconds: 20) else { return nil }
if let expectedMimeType {
guard chunk.mimeType == expectedMimeType,
b64.utf8.count + chunk.data.utf8.count <= Self.generatedMaxBytes / 3 * 4 else { return nil }
}
name = chunk.name
b64 += chunk.data
done = chunk.done
if done { break }
guard chunk.nextOffset > offset else { return nil }
offset = chunk.nextOffset
}
guard done, let data = Data(base64Encoded: b64), let image = UIImage(data: data) else {
return nil
guard done, let data = Data(base64Encoded: b64) else { return nil }
let displayName = name.isEmpty ? nameFromPath(path) : name
if let expectedMimeType {
// ImageIO decompression must not block scrolling on the main actor.
guard let loaded = await Task.detached(priority: .utility, operation: {
Self.decodeGeneratedImage(data, mimeType: expectedMimeType)
}).value else { return nil }
return (displayName, loaded.image, loaded.bytes)
}
return (name.isEmpty ? nameFromPath(path) : name, image, data.count)
guard let image = UIImage(data: data) else { return nil }
return (displayName, image, data.count)
}

nonisolated static let generatedMaxBytes = 24 * 1024 * 1024

/// Decode one static frame with bounded dimensions, then downsample for
/// mobile. The source metadata is read without allocating the full raster.
nonisolated static func decodeGeneratedImage(_ data: Data, mimeType: String)
-> (image: UIImage, bytes: Int)? {
guard data.count <= generatedMaxBytes,
GeneratedImageReference.supportedMimeTypes.contains(mimeType),
let source = CGImageSourceCreateWithData(data as CFData,
[kCGImageSourceShouldCache: false] as CFDictionary),
let type = CGImageSourceGetType(source),
UTType(type as String)?.preferredMIMEType == mimeType,
let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any],
let width = properties[kCGImagePropertyPixelWidth] as? NSNumber,
let height = properties[kCGImagePropertyPixelHeight] as? NSNumber,
(1...4096).contains(width.intValue), (1...4096).contains(height.intValue),
let image = CGImageSourceCreateThumbnailAtIndex(source, 0, [
kCGImageSourceCreateThumbnailFromImageAlways: true,
kCGImageSourceCreateThumbnailWithTransform: true,
kCGImageSourceShouldCacheImmediately: true,
kCGImageSourceThumbnailMaxPixelSize: 2048,
] as CFDictionary) else { return nil }
return (UIImage(cgImage: image), image.bytesPerRow * image.height)
}
}

Expand Down Expand Up @@ -550,6 +600,117 @@ struct AttachmentThumbView: View {
}
}

// MARK: - Generated images

/// Inline generated output. Reuses the attachment cache/relay and full-screen
/// viewer while keeping the original aspect ratio instead of a cropped thumb.
struct GeneratedImageView: View {
let owner: String
let host: String
let reference: GeneratedImageReference
var onResize: () -> Void = {}

private let cache = AttachmentImageCache.shared
@State private var preview: AttachmentPreview?

static func devices(owner: String, host: String) -> [String] {
var ids: [String] = []
for id in [owner, host] where !id.isEmpty && !ids.contains(id) { ids.append(id) }
return ids
}

private var devices: [String] { Self.devices(owner: owner, host: host) }

private var source: (device: String, snapshot: AttachmentImageCache.Snapshot)? {
var pending: (String, AttachmentImageCache.Snapshot)?
for device in devices {
let snapshot = cache.snapshot(deviceId: device, path: reference.path,
expectedMimeType: reference.mimeType)
switch snapshot {
case .loaded: return (device, snapshot)
case .loading: if pending == nil { pending = (device, snapshot) }
case .error: break
}
}
return pending ?? devices.first.map { ($0, .error) }
}

private var isLoaded: Bool {
if let source, case .loaded = source.snapshot { return true }
return false
}

private struct LoadRequest: Hashable {
var devices: [String]
var reference: GeneratedImageReference
var needsLoad: Bool
}

var body: some View {
Group {
switch source?.snapshot ?? .error {
case .loaded(_, let image):
Button {
preview = AttachmentPreview(name: reference.name, image: image)
} label: {
Image(uiImage: image)
.resizable()
.aspectRatio(contentMode: .fit)
.clipShape(RoundedRectangle(cornerRadius: 12))
.frame(maxWidth: 512, maxHeight: 420)
}
.buttonStyle(.plain)
.accessibilityLabel("Preview generated image")
.accessibilityHint(reference.name)
case .loading:
ProgressView()
.tint(Theme.textFaint)
.frame(maxWidth: .infinity)
.frame(height: 220)
.background(whiteAlpha(0.035), in: RoundedRectangle(cornerRadius: 12))
.accessibilityLabel("Loading generated image")
case .error:
Button {
if let device = source?.device {
cache.load(deviceId: device, path: reference.path,
expectedMimeType: reference.mimeType)
}
} label: {
Label("Generated image unavailable", systemImage: "photo.badge.exclamationmark")
.font(.footnote)
.foregroundStyle(Theme.textFaint)
.frame(maxWidth: .infinity)
.frame(height: 220)
.background(whiteAlpha(0.035), in: RoundedRectangle(cornerRadius: 12))
}
.buttonStyle(.plain)
.accessibilityHint("Tap to retry")
}
}
.frame(maxWidth: 512)
.frame(maxWidth: .infinity)
.onGeometryChange(for: CGSize.self) { $0.size } action: { _ in onResize() }
.task(id: LoadRequest(devices: devices, reference: reference, needsLoad: !isLoaded)) {
// Retry while visible; disappearing or changing sources cancels
// the timer. A cached image needs no timer or relay request.
while !Task.isCancelled && !isLoaded {
guard !devices.isEmpty else { return }
for device in devices {
let snapshot = cache.snapshot(deviceId: device, path: reference.path,
expectedMimeType: reference.mimeType)
cache.load(deviceId: device, path: reference.path,
expectedMimeType: reference.mimeType)
// Give an untried owner the first attempt. Once it has
// failed, also try the host while the owner backs off.
if case .loading = snapshot { break }
}
do { try await Task.sleep(for: .seconds(2)) } catch { return }
}
}
.fullScreenCover(item: $preview) { AttachmentLightbox(preview: $0) }
}
}

// MARK: - Lightbox (attachments.rs lightbox: dim scrim, image ≤85vh/90vw,
// name under it, any tap closes)

Expand Down
17 changes: 16 additions & 1 deletion apps/ios/Zeron/Models/Entities.swift
Original file line number Diff line number Diff line change
Expand Up @@ -246,15 +246,30 @@ struct RenderToolCall: Hashable {
var string: (String) -> String? { { key in self.fields[key] as? String } }
}

/// Durable metadata only; bytes remain on the message owner's device.
struct GeneratedImageReference: Hashable {
var path: String
var name: String
var mimeType: String

static let supportedMimeTypes: Set<String> = ["image/png", "image/jpeg", "image/webp", "image/gif"]

var isValid: Bool {
path.hasPrefix("/") && !path.contains("\0") && !name.isEmpty
&& Self.supportedMimeTypes.contains(mimeType)
}
}

enum MessagePart: Hashable, Identifiable {
case image(id: String, reference: GeneratedImageReference)
case text(id: String, text: String)
case tool(id: String, call: RenderToolCall, isError: Bool, resolved: Bool)
case input(id: String, requestId: String, questions: [UserInputQuestion], resolved: Bool)
case error(id: String, message: String)

var id: String {
switch self {
case .text(let id, _), .tool(let id, _, _, _), .input(let id, _, _, _), .error(let id, _):
case .text(let id, _), .image(let id, _), .tool(let id, _, _, _), .input(let id, _, _, _), .error(let id, _):
return id
}
}
Expand Down
10 changes: 9 additions & 1 deletion apps/ios/Zeron/Sync/SessionStore.swift
Original file line number Diff line number Diff line change
Expand Up @@ -457,13 +457,21 @@ final class SessionStore {
continuationOf: m["continuationOf"]?.stringValue)
}

nonisolated private static func partFrom(_ value: LoroValue) -> MessagePart? {
nonisolated static func partFrom(_ value: LoroValue) -> MessagePart? {
guard let m = value.mapValue,
let id = m["id"]?.stringValue,
let kind = m["kind"]?.stringValue else { return nil }
switch kind {
case "text":
return .text(id: id, text: m["text"]?.stringValue ?? "")
case "image":
let reference = GeneratedImageReference(path: m["path"]?.stringValue ?? "",
name: m["name"]?.stringValue ?? "",
mimeType: m["mimeType"]?.stringValue ?? "")
guard reference.isValid else {
return .error(id: id, message: "Generated image unavailable")
}
return .image(id: id, reference: reference)
case "tool":
guard let callMap = m["call"]?.mapValue else { return nil }
let tag = callMap["kind"]?.stringValue ?? "unknown"
Expand Down
13 changes: 13 additions & 0 deletions apps/ios/Zeron/Transcript/TranscriptRows.swift
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import Foundation

enum RowKind {
case generatedImage(owner: String, reference: GeneratedImageReference)
case user(text: String)
case markdown(block: MDBlock, streaming: Bool)
case toolGroup(tools: [ToolItem], autoOpen: Bool)
Expand Down Expand Up @@ -177,6 +178,18 @@ enum TranscriptRowBuilder {
first = false
}

case .image(let partId, let reference):
flushTools(lastIx: ix - 1)
var version = fnv1a("\(entry.deviceId)\0\(reference.path)\0\(reference.name)\0\(reference.mimeType)")
if settled, ix == lastPartIx { version ^= 1 << 62 }
rows.append(TranscriptRow(id: "\(entry.id)#\(partId)", version: version,
turnStart: first,
kind: .generatedImage(owner: entry.deviceId, reference: reference),
entryId: entry.id,
timestamp: settled && ix == lastPartIx ? entry.createdAt : nil,
partKey: nil))
first = false

case .input(let partId, _, let questions, let resolved):
flushTools(lastIx: ix - 1)
let header = questions.first?.header ?? "Question"
Expand Down
3 changes: 3 additions & 0 deletions apps/ios/Zeron/Transcript/TranscriptView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ struct TranscriptView: View {
userExpansionHeights[row.entryId] = height
scroll.refreshLayout?()
})
case .generatedImage(let owner, let reference):
GeneratedImageView(owner: owner, host: store.hostDeviceId ?? "", reference: reference,
onResize: { scroll.refreshLayout?() })
case .markdown(let block, let streaming):
MarkdownRowView(row: row, block: block, streaming: streaming, veils: veils)
case .toolGroup(let tools, let autoOpen):
Expand Down
Loading
Loading