From ee92de18a0be68666553509cc1fe5aad48c1a552 Mon Sep 17 00:00:00 2001 From: Patrick Schiller Date: Fri, 14 Aug 2026 15:36:26 +0200 Subject: [PATCH] Capture Safari article images Signed-off-by: Patrick Schiller --- README.de.md | 4 +- README.md | 6 +- ios/README.md | 6 +- ios/SourceBraid/App/ContentView.swift | 2 +- ios/SourceBraid/Shared/ClipBuilder.swift | 142 +++++++++++++++++- ios/SourceBraid/Shared/GitHubClient.swift | 7 + ios/SourceBraidShare/ShareInputResolver.swift | 19 ++- ios/SourceBraidShare/SharePreprocessing.js | 72 ++++++++- ios/SourceBraidShare/ShareViewModel.swift | 54 ++++++- ios/SourceBraidTests/ClipBuilderTests.swift | 37 +++++ tests/test_ios_share_preprocessing.py | 21 ++- 11 files changed, 352 insertions(+), 18 deletions(-) diff --git a/README.de.md b/README.de.md index 10e615a..2359783 100644 --- a/README.de.md +++ b/README.de.md @@ -343,8 +343,8 @@ zur lokalen Codex-Einrichtung und zum späteren ChatGPT-Endpunkt stehen in Die native iOS-App und Share Extension liegen unter [`ios/`](ios/README.md). Nach einmaliger Einrichtung von Repository und Token können URLs, ausgewählter -Text, Safari-Artikel, PDFs und andere Dateien über das Teilen-Menü an das -konfigurierte private Archiv gesendet werden. +Text, Safari-Artikel mit relevanten Bild-Assets, PDFs und andere Dateien über +das Teilen-Menü an das konfigurierte private Archiv gesendet werden. ## Android-Roadmap diff --git a/README.md b/README.md index 5518ba7..6a59159 100644 --- a/README.md +++ b/README.md @@ -313,9 +313,9 @@ for the local Codex setup and the later ChatGPT endpoint. ## iOS The native iOS app and Share Extension live under [`ios/`](ios/README.md). -After one-time repository and token setup, URLs, selected text, Safari articles, -PDFs, and other files can be sent to the configured private archive through the -system share sheet. +After one-time repository and token setup, URLs, selected text, Safari articles +with relevant image assets, PDFs, and other files can be sent to the configured +private archive through the system share sheet. ## Android roadmap diff --git a/ios/README.md b/ios/README.md index 6d1b51b..6992959 100644 --- a/ios/README.md +++ b/ios/README.md @@ -1,6 +1,6 @@ # SourceBraid for iOS -The iOS project contains a SwiftUI configuration app and a native Share Extension. It saves shared URLs, readable Safari content, selected text, PDFs, and other files into the same GitHub repository and URL-hash-sharded `web-clips/index/*.jsonl` format as the Chrome extension. +The iOS project contains a SwiftUI configuration app and a native Share Extension. It saves shared URLs, readable Safari content and its relevant image assets, selected text, PDFs, and other files into the same GitHub repository and URL-hash-sharded `web-clips/index/*.jsonl` format as the Chrome extension. ## Open and sign @@ -63,13 +63,13 @@ python3 ../scripts/validate_ios_release.py \ ## Use -In FAZ, Safari, Files, or another app: +In Safari, Files, or another app: 1. Tap **Share**. 2. Choose **SourceBraid**. If it is hidden, use **More** to enable it. 3. Edit the title, add optional tags or a note, and tap **Save**. -Apps such as Chrome and FAZ generally share only a URL. SourceBraid loads public web URLs in an isolated web view and converts the readable page content to Markdown before saving, recording the same extraction methods as the browser extension (including specialized DeepMind captures). Safari can additionally supply its already visible page text through the extension's preprocessing script. Shared PDFs are queued as `pdf-docling-pending`; the PDF is pushed last so the existing GitHub Actions workflow can safely convert it after its Markdown and index metadata exist. Pages that require an authenticated browser session or block the isolated request are saved as clearly labeled link-only clips. +Apps such as Chrome generally share only a URL. SourceBraid loads public web URLs in an isolated web view and converts the readable page content to Markdown before saving, recording the same extraction methods as the browser extension (including specialized DeepMind captures). Safari can additionally supply its already visible page text and relevant article images through the extension's preprocessing script. Images are copied into the clip's asset folder and linked relatively in Markdown. Shared PDFs are queued as `pdf-docling-pending`; the PDF is pushed last so the existing GitHub Actions workflow can safely convert it after its Markdown and index metadata exist. Pages that require an authenticated browser session or block the isolated request are saved as clearly labeled link-only clips. ## Build without signing diff --git a/ios/SourceBraid/App/ContentView.swift b/ios/SourceBraid/App/ContentView.swift index 2fb8623..46a551a 100644 --- a/ios/SourceBraid/App/ContentView.swift +++ b/ios/SourceBraid/App/ContentView.swift @@ -14,7 +14,7 @@ struct ContentView: View { ) .foregroundStyle(model.isConfigured ? Color.green : Color.orange) - Text("In FAZ or any other app, tap Share and choose “SourceBraid”.") + Text("Weave the web into durable Markdown in your own GitHub repository.") .font(.subheadline) .foregroundStyle(.secondary) } header: { diff --git a/ios/SourceBraid/Shared/ClipBuilder.swift b/ios/SourceBraid/Shared/ClipBuilder.swift index 2da1a7e..9b1e459 100644 --- a/ios/SourceBraid/Shared/ClipBuilder.swift +++ b/ios/SourceBraid/Shared/ClipBuilder.swift @@ -10,6 +10,7 @@ struct CaptureInput { var fileData: Data? var filename: String? var mimeType: String? + var images: [CaptureImage] = [] static let empty = CaptureInput( url: nil, @@ -22,17 +23,51 @@ struct CaptureInput { ) } +struct CaptureImage: Equatable { + let url: URL + let alt: String + let caption: String +} + +struct CapturedImageAsset { + let image: CaptureImage + let data: Data + let mimeType: String +} + struct CaptureAttachment { let path: String let data: Data } +struct CapturedImageAttachment { + let image: CaptureImage + let attachment: CaptureAttachment + let mimeType: String +} + struct CaptureDraft { let title: String let path: String let markdown: String let indexEntry: SourceBraidIndexEntry let attachment: CaptureAttachment? + let imageAttachments: [CapturedImageAttachment] +} + +struct SourceBraidImageIndexEntry: Encodable { + let url: String + let path: String + let relativePath: String + let alt: String + let caption: String + let contentType: String + + enum CodingKeys: String, CodingKey { + case url, path, alt, caption + case relativePath = "relative_path" + case contentType = "content_type" + } } struct SourceBraidIndexEntry: Encodable { @@ -50,6 +85,7 @@ struct SourceBraidIndexEntry: Encodable { let capturedAt: String let attachmentPath: String? let pdfPath: String? + let images: [SourceBraidImageIndexEntry]? enum CodingKeys: String, CodingKey { case title, url, path, date, tags, source @@ -61,6 +97,7 @@ struct SourceBraidIndexEntry: Encodable { case capturedAt = "captured_at" case attachmentPath = "attachment_path" case pdfPath = "pdf_path" + case images } } @@ -73,11 +110,12 @@ enum ClipBuilder { tags: [String], notes: String, configuration: SourceBraidConfiguration, + imageAssets: [CapturedImageAsset] = [], now: Date = Date() ) throws -> CaptureDraft { let normalizedConfiguration = configuration.normalized() let hasText = !input.sharedText.trimmed.isEmpty || !input.articleText.trimmed.isEmpty - guard input.url != nil || hasText || input.fileData != nil else { + guard input.url != nil || hasText || input.fileData != nil || !input.images.isEmpty else { throw ClipBuilderError.emptyInput } if let data = input.fileData, data.count > maximumAttachmentBytes { @@ -114,6 +152,10 @@ enum ClipBuilder { captureMethod = "ios-share-url" sourceType = "article" contentFormat = "link" + } else if !input.images.isEmpty { + captureMethod = "ios-share-images" + sourceType = "article" + contentFormat = "images" } else { captureMethod = "ios-share-text" sourceType = "note" @@ -131,6 +173,25 @@ enum ClipBuilder { attachment = nil } + let imageAttachments = queuesPDFConversion + ? [] + : buildImageAttachments( + imageAssets, + rootFolder: normalizedConfiguration.rootFolder, + captureDate: captureDate, + markdownPath: path + ) + let indexImages = imageAttachments.map { image in + SourceBraidImageIndexEntry( + url: image.image.url.absoluteString, + path: image.attachment.path, + relativePath: relativePath(from: path, to: image.attachment.path), + alt: image.image.alt, + caption: image.image.caption, + contentType: image.mimeType + ) + } + let markdown = buildMarkdown( input: input, title: title, @@ -145,6 +206,7 @@ enum ClipBuilder { tags: tags, notes: notes, attachmentPath: attachment?.path, + imageAttachments: imageAttachments, markdownPath: path ) let entry = SourceBraidIndexEntry( @@ -161,9 +223,17 @@ enum ClipBuilder { converter: queuesPDFConversion ? "docling" : nil, capturedAt: capturedAt, attachmentPath: queuesPDFConversion ? nil : attachment?.path, - pdfPath: queuesPDFConversion ? attachment?.path : nil + pdfPath: queuesPDFConversion ? attachment?.path : nil, + images: indexImages.isEmpty ? nil : indexImages + ) + return CaptureDraft( + title: title, + path: path, + markdown: markdown, + indexEntry: entry, + attachment: attachment, + imageAttachments: imageAttachments ) - return CaptureDraft(title: title, path: path, markdown: markdown, indexEntry: entry, attachment: attachment) } static func parseTags(_ value: String) -> [String] { @@ -188,6 +258,7 @@ enum ClipBuilder { tags: [String], notes: String, attachmentPath: String?, + imageAttachments: [CapturedImageAttachment], markdownPath: String ) -> String { var lines = [ @@ -243,9 +314,46 @@ enum ClipBuilder { if !sharedText.isEmpty, sharedText != input.url?.absoluteString, sharedText != articleText { lines.append(contentsOf: ["## Shared text", "", sharedText, ""]) } + if !imageAttachments.isEmpty { + lines.append(contentsOf: ["## Images", ""]) + for image in imageAttachments { + let alt = markdownImageAlt(image.image.alt) + let imagePath = relativePath(from: markdownPath, to: image.attachment.path) + lines.append("![\(alt)](\(imagePath))") + let caption = image.image.caption.trimmingCharacters(in: .whitespacesAndNewlines) + if !caption.isEmpty, caption != image.image.alt { + lines.append(caption) + } + lines.append("") + } + } return lines.joined(separator: "\n") + "\n" } + private static func buildImageAttachments( + _ imageAssets: [CapturedImageAsset], + rootFolder: String, + captureDate: String, + markdownPath: String + ) -> [CapturedImageAttachment] { + let clipSlug = markdownPath + .split(separator: "/") + .last + .map(String.init)? + .replacingOccurrences(of: ".md", with: "") ?? "document" + return imageAssets.enumerated().map { index, asset in + let filename = String(format: "%02d", index + 1) + + "." + + imageFileExtension(url: asset.image.url, mimeType: asset.mimeType) + let path = "\(rootFolder)/\(captureDate.prefix(4))/\(captureDate.dropFirst(5).prefix(2))/assets/\(clipSlug)/\(filename)" + return CapturedImageAttachment( + image: asset.image, + attachment: CaptureAttachment(path: path, data: asset.data), + mimeType: asset.mimeType + ) + } + } + private static func normalizedTitle(_ suppliedTitle: String, input: CaptureInput) -> String { let candidates: [String?] = [suppliedTitle, input.suggestedTitle, input.filename, input.url.flatMap(hostname), "Saved item"] return candidates.compactMap { $0?.trimmed }.first { !$0.isEmpty } ?? "Saved item" @@ -298,6 +406,34 @@ enum ClipBuilder { return mimeType == "application/pdf" ? "pdf" : "bin" } + private static func imageFileExtension(url: URL, mimeType: String) -> String { + let extensions = [ + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/webp": "webp", + "image/gif": "gif", + "image/svg+xml": "svg", + "image/avif": "avif" + ] + if let value = extensions[mimeType.lowercased()] { + return value + } + let value = url.pathExtension.lowercased() + if value.range(of: "^[a-z0-9]{1,8}$", options: .regularExpression) != nil { + return value + } + return "jpg" + } + + private static func markdownImageAlt(_ value: String) -> String { + value + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "[", with: "\\[") + .replacingOccurrences(of: "]", with: "\\]") + .replacingOccurrences(of: "\n", with: " ") + } + private static func relativePath(from markdownPath: String, to attachmentPath: String) -> String { var from = markdownPath.split(separator: "/").dropLast().map(String.init) var to = attachmentPath.split(separator: "/").map(String.init) diff --git a/ios/SourceBraid/Shared/GitHubClient.swift b/ios/SourceBraid/Shared/GitHubClient.swift index 1d0004e..2215173 100644 --- a/ios/SourceBraid/Shared/GitHubClient.swift +++ b/ios/SourceBraid/Shared/GitHubClient.swift @@ -21,6 +21,13 @@ struct GitHubClient { try await putReplacing(path: attachment.path, data: attachment.data, message: "Queue SourceBraid PDF: \(draft.title)") return } + for image in draft.imageAttachments { + try await putReplacing( + path: image.attachment.path, + data: image.attachment.data, + message: "Save SourceBraid image: \(draft.title)" + ) + } if let attachment = draft.attachment { try await putReplacing(path: attachment.path, data: attachment.data, message: "Save SourceBraid attachment: \(draft.title)") } diff --git a/ios/SourceBraidShare/ShareInputResolver.swift b/ios/SourceBraidShare/ShareInputResolver.swift index ac9d059..c7f9593 100644 --- a/ios/SourceBraidShare/ShareInputResolver.swift +++ b/ios/SourceBraidShare/ShareInputResolver.swift @@ -49,7 +49,7 @@ enum ShareInputResolver { input.url = url } } - guard input.url != nil || input.fileData != nil || !input.sharedText.isEmpty || !input.articleText.isEmpty else { + guard input.url != nil || input.fileData != nil || !input.sharedText.isEmpty || !input.articleText.isEmpty || !input.images.isEmpty else { throw ShareInputError.unsupported } return input @@ -69,6 +69,23 @@ enum ShareInputResolver { if input.articleText.isEmpty, let value = result["articleText"] as? String { input.articleText = limited(value) } + if let values = result["images"] as? [Any] { + var seen = Set() + input.images = values.compactMap { value in + guard let image = value as? [String: Any], + let urlValue = image["url"] as? String, + let url = URL(string: urlValue), + ["http", "https"].contains(url.scheme?.lowercased() ?? ""), + seen.insert(url.absoluteString).inserted else { + return nil + } + return CaptureImage( + url: url, + alt: limited(image["alt"] as? String ?? ""), + caption: limited(image["caption"] as? String ?? "") + ) + } + } } private static func supportedFileType(_ provider: NSItemProvider) -> String? { diff --git a/ios/SourceBraidShare/SharePreprocessing.js b/ios/SourceBraidShare/SharePreprocessing.js index fe479d5..785761f 100644 --- a/ios/SourceBraidShare/SharePreprocessing.js +++ b/ios/SourceBraidShare/SharePreprocessing.js @@ -11,7 +11,77 @@ var ExtensionPreprocessingJS = { url: document.location.href, title: document.title || "", selectedText: selection.slice(0, 500000), - articleText: articleText + articleText: articleText, + images: sourceBraidImages(article) }); } }; + +function sourceBraidImages(article) { + if (!article || typeof article.querySelectorAll !== "function") { + return []; + } + + var images = []; + var seen = {}; + var nodes = article.querySelectorAll("img"); + for (var index = 0; index < nodes.length && images.length < 12; index += 1) { + var image = nodes[index]; + var url = sourceBraidImageURL(image); + if (!url || seen[url] || sourceBraidShouldSkipImage(url, image)) { + continue; + } + seen[url] = true; + + var figure = typeof image.closest === "function" ? image.closest("figure") : null; + var captionNode = figure && typeof figure.querySelector === "function" + ? figure.querySelector("figcaption") + : null; + images.push({ + url: url, + alt: sourceBraidLimitedText(image.alt || image.getAttribute("alt") || image.getAttribute("title") || ""), + caption: sourceBraidLimitedText(captionNode && captionNode.innerText ? captionNode.innerText : "") + }); + } + return images; +} + +function sourceBraidImageURL(image) { + var candidates = [ + image.currentSrc, + image.src, + image.getAttribute("src"), + image.getAttribute("data-src"), + image.getAttribute("data-lazy-src"), + sourceBraidLargestSrcset(image.getAttribute("srcset") || image.getAttribute("data-srcset")) + ]; + for (var index = 0; index < candidates.length; index += 1) { + var candidate = candidates[index]; + if (typeof candidate === "string" && /^https?:\/\//i.test(candidate)) { + return candidate; + } + } + return ""; +} + +function sourceBraidLargestSrcset(value) { + if (!value) { + return ""; + } + var candidates = value.split(","); + var last = candidates[candidates.length - 1] || ""; + return last.trim().split(/\s+/)[0] || ""; +} + +function sourceBraidShouldSkipImage(url, image) { + var width = Number(image.getAttribute("width") || image.naturalWidth || 0); + var height = Number(image.getAttribute("height") || image.naturalHeight || 0); + if (width > 0 && height > 0 && (width < 80 || height < 80)) { + return true; + } + return /\/(?:avatar|logo|icon|spinner|tracking|pixel)[^/]*\.(?:gif|png|jpe?g|webp|svg)(?:[?#].*)?$/i.test(url); +} + +function sourceBraidLimitedText(value) { + return String(value || "").trim().slice(0, 2000); +} diff --git a/ios/SourceBraidShare/ShareViewModel.swift b/ios/SourceBraidShare/ShareViewModel.swift index 7630dc2..edab068 100644 --- a/ios/SourceBraidShare/ShareViewModel.swift +++ b/ios/SourceBraidShare/ShareViewModel.swift @@ -46,12 +46,14 @@ final class ShareViewModel: ObservableObject { guard !token.isEmpty else { throw ShareSaveError.notConfigured } + let imageAssets = await ImageAssetDownloader.download(input.images) let draft = try ClipBuilder.build( input: input, title: title, tags: ClipBuilder.parseTags(tags), notes: notes, - configuration: configuration + configuration: configuration, + imageAssets: imageAssets ) try await GitHubClient(configuration: configuration, token: token).save(draft) RecentCaptureStore.add(RecentCapture(title: draft.title, path: draft.path)) @@ -119,6 +121,56 @@ final class ShareViewModel: ObservableObject { } } +private enum ImageAssetDownloader { + private static let maximumImageCount = 12 + private static let maximumImageBytes = 8 * 1024 * 1024 + private static let maximumTotalBytes = 25 * 1024 * 1024 + + static func download(_ images: [CaptureImage]) async -> [CapturedImageAsset] { + var assets: [CapturedImageAsset] = [] + var totalBytes = 0 + + for image in images.prefix(maximumImageCount) { + guard let asset = try? await fetch(image), + totalBytes + asset.data.count <= maximumTotalBytes else { + continue + } + totalBytes += asset.data.count + assets.append(asset) + } + return assets + } + + private static func fetch(_ image: CaptureImage) async throws -> CapturedImageAsset { + var request = URLRequest(url: image.url) + request.timeoutInterval = 20 + request.cachePolicy = .reloadIgnoringLocalCacheData + request.setValue("image/*", forHTTPHeaderField: "Accept") + + let (data, response) = try await URLSession.shared.data(for: request) + guard let response = response as? HTTPURLResponse, + (200...299).contains(response.statusCode), + !data.isEmpty, + data.count <= maximumImageBytes else { + throw ImageDownloadError.invalidResponse + } + let mimeType = ( + (response.value(forHTTPHeaderField: "Content-Type") ?? "") + .split(separator: ";", maxSplits: 1) + .first + .map(String.init) ?? "" + ).lowercased() + guard mimeType.hasPrefix("image/") else { + throw ImageDownloadError.invalidResponse + } + return CapturedImageAsset(image: image, data: data, mimeType: mimeType) + } + + private enum ImageDownloadError: Error { + case invalidResponse + } +} + enum ShareSaveError: LocalizedError { case notConfigured case cancelled diff --git a/ios/SourceBraidTests/ClipBuilderTests.swift b/ios/SourceBraidTests/ClipBuilderTests.swift index a776202..8ae9b6f 100644 --- a/ios/SourceBraidTests/ClipBuilderTests.swift +++ b/ios/SourceBraidTests/ClipBuilderTests.swift @@ -96,6 +96,43 @@ final class ClipBuilderTests: XCTestCase { XCTAssertFalse(draft.markdown.contains("## Shared text")) } + func testSafariImagesAreStoredAsLocalAssetsAndLinkedFromMarkdown() throws { + let image = CaptureImage( + url: URL(string: "https://images.example.com/cover.webp")!, + alt: "Example cover", + caption: "A source-provided caption" + ) + let input = CaptureInput( + url: URL(string: "https://www.example.com/article")!, + suggestedTitle: "Example Article", + sharedText: "", + articleText: "Article body", + fileData: nil, + filename: nil, + mimeType: nil, + images: [image] + ) + let date = ISO8601DateFormatter().date(from: "2026-08-14T12:00:00Z")! + let draft = try ClipBuilder.build( + input: input, + title: "Example Article", + tags: [], + notes: "", + configuration: configuration, + imageAssets: [ + CapturedImageAsset(image: image, data: Data("image".utf8), mimeType: "image/webp") + ], + now: date + ) + + let attachment = try XCTUnwrap(draft.imageAttachments.first) + XCTAssertTrue(attachment.attachment.path.hasSuffix("/assets/2026-08-14-example.com-example-article-f32a89/01.webp")) + XCTAssertTrue(draft.markdown.contains("![Example cover](assets/2026-08-14-example.com-example-article-f32a89/01.webp)")) + XCTAssertTrue(draft.markdown.contains("A source-provided caption")) + XCTAssertEqual(draft.indexEntry.images?.first?.path, attachment.attachment.path) + XCTAssertEqual(draft.indexEntry.images?.first?.url, image.url.absoluteString) + } + func testTagParsingTrimsAndDeduplicates() { XCTAssertEqual(ClipBuilder.parseTags("AI, reading, ai, research "), ["AI", "reading", "research"]) } diff --git a/tests/test_ios_share_preprocessing.py b/tests/test_ios_share_preprocessing.py index be6b904..6901ef9 100644 --- a/tests/test_ios_share_preprocessing.py +++ b/tests/test_ios_share_preprocessing.py @@ -14,6 +14,18 @@ def test_exposes_a_global_object_that_returns_page_details(self): const fs = require("fs"); const vm = require("vm"); const source = fs.readFileSync(process.argv[1], "utf8"); +const image = { + currentSrc: "https://cdn.example.com/cover.jpg", + alt: "Article cover", + naturalWidth: 1200, + naturalHeight: 800, + getAttribute: () => "", + closest: () => ({ querySelector: () => ({ innerText: "Photo caption" }) }) +}; +const article = { + innerText: "Readable page text", + querySelectorAll: () => [image] +}; const sandbox = { window: { getSelection: () => ({ toString: () => "Selected excerpt" }), @@ -22,9 +34,7 @@ def test_exposes_a_global_object_that_returns_page_details(self): document: { title: "Example article", location: { href: "https://example.com/article" }, - querySelector: (selector) => selector === "article" - ? { innerText: "Readable page text" } - : null + querySelector: (selector) => selector === "article" ? article : null } }; vm.createContext(sandbox); @@ -50,6 +60,11 @@ def test_exposes_a_global_object_that_returns_page_details(self): "title": "Example article", "selectedText": "Selected excerpt", "articleText": "Readable page text", + "images": [{ + "url": "https://cdn.example.com/cover.jpg", + "alt": "Article cover", + "caption": "Photo caption", + }], }, )