From 5354a8d75d5145dbaf0f696337296ba2f84bfb76 Mon Sep 17 00:00:00 2001 From: Patrick Schiller Date: Fri, 31 Jul 2026 22:51:22 +0200 Subject: [PATCH] Fix repository errors and DeepMind capture Signed-off-by: Patrick Schiller --- README.de.md | 8 ++-- README.md | 10 ++-- background.js | 15 ++++++ content.js | 64 ++++++++++++++++++++++++++ tests/capture-utils.test.js | 91 +++++++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 6 deletions(-) diff --git a/README.de.md b/README.de.md index a0c9f76..d720361 100644 --- a/README.de.md +++ b/README.de.md @@ -62,6 +62,7 @@ Die Trennung zwischen GitHub-Archiv und lokalem Such-Cache ist für größere Sa | **Forem / DEV** | Forem API mit Quell-Markdown | Markdown wird normalisiert und ohne Seiten-Chrome gespeichert | Relevante Bilder werden lokal gespeichert | Sichtbarer Seiteninhalt | | **Ghost** | Konfigurierte Ghost Content API | Strukturierter Post-Inhalt; die kanonische URL wird vor der Übernahme geprüft | Relevante Bilder werden lokal gespeichert | Sichtbarer Seiteninhalt | | **Blogger** | Blogger API anhand erkannter Blog- und Post-IDs | Strukturierter Artikelinhalt | Relevante Bilder werden lokal gespeichert | Sichtbarer Seiteninhalt | +| **Google-DeepMind-Blog** | Artikelsektionen aus dem Seiten-DOM | Vollständiger Beitrag ohne Cover-Bedienelemente und Karten verwandter Beiträge | Artikelbilder werden lokal gespeichert | Allgemeine Extraktion des sichtbaren Seiteninhalts | | **JSON Feed, RSS oder Atom** | Im HTML angekündigter Feed | Vollständiger Feed-Inhalt, sofern vorhanden | Relevante Bilder werden lokal gespeichert | Sichtbarer Seiteninhalt | | **Allgemeine HTML-Seite** | Sichtbarer DOM, bevorzugt `article`, `main` oder `[role="main"]` | Überschriften, Absätze, Links, Listen, Zitate, Codeblöcke und Tabellen | Inhaltlich relevante Bilder werden lokal gespeichert | `body` als letzte Rückfallstufe | @@ -77,8 +78,9 @@ SourceBraid verwendet immer die inhaltlich hochwertigste verfügbare Quelle. Bei 6. Forem / DEV API 7. Ghost Content API 8. Blogger API -9. JSON Feed, RSS oder Atom -10. sichtbarer DOM +9. Google-DeepMind-Blog-DOM +10. JSON Feed, RSS oder Atom +11. sichtbarer DOM Die erste passende und validierte Quelle gewinnt. Anschließend normalisiert SourceBraid das Markdown, lädt Bilder herunter, schreibt das YAML-Frontmatter und aktualisiert den Index. @@ -161,7 +163,7 @@ Bei einem Gist wird eine einzelne Markdown-Datei direkt als Dokumentinhalt gespe 5. Eine unterstützte Quelle öffnen und auf das **SourceBraid**-Symbol klicken. 6. GitHub-Repository konfigurieren, optional Tags oder Notizen ergänzen und **Save to GitHub** wählen. -Nach der ersten Einrichtung bleiben die GitHub-Einstellungen hinter dem Einstellungssymbol im Popup verborgen. Scheitert nur der GitHub-Upload nach einer erfolgreichen Extraktion, steht im Popup **Download Fallback** zur Verfügung. +Nach der ersten Einrichtung bleiben die GitHub-Einstellungen hinter dem Einstellungssymbol im Popup verborgen. Scheitert nur der GitHub-Upload nach einer erfolgreichen Extraktion, steht im Popup **Download Fallback** zur Verfügung. Vor dem Upload prüft SourceBraid, ob das konfigurierte Repository existiert und für den Token erreichbar ist; bei `404 Not Found` zeigt das Popup einen eindeutigen Fehler an. ## GitHub-Token diff --git a/README.md b/README.md index 61cc43e..c8a521d 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ index repair. | **Forem / DEV** | Forem API with source Markdown | Normalized Markdown without site chrome | Relevant images stored locally | Visible page content | | **Ghost** | Configured Ghost Content API | Structured post content with canonical URL validation | Relevant images stored locally | Visible page content | | **Blogger** | Blogger API using detected blog and post IDs | Structured article content | Relevant images stored locally | Visible page content | +| **Google DeepMind blog** | Article sections from the page DOM | Full post without the cover controls or related-post cards | Article images stored locally | Generic visible-page extraction | | **JSON Feed, RSS, or Atom** | Feed announced by the HTML page | Full feed content when available | Relevant images stored locally | Visible page content | | **Generic HTML page** | Visible DOM, preferring `article`, `main`, or `[role="main"]` | Headings, paragraphs, links, lists, quotes, code, and tables | Content-relevant images stored locally | `body` as the final fallback | @@ -109,8 +110,9 @@ adapters run in this order: 6. Forem / DEV API 7. Ghost Content API 8. Blogger API -9. JSON Feed, RSS, or Atom -10. Visible DOM +9. Google DeepMind blog DOM +10. JSON Feed, RSS, or Atom +11. Visible DOM The first matching, validated source wins. SourceBraid then normalizes the Markdown, downloads images, writes YAML frontmatter, and updates the index. @@ -225,7 +227,9 @@ be loaded through the still-open Gist tab. After setup, GitHub settings stay collapsed behind the settings icon. If only the GitHub upload fails after successful extraction, the popup offers a -**Download Fallback**. +**Download Fallback**. Before an upload starts, SourceBraid verifies that the +configured repository exists and is accessible to the token; the popup reports +an explicit error when GitHub returns `404 Not Found`. ## GitHub token diff --git a/background.js b/background.js index 0eab5c4..72c8957 100644 --- a/background.js +++ b/background.js @@ -248,6 +248,7 @@ function downloadMarkdown(message, sendResponse) { async function saveToGitHub(settings, clip) { validateGitHubSave(settings, clip); + await ensureGitHubRepository(settings); const indexPath = Core.buildIndexPath({ rootFolder: settings.rootFolder, url: clip.indexEntry?.url }); const existingIndex = await getContent(settings, indexPath); @@ -286,6 +287,7 @@ async function saveToGitHub(settings, clip) { async function savePdfToGitHub(settings, pdf) { validatePdfSave(settings, pdf); + await ensureGitHubRepository(settings); await ensurePdfWorkflow(settings); const source = Core.parsePdfSourceUrl(pdf.url); @@ -790,6 +792,19 @@ function githubContentsUrl(settings, path, query) { return query ? `${base}?${query}` : base; } +async function ensureGitHubRepository(settings) { + const repository = `${settings.owner}/${settings.repo}`; + const url = `https://api.github.com/repos/${encodeURIComponent(settings.owner)}/${encodeURIComponent(settings.repo)}`; + const response = await githubFetch(settings, url, { method: "GET" }); + + if (response.status === 404) { + throw new Error(`GitHub repository ${repository} was not found or the token cannot access it.`); + } + if (!response.ok) { + throw new Error(await githubError(response, `Could not access GitHub repository ${repository}`)); + } +} + function githubFetch(settings, url, options = {}) { return fetch(url, { ...options, diff --git a/content.js b/content.js index 5b78851..56da2a8 100644 --- a/content.js +++ b/content.js @@ -72,6 +72,10 @@ async function captureMarkdown(options) { article = await tryBloggerApi(metadata, adapterSettings); } + if (!article) { + article = tryGoogleDeepMindBlog(metadata); + } + if (!article) { article = await trySyndicationFeeds(metadata); } @@ -765,6 +769,66 @@ async function tryBloggerApi(metadata, settings) { } } +function tryGoogleDeepMindBlog(metadata) { + let url; + try { + url = new URL(metadata.pageUrl); + } catch (_error) { + return null; + } + if (url.hostname.toLowerCase() !== "deepmind.google" || !url.pathname.startsWith("/blog/")) { + return null; + } + + const main = document.querySelector("main"); + if (!main) { + return null; + } + + const content = document.createElement("div"); + for (const section of Array.from(main.children)) { + if (section.tagName?.toLowerCase() !== "section") { + continue; + } + + const heading = cleanText(section.querySelector("h1, h2, h3")?.textContent || ""); + if (/^related posts$/i.test(heading)) { + break; + } + if (section.matches(".section-cover") || section.querySelector("h1")) { + continue; + } + + const clone = section.cloneNode(true); + clone.querySelectorAll([ + "script", + "style", + "nav", + "aside", + "form", + "iframe", + "noscript", + "svg", + "button", + "[aria-label='Share']" + ].join(",")).forEach((node) => node.remove()); + if (cleanText(clone.textContent).length >= 40 || clone.querySelector("img, video")) { + content.append(clone); + } + } + + if (cleanText(content.textContent).length < 500) { + return null; + } + + return { + ...metadata, + captureMethod: "google-deepmind-dom", + maxImages: 30, + html: content.innerHTML + }; +} + async function trySyndicationFeeds(metadata) { for (const feed of metadata.feedUrls.slice(0, 4)) { try { diff --git a/tests/capture-utils.test.js b/tests/capture-utils.test.js index 0a26cf6..8ca5b69 100644 --- a/tests/capture-utils.test.js +++ b/tests/capture-utils.test.js @@ -1,5 +1,8 @@ const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); const test = require("node:test"); +const vm = require("node:vm"); const Core = require("../capture-utils.js"); @@ -205,3 +208,91 @@ test("buildDocument emits wiki and gist metadata", () => { assert.match(markdown, /wiki_path: "\/Docs\/Page"/); assert.match(markdown, /gist_files: \["README\.md", "example\.js"\]/); }); + +test("GitHub saves reject an unavailable target repository before reading archive paths", async () => { + const requests = []; + const context = { + TextEncoder, + URL, + chrome: { runtime: { onMessage: { addListener() {} } } }, + fetch: async (url) => { + requests.push(url); + return { + ok: false, + status: 404, + statusText: "Not Found", + async json() { return { message: "Not Found" }; } + }; + }, + globalThis: null, + importScripts() {} + }; + context.globalThis = context; + context.SourceBraidCore = Core; + vm.runInNewContext(fs.readFileSync(path.join(__dirname, "..", "background.js"), "utf8"), context); + + await assert.rejects( + context.ensureGitHubRepository({ owner: "missing-owner", repo: "missing-repo", token: "test-token" }), + /GitHub repository missing-owner\/missing-repo was not found or the token cannot access it\./ + ); + assert.deepEqual(requests, ["https://api.github.com/repos/missing-owner/missing-repo"]); +}); + +test("DeepMind blog capture excludes cover and related-post cards", () => { + function section({ heading = "", text = "", classes = "", media = false } = {}) { + const clone = { + textContent: text, + outerHTML: `
${text}
`, + querySelector(selector) { return selector === "img, video" && media ? {} : null; }, + querySelectorAll() { return []; } + }; + return { + tagName: "SECTION", + textContent: text, + matches(selector) { return selector === ".section-cover" && classes.includes("section-cover"); }, + querySelector(selector) { + if (selector === "h1, h2, h3" && heading) return { textContent: heading }; + if (selector === "h1" && classes.includes("has-h1")) return {}; + return null; + }, + cloneNode() { return { ...clone }; } + }; + } + + const content = { + nodes: [], + append(node) { this.nodes.push(node); }, + get textContent() { return this.nodes.map((node) => node.textContent).join(" "); }, + get innerHTML() { return this.nodes.map((node) => node.outerHTML).join(""); } + }; + const articleText = "Article paragraph. ".repeat(40); + const context = { + URL, + Node: { TEXT_NODE: 3, ELEMENT_NODE: 1 }, + chrome: { runtime: { onMessage: { addListener() {} } } }, + document: { + querySelector(selector) { + return selector === "main" ? { + children: [ + section({ heading: "Example", text: "Cover", classes: "section-cover has-h1" }), + section({ text: articleText }), + section({ heading: "Related posts", text: "Related card" }) + ] + } : null; + }, + createElement() { return content; } + }, + globalThis: null + }; + context.globalThis = context; + context.SourceBraidCore = Core; + vm.runInNewContext(fs.readFileSync(path.join(__dirname, "..", "content.js"), "utf8"), context); + + const result = context.tryGoogleDeepMindBlog({ + pageUrl: "https://deepmind.google/blog/example/", + title: "Example" + }); + assert.equal(result.captureMethod, "google-deepmind-dom"); + assert.match(result.html, /Article paragraph/); + assert.doesNotMatch(result.html, /Cover|Related card/); +});