diff --git a/db/schema.ts b/db/schema.ts index 9d22983..9ab308a 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -53,7 +53,19 @@ export type SourceType = | "codeberg" | "sourceforge" | "play_store" - | "direct"; // APK from website, etc. + | "direct" // APK from website, etc. + // Third-party F-Droid-format repos (see db/seed/data/fdroid-repos.ts) + | "guardian" + | "microg" + | "molly" + | "cromite" + | "bitwarden" + | "threema" + | "session" + | "briar" + | "simplex" + | "newpipe" + | "calyx"; export type AppSourceMetadata = { apkFilterRegex?: string; diff --git a/db/seed/data/fdroid-repos.ts b/db/seed/data/fdroid-repos.ts new file mode 100644 index 0000000..275da6e --- /dev/null +++ b/db/seed/data/fdroid-repos.ts @@ -0,0 +1,100 @@ +import type { SourceType } from "../../schema"; + +/** + * Third-party F-Droid-format repositories (index-v2), typically run by the + * app developers themselves. Each entry becomes its own source type so an + * app can carry several repo sources without colliding on the + * (app_id, source) unique constraint. + */ +export type ThirdPartyFdroidRepo = { + /** Source type slug stored in app_sources.source */ + source: SourceType; + /** Display name — also used as the Obtainium config author */ + name: string; + /** Base repo URL; the index lives at `${repoUrl}/index-v2.json` */ + repoUrl: string; + /** Human-facing page linked from source badges */ + webUrl: string; + /** Filename inside .cache/ */ + cacheFile: string; +}; + +export const thirdPartyFdroidRepos: ThirdPartyFdroidRepo[] = [ + { + source: "guardian", + name: "Guardian Project", + repoUrl: "https://guardianproject.info/fdroid/repo", + webUrl: "https://guardianproject.info/fdroid/", + cacheFile: "guardian-index.json", + }, + { + source: "microg", + name: "microG", + repoUrl: "https://microg.org/fdroid/repo", + webUrl: "https://microg.org/download.html", + cacheFile: "microg-index.json", + }, + { + source: "molly", + name: "Molly", + repoUrl: "https://molly.im/fdroid/foss/fdroid/repo", + webUrl: "https://molly.im/download/fdroid/", + cacheFile: "molly-index.json", + }, + { + source: "cromite", + name: "Cromite", + repoUrl: "https://www.cromite.org/fdroid/repo", + webUrl: "https://www.cromite.org/", + cacheFile: "cromite-index.json", + }, + { + source: "bitwarden", + name: "Bitwarden", + repoUrl: "https://mobileapp.bitwarden.com/fdroid/repo", + webUrl: "https://bitwarden.com/download/", + cacheFile: "bitwarden-index.json", + }, + { + source: "threema", + name: "Threema Libre", + repoUrl: "https://releases.threema.ch/fdroid/repo", + webUrl: "https://threema.ch/en/faq/threema_libre", + cacheFile: "threema-index.json", + }, + { + source: "session", + name: "Session", + repoUrl: "https://fdroid.getsession.org/fdroid/repo", + webUrl: "https://getsession.org/download", + cacheFile: "session-index.json", + }, + { + source: "briar", + name: "Briar", + repoUrl: "https://briarproject.org/fdroid/repo", + webUrl: "https://briarproject.org/download-briar/", + cacheFile: "briar-index.json", + }, + { + source: "simplex", + name: "SimpleX Chat", + repoUrl: "https://app.simplex.chat/fdroid/repo", + webUrl: "https://simplex.chat/downloads/", + cacheFile: "simplex-index.json", + }, + { + source: "newpipe", + name: "NewPipe", + repoUrl: "https://archive.newpipe.net/fdroid/repo", + webUrl: "https://newpipe.net/", + cacheFile: "newpipe-index.json", + }, + { + source: "calyx", + name: "Calyx Institute", + repoUrl: "https://calyxos.gitlab.io/calyx-fdroid-repo/fdroid/repo", + webUrl: "https://calyxinstitute.org/", + cacheFile: "calyx-index.json", + }, +]; diff --git a/db/seed/fetch.ts b/db/seed/fetch.ts index 3f7057e..4f0faa0 100644 --- a/db/seed/fetch.ts +++ b/db/seed/fetch.ts @@ -2,6 +2,7 @@ import "dotenv/config"; import { execFileSync } from "node:child_process"; import * as fs from "node:fs"; import * as path from "node:path"; +import { thirdPartyFdroidRepos } from "./data/fdroid-repos"; const CACHE_DIR = path.resolve(process.cwd(), ".cache"); @@ -79,6 +80,22 @@ async function main() { "IzzyOnDroid index", ); + // Third-party repos are best-effort: a repo being down shouldn't + // block the whole fetch (existing cached copies keep being used). + for (const repo of thirdPartyFdroidRepos) { + try { + await fetchFile( + `${repo.repoUrl}/index-v2.json`, + path.join(CACHE_DIR, repo.cacheFile), + `${repo.name} repo index`, + ); + } catch (err) { + console.warn( + ` [skip] ${repo.name}: ${err instanceof Error ? err.message : err}`, + ); + } + } + fetchGitRepo( SOURCES.obtainium.repo, path.join(CACHE_DIR, SOURCES.obtainium.dir), diff --git a/db/seed/import.ts b/db/seed/import.ts index 0c21960..56a4c91 100644 --- a/db/seed/import.ts +++ b/db/seed/import.ts @@ -8,6 +8,7 @@ import * as schema from "../schema"; import { appSources, apps, proprietaryApps, tags } from "../schema"; import { alternativeMappings } from "./data/alternatives"; import { appOverrides } from "./data/app-overrides"; +import { thirdPartyFdroidRepos } from "./data/fdroid-repos"; import { proprietaryApps as proprietaryAppSeeds } from "./data/proprietary-apps"; import { fdroidAntiFeatureMap, fdroidCategoryMap, tagSeeds } from "./data/tags"; import { webApps as webAppSeeds } from "./data/web-apps"; @@ -31,6 +32,7 @@ const db = drizzle(client, { schema }); const stats = { fdroidParsed: 0, izzyParsed: 0, + thirdPartyParsed: 0, obtainiumParsed: 0, uniqueApps: 0, mergedSources: 0, @@ -69,6 +71,20 @@ function parseAllSources(): ParsedApp[] { console.warn("IzzyOnDroid index not found — run seed:fetch first"); } + for (const repo of thirdPartyFdroidRepos) { + const repoPath = path.join(CACHE_DIR, repo.cacheFile); + if (!fs.existsSync(repoPath)) { + console.warn(`${repo.name} index not found — run seed:fetch first`); + continue; + } + console.log(`Parsing ${repo.name} repo index...`); + const repoIndex = JSON.parse(fs.readFileSync(repoPath, "utf-8")); + const repoApps = parseFDroidIndex(repoIndex, repo); + stats.thirdPartyParsed += repoApps.length; + console.log(` ${repoApps.length} apps parsed`); + allApps.push(...repoApps); + } + const obtainiumDir = path.join(CACHE_DIR, "obtainium"); if (fs.existsSync(obtainiumDir)) { console.log("Parsing Obtainium configs..."); @@ -142,7 +158,14 @@ async function upsertApps(dedupedApps: ParsedApp[]) { license = COALESCE(excluded.license, apps.license), website_url = COALESCE(excluded.website_url, apps.website_url), repository_url = COALESCE(excluded.repository_url, apps.repository_url), - updated_at = excluded.updated_at`, + updated_at = CASE WHEN + apps.name IS NOT excluded.name + OR COALESCE(excluded.description, apps.description) IS NOT apps.description + OR COALESCE(excluded.icon_url, apps.icon_url) IS NOT apps.icon_url + OR COALESCE(excluded.license, apps.license) IS NOT apps.license + OR COALESCE(excluded.website_url, apps.website_url) IS NOT apps.website_url + OR COALESCE(excluded.repository_url, apps.repository_url) IS NOT apps.repository_url + THEN excluded.updated_at ELSE apps.updated_at END`, args: [ appId, parsed.name, @@ -306,7 +329,14 @@ async function upsertProprietaryApps() { ON CONFLICT (slug) DO UPDATE SET name = excluded.name, description = excluded.description, icon_url = excluded.icon_url, website_url = excluded.website_url, - package_name = excluded.package_name, updated_at = excluded.updated_at`, + package_name = excluded.package_name, + updated_at = CASE WHEN + proprietary_apps.name IS NOT excluded.name + OR proprietary_apps.description IS NOT excluded.description + OR proprietary_apps.icon_url IS NOT excluded.icon_url + OR proprietary_apps.website_url IS NOT excluded.website_url + OR proprietary_apps.package_name IS NOT excluded.package_name + THEN excluded.updated_at ELSE proprietary_apps.updated_at END`, args: [ propId, seed.name, @@ -368,7 +398,12 @@ async function upsertWebApps() { description = COALESCE(excluded.description, apps.description), website_url = COALESCE(excluded.website_url, apps.website_url), repository_url = COALESCE(excluded.repository_url, apps.repository_url), - updated_at = excluded.updated_at`, + updated_at = CASE WHEN + apps.name IS NOT excluded.name + OR COALESCE(excluded.description, apps.description) IS NOT apps.description + OR COALESCE(excluded.website_url, apps.website_url) IS NOT apps.website_url + OR COALESCE(excluded.repository_url, apps.repository_url) IS NOT apps.repository_url + THEN excluded.updated_at ELSE apps.updated_at END`, args: [ generateId(), seed.name, @@ -505,7 +540,7 @@ async function main() { console.log(`\n${"═".repeat(50)}`); console.log("Import complete:"); console.log( - ` Parsed: ${stats.fdroidParsed} F-Droid | ${stats.izzyParsed} Izzy | ${stats.obtainiumParsed} Obtainium`, + ` Parsed: ${stats.fdroidParsed} F-Droid | ${stats.izzyParsed} Izzy | ${stats.thirdPartyParsed} third-party repos | ${stats.obtainiumParsed} Obtainium`, ); console.log( ` Deduped: ${stats.uniqueApps} unique (${stats.mergedSources} merged)`, diff --git a/db/seed/parsers/fdroid.ts b/db/seed/parsers/fdroid.ts index 4b1b2a9..50a2964 100644 --- a/db/seed/parsers/fdroid.ts +++ b/db/seed/parsers/fdroid.ts @@ -1,4 +1,5 @@ -import type { ParsedApp } from "../lib/types"; +import type { ThirdPartyFdroidRepo } from "../data/fdroid-repos"; +import type { ParsedApp, ParsedAppSource } from "../lib/types"; type FDroidIndex = { repo: { address: string }; @@ -37,11 +38,19 @@ function extractAntiFeatures( export type FDroidSourceType = "fdroid" | "izzyondroid"; +/** + * Parse any F-Droid-format index-v2 file. Pass "fdroid"/"izzyondroid" for + * the two main repos (which have per-app web pages), or a + * ThirdPartyFdroidRepo entry for a developer-run repo — those get the + * repo's human-facing page as the source URL plus a full Obtainium + * FDroidRepo config so install deep links work. + */ export function parseFDroidIndex( indexJson: FDroidIndex, - sourceType: FDroidSourceType, + sourceType: FDroidSourceType | ThirdPartyFdroidRepo, ): ParsedApp[] { - const repoAddress = indexJson.repo?.address || ""; + const repo = typeof sourceType === "string" ? undefined : sourceType; + const repoAddress = indexJson.repo?.address || repo?.repoUrl || ""; const apps: ParsedApp[] = []; for (const [packageName, pkg] of Object.entries(indexJson.packages)) { @@ -58,10 +67,31 @@ export function parseFDroidIndex( iconUrl = `${repoAddress}${iconPath}`; } - const sourceUrl = - sourceType === "fdroid" - ? `https://f-droid.org/packages/${packageName}/` - : `https://apt.izzysoft.de/fdroid/index/apk/${packageName}`; + let source: ParsedAppSource; + if (repo) { + source = { + source: repo.source, + url: repo.webUrl, + metadata: { + obtainiumConfig: { + id: packageName, + url: repoAddress, + name, + author: repo.name, + additionalSettings: JSON.stringify({ appIdOrName: packageName }), + overrideSource: "FDroidRepo", + }, + }, + }; + } else { + source = { + source: sourceType as FDroidSourceType, + url: + sourceType === "fdroid" + ? `https://f-droid.org/packages/${packageName}/` + : `https://apt.izzysoft.de/fdroid/index/apk/${packageName}`, + }; + } apps.push({ packageName, @@ -74,7 +104,7 @@ export function parseFDroidIndex( repositoryUrl: meta.sourceCode || undefined, categories: meta.categories || [], antiFeatures: extractAntiFeatures(meta.antiFeatures), - sources: [{ source: sourceType, url: sourceUrl }], + sources: [source], }); } diff --git a/src/components/source-badge.tsx b/src/components/source-badge.tsx index 47528f3..279f634 100644 --- a/src/components/source-badge.tsx +++ b/src/components/source-badge.tsx @@ -9,6 +9,17 @@ const sourceStyles: Record = { sourceforge: "border-amber-500/30 bg-amber-500/10 text-amber-400", direct: "border-purple-500/30 bg-purple-500/10 text-purple-400", play_store: "border-emerald-500/30 bg-emerald-500/10 text-emerald-400", + guardian: "border-indigo-500/30 bg-indigo-500/10 text-indigo-400", + microg: "border-sky-500/30 bg-sky-500/10 text-sky-400", + molly: "border-rose-500/30 bg-rose-500/10 text-rose-400", + cromite: "border-cyan-500/30 bg-cyan-500/10 text-cyan-400", + bitwarden: "border-blue-500/30 bg-blue-500/10 text-blue-300", + threema: "border-lime-500/30 bg-lime-500/10 text-lime-400", + session: "border-violet-500/30 bg-violet-500/10 text-violet-400", + briar: "border-fuchsia-500/30 bg-fuchsia-500/10 text-fuchsia-400", + simplex: "border-yellow-500/30 bg-yellow-500/10 text-yellow-400", + newpipe: "border-red-500/30 bg-red-500/10 text-red-400", + calyx: "border-stone-400/30 bg-stone-400/10 text-stone-300", }; const sourceLabels: Record = { @@ -20,6 +31,17 @@ const sourceLabels: Record = { sourceforge: "SourceForge", direct: "Direct", play_store: "Play Store", + guardian: "Guardian Project", + microg: "microG", + molly: "Molly", + cromite: "Cromite", + bitwarden: "Bitwarden", + threema: "Threema", + session: "Session", + briar: "Briar", + simplex: "SimpleX", + newpipe: "NewPipe", + calyx: "Calyx", }; interface SourceBadgeProps { diff --git a/src/lib/obtainium.ts b/src/lib/obtainium.ts index 0c1de83..ed89984 100644 --- a/src/lib/obtainium.ts +++ b/src/lib/obtainium.ts @@ -9,6 +9,7 @@ type ObtainiumSource = { source: string; label: string; link: string; + hasConfig: boolean; }; const SOURCE_PRIORITY: Record = { @@ -17,8 +18,20 @@ const SOURCE_PRIORITY: Record = { codeberg: 2, fdroid: 3, izzyondroid: 4, - sourceforge: 5, - direct: 6, + // Developer-run F-Droid repos + guardian: 5, + microg: 5, + molly: 5, + cromite: 5, + bitwarden: 5, + threema: 5, + session: 5, + briar: 5, + simplex: 5, + newpipe: 5, + calyx: 5, + sourceforge: 6, + direct: 7, }; const SOURCE_LABELS: Record = { @@ -29,6 +42,17 @@ const SOURCE_LABELS: Record = { izzyondroid: "via IzzyOnDroid", sourceforge: "via SourceForge", direct: "Direct download", + guardian: "via Guardian Project repo", + microg: "via microG repo", + molly: "via Molly repo", + cromite: "via Cromite repo", + bitwarden: "via Bitwarden repo", + threema: "via Threema repo", + session: "via Session repo", + briar: "via Briar repo", + simplex: "via SimpleX repo", + newpipe: "via NewPipe repo", + calyx: "via Calyx repo", }; /** @@ -92,18 +116,15 @@ export function getObtainiumSources(sources: SourceInfo[]): ObtainiumSource[] { results.push({ source: source.source, - label: meta?.obtainiumConfig - ? "Obtainium config" - : (SOURCE_LABELS[source.source] ?? source.source), + label: SOURCE_LABELS[source.source] ?? source.source, link, + hasConfig: Boolean(meta?.obtainiumConfig), }); } // Sources with obtainiumConfig get top priority, then by source type return results.sort((a, b) => { - const aHasConfig = a.label === "Obtainium config" ? 0 : 1; - const bHasConfig = b.label === "Obtainium config" ? 0 : 1; - if (aHasConfig !== bHasConfig) return aHasConfig - bHasConfig; + if (a.hasConfig !== b.hasConfig) return a.hasConfig ? -1 : 1; return ( (SOURCE_PRIORITY[a.source] ?? 99) - (SOURCE_PRIORITY[b.source] ?? 99) ); diff --git a/src/routes/-worker-entry.ts b/src/routes/-worker-entry.ts index 6cf698c..ba67b1c 100644 --- a/src/routes/-worker-entry.ts +++ b/src/routes/-worker-entry.ts @@ -286,6 +286,17 @@ const ALLOWED_ICON_HOSTS = [ "assets-prod.sumo.prod.webservices.mozgcp.net", "brave.com", "play-lh.googleusercontent.com", + // Third-party F-Droid repo hosts (db/seed/data/fdroid-repos.ts) + "microg.org", + "molly.im", + "cromite.org", + "mobileapp.bitwarden.com", + "releases.threema.ch", + "fdroid.getsession.org", + "briarproject.org", + "app.simplex.chat", + "archive.newpipe.net", + "calyxos.gitlab.io", ]; const ICON_CACHE_SECONDS = 7 * 24 * 60 * 60; // 7 days