diff --git a/package.json b/package.json index 9dd8746..d37b1c6 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "translate:docs": "tsx ./tools/translate-docs.ts", "fill:docs-verses": "tsx ./tools/fill-docs-verses.ts", "rename:cuv:audio": "tsx ./tools/rename-audio.ts", + "fetch:wordproject:audio": "tsx ./tools/fetch-wordproject-audio.ts", "generate:kotlin": "npm run cli generate-kotlin-models", "cli": "npm run build && node --max-old-space-size=8192 ./packages/helloao-cli/dist/esm/cli.js", "db:clone": "npm run cli init ./free-use-bible-api.download.db -- --source https://bible.helloao.org/bible.db", @@ -59,6 +60,7 @@ "@types/markdown-it": "^12.2.3", "@types/node": "^22.7.4", "@types/papaparse": "5.3.15", + "@zip.js/zip.js": "^2.7.62", "del": "7.0.0", "esbuild": "0.24.0", "fast-glob": "^3.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 39f6cfd..c0c75cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,6 +31,9 @@ importers: '@types/papaparse': specifier: 5.3.15 version: 5.3.15 + '@zip.js/zip.js': + specifier: ^2.7.62 + version: 2.7.62 del: specifier: 7.0.0 version: 7.0.0 diff --git a/tools/fetch-wordproject-audio.ts b/tools/fetch-wordproject-audio.ts new file mode 100644 index 0000000..9002257 --- /dev/null +++ b/tools/fetch-wordproject-audio.ts @@ -0,0 +1,691 @@ +/** + * Downloads the Wordproject audio Bibles (https://www.wordproject.org/bibles/audio/index.htm) + * one zip per book, and unzips them into the folder structure the Free Use Bible API serves + * audio from: + * + * {output}/api/{translation}/{bookId}/{chapter}/audio/{reader}.mp3 + * + * The reader is "wordproject" unless a set says otherwise (sets that share a translation with + * another set, like the Cantonese reading of the Chinese Union Version, get their own reader). + * + * Which translation each audio Bible is written to, and how well it matches, is defined in + * ./wordproject-audio.ts. Run with --list to see it. + * + * Usage: + * + * npm run fetch:wordproject:audio -- --output ./audio-out + * npm run fetch:wordproject:audio -- --output ./audio-out --set 17_amharic --set 01_english + * npm run fetch:wordproject:audio -- --list + * + * Downloaded zips are kept in --cache so that a second run only downloads what is missing. + */ + +import { existsSync } from 'node:fs'; +import { + mkdir, + readFile, + rename, + rm, + unlink, + writeFile, +} from 'node:fs/promises'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; +import { + configure, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, +} from '@zip.js/zip.js'; +import { + bookChapterCountMap, + bookOrderMap, +} from '../packages/helloao-tools/generation/book-order.ts'; +import { + DEFAULT_READER, + getBookNumbers, + getChapterFromEntryName, + getZipUrls, + WORDPROJECT_AUDIO_SETS, + WORDPROJECT_MATCH_LEVELS, + type WordprojectAudioSet, + type WordprojectMatch, +} from './wordproject-audio.ts'; + +configure({ useWebWorkers: false }); + +const DEFAULT_CACHE = './sources/wordproject-audio'; +const DEFAULT_MIN_MATCH: WordprojectMatch = 'poor'; +const DOWNLOAD_ATTEMPTS = 2; + +const HELP = `Usage: npm run fetch:wordproject:audio -- --output [options] + +Downloads the Wordproject audio Bibles one zip per book and unzips them into +{output}/api/{translation}/{bookId}/{chapter}/audio/{reader}.mp3 + +Options: + -o, --output Directory to write the api/ tree into (required unless --list) + -s, --set Only fetch this audio Bible, e.g. 17_amharic. Can be repeated + or comma separated. Defaults to every set. + --min-match Skip sets whose translation match is worse than this: + ${WORDPROJECT_MATCH_LEVELS.filter((m) => m !== 'none').join(', ')} (default: ${DEFAULT_MIN_MATCH}) + --map = Write a set to these translations instead of the built-in ones, + e.g. --map 35_zulu=zul_bib. Comma separate several translations. + --all-translations Write a copy to every matching translation instead of only the first + --cache Where to keep downloaded zips (default: ${DEFAULT_CACHE}) + --delete-zips Delete each zip after it is unzipped + --mirror Another host to try when a download fails. Can be repeated. + --concurrency Number of books to download at once (default: 3) + --overwrite Replace audio files that already exist + --dry-run Print what would be downloaded and written, without doing it + --list Print every audio Bible and its translations, then exit + -h, --help Show this help + +Behind an HTTP proxy, set NODE_USE_ENV_PROXY=1 so Node's fetch uses HTTPS_PROXY.`; + +/** Maps a book number (1 = GEN) to its book ID. */ +const bookIdByNumber = new Map( + [...bookOrderMap].map(([id, order]) => [order, id]) +); + +interface Options { + output: string; + cache: string; + allTranslations: boolean; + deleteZips: boolean; + mirrors: string[]; + overwrite: boolean; + dryRun: boolean; +} + +interface BookTask { + set: WordprojectAudioSet; + reader: string; + translations: string[]; + bookNumber: number; + bookId: string; +} + +interface BookResult { + task: BookTask; + error?: string; + written: number; + existing: number; + chapters: Set; + problems: string[]; +} + +async function main() { + const { values } = parseArgs({ + options: { + output: { type: 'string', short: 'o' }, + set: { type: 'string', short: 's', multiple: true }, + 'min-match': { type: 'string', default: DEFAULT_MIN_MATCH }, + map: { type: 'string', multiple: true }, + 'all-translations': { type: 'boolean', default: false }, + cache: { type: 'string', default: DEFAULT_CACHE }, + 'delete-zips': { type: 'boolean', default: false }, + mirror: { type: 'string', multiple: true }, + concurrency: { type: 'string', default: '3' }, + overwrite: { type: 'boolean', default: false }, + 'dry-run': { type: 'boolean', default: false }, + list: { type: 'boolean', default: false }, + help: { type: 'boolean', short: 'h', default: false }, + }, + }); + + if (values.help) { + console.log(HELP); + return; + } + + if (values.list) { + printSets(WORDPROJECT_AUDIO_SETS); + return; + } + + if (!values.output) { + console.error('Missing required option: --output\n'); + console.error(HELP); + process.exitCode = 1; + return; + } + + const minMatch = values['min-match'] as WordprojectMatch; + const minMatchIndex = WORDPROJECT_MATCH_LEVELS.indexOf(minMatch); + if (minMatchIndex < 0 || minMatch === 'none') { + throw new Error( + `--min-match must be one of ${WORDPROJECT_MATCH_LEVELS.filter((m) => m !== 'none').join(', ')}` + ); + } + + const concurrency = parseInt(values.concurrency!, 10); + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new Error('--concurrency must be a positive whole number'); + } + + const overrides = parseMapOptions(values.map ?? []); + const sets = selectSets(values.set ?? [], overrides); + + const options: Options = { + output: path.resolve(values.output), + cache: path.resolve(values.cache!), + allTranslations: values['all-translations']!, + deleteZips: values['delete-zips']!, + mirrors: values.mirror ?? [], + overwrite: values.overwrite!, + dryRun: values['dry-run']!, + }; + + const tasks: BookTask[] = []; + const skipped: string[] = []; + const destinations = new Map(); + + for (const set of sets) { + const overridden = overrides.get(set.id); + const translations = overridden ?? set.translations; + if (translations.length === 0) { + skipped.push( + `${set.id}: no matching translation in the API (use --map ${set.id}= to write it somewhere)` + ); + continue; + } + if ( + !overridden && + WORDPROJECT_MATCH_LEVELS.indexOf(set.match) > minMatchIndex + ) { + skipped.push( + `${set.id}: match is "${set.match}", below --min-match ${minMatch}` + ); + continue; + } + + const reader = set.reader ?? DEFAULT_READER; + const targets = options.allTranslations + ? translations + : translations.slice(0, 1); + + for (const translation of targets) { + for (const value of [translation, reader]) { + if (/[\\/]/.test(value) || value === '.' || value === '..') { + throw new Error( + `${set.id}: "${value}" must not contain path separators` + ); + } + } + const key = `${translation}/${reader}`; + const previous = destinations.get(key); + if (previous) { + throw new Error( + `${set.id} and ${previous} would both write ${translation}/.../audio/${reader}.mp3. Give one of them a different reader in tools/wordproject-audio.ts or a different --map.` + ); + } + destinations.set(key, set.id); + } + + for (const bookNumber of getBookNumbers(set)) { + tasks.push({ + set, + reader, + translations: targets, + bookNumber, + bookId: bookIdByNumber.get(bookNumber)!, + }); + } + } + + if (skipped.length > 0) { + console.warn(`Skipping ${skipped.length} audio Bible(s):`); + for (const s of skipped) console.warn(` ${s}`); + console.warn(''); + } + + const selected = [...new Set(tasks.map((t) => t.set))]; + for (const set of selected) { + if (set.match === 'poor' && !overrides.has(set.id)) { + console.warn( + `warning: ${set.id} is a poor match for ${set.translations[0]}: ${set.notes ?? ''}` + ); + } + } + + console.log( + `${options.dryRun ? '[dry-run] ' : ''}Fetching ${tasks.length} book(s) from ${selected.length} audio Bible(s) into ${path.join(options.output, 'api')}\n` + ); + + const results = await runPool(tasks, concurrency, (task) => + processBook(task, options) + ); + + printSummary(results, options); +} + +/** + * Parses --map options like "35_zulu=zul_bib,zul_abc". + */ +function parseMapOptions(values: string[]): Map { + const map = new Map(); + for (const value of values) { + const [id, list] = value.split('='); + if (!id || !list) { + throw new Error( + `--map must look like =, got "${value}"` + ); + } + const set = findSet(id.trim()); + map.set( + set.id, + list + .split(',') + .map((t) => t.trim()) + .filter(Boolean) + ); + } + return map; +} + +/** + * Finds the sets with the given folder (e.g. 04_chinese), number (e.g. 04 or 4) or zip ID (e.g. 2_BR). + * A number can match more than one set, e.g. 02 matches both Portuguese narrations. + */ +function findSets(id: string): WordprojectAudioSet[] { + const exact = WORDPROJECT_AUDIO_SETS.filter( + (s) => s.id === id || s.zipId === id + ); + const sets = + exact.length > 0 + ? exact + : WORDPROJECT_AUDIO_SETS.filter( + (s) => + /^\d+$/.test(id) && + parseInt(s.id, 10) === parseInt(id, 10) + ); + if (sets.length === 0) { + throw new Error( + `Unknown audio Bible "${id}". Run with --list to see the IDs.` + ); + } + return sets; +} + +function findSet(id: string): WordprojectAudioSet { + const sets = findSets(id); + if (sets.length > 1) { + throw new Error( + `"${id}" matches ${sets.map((s) => s.id).join(' and ')}; use the full ID.` + ); + } + return sets[0]; +} + +function selectSets( + values: string[], + overrides: Map +): WordprojectAudioSet[] { + const ids = values + .flatMap((v) => v.split(',')) + .map((v) => v.trim()) + .filter(Boolean); + if (ids.length === 0) { + return WORDPROJECT_AUDIO_SETS; + } + const sets = new Set(ids.flatMap(findSets)); + // A --map for a set that wasn't selected is almost certainly a mistake. + for (const id of overrides.keys()) { + if (![...sets].some((s) => s.id === id)) { + console.warn( + `warning: --map ${id} is ignored because --set doesn't include it` + ); + } + } + return WORDPROJECT_AUDIO_SETS.filter((s) => sets.has(s)); +} + +async function processBook( + task: BookTask, + options: Options +): Promise { + const result: BookResult = { + task, + written: 0, + existing: 0, + chapters: new Set(), + problems: [], + }; + const { set, bookNumber, bookId } = task; + const urls = getZipUrls(set, bookNumber, options.mirrors); + const zipPath = path.join(options.cache, set.id, `${bookNumber}.zip`); + const label = `${set.id} ${bookId}`; + + if (options.dryRun) { + console.log(`[dry-run] ${label}: ${urls[0]}`); + for (const translation of task.translations) { + console.log( + ` -> ${path.relative(options.output, destinationPath(options.output, translation, bookId, '{chapter}', task.reader))}` + ); + } + return result; + } + + try { + if (!existsSync(zipPath)) { + await downloadZip(urls, zipPath); + } + } catch (err) { + result.error = errorMessage(err); + console.error(`failed: ${label}: ${result.error}`); + return result; + } + + try { + await extractBook(task, zipPath, options, result); + } catch (err) { + result.error = `could not unzip ${zipPath}: ${errorMessage(err)}`; + console.error(`failed: ${label}: ${result.error}`); + // Remove the broken zip so the next run downloads it again. + await rm(zipPath, { force: true }); + return result; + } + + if (options.deleteZips) { + await rm(zipPath, { force: true }); + } + + console.log( + `done: ${label}: ${result.chapters.size} chapter(s)` + + (result.existing > 0 ? `, ${result.existing} already existed` : '') + ); + return result; +} + +/** + * Downloads the first URL that returns a zip file. + */ +async function downloadZip(urls: string[], zipPath: string) { + const failures: string[] = []; + for (const url of urls) { + for (let attempt = 1; attempt <= DOWNLOAD_ATTEMPTS; attempt++) { + try { + const response = await fetch(url); + if (!response.ok) { + failures.push(`${url}: HTTP ${response.status}`); + // A 404 won't change on retry. + break; + } + const data = new Uint8Array(await response.arrayBuffer()); + // Some hosts answer missing files with an HTML page. + if (data.length < 4 || data[0] !== 0x50 || data[1] !== 0x4b) { + failures.push(`${url}: not a zip file`); + break; + } + await mkdir(path.dirname(zipPath), { recursive: true }); + const partPath = `${zipPath}.part`; + await writeFile(partPath, data); + await rename(partPath, zipPath); + return; + } catch (err) { + if (attempt < DOWNLOAD_ATTEMPTS) { + await delay(1000 * attempt); + } else { + failures.push(`${url}: ${errorMessage(err)}`); + } + } + } + } + throw new Error( + `no zip could be downloaded:\n ${failures.join('\n ')}` + ); +} + +async function extractBook( + task: BookTask, + zipPath: string, + options: Options, + result: BookResult +) { + const chapterCount = bookChapterCountMap.get(task.bookId)!; + const zip = new ZipReader( + new Uint8ArrayReader(new Uint8Array(await readFile(zipPath))) + ); + try { + const entries = (await zip.getEntries()).filter( + (e) => + !e.directory && + /\.mp3$/i.test(e.filename) && + !e.filename.startsWith('__MACOSX/') && + !path.posix.basename(e.filename).startsWith('._') + ); + + const sources = new Map(); + for (const entry of entries) { + const chapter = getChapterFromEntryName( + entry.filename, + chapterCount, + entries.length + ); + if (chapter === null) { + result.problems.push( + `${entry.filename}: no chapter number in the name` + ); + continue; + } + if (chapter < 1 || chapter > chapterCount) { + result.problems.push( + `${entry.filename}: chapter ${chapter} but ${task.bookId} has ${chapterCount}` + ); + continue; + } + const previous = sources.get(chapter); + if (previous) { + result.problems.push( + `${entry.filename}: chapter ${chapter} already came from ${previous}` + ); + continue; + } + sources.set(chapter, entry.filename); + + const destinations = task.translations.map((translation) => + destinationPath( + options.output, + translation, + task.bookId, + chapter.toString(), + task.reader + ) + ); + const pending = options.overwrite + ? destinations + : destinations.filter((d) => !existsSync(d)); + result.existing += destinations.length - pending.length; + result.chapters.add(chapter); + if (pending.length === 0) { + continue; + } + + const data = await entry.getData!(new Uint8ArrayWriter()); + for (const destination of pending) { + await mkdir(path.dirname(destination), { recursive: true }); + const partPath = `${destination}.part`; + await writeFile(partPath, data); + await rename(partPath, destination).catch(async (err) => { + await unlink(partPath).catch(() => {}); + throw err; + }); + result.written++; + } + } + } finally { + await zip.close(); + } +} + +function destinationPath( + output: string, + translation: string, + bookId: string, + chapter: string, + reader: string +) { + return path.join( + output, + 'api', + translation, + bookId, + chapter, + 'audio', + `${reader}.mp3` + ); +} + +/** + * Runs the given function over the items with at most `limit` running at once. + */ +async function runPool( + items: T[], + limit: number, + fn: (item: T) => Promise +): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker() { + while (next < items.length) { + const index = next++; + results[index] = await fn(items[index]); + } + } + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, worker) + ); + return results; +} + +function printSummary(results: BookResult[], options: Options) { + if (options.dryRun) { + return; + } + + const bySet = new Map(); + for (const r of results) { + const list = bySet.get(r.task.set) ?? []; + list.push(r); + bySet.set(r.task.set, list); + } + + let failedBooks = 0; + let written = 0; + let existing = 0; + let incomplete = 0; + + console.log('\nSummary:'); + for (const [set, list] of bySet) { + const failures = list.filter((r) => r.error); + const gaps: string[] = []; + const problems: string[] = []; + for (const r of list) { + written += r.written; + existing += r.existing; + problems.push(...r.problems.map((p) => `${r.task.bookId}: ${p}`)); + if (r.error) continue; + const count = bookChapterCountMap.get(r.task.bookId)!; + const missing: number[] = []; + for (let c = 1; c <= count; c++) { + if (!r.chapters.has(c)) missing.push(c); + } + if (missing.length > 0) { + gaps.push(`${r.task.bookId} ${missing.join(', ')}`); + } + } + failedBooks += failures.length; + if (gaps.length > 0 || problems.length > 0) incomplete++; + + const ok = list.length - failures.length; + const translations = list[0].task.translations.join(', '); + console.log( + ` ${set.id} -> ${translations} (${list[0].task.reader}, ${set.match}): ${ok}/${list.length} books` + ); + if (failures.length > 0) { + // The full reasons were printed as each book failed. + console.log( + ` failed (${failures.length}): ${failures.map((f) => f.task.bookId).join(', ')}` + ); + } + if (gaps.length > 0) { + console.log(` missing chapters: ${gaps.join('; ')}`); + } + for (const p of problems) { + console.log(` ${p}`); + } + } + + console.log( + `\nWrote ${written} file(s)` + + (existing > 0 + ? `, ${existing} already existed (use --overwrite to replace)` + : '') + + (failedBooks > 0 ? `, ${failedBooks} book(s) failed` : '') + + (incomplete > 0 + ? `, ${incomplete} audio Bible(s) have missing or unexpected files` + : '') + ); + + if (failedBooks > 0) { + process.exitCode = 1; + } +} + +function printSets(sets: WordprojectAudioSet[]) { + const rows = sets.map((s) => [ + s.id, + s.name, + s.books === 'nt' ? 'NT' : 'all', + s.match, + s.translations.join(', ') || '-', + s.reader ?? DEFAULT_READER, + ]); + const header = [ + 'id', + 'language', + 'books', + 'match', + 'translations', + 'reader', + ]; + const widths = header.map((h, i) => + Math.max(h.length, ...rows.map((r) => r[i].length)) + ); + const line = (r: string[]) => + r + .map((c, i) => c.padEnd(widths[i])) + .join(' ') + .trimEnd(); + console.log(line(header)); + console.log(line(widths.map((w) => '-'.repeat(w)))); + for (const r of rows) console.log(line(r)); + + const flagged = sets.filter((s) => s.match !== 'exact' && s.notes); + if (flagged.length > 0) { + console.log('\nNotes:'); + for (const s of flagged) { + console.log(` ${s.id} (${s.match}): ${s.notes}`); + } + } +} + +function errorMessage(err: unknown): string { + if (err instanceof Error) { + const cause = (err as any).cause; + return cause instanceof Error + ? `${err.message} (${cause.message})` + : err.message; + } + return String(err); +} + +function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exitCode = 1; +}); diff --git a/tools/wordproject-audio.ts b/tools/wordproject-audio.ts new file mode 100644 index 0000000..f684383 --- /dev/null +++ b/tools/wordproject-audio.ts @@ -0,0 +1,801 @@ +/** + * The audio Bibles published by Wordproject (https://www.wordproject.org/bibles/audio/index.htm) + * and the Free Use Bible API translations each one matches. + * + * Wordproject offers one zip per book, named `{zipId}_{bookNumber}.zip`, for example + * https://www.wordproaudio.net/bibles/app/audio/17_1.zip (Amharic, Genesis). + * Book numbers follow the 66 book Protestant order (1 = GEN ... 66 = REV). + * + * The URLs below come from the book pages on wordproject.org. Some book pages link to the + * wrong zip (for example Somali Psalms links to 36_18.zip, which is Job), so the URLs are + * built from the naming pattern rather than copied from the pages. + * + * ## How the translations were matched + * + * Each recording narrates the text Wordproject publishes for that language. The text of + * John 1 and Genesis 1 on wordproject.org was compared with every API translation in the + * same language (character similarity after removing punctuation and diacritics). + * + * - `exact`: the API has the same text (similarity of about 0.95 or more). + * - `close`: the same translation in a revised edition, another script, or a partial copy, + * or the same text read in a different spoken language. + * - `unverified`: the language matches but Wordproject has no text to compare with. + * - `poor`: the API has no copy of the narrated translation, so this is the closest + * translation in the same language. Listeners will hear different wording than they read. + * - `none`: the API has no translation in this language. Nothing is written for these + * unless a translation is given with `--map` on the command line. + */ + +export type WordprojectMatch = + | 'exact' + | 'close' + | 'unverified' + | 'poor' + | 'none'; + +/** + * Match levels from best to worst. + */ +export const WORDPROJECT_MATCH_LEVELS: readonly WordprojectMatch[] = [ + 'exact', + 'close', + 'unverified', + 'poor', + 'none', +]; + +export interface WordprojectAudioSet { + /** + * The folder of the audio Bible on wordproject.org, e.g. `17_amharic`. + */ + id: string; + + /** + * The ISO 639-3 code of the spoken language. + */ + language: string; + + /** + * The English name of the audio Bible. + */ + name: string; + + /** + * The translation that is narrated, as far as Wordproject says. + */ + narratedText: string; + + /** + * The prefix of the zip file names, e.g. `17` for `17_1.zip`. + */ + zipId: string; + + /** + * The URL that the zip files live under. Defaults to {@link DEFAULT_HOST}. + */ + host?: string; + + /** + * Which books have audio. Defaults to `all`. + */ + books?: 'all' | 'nt'; + + /** + * Zip URLs for books that don't follow the `{host}{zipId}_{book}.zip` pattern. + */ + zipUrls?: Record; + + /** + * The Free Use Bible API translations that this audio matches, best first. + * The audio is written to the first one unless `--all-translations` is passed. + */ + translations: string[]; + + /** + * How well the translations match the narration. + */ + match: WordprojectMatch; + + /** + * The reader ID used for the file name. Defaults to {@link DEFAULT_READER}. + * Sets that share a translation with another set get their own reader so the files don't collide. + */ + reader?: string; + + /** + * Why the translations were picked and anything to watch out for. + */ + notes?: string; +} + +export const DEFAULT_READER = 'wordproject'; + +export const DEFAULT_HOST = 'https://www.wordproaudio.net/bibles/app/audio/'; + +/** + * Hosts that Wordproject serves the same zip files from. + * When a download fails the script tries the same file name on these hosts. + */ +export const MIRROR_HOSTS = [ + 'https://www.wordproaudio.net/bibles/app/audio/', + 'https://www.wordproaudio.org/bibles/app/audio/', + 'http://audio5.wordfree.net/bibles/app/audio/', +]; + +const AUDIO5_HOST = 'http://audio5.wordfree.net/bibles/app/audio/'; +const WORDPROAUDIO_ORG_HOST = 'https://www.wordproaudio.org/bibles/app/audio/'; + +/** + * The Modern Greek Old Testament zips are named after the book instead of numbered. + */ +const MODERN_GREEK_OT_ZIPS = [ + 'genesis', + 'exodus', + 'leviticus', + 'numbers', + 'deuterenomy', + 'joshua', + 'judges', + 'ruth', + '1samuel', + '2samuel', + '1kings', + '2kings', + '1chron', + '2chron', + 'ezra', + 'nehemiah', + 'esther', + 'job', + 'psalms', + 'proverbs', + 'eccles', + 'songsos', + 'isaiah', + 'jeremiah', + 'lament', + 'ezekiel', + 'dan', + 'hosea', + 'joel', + 'amos', + 'obad', + 'jonah', + 'micah', + 'nahum', + 'habak', + 'zepha', + 'hagg', + 'zech', + 'malac', +]; + +export const WORDPROJECT_AUDIO_SETS: WordprojectAudioSet[] = [ + { + id: '01_english', + language: 'eng', + name: 'English', + narratedText: 'King James Version', + zipId: '1', + translations: ['eng_kjv', 'eng_kja', 'eng_cpb'], + match: 'exact', + }, + { + id: '02_portuguese', + language: 'por', + name: 'Portuguese (European narration)', + narratedText: 'João Ferreira de Almeida (Revista e Corrigida)', + zipId: '2', + translations: ['por_blj', 'por_bsl'], + match: 'poor', + notes: 'The API has no Almeida translation. Bíblia Livre is the closest (0.85 in Genesis 1, 0.78 in John 1).', + }, + { + id: '02_portuguese-BR', + language: 'por', + name: 'Portuguese (Brazilian narration)', + narratedText: 'João Ferreira de Almeida (Revista e Corrigida)', + zipId: '2_BR', + host: WORDPROAUDIO_ORG_HOST, + translations: ['por_blj', 'por_bsl'], + match: 'poor', + reader: 'wordproject_br', + notes: 'Same text as 02_portuguese read by a Brazilian narrator, so it gets its own reader. The API has no Almeida translation.', + }, + { + id: '03_hindi', + language: 'hin', + name: 'Hindi', + narratedText: 'Hindi Indian Revised Version', + zipId: '3', + translations: ['HINIRV'], + match: 'close', + notes: 'John 1 is identical (0.98) but Genesis 1 differs more (0.83), so Wordproject may use an older edition of the IRV Old Testament.', + }, + { + id: '04_chinese', + language: 'cmn', + name: 'Chinese (Mandarin)', + narratedText: 'Chinese Union Version', + zipId: '4', + translations: ['cmn_cuv', 'cmn_cu1', 'cmn_cuvmps', 'cmn_cuvmpt'], + match: 'exact', + notes: 'Same text in simplified and traditional characters. The API already serves a "mandarin" CUV reader for these translations.', + }, + { + id: '05_swahili', + language: 'swh', + name: 'Swahili', + narratedText: 'Swahili Union Version', + zipId: '5', + translations: ['swh_bib', 'swh_ulb', 'swh_onmm'], + match: 'poor', + notes: 'The API has no Swahili Union Version. The closest are about 0.7 similar.', + }, + { + id: '06_spanish', + language: 'spa', + name: 'Spanish', + narratedText: 'Reina-Valera 1909', + zipId: '6', + translations: ['spa_r09'], + match: 'exact', + }, + { + id: '07_french', + language: 'fra', + name: 'French', + narratedText: 'Louis Segond 1910', + zipId: '7', + translations: ['fra_lsg'], + match: 'exact', + }, + { + id: '08_russian', + language: 'rus', + name: 'Russian', + narratedText: 'Synodal Translation', + zipId: '8', + translations: ['rus_syn'], + match: 'close', + notes: 'Same translation: John 1 is identical (1.00) but Genesis 1 differs more (0.88), probably spelling or edition differences.', + }, + { + id: '09_german', + language: 'deu', + name: 'German', + narratedText: 'Luther 1912', + zipId: '9', + translations: ['deu_l12'], + match: 'exact', + }, + { + id: '10_italian', + language: 'ita', + name: 'Italian', + narratedText: 'Riveduta 1927', + zipId: '10', + translations: ['ita_riv'], + match: 'exact', + }, + { + id: '11_korean', + language: 'kor', + name: 'Korean', + narratedText: 'Korean Revised Version', + zipId: '11', + translations: ['kor_old'], + match: 'exact', + }, + { + id: '12_japanese', + language: 'jpn', + name: 'Japanese', + narratedText: 'Kougo-yaku (1954/1955)', + zipId: '12', + translations: ['jpn_kou'], + match: 'exact', + }, + { + id: '13_cantonese', + language: 'yue', + name: 'Chinese (Cantonese)', + narratedText: 'Chinese Union Version', + zipId: '13', + translations: ['cmn_cuv', 'cmn_cuvmpt', 'cmn_cu1', 'cmn_cuvmps'], + match: 'close', + reader: 'wordproject_cantonese', + notes: 'The Chinese Union Version read in Cantonese. The API lists the CUV as Mandarin (cmn), so it gets its own reader.', + }, + { + id: '14_indonesian', + language: 'ind', + name: 'Indonesian', + narratedText: 'Terjemahan Baru', + zipId: '14', + translations: ['ind_ayt'], + match: 'poor', + notes: 'The API has no Terjemahan Baru. Alkitab Yang Terbuka is the closest (about 0.75).', + }, + { + id: '15_turkish', + language: 'tur', + name: 'Turkish', + narratedText: 'Kutsal Kitap Yeni Çeviri', + zipId: '15', + translations: ['tur_ytc'], + match: 'poor', + notes: 'The API has no Yeni Çeviri. Yorumsuz Türkçe Çeviri is the closest (0.84 in John 1, 0.74 in Genesis 1).', + }, + { + id: '16_arabic', + language: 'arb', + name: 'Arabic', + narratedText: 'Van Dyck', + zipId: '16', + translations: ['arb_vdv'], + match: 'close', + notes: 'Same translation (0.92 to 0.94); the small differences look like edition and spelling differences.', + }, + { + id: '17_amharic', + language: 'amh', + name: 'Amharic', + narratedText: 'Amharic Bible (1962, Haile Selassie)', + zipId: '17', + translations: ['amh_amh'], + match: 'close', + notes: 'Same text (0.95 in John 1), but the API translation only has the New Testament, so the Old Testament audio has no text to go with it.', + }, + { + id: '18_thai', + language: 'tha', + name: 'Thai', + narratedText: 'Thai KJV', + zipId: '18', + translations: ['tha_kjv'], + match: 'exact', + }, + { + id: '19_vietnamese', + language: 'vie', + name: 'Vietnamese', + narratedText: 'Vietnamese Bible 1934', + zipId: '19', + translations: ['vie_1934'], + match: 'exact', + }, + { + id: '20_farsi', + language: 'pes', + name: 'Farsi (Persian)', + narratedText: 'Old Persian Version', + zipId: '20', + translations: ['pes_pbs'], + match: 'exact', + }, + { + id: '21_urdu', + language: 'urd', + name: 'Urdu', + narratedText: 'Urdu Bible (Revised Version)', + zipId: '21', + translations: ['urd_oucv', 'urd_gvu'], + match: 'poor', + notes: 'The API has no copy of this Urdu text. The closest are about 0.7 similar.', + }, + { + id: '22_bengali', + language: 'ben', + name: 'Bengali', + narratedText: 'Bengali Bible (Old Version)', + zipId: '22', + translations: ['ben_irv'], + match: 'poor', + notes: 'The IRV is a revision of the old version but reads quite differently (0.3 to 0.6).', + }, + { + id: '23_gujarati', + language: 'guj', + name: 'Gujarati', + narratedText: 'Gujarati Bible (Old Version)', + zipId: '23', + translations: ['guj_irv'], + match: 'poor', + notes: 'The IRV is a revision of the old version but reads quite differently (about 0.4).', + }, + { + id: '24_kannada', + language: 'kan', + name: 'Kannada', + narratedText: 'Kannada Bible (Old Version)', + zipId: '24', + translations: ['kan_irv'], + match: 'poor', + notes: 'The IRV is a revision of the old version (0.58 to 0.70).', + }, + { + id: '25_malayalam', + language: 'mal', + name: 'Malayalam', + narratedText: 'Sathyavedapusthakam (1910)', + zipId: '25', + translations: ['mal_fbf', 'mal_irv'], + match: 'exact', + notes: 'mal_fbf is the same text; mal_irv is a close revision (about 0.92).', + }, + { + id: '26_oriya', + language: 'ory', + name: 'Oriya', + narratedText: 'Oriya Bible (Old Version)', + zipId: '26', + translations: ['ory_irv'], + match: 'poor', + notes: 'The IRV is a revision of the old version but reads quite differently (about 0.4).', + }, + { + id: '27_punjabi', + language: 'pan', + name: 'Punjabi', + narratedText: 'Punjabi Bible (Old Version)', + zipId: '27', + translations: ['pan_irv'], + match: 'poor', + notes: 'The IRV is a revision of the old version (0.57 in John 1, 0.82 in Genesis 1).', + }, + { + id: '28_marathi', + language: 'mar', + name: 'Marathi', + narratedText: 'Marathi Bible (Old Version)', + zipId: '28', + translations: ['mar_irv'], + match: 'poor', + notes: 'The IRV is a revision of the old version (0.46 to 0.72).', + }, + { + id: '29_telugu', + language: 'tel', + name: 'Telugu', + narratedText: 'Telugu Bible (Old Version)', + zipId: '29', + translations: ['tel_irv'], + match: 'poor', + notes: 'The IRV is a revision of the old version but reads quite differently (about 0.5).', + }, + { + id: '30_tamil', + language: 'tam', + name: 'Tamil', + narratedText: 'Tamil Bible (Old Version)', + zipId: '30', + translations: ['tam_irv'], + match: 'close', + notes: 'The IRV is a light revision of the old version (0.86 to 0.89).', + }, + { + id: '31_serbian', + language: 'srp', + name: 'Serbian', + narratedText: 'Daničić-Karadžić', + zipId: '31', + translations: ['srp_dkl', 'srp_dkc'], + match: 'exact', + notes: 'srp_dkl is the same text in Latin script, srp_dkc in Cyrillic.', + }, + { + id: '32_czech', + language: 'ces', + name: 'Czech', + narratedText: 'Bible Kralická', + zipId: '32', + translations: ['ces_bkr'], + match: 'exact', + }, + { + id: '33_polish', + language: 'pol', + name: 'Polish', + narratedText: 'Biblia Gdańska (1632)', + zipId: '33', + translations: ['pol_ubg'], + match: 'poor', + notes: 'The Updated Gdańsk Bible modernises the wording heavily (0.84 in John 1, 0.65 in Genesis 1).', + }, + { + id: '34_romanian', + language: 'ron', + name: 'Romanian', + narratedText: 'Cornilescu 1924', + zipId: '34', + translations: ['ron_924'], + match: 'close', + notes: 'ron_924 is the Cornilescu text transliterated into Cyrillic script, so the words match the audio but the script does not. The only Latin script option, ron_btf, is a different translation (about 0.8).', + }, + { + id: '35_zulu', + language: 'zul', + name: 'Zulu', + narratedText: 'Zulu Bible (1959)', + zipId: '35', + translations: [], + match: 'none', + notes: 'The API has no Zulu translation.', + }, + { + id: '36_somali', + language: 'som', + name: 'Somali', + narratedText: 'Kitaabka Quduuska Ah', + zipId: '36', + translations: ['som_sim'], + match: 'exact', + notes: 'The Psalms, Obadiah and Colossians pages link to the wrong zip; the pattern URL is used instead.', + }, + { + id: '39_bulgarian', + language: 'bul', + name: 'Bulgarian', + narratedText: 'Bulgarian Bible', + zipId: '39', + translations: [], + match: 'none', + notes: 'The API has no Bulgarian translation.', + }, + { + id: '40_mod_greek_2', + language: 'ell', + name: 'Modern Greek', + narratedText: 'Modern Greek Bible', + zipId: '40', + zipUrls: Object.fromEntries( + MODERN_GREEK_OT_ZIPS.map((name, i) => [ + i + 1, + `http://audio4.wordfree.net/bibles/app/audio/40_${String(i + 1).padStart(2, '0')}_${name}.zip`, + ]) + ), + translations: [], + match: 'none', + notes: 'The API has no Modern Greek translation. The Old Testament zips are named after the book and hosted on audio4.wordfree.net.', + }, + { + id: '41_latin', + language: 'lat', + name: 'Latin', + narratedText: 'Vulgate', + zipId: '41', + books: 'nt', + translations: ['lat_clv'], + match: 'exact', + notes: 'New Testament only.', + }, + { + id: '42_luganda', + language: 'lug', + name: 'Luganda', + narratedText: 'Luganda Bible (Bible Society of Uganda)', + zipId: '42', + translations: ['lug_bib'], + match: 'poor', + notes: 'The API translation is a newer Luganda text (0.63 to 0.75).', + }, + { + id: '43_burmese', + language: 'mya', + name: 'Burmese', + narratedText: 'Judson Bible', + zipId: '43', + translations: ['mya_jvb'], + match: 'exact', + }, + { + id: '44_hebrew', + language: 'heb', + name: 'Hebrew', + narratedText: + 'Modern Hebrew Bible (Masoretic Old Testament, modern Hebrew New Testament)', + zipId: '44', + translations: ['heb_mod', 'heb_wlc'], + match: 'exact', + notes: 'heb_mod is the same text. heb_wlc only has the Old Testament, which matches (0.99).', + }, + { + id: '45_hungarian', + language: 'hun', + name: 'Hungarian', + narratedText: 'Károli', + zipId: '45', + translations: ['hun_bib'], + match: 'close', + notes: 'Close wording in John 1 (0.94), but the API translation only has the New Testament, so the Old Testament audio has no text to go with it.', + }, + { + id: '46_tagalog', + language: 'tgl', + name: 'Tagalog', + narratedText: 'Ang Dating Biblia (1905)', + zipId: '46', + translations: ['tgl_bib', 'tgl_ulb'], + match: 'poor', + notes: 'The API has no Ang Dating Biblia. The closest are about 0.5 to 0.65 similar.', + }, + { + id: '47_khmer', + language: 'khm', + name: 'Khmer', + narratedText: 'Khmer Bible', + zipId: '47', + host: WORDPROAUDIO_ORG_HOST, + books: 'nt', + translations: [], + match: 'none', + notes: 'New Testament only. The API has no Khmer translation.', + }, + { + id: '58_greek', + language: 'grc', + name: 'Koine Greek', + narratedText: "Scrivener's Textus Receptus (1894)", + zipId: '58', + books: 'nt', + translations: ['grc_gtr'], + match: 'exact', + notes: 'New Testament only. The Old Testament pages are leftovers that link to Modern Greek audio.', + }, + { + id: '61_chinese_cat', + language: 'cmn', + name: 'Chinese Catholic (Mandarin)', + narratedText: 'Si Gao Version (思高本)', + zipId: '61', + translations: [], + match: 'none', + notes: 'The API has no Catholic Chinese translation. Every Protestant Chinese translation scores 0.6 or less, and the canon differs.', + }, + { + id: '62_cantonese_cath', + language: 'yue', + name: 'Chinese Catholic (Cantonese)', + narratedText: 'Si Gao Version (思高本)', + zipId: '62', + translations: [], + match: 'none', + notes: 'The API has no Catholic Chinese translation.', + }, + { + id: '63_finnish', + language: 'fin', + name: 'Finnish', + narratedText: 'Raamattu (1933/1938)', + zipId: '63', + books: 'nt', + translations: ['fin_bib'], + match: 'poor', + notes: 'New Testament only. The only Finnish translation in the API is a modern paraphrase (about 0.4).', + }, + { + id: '64_swedish', + language: 'swe', + name: 'Swedish', + narratedText: 'Bibeln 1917', + zipId: '64', + host: AUDIO5_HOST, + books: 'nt', + translations: ['swe_fol'], + match: 'poor', + notes: 'New Testament only. The API has no 1917 translation; Folkbibeln is the closest (0.76).', + }, + { + id: '65_turkmen', + language: 'tuk', + name: 'Turkmen', + narratedText: 'Turkmen New Testament (IBT)', + zipId: '65', + books: 'nt', + translations: [], + match: 'none', + notes: 'New Testament only. The API has no Turkmen translation.', + }, + { + id: '67_ukrainian', + language: 'ukr', + name: 'Ukrainian', + narratedText: 'Ohienko (1962)', + zipId: '67', + translations: ['ukr_pan', 'ukr_1996'], + match: 'poor', + notes: 'The API has no Ohienko translation; the Kulish translations are about 0.55 to 0.6 similar. Wordproject does not say which text the audio reads, so this assumes Ohienko like its text Bible.', + }, + { + id: '100_uhygur', + language: 'uig', + name: 'Uyghur', + narratedText: 'Uyghur New Testament', + zipId: '100', + host: AUDIO5_HOST, + books: 'nt', + translations: ['uig_ara', 'uig_lat', 'uig_cyr', 'uig_pin'], + match: 'unverified', + notes: 'New Testament only. Wordproject has no Uyghur text to compare with. The four API translations are the same text in different scripts; uig_ara uses the Arabic script most Uyghur readers use.', + }, + { + id: '101_makhuwa', + language: 'vmw', + name: 'Makhuwa (Macua)', + narratedText: 'Makhuwa New Testament', + zipId: '101', + host: AUDIO5_HOST, + books: 'nt', + translations: ['mgh_wft'], + match: 'poor', + notes: 'New Testament only. The API only has Makhuwa-Meetto (mgh), a related but different language from Makhuwa (vmw).', + }, +]; + +/** + * Gets the book numbers (1 = GEN ... 66 = REV) that the given set has audio for. + */ +export function getBookNumbers(set: WordprojectAudioSet): number[] { + const first = set.books === 'nt' ? 40 : 1; + const numbers: number[] = []; + for (let n = first; n <= 66; n++) { + numbers.push(n); + } + return numbers; +} + +/** + * Gets the URLs to try, in order, for the zip of the given book. + * @param set The audio set. + * @param bookNumber The book number (1 = GEN ... 66 = REV). + * @param extraHosts Hosts to try after the built-in ones. + */ +export function getZipUrls( + set: WordprojectAudioSet, + bookNumber: number, + extraHosts: string[] = [] +): string[] { + const override = set.zipUrls?.[bookNumber]; + const fileName = override + ? override.slice(override.lastIndexOf('/') + 1) + : `${set.zipId}_${bookNumber}.zip`; + const primary = override ?? `${set.host ?? DEFAULT_HOST}${fileName}`; + + const urls = [primary]; + for (const host of [...MIRROR_HOSTS, ...extraHosts]) { + const url = `${host.endsWith('/') ? host : host + '/'}${fileName}`; + if (!urls.includes(url)) { + urls.push(url); + } + } + return urls; +} + +/** + * Works out the chapter number of an audio file from its name inside a zip. + * Uses the last number in the file name, so `1.mp3`, `17/1/1.mp3` and + * `Bible_gk_19_psalm_001.mp3` give 1. + * @param entryName The path of the file inside the zip. + * @param chapterCount The number of chapters in the book. + * @param mp3Count The number of mp3 files in the zip. + * @returns The chapter number, or null if it can't be worked out. + */ +export function getChapterFromEntryName( + entryName: string, + chapterCount: number, + mp3Count: number +): number | null { + // Single chapter books are sometimes named after the book only, e.g. "31##obadiah.mp3" or "36_31.mp3" + if (chapterCount === 1 && mp3Count === 1) { + return 1; + } + const baseName = entryName + .slice(entryName.lastIndexOf('/') + 1) + .replace(/\.mp3$/i, ''); + const numbers = baseName.match(/\d+/g); + if (numbers && numbers.length > 0) { + return parseInt(numbers[numbers.length - 1], 10); + } + return null; +}