diff --git a/packages/helloao-cli/actions.ts b/packages/helloao-cli/actions.ts index 16a7192..c8e7545 100644 --- a/packages/helloao-cli/actions.ts +++ b/packages/helloao-cli/actions.ts @@ -69,6 +69,8 @@ import { findBibleMultiConverterJar, promptForBibleMultiConverter, convertUsfmToUsx3, + isUsfmPreprocessingOutdated, + USFM_PREPROCESSING_VERSION, printConversionInstructions, } from './conversion.js'; import { loadMetadataOverride } from './meta.js'; @@ -1756,16 +1758,18 @@ export async function sourceTranslations( logger.log('Connecting to database for download tracking...'); db = await database.getDb(options.db); sourceExists = db.prepare( - 'SELECT usfmZipEtag, usfmDownloadDate FROM EBibleSource WHERE id = @id AND sha256 = @sha256;' + 'SELECT usfmZipEtag, usfmDownloadDate, usfmPreprocessingVersion FROM EBibleSource WHERE id = @id AND sha256 = @sha256;' ); let skippedByDatabase = 0; let sourcesChecked = 0; let notDownloaded = 0; + let outdatedConversion = 0; filteredSources = filteredSources.filter((source) => { const existingSource = sourceExists.get(source) as { usfmZipEtag: string; usfmDownloadDate: string; + usfmPreprocessingVersion: number | null; }; sourcesChecked++; if (existingSource) { @@ -1781,6 +1785,17 @@ export async function sourceTranslations( source.usfmDownloadDate = existingSource.usfmDownloadDate as any; + if ( + convertToUsx3 && + isUsfmPreprocessingOutdated( + existingSource.usfmPreprocessingVersion + ) + ) { + // Converted before the current USFM preprocessing (or not converted at all) + outdatedConversion++; + return true; + } + // If specific translations were requested, include sources that match if ( translations && @@ -1809,8 +1824,9 @@ export async function sourceTranslations( logger.log(` • Checked: ${sourcesChecked}`); logger.log(` • Skipped: ${skippedByDatabase}`); logger.log(` • New: ${notDownloaded}`); + logger.log(` • Outdated USX3 conversion: ${outdatedConversion}`); logger.log( - ` • Needs Update: ${filteredSources.length - notDownloaded}` + ` • Needs Update: ${filteredSources.length - notDownloaded - outdatedConversion}` ); sourceUpsert = db.prepare(`INSERT INTO EBibleSource( @@ -1819,14 +1835,14 @@ export async function sourceTranslations( newTestamentBooks, newTestamentChapters, newTestamentVerses, apocryphaBooks, apocryphaChapters, apocryphaVerses, redistributable, sourceDate, updateDate, usfmDownloadDate, - usfmDownloadPath, sha256, usfmZipUrl, usfmZipEtag, FCBHID + usfmDownloadPath, sha256, usfmZipUrl, usfmZipEtag, FCBHID, usfmPreprocessingVersion ) VALUES ( @id, @translationId, @title, @shortTitle, @languageCode, @textDirection, @copyright, @description, @oldTestamentBooks, @oldTestamentChapters, @oldTestamentVerses, @newTestamentBooks, @newTestamentChapters, @newTestamentVerses, @apocryphaBooks, @apocryphaChapters, @apocryphaVerses, @redistributable, @sourceDate, @updateDate, @usfmDownloadDate, - @usfmDownloadPath, @sha256, @usfmZipUrl, @usfmZipEtag, @FCBHID + @usfmDownloadPath, @sha256, @usfmZipUrl, @usfmZipEtag, @FCBHID, @usfmPreprocessingVersion ) ON CONFLICT(id) DO UPDATE SET translationId = excluded.translationId, title = excluded.title, shortTitle = excluded.shortTitle, languageCode = excluded.languageCode, textDirection = excluded.textDirection, copyright = excluded.copyright, @@ -1839,7 +1855,7 @@ export async function sourceTranslations( updateDate = excluded.updateDate, usfmDownloadDate = excluded.usfmDownloadDate, usfmDownloadPath = excluded.usfmDownloadPath, sha256 = excluded.sha256, usfmZipUrl = excluded.usfmZipUrl, usfmZipEtag = excluded.usfmZipEtag, - FCBHID = excluded.FCBHID; + FCBHID = excluded.FCBHID, usfmPreprocessingVersion = excluded.usfmPreprocessingVersion; `); } else { logger.log('Database connection skipped (tracking disabled)'); @@ -2072,6 +2088,7 @@ export async function sourceTranslations( source.usfmDownloadDate = DateTime.utc().toISO() as any; source.usfmDownloadPath = outputDir; + source.usfmPreprocessingVersion = null; if (sourceUpsert) { sourceUpsert.run(source); } @@ -2124,6 +2141,8 @@ export async function sourceTranslations( source.usfmDownloadDate = DateTime.utc().toISO() as any; source.usfmDownloadPath = outputPath; + source.usfmPreprocessingVersion = + USFM_PREPROCESSING_VERSION; sourceUpsert.run(source); } numConverted++; @@ -2319,11 +2338,12 @@ export async function sourceFetchBibleTranslations( let sourceUpsert: Sql.Statement | null = null; if (db) { const sourceExists = db.prepare( - 'SELECT sha256, gitTreeSha, downloadDate, downloadPath FROM FetchBibleSource WHERE id = @id;' + 'SELECT sha256, gitTreeSha, downloadDate, downloadPath, usfmPreprocessingVersion FROM FetchBibleSource WHERE id = @id;' ); let unchanged = 0; let newSources = 0; let changedSources = 0; + let outdatedConversion = 0; selected = selected.filter((entry) => { const existing = sourceExists.get({ id: entry.source.id }) as @@ -2332,6 +2352,7 @@ export async function sourceFetchBibleTranslations( gitTreeSha: string | null; downloadDate: string | null; downloadPath: string | null; + usfmPreprocessingVersion: number | null; } | undefined; const outputPath = path.resolve( @@ -2350,6 +2371,16 @@ export async function sourceFetchBibleTranslations( } else if (hasFetchBibleSourceChanged(existing, entry.source)) { changedSources++; return true; + } else if ( + convertToUsx3 && + entry.source.sourceFormat === 'usfm' && + isUsfmPreprocessingOutdated( + existing.usfmPreprocessingVersion + ) + ) { + // Converted before the current USFM preprocessing (or not converted at all) + outdatedConversion++; + return true; } else if (hasTranslations) { // Explicitly requested translations are always re-downloaded return true; @@ -2363,15 +2394,18 @@ export async function sourceFetchBibleTranslations( logger.log(` • Skipped (unchanged): ${unchanged}`); logger.log(` • New: ${newSources}`); logger.log(` • Needs Update: ${changedSources}`); + logger.log(` • Outdated USX3 conversion: ${outdatedConversion}`); sourceUpsert = db.prepare(`INSERT INTO FetchBibleSource( id, translationId, languageCode, name, englishName, abbreviation, textDirection, year, ebibleId, sourceService, sourceFormat, sourceUpdated, sourceRevision, - attribution, attributionUrl, gitCommitSha, gitTreeSha, downloadDate, downloadPath, sha256 + attribution, attributionUrl, gitCommitSha, gitTreeSha, downloadDate, downloadPath, sha256, + usfmPreprocessingVersion ) VALUES ( @id, @translationId, @languageCode, @name, @englishName, @abbreviation, @textDirection, @year, @ebibleId, @sourceService, @sourceFormat, @sourceUpdated, @sourceRevision, - @attribution, @attributionUrl, @gitCommitSha, @gitTreeSha, @downloadDate, @downloadPath, @sha256 + @attribution, @attributionUrl, @gitCommitSha, @gitTreeSha, @downloadDate, @downloadPath, @sha256, + @usfmPreprocessingVersion ) ON CONFLICT(id) DO UPDATE SET translationId = excluded.translationId, languageCode = excluded.languageCode, name = excluded.name, englishName = excluded.englishName, abbreviation = excluded.abbreviation, @@ -2381,7 +2415,7 @@ export async function sourceFetchBibleTranslations( attribution = excluded.attribution, attributionUrl = excluded.attributionUrl, gitCommitSha = excluded.gitCommitSha, gitTreeSha = excluded.gitTreeSha, downloadDate = excluded.downloadDate, downloadPath = excluded.downloadPath, - sha256 = excluded.sha256; + sha256 = excluded.sha256, usfmPreprocessingVersion = excluded.usfmPreprocessingVersion; `); } else if (!overwrite) { const before = selected.length; @@ -2420,10 +2454,14 @@ export async function sourceFetchBibleTranslations( const recordDownload = ( entry: FetchBibleCatalogEntry, - outputPath: string + outputPath: string, + converted: boolean ) => { entry.source.downloadDate = DateTime.utc().toISO() as any; entry.source.downloadPath = outputPath; + entry.source.usfmPreprocessingVersion = converted + ? USFM_PREPROCESSING_VERSION + : null; sourceUpsert?.run(entry.source); result.downloaded.push(entry.source.translationId); }; @@ -2472,7 +2510,7 @@ export async function sourceFetchBibleTranslations( entry, }); } else { - recordDownload(entry, outputPath); + recordDownload(entry, outputPath, false); } } catch (err) { logger.error(`Error processing ${source.translationId}:`, err); @@ -2509,7 +2547,7 @@ export async function sourceFetchBibleTranslations( jarPath! ); if (success) { - recordDownload(entry, outputPath); + recordDownload(entry, outputPath, true); numConverted++; } else { result.failed.push(entry.source.translationId); diff --git a/packages/helloao-cli/bsb.spec.ts b/packages/helloao-cli/bsb.spec.ts index 1c45653..290d7dd 100644 --- a/packages/helloao-cli/bsb.spec.ts +++ b/packages/helloao-cli/bsb.spec.ts @@ -16,6 +16,7 @@ jest.mock('./conversion.js', () => ({ findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'), promptForBibleMultiConverter: jest.fn(async () => null), convertUsfmToUsx3: jest.fn(), + USFM_PREPROCESSING_VERSION: 1, })); const encoder = new TextEncoder(); diff --git a/packages/helloao-cli/bsb.ts b/packages/helloao-cli/bsb.ts index bd78cec..c21fd98 100644 --- a/packages/helloao-cli/bsb.ts +++ b/packages/helloao-cli/bsb.ts @@ -16,6 +16,7 @@ import { log } from '@helloao/tools'; import { InputTranslationMetadata } from '@helloao/tools/generation/common-types.js'; import { convertUsfmToUsx3, + USFM_PREPROCESSING_VERSION, findBibleMultiConverterJar, printConversionInstructions, promptForBibleMultiConverter, @@ -147,7 +148,9 @@ export async function fetchBsbTranslation( const content = new Uint8Array(await response.arrayBuffer()); const sha256 = createHash('sha256').update(content).digest('hex'); const marker = `Source: ${url}\nSHA-256: ${sha256}\nFormat: ${ - convertToUsx3 ? 'USX3' : 'USFM' + convertToUsx3 + ? `USX3\nPreprocessing: ${USFM_PREPROCESSING_VERSION}` + : 'USFM' }\n`; if ( diff --git a/packages/helloao-cli/conversion.spec.ts b/packages/helloao-cli/conversion.spec.ts new file mode 100644 index 0000000..4aa075d --- /dev/null +++ b/packages/helloao-cli/conversion.spec.ts @@ -0,0 +1,90 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { preprocessUsfm, preprocessUsfmDirectory } from './conversion.js'; + +describe('preprocessUsfm()', () => { + it('should replace \\ref with its display text', () => { + expect( + preprocessUsfm( + '\\f + \\fr 4:6 \\ft See \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\\f*' + ) + ).toBe('\\f + \\fr 4:6 \\ft See Psalm 91:11–12\\f*'); + }); + + it('should not add a space after an opening parenthesis', () => { + expect(preprocessUsfm('\\ft (\\ref 1 Enoch 1:9|1:9\\ref*)')).toBe( + '\\ft (1 Enoch 1:9)' + ); + }); + + it('should support \\ref without a target', () => { + expect(preprocessUsfm('See \\ref Genesis 1:1\\ref*.')).toBe( + 'See Genesis 1:1.' + ); + }); + + it('should support attribute targets and nested \\+ref', () => { + expect( + preprocessUsfm( + 'A \\ref John 3:16|loc="JHN 3:16"\\ref* and \\+ref Mark 1:1|MRK 1:1\\+ref*' + ) + ).toBe('A John 3:16 and Mark 1:1'); + }); + + it('should handle multiple references on one line', () => { + expect( + preprocessUsfm( + '\\ref Isaiah 40:3|ISA 40:3\\ref*; \\ref Malachi 3:1|MAL 3:1\\ref*' + ) + ).toBe('Isaiah 40:3; Malachi 3:1'); + }); + + it('should leave other markers unchanged', () => { + const usfm = + '\\v 1 In the beginning \\f + \\fr 1:1 \\ft text|with pipe\\f*\n\\rem \\refx'; + expect(preprocessUsfm(usfm)).toBe(usfm); + }); +}); + +describe('preprocessUsfmDirectory()', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), 'conversion-spec-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('should write preprocessed USFM files to the output directory without changing the input', async () => { + const input = path.join(dir, 'in'); + const output = path.join(dir, 'out'); + await mkdir(input); + const original = + '\\id MAT\n\\v 6 \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\n'; + await writeFile(path.join(input, 'MAT.usfm'), original); + await writeFile(path.join(input, 'GEN.usfm'), '\\id GEN\n'); + await writeFile( + path.join(input, 'notes.json'), + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + + const changed = await preprocessUsfmDirectory(input, output); + + expect(changed).toBe(1); + expect(await readFile(path.join(output, 'MAT.usfm'), 'utf-8')).toBe( + '\\id MAT\n\\v 6 Psalm 91:11–12\n' + ); + expect(await readFile(path.join(output, 'GEN.usfm'), 'utf-8')).toBe( + '\\id GEN\n' + ); + expect(await readFile(path.join(output, 'notes.json'), 'utf-8')).toBe( + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + expect(await readFile(path.join(input, 'MAT.usfm'), 'utf-8')).toBe( + original + ); + }); +}); diff --git a/packages/helloao-cli/conversion.ts b/packages/helloao-cli/conversion.ts index ea6309f..4a90672 100644 --- a/packages/helloao-cli/conversion.ts +++ b/packages/helloao-cli/conversion.ts @@ -1,6 +1,14 @@ -import { mkdir, readdir } from 'node:fs/promises'; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; import { existsSync } from 'fs'; -import path from 'path'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { execFile as execFileCallback } from 'child_process'; import { promisify } from 'util'; import { confirm, input } from '@inquirer/prompts'; @@ -92,7 +100,88 @@ export async function promptForBibleMultiConverter(): Promise { } /** - * Converts USFM directory to USX3 using BibleMultiConverter + * The version of the USFM preprocessing that convertUsfmToUsx3() does. + * Increase this when preprocessUsfm() changes so that fetchers that skip unchanged sources + * (e.g. fetchBsbTranslation()) convert them again. + */ +export const USFM_PREPROCESSING_VERSION = 1; + +/** + * Determines whether a source that was converted to USX3 with the given preprocessing version needs to be converted again, + * because it was converted before the current USFM preprocessing (or wasn't converted at all). + * @param version The preprocessing version that the source was converted with. Null or undefined if it wasn't converted. + */ +export function isUsfmPreprocessingOutdated( + version: number | null | undefined +): boolean { + return version !== USFM_PREPROCESSING_VERSION; +} + +/** + * Matches a USFM 3 reference (`\\ref TEXT|TARGET\\ref*`), including the nested form (`\\+ref ...\\+ref*`). + * Group 1 is the "+" of a nested marker, group 2 is the display text, and group 3 is the optional link target + * (e.g. `PSA 91:11-12` or `loc="PSA 91:11-12"`). + */ +const USFM_REF_REGEX = + /\\(\+?)ref\s((?:(?!\\\+?ref\*)[^|])*?)(?:\|((?:(?!\\\+?ref\*)[\s\S])*?))?\\\1ref\*/g; + +/** + * The extensions of the files that are treated as USFM by preprocessUsfmDirectory(). + */ +const USFM_EXTENSIONS = new Set(['.usfm', '.sfm']); + +/** + * Rewrites the USFM markup that BibleMultiConverter doesn't understand so that the converted USX3 contains + * only the text that readers should see. + * + * - `\\ref TEXT|TARGET\\ref*` is replaced by `TEXT`. BibleMultiConverter doesn't know the `\\ref` marker, so it drops + * the markers and keeps the text on both sides of the pipe (e.g. "Psalm 91:11–12|PSA 91:11-12"), and it adds a space + * where the opening marker was (e.g. "( 1 Enoch 1:9|1:9)"). See https://github.com/HelloAOLab/bible-api/issues/144. + * + * @param usfm The USFM content. + * @returns The preprocessed USFM content. + */ +export function preprocessUsfm(usfm: string): string { + return usfm.replace(USFM_REF_REGEX, (_match, _plus, text: string) => text); +} + +/** + * Copies the files in the given directory to the given output directory and runs preprocessUsfm() on every USFM file. + * Other files are copied unchanged. + * @param inputDir The directory that contains the USFM files. + * @param outputDir The directory that the preprocessed files should be written to. + * @returns The number of USFM files that were changed. + */ +export async function preprocessUsfmDirectory( + inputDir: string, + outputDir: string +): Promise { + await mkdir(outputDir, { recursive: true }); + let changed = 0; + for (const entry of await readdir(inputDir, { withFileTypes: true })) { + if (!entry.isFile()) { + continue; + } + const inputPath = path.join(inputDir, entry.name); + const outputPath = path.join(outputDir, entry.name); + const content = await readFile(inputPath); + if (USFM_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) { + const text = content.toString('utf-8'); + const processed = preprocessUsfm(text); + if (processed !== text) { + changed += 1; + await writeFile(outputPath, processed, 'utf-8'); + continue; + } + } + await writeFile(outputPath, content); + } + return changed; +} + +/** + * Converts USFM directory to USX3 using BibleMultiConverter. + * The USFM files are first preprocessed with preprocessUsfm() in a temporary directory, so the input directory is not modified. */ export async function convertUsfmToUsx3( inputDir: string, @@ -100,9 +189,21 @@ export async function convertUsfmToUsx3( jarPath: string ): Promise { const logger = log.getLogger(); + let preprocessedDir: string | null = null; try { logger.log(`Converting USFM files from ${inputDir} to USX3 format...`); + preprocessedDir = await mkdtemp( + path.join(tmpdir(), 'bible-api-usfm-preprocessed-') + ); + const changed = await preprocessUsfmDirectory( + inputDir, + preprocessedDir + ); + if (changed > 0) { + logger.log(`Preprocessed ${changed} USFM files before conversion.`); + } + // Handle overwrite logic for output directory // if (overwrite && existsSync(outputDir)) { // logger.log(`Overwriting existing USX3 directory: ${outputDir}`); @@ -118,7 +219,7 @@ export async function convertUsfmToUsx3( jarPath, 'ParatextConverter', 'USFM', - inputDir, + preprocessedDir, 'USX3', outputDir, '*.usx', @@ -147,6 +248,10 @@ export async function convertUsfmToUsx3( } catch (error: any) { logger.error('Conversion failed:', error.message); return false; + } finally { + if (preprocessedDir) { + await rm(preprocessedDir, { recursive: true, force: true }); + } } } diff --git a/packages/helloao-cli/ebible.ts b/packages/helloao-cli/ebible.ts index a4c5044..855b015 100644 --- a/packages/helloao-cli/ebible.ts +++ b/packages/helloao-cli/ebible.ts @@ -92,6 +92,7 @@ export async function fetchEBibleMetadata(): Promise { usfmZipUrl: null, usfmZipEtag: null, sha256: null, + usfmPreprocessingVersion: null, }; source.sha256 = sha256() diff --git a/packages/helloao-cli/fetch-bible.spec.ts b/packages/helloao-cli/fetch-bible.spec.ts index 88e5b92..68a50fc 100644 --- a/packages/helloao-cli/fetch-bible.spec.ts +++ b/packages/helloao-cli/fetch-bible.spec.ts @@ -189,6 +189,7 @@ describe('createFetchBibleSource()', () => { sourceRevision: 0, downloadDate: null, downloadPath: null, + usfmPreprocessingVersion: null, }); expect(source.sha256).toMatch(/^[0-9a-f]{64}$/); }); diff --git a/packages/helloao-cli/fetch-bible.ts b/packages/helloao-cli/fetch-bible.ts index 8aafec9..84d3565 100644 --- a/packages/helloao-cli/fetch-bible.ts +++ b/packages/helloao-cli/fetch-bible.ts @@ -469,6 +469,7 @@ export function createFetchBibleSource( gitTreeSha: git.treeSha ?? null, downloadDate: null, downloadPath: null, + usfmPreprocessingVersion: null, sha256: null, }; diff --git a/packages/helloao-cli/migrations/20261119000000_add_usfm_preprocessing_version/migration.sql b/packages/helloao-cli/migrations/20261119000000_add_usfm_preprocessing_version/migration.sql new file mode 100644 index 0000000..720e4ac --- /dev/null +++ b/packages/helloao-cli/migrations/20261119000000_add_usfm_preprocessing_version/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "EBibleSource" ADD COLUMN "usfmPreprocessingVersion" INTEGER; +ALTER TABLE "FetchBibleSource" ADD COLUMN "usfmPreprocessingVersion" INTEGER; diff --git a/packages/helloao-cli/schema.prisma b/packages/helloao-cli/schema.prisma index e5ac2bb..8f680b8 100644 --- a/packages/helloao-cli/schema.prisma +++ b/packages/helloao-cli/schema.prisma @@ -98,6 +98,10 @@ model EBibleSource { // The SHA-256 hash of the source // Can be used to determine if the source has changed sha256 String? + + // The version of the USFM preprocessing (USFM_PREPROCESSING_VERSION) that the source was converted to USX3 with. + // Null if the source was not converted. Sources converted with an older version are downloaded again. + usfmPreprocessingVersion Int? } // A translation that is available from the fetch.bible collection @@ -145,6 +149,10 @@ model FetchBibleSource { // The SHA-256 hash of the source metadata // Can be used to determine if the source has changed sha256 String? + + // The version of the USFM preprocessing (USFM_PREPROCESSING_VERSION) that the source was converted to USX3 with. + // Null if the source was not converted. Sources converted with an older version are downloaded again. + usfmPreprocessingVersion Int? } // Files that were input into the API generator diff --git a/packages/helloao-cli/source-preprocessing.spec.ts b/packages/helloao-cli/source-preprocessing.spec.ts new file mode 100644 index 0000000..aa82a6f --- /dev/null +++ b/packages/helloao-cli/source-preprocessing.spec.ts @@ -0,0 +1,289 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from '@zip.js/zip.js'; +import { sourceFetchBibleTranslations, sourceTranslations } from './actions.js'; +import { convertUsfmToUsx3, USFM_PREPROCESSING_VERSION } from './conversion.js'; +import { getDb } from './db.js'; +import { FetchBibleCatalogEntry } from './fetch-bible.js'; +import { EBibleSource, FetchBibleSource } from './prisma-gen/client.js'; + +jest.mock('./conversion.js', () => ({ + ...jest.requireActual('./conversion.js'), + findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'), + promptForBibleMultiConverter: jest.fn(async () => null), + convertUsfmToUsx3: jest.fn(async () => true), +})); + +const encoder = new TextEncoder(); +const USFM = '\\id MAT\n\\c 1\n\\p\n\\v 1 Text.\n'; + +async function createZip(files: Record): Promise { + const writer = new Uint8ArrayWriter(); + const zip = new ZipWriter(writer); + for (const [file, content] of Object.entries(files)) { + await zip.add(file, new Uint8ArrayReader(encoder.encode(content))); + } + await zip.close(); + return await writer.getData(); +} + +function ebibleSource(): EBibleSource { + return { + id: 'engtest', + translationId: 'eng_test', + title: 'Test Bible', + shortTitle: 'Test', + textDirection: 'ltr', + languageCode: 'eng', + copyright: 'Public Domain', + description: 'Test', + oldTestamentBooks: 0, + oldTestamentChapters: 0, + oldTestamentVerses: 0, + newTestamentBooks: 1, + newTestamentChapters: 1, + newTestamentVerses: 1, + apocryphaBooks: 0, + apocryphaChapters: 0, + apocryphaVerses: 0, + redistributable: 'TRUE' as any, + FCBHID: 'ENGTST', + sourceDate: '2024-01-01T00:00:00.000Z' as any, + updateDate: '2024-01-01T00:00:00.000Z' as any, + usfmDownloadDate: null, + usfmDownloadPath: null, + usfmZipUrl: null, + usfmZipEtag: null, + sha256: 'abc123', + usfmPreprocessingVersion: null, + }; +} + +function fetchBibleEntry(): FetchBibleCatalogEntry { + const source: FetchBibleSource = { + id: 'tst_test', + translationId: 'tst_test', + languageCode: 'tst', + name: 'Test', + englishName: 'Test', + abbreviation: 'TST', + textDirection: 'ltr', + year: 2024, + ebibleId: null, + sourceService: 'manual', + sourceFormat: 'usfm', + sourceUpdated: null, + sourceRevision: null, + attribution: null, + attributionUrl: null, + gitCommitSha: null, + gitTreeSha: 'tree', + downloadDate: null, + downloadPath: null, + sha256: 'def456', + usfmPreprocessingVersion: null, + }; + return { + source, + item: { + name: { + local: 'Test', + local_abbrev: '', + english: 'Test', + english_abbrev: 'TST', + }, + year: 2024, + books_ot: [], + books_nt: ['mat'], + direction: 'ltr', + copyright: { + licenses: [{ license: 'public', url: null }], + attribution: '', + attribution_url: '', + }, + tags: [], + } as any, + meta: {} as any, + books: ['mat'], + collectionUrl: 'https://example.com/collection', + }; +} + +describe('USFM preprocessing version tracking', () => { + const originalFetch = global.fetch; + let dir: string; + let dbPath: string; + let fetchedUrls: string[]; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), 'source-preprocessing-')); + dbPath = path.join(dir, 'bible-api.db'); + fetchedUrls = []; + const zip = await createZip({ 'MAT.usfm': USFM }); + global.fetch = jest.fn(async (url: any) => { + const u = String(url); + fetchedUrls.push(u); + if (u.includes('details.php')) { + return new Response('engtest_usfm.zip'); + } + if (u.endsWith('_usfm.zip')) { + return new Response(zip); + } + return new Response(USFM); + }) as any; + (convertUsfmToUsx3 as jest.Mock).mockClear(); + }); + + afterEach(async () => { + global.fetch = originalFetch; + await rm(dir, { recursive: true, force: true }); + }); + + async function getVersion(table: string, id: string) { + const db = await getDb(dbPath); + try { + return ( + db + .prepare( + `SELECT usfmPreprocessingVersion FROM ${table} WHERE id = ?;` + ) + .get(id) as { usfmPreprocessingVersion: number | null } + ).usfmPreprocessingVersion; + } finally { + db.close(); + } + } + + async function setVersion( + table: string, + id: string, + version: number | null + ) { + const db = await getDb(dbPath); + try { + db.prepare( + `UPDATE ${table} SET usfmPreprocessingVersion = ? WHERE id = ?;` + ).run(version, id); + } finally { + db.close(); + } + } + + describe('sourceTranslations()', () => { + const source = (o = {}) => [{ ...ebibleSource(), ...o }]; + const options = () => ({ + db: dbPath, + convertToUsx3: true, + sources: source(), + ignorelist: [], + }); + + it('should record the preprocessing version and skip sources converted with it', async () => { + const first = await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(first.downloaded).toEqual(['eng_test']); + expect(await getVersion('EBibleSource', 'engtest')).toBe( + USFM_PREPROCESSING_VERSION + ); + + fetchedUrls = []; + const second = await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(second.downloaded).toEqual([]); + expect(fetchedUrls).toEqual([]); + }); + + it('should download sources converted with an older preprocessing version again', async () => { + await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + await setVersion('EBibleSource', 'engtest', null); + + fetchedUrls = []; + const result = await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(result.downloaded).toEqual(['eng_test']); + expect(fetchedUrls.length).toBeGreaterThan(0); + expect(await getVersion('EBibleSource', 'engtest')).toBe( + USFM_PREPROCESSING_VERSION + ); + }); + + it('should not download unconverted sources again when not converting', async () => { + const opts = { ...options(), convertToUsx3: false }; + await sourceTranslations(path.join(dir, 'out'), undefined, opts); + expect(await getVersion('EBibleSource', 'engtest')).toBeNull(); + + fetchedUrls = []; + const result = await sourceTranslations( + path.join(dir, 'out'), + undefined, + opts + ); + expect(result.downloaded).toEqual([]); + expect(fetchedUrls).toEqual([]); + }); + }); + + describe('sourceFetchBibleTranslations()', () => { + const options = () => ({ + db: dbPath, + convertToUsx3: true, + includeEBible: true, + catalog: [fetchBibleEntry()], + ignorelist: [], + }); + + it('should record the preprocessing version and skip sources converted with it', async () => { + const first = await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(first.downloaded).toEqual(['tst_test']); + expect(await getVersion('FetchBibleSource', 'tst_test')).toBe( + USFM_PREPROCESSING_VERSION + ); + + fetchedUrls = []; + const second = await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(second.downloaded).toEqual([]); + expect(fetchedUrls).toEqual([]); + }); + + it('should download sources converted with an older preprocessing version again', async () => { + await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + await setVersion('FetchBibleSource', 'tst_test', null); + + const result = await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(result.downloaded).toEqual(['tst_test']); + expect(await getVersion('FetchBibleSource', 'tst_test')).toBe( + USFM_PREPROCESSING_VERSION + ); + }); + }); +});