From 80f508b7c6cc748c07218a4923e4dd275fb87136 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 16:15:30 +0000 Subject: [PATCH 1/2] Strip USFM \ref markers before converting USFM to USX3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BibleMultiConverter 0.0.9 doesn't know \ref, so it kept both sides of the pipe ("Psalm 91:11–12|PSA 91:11-12") and added a space where the marker was ("( 1 Enoch 1:9|1:9)"). convertUsfmToUsx3() now copies the USFM files to a temporary directory and replaces \ref TEXT|TARGET\ref* (and \+ref) with TEXT before running the converter, so every fetcher that converts USFM to USX3 gets the fix. The BSB marker file now records the preprocessing version so existing BSB downloads are converted again. Fixes #144 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019LsURyVgHxWYTXpbFoYSPV --- packages/helloao-cli/bsb.spec.ts | 1 + packages/helloao-cli/bsb.ts | 5 +- packages/helloao-cli/conversion.spec.ts | 90 +++++++++++++++++++++ packages/helloao-cli/conversion.ts | 101 +++++++++++++++++++++++- 4 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 packages/helloao-cli/conversion.spec.ts diff --git a/packages/helloao-cli/bsb.spec.ts b/packages/helloao-cli/bsb.spec.ts index 1c45653..290d7dd 100644 --- a/packages/helloao-cli/bsb.spec.ts +++ b/packages/helloao-cli/bsb.spec.ts @@ -16,6 +16,7 @@ jest.mock('./conversion.js', () => ({ findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'), promptForBibleMultiConverter: jest.fn(async () => null), convertUsfmToUsx3: jest.fn(), + USFM_PREPROCESSING_VERSION: 1, })); const encoder = new TextEncoder(); diff --git a/packages/helloao-cli/bsb.ts b/packages/helloao-cli/bsb.ts index bd78cec..c21fd98 100644 --- a/packages/helloao-cli/bsb.ts +++ b/packages/helloao-cli/bsb.ts @@ -16,6 +16,7 @@ import { log } from '@helloao/tools'; import { InputTranslationMetadata } from '@helloao/tools/generation/common-types.js'; import { convertUsfmToUsx3, + USFM_PREPROCESSING_VERSION, findBibleMultiConverterJar, printConversionInstructions, promptForBibleMultiConverter, @@ -147,7 +148,9 @@ export async function fetchBsbTranslation( const content = new Uint8Array(await response.arrayBuffer()); const sha256 = createHash('sha256').update(content).digest('hex'); const marker = `Source: ${url}\nSHA-256: ${sha256}\nFormat: ${ - convertToUsx3 ? 'USX3' : 'USFM' + convertToUsx3 + ? `USX3\nPreprocessing: ${USFM_PREPROCESSING_VERSION}` + : 'USFM' }\n`; if ( diff --git a/packages/helloao-cli/conversion.spec.ts b/packages/helloao-cli/conversion.spec.ts new file mode 100644 index 0000000..4aa075d --- /dev/null +++ b/packages/helloao-cli/conversion.spec.ts @@ -0,0 +1,90 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { preprocessUsfm, preprocessUsfmDirectory } from './conversion.js'; + +describe('preprocessUsfm()', () => { + it('should replace \\ref with its display text', () => { + expect( + preprocessUsfm( + '\\f + \\fr 4:6 \\ft See \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\\f*' + ) + ).toBe('\\f + \\fr 4:6 \\ft See Psalm 91:11–12\\f*'); + }); + + it('should not add a space after an opening parenthesis', () => { + expect(preprocessUsfm('\\ft (\\ref 1 Enoch 1:9|1:9\\ref*)')).toBe( + '\\ft (1 Enoch 1:9)' + ); + }); + + it('should support \\ref without a target', () => { + expect(preprocessUsfm('See \\ref Genesis 1:1\\ref*.')).toBe( + 'See Genesis 1:1.' + ); + }); + + it('should support attribute targets and nested \\+ref', () => { + expect( + preprocessUsfm( + 'A \\ref John 3:16|loc="JHN 3:16"\\ref* and \\+ref Mark 1:1|MRK 1:1\\+ref*' + ) + ).toBe('A John 3:16 and Mark 1:1'); + }); + + it('should handle multiple references on one line', () => { + expect( + preprocessUsfm( + '\\ref Isaiah 40:3|ISA 40:3\\ref*; \\ref Malachi 3:1|MAL 3:1\\ref*' + ) + ).toBe('Isaiah 40:3; Malachi 3:1'); + }); + + it('should leave other markers unchanged', () => { + const usfm = + '\\v 1 In the beginning \\f + \\fr 1:1 \\ft text|with pipe\\f*\n\\rem \\refx'; + expect(preprocessUsfm(usfm)).toBe(usfm); + }); +}); + +describe('preprocessUsfmDirectory()', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), 'conversion-spec-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('should write preprocessed USFM files to the output directory without changing the input', async () => { + const input = path.join(dir, 'in'); + const output = path.join(dir, 'out'); + await mkdir(input); + const original = + '\\id MAT\n\\v 6 \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\n'; + await writeFile(path.join(input, 'MAT.usfm'), original); + await writeFile(path.join(input, 'GEN.usfm'), '\\id GEN\n'); + await writeFile( + path.join(input, 'notes.json'), + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + + const changed = await preprocessUsfmDirectory(input, output); + + expect(changed).toBe(1); + expect(await readFile(path.join(output, 'MAT.usfm'), 'utf-8')).toBe( + '\\id MAT\n\\v 6 Psalm 91:11–12\n' + ); + expect(await readFile(path.join(output, 'GEN.usfm'), 'utf-8')).toBe( + '\\id GEN\n' + ); + expect(await readFile(path.join(output, 'notes.json'), 'utf-8')).toBe( + '{"a":"\\\\ref x|y\\\\ref*"}' + ); + expect(await readFile(path.join(input, 'MAT.usfm'), 'utf-8')).toBe( + original + ); + }); +}); diff --git a/packages/helloao-cli/conversion.ts b/packages/helloao-cli/conversion.ts index b01fb8a..f7342ea 100644 --- a/packages/helloao-cli/conversion.ts +++ b/packages/helloao-cli/conversion.ts @@ -1,5 +1,14 @@ -import { mkdir, readdir } from 'node:fs/promises'; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; import { existsSync } from 'fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { execFile as execFileCallback } from 'child_process'; import { promisify } from 'util'; import { confirm, input } from '@inquirer/prompts'; @@ -84,7 +93,77 @@ export async function promptForBibleMultiConverter(): Promise { } /** - * Converts USFM directory to USX3 using BibleMultiConverter + * The version of the USFM preprocessing that convertUsfmToUsx3() does. + * Increase this when preprocessUsfm() changes so that fetchers that skip unchanged sources + * (e.g. fetchBsbTranslation()) convert them again. + */ +export const USFM_PREPROCESSING_VERSION = 1; + +/** + * Matches a USFM 3 reference (`\\ref TEXT|TARGET\\ref*`), including the nested form (`\\+ref ...\\+ref*`). + * Group 1 is the "+" of a nested marker, group 2 is the display text, and group 3 is the optional link target + * (e.g. `PSA 91:11-12` or `loc="PSA 91:11-12"`). + */ +const USFM_REF_REGEX = + /\\(\+?)ref\s((?:(?!\\\+?ref\*)[^|])*?)(?:\|((?:(?!\\\+?ref\*)[\s\S])*?))?\\\1ref\*/g; + +/** + * The extensions of the files that are treated as USFM by preprocessUsfmDirectory(). + */ +const USFM_EXTENSIONS = new Set(['.usfm', '.sfm']); + +/** + * Rewrites the USFM markup that BibleMultiConverter doesn't understand so that the converted USX3 contains + * only the text that readers should see. + * + * - `\\ref TEXT|TARGET\\ref*` is replaced by `TEXT`. BibleMultiConverter doesn't know the `\\ref` marker, so it drops + * the markers and keeps the text on both sides of the pipe (e.g. "Psalm 91:11–12|PSA 91:11-12"), and it adds a space + * where the opening marker was (e.g. "( 1 Enoch 1:9|1:9)"). See https://github.com/HelloAOLab/bible-api/issues/144. + * + * @param usfm The USFM content. + * @returns The preprocessed USFM content. + */ +export function preprocessUsfm(usfm: string): string { + return usfm.replace(USFM_REF_REGEX, (_match, _plus, text: string) => text); +} + +/** + * Copies the files in the given directory to the given output directory and runs preprocessUsfm() on every USFM file. + * Other files are copied unchanged. + * @param inputDir The directory that contains the USFM files. + * @param outputDir The directory that the preprocessed files should be written to. + * @returns The number of USFM files that were changed. + */ +export async function preprocessUsfmDirectory( + inputDir: string, + outputDir: string +): Promise { + await mkdir(outputDir, { recursive: true }); + let changed = 0; + for (const entry of await readdir(inputDir, { withFileTypes: true })) { + if (!entry.isFile()) { + continue; + } + const inputPath = path.join(inputDir, entry.name); + const outputPath = path.join(outputDir, entry.name); + const content = await readFile(inputPath); + if (USFM_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) { + const text = content.toString('utf-8'); + const processed = preprocessUsfm(text); + if (processed !== text) { + changed += 1; + await writeFile(outputPath, processed, 'utf-8'); + continue; + } + } + await writeFile(outputPath, content); + } + return changed; +} + +/** + * Converts USFM directory to USX3 using BibleMultiConverter. + * The USFM files are first preprocessed with preprocessUsfm() in a temporary directory, so the input directory is not modified. */ export async function convertUsfmToUsx3( inputDir: string, @@ -92,9 +171,21 @@ export async function convertUsfmToUsx3( jarPath: string ): Promise { const logger = log.getLogger(); + let preprocessedDir: string | null = null; try { logger.log(`Converting USFM files from ${inputDir} to USX3 format...`); + preprocessedDir = await mkdtemp( + path.join(tmpdir(), 'bible-api-usfm-preprocessed-') + ); + const changed = await preprocessUsfmDirectory( + inputDir, + preprocessedDir + ); + if (changed > 0) { + logger.log(`Preprocessed ${changed} USFM files before conversion.`); + } + // Handle overwrite logic for output directory // if (overwrite && existsSync(outputDir)) { // logger.log(`Overwriting existing USX3 directory: ${outputDir}`); @@ -110,7 +201,7 @@ export async function convertUsfmToUsx3( jarPath, 'ParatextConverter', 'USFM', - inputDir, + preprocessedDir, 'USX3', outputDir, '*.usx', @@ -139,6 +230,10 @@ export async function convertUsfmToUsx3( } catch (error: any) { logger.error('Conversion failed:', error.message); return false; + } finally { + if (preprocessedDir) { + await rm(preprocessedDir, { recursive: true, force: true }); + } } } From 4ea311dfb3406e00ac4d46be1c64d7d8b49c7647 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 19:42:28 +0000 Subject: [PATCH 2/2] Download sources again when their USX3 conversion is outdated The eBible and fetch.bible fetchers skip sources that were already downloaded, so translations converted before the \ref preprocessing would never be converted again. Both source tables now record the USFM_PREPROCESSING_VERSION that each source was converted with, and when converting to USX3 the fetchers download sources again if they were converted with a different version (or not converted at all). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019LsURyVgHxWYTXpbFoYSPV --- packages/helloao-cli/actions.ts | 62 +++- packages/helloao-cli/conversion.ts | 11 + packages/helloao-cli/ebible.ts | 1 + packages/helloao-cli/fetch-bible.spec.ts | 1 + packages/helloao-cli/fetch-bible.ts | 1 + .../migration.sql | 3 + packages/helloao-cli/schema.prisma | 8 + .../helloao-cli/source-preprocessing.spec.ts | 289 ++++++++++++++++++ 8 files changed, 364 insertions(+), 12 deletions(-) create mode 100644 packages/helloao-cli/migrations/20261119000000_add_usfm_preprocessing_version/migration.sql create mode 100644 packages/helloao-cli/source-preprocessing.spec.ts diff --git a/packages/helloao-cli/actions.ts b/packages/helloao-cli/actions.ts index cc3e19b..93fdb08 100644 --- a/packages/helloao-cli/actions.ts +++ b/packages/helloao-cli/actions.ts @@ -69,6 +69,8 @@ import { findBibleMultiConverterJar, promptForBibleMultiConverter, convertUsfmToUsx3, + isUsfmPreprocessingOutdated, + USFM_PREPROCESSING_VERSION, printConversionInstructions, } from './conversion.js'; import { loadMetadataOverride } from './meta.js'; @@ -1750,16 +1752,18 @@ export async function sourceTranslations( logger.log('Connecting to database for download tracking...'); db = await database.getDb(options.db); sourceExists = db.prepare( - 'SELECT usfmZipEtag, usfmDownloadDate FROM EBibleSource WHERE id = @id AND sha256 = @sha256;' + 'SELECT usfmZipEtag, usfmDownloadDate, usfmPreprocessingVersion FROM EBibleSource WHERE id = @id AND sha256 = @sha256;' ); let skippedByDatabase = 0; let sourcesChecked = 0; let notDownloaded = 0; + let outdatedConversion = 0; filteredSources = filteredSources.filter((source) => { const existingSource = sourceExists.get(source) as { usfmZipEtag: string; usfmDownloadDate: string; + usfmPreprocessingVersion: number | null; }; sourcesChecked++; if (existingSource) { @@ -1775,6 +1779,17 @@ export async function sourceTranslations( source.usfmDownloadDate = existingSource.usfmDownloadDate as any; + if ( + convertToUsx3 && + isUsfmPreprocessingOutdated( + existingSource.usfmPreprocessingVersion + ) + ) { + // Converted before the current USFM preprocessing (or not converted at all) + outdatedConversion++; + return true; + } + // If specific translations were requested, include sources that match if ( translations && @@ -1803,8 +1818,9 @@ export async function sourceTranslations( logger.log(` • Checked: ${sourcesChecked}`); logger.log(` • Skipped: ${skippedByDatabase}`); logger.log(` • New: ${notDownloaded}`); + logger.log(` • Outdated USX3 conversion: ${outdatedConversion}`); logger.log( - ` • Needs Update: ${filteredSources.length - notDownloaded}` + ` • Needs Update: ${filteredSources.length - notDownloaded - outdatedConversion}` ); sourceUpsert = db.prepare(`INSERT INTO EBibleSource( @@ -1813,14 +1829,14 @@ export async function sourceTranslations( newTestamentBooks, newTestamentChapters, newTestamentVerses, apocryphaBooks, apocryphaChapters, apocryphaVerses, redistributable, sourceDate, updateDate, usfmDownloadDate, - usfmDownloadPath, sha256, usfmZipUrl, usfmZipEtag, FCBHID + usfmDownloadPath, sha256, usfmZipUrl, usfmZipEtag, FCBHID, usfmPreprocessingVersion ) VALUES ( @id, @translationId, @title, @shortTitle, @languageCode, @textDirection, @copyright, @description, @oldTestamentBooks, @oldTestamentChapters, @oldTestamentVerses, @newTestamentBooks, @newTestamentChapters, @newTestamentVerses, @apocryphaBooks, @apocryphaChapters, @apocryphaVerses, @redistributable, @sourceDate, @updateDate, @usfmDownloadDate, - @usfmDownloadPath, @sha256, @usfmZipUrl, @usfmZipEtag, @FCBHID + @usfmDownloadPath, @sha256, @usfmZipUrl, @usfmZipEtag, @FCBHID, @usfmPreprocessingVersion ) ON CONFLICT(id) DO UPDATE SET translationId = excluded.translationId, title = excluded.title, shortTitle = excluded.shortTitle, languageCode = excluded.languageCode, textDirection = excluded.textDirection, copyright = excluded.copyright, @@ -1833,7 +1849,7 @@ export async function sourceTranslations( updateDate = excluded.updateDate, usfmDownloadDate = excluded.usfmDownloadDate, usfmDownloadPath = excluded.usfmDownloadPath, sha256 = excluded.sha256, usfmZipUrl = excluded.usfmZipUrl, usfmZipEtag = excluded.usfmZipEtag, - FCBHID = excluded.FCBHID; + FCBHID = excluded.FCBHID, usfmPreprocessingVersion = excluded.usfmPreprocessingVersion; `); } else { logger.log('Database connection skipped (tracking disabled)'); @@ -2064,6 +2080,7 @@ export async function sourceTranslations( source.usfmDownloadDate = DateTime.utc().toISO() as any; source.usfmDownloadPath = outputDir; + source.usfmPreprocessingVersion = null; if (sourceUpsert) { sourceUpsert.run(source); } @@ -2116,6 +2133,8 @@ export async function sourceTranslations( source.usfmDownloadDate = DateTime.utc().toISO() as any; source.usfmDownloadPath = outputPath; + source.usfmPreprocessingVersion = + USFM_PREPROCESSING_VERSION; sourceUpsert.run(source); } numConverted++; @@ -2311,11 +2330,12 @@ export async function sourceFetchBibleTranslations( let sourceUpsert: Sql.Statement | null = null; if (db) { const sourceExists = db.prepare( - 'SELECT sha256, gitTreeSha, downloadDate, downloadPath FROM FetchBibleSource WHERE id = @id;' + 'SELECT sha256, gitTreeSha, downloadDate, downloadPath, usfmPreprocessingVersion FROM FetchBibleSource WHERE id = @id;' ); let unchanged = 0; let newSources = 0; let changedSources = 0; + let outdatedConversion = 0; selected = selected.filter((entry) => { const existing = sourceExists.get({ id: entry.source.id }) as @@ -2324,6 +2344,7 @@ export async function sourceFetchBibleTranslations( gitTreeSha: string | null; downloadDate: string | null; downloadPath: string | null; + usfmPreprocessingVersion: number | null; } | undefined; const outputPath = path.resolve( @@ -2342,6 +2363,16 @@ export async function sourceFetchBibleTranslations( } else if (hasFetchBibleSourceChanged(existing, entry.source)) { changedSources++; return true; + } else if ( + convertToUsx3 && + entry.source.sourceFormat === 'usfm' && + isUsfmPreprocessingOutdated( + existing.usfmPreprocessingVersion + ) + ) { + // Converted before the current USFM preprocessing (or not converted at all) + outdatedConversion++; + return true; } else if (hasTranslations) { // Explicitly requested translations are always re-downloaded return true; @@ -2355,15 +2386,18 @@ export async function sourceFetchBibleTranslations( logger.log(` • Skipped (unchanged): ${unchanged}`); logger.log(` • New: ${newSources}`); logger.log(` • Needs Update: ${changedSources}`); + logger.log(` • Outdated USX3 conversion: ${outdatedConversion}`); sourceUpsert = db.prepare(`INSERT INTO FetchBibleSource( id, translationId, languageCode, name, englishName, abbreviation, textDirection, year, ebibleId, sourceService, sourceFormat, sourceUpdated, sourceRevision, - attribution, attributionUrl, gitCommitSha, gitTreeSha, downloadDate, downloadPath, sha256 + attribution, attributionUrl, gitCommitSha, gitTreeSha, downloadDate, downloadPath, sha256, + usfmPreprocessingVersion ) VALUES ( @id, @translationId, @languageCode, @name, @englishName, @abbreviation, @textDirection, @year, @ebibleId, @sourceService, @sourceFormat, @sourceUpdated, @sourceRevision, - @attribution, @attributionUrl, @gitCommitSha, @gitTreeSha, @downloadDate, @downloadPath, @sha256 + @attribution, @attributionUrl, @gitCommitSha, @gitTreeSha, @downloadDate, @downloadPath, @sha256, + @usfmPreprocessingVersion ) ON CONFLICT(id) DO UPDATE SET translationId = excluded.translationId, languageCode = excluded.languageCode, name = excluded.name, englishName = excluded.englishName, abbreviation = excluded.abbreviation, @@ -2373,7 +2407,7 @@ export async function sourceFetchBibleTranslations( attribution = excluded.attribution, attributionUrl = excluded.attributionUrl, gitCommitSha = excluded.gitCommitSha, gitTreeSha = excluded.gitTreeSha, downloadDate = excluded.downloadDate, downloadPath = excluded.downloadPath, - sha256 = excluded.sha256; + sha256 = excluded.sha256, usfmPreprocessingVersion = excluded.usfmPreprocessingVersion; `); } else if (!overwrite) { const before = selected.length; @@ -2412,10 +2446,14 @@ export async function sourceFetchBibleTranslations( const recordDownload = ( entry: FetchBibleCatalogEntry, - outputPath: string + outputPath: string, + converted: boolean ) => { entry.source.downloadDate = DateTime.utc().toISO() as any; entry.source.downloadPath = outputPath; + entry.source.usfmPreprocessingVersion = converted + ? USFM_PREPROCESSING_VERSION + : null; sourceUpsert?.run(entry.source); result.downloaded.push(entry.source.translationId); }; @@ -2464,7 +2502,7 @@ export async function sourceFetchBibleTranslations( entry, }); } else { - recordDownload(entry, outputPath); + recordDownload(entry, outputPath, false); } } catch (err) { logger.error(`Error processing ${source.translationId}:`, err); @@ -2501,7 +2539,7 @@ export async function sourceFetchBibleTranslations( jarPath! ); if (success) { - recordDownload(entry, outputPath); + recordDownload(entry, outputPath, true); numConverted++; } else { result.failed.push(entry.source.translationId); diff --git a/packages/helloao-cli/conversion.ts b/packages/helloao-cli/conversion.ts index f7342ea..0b04796 100644 --- a/packages/helloao-cli/conversion.ts +++ b/packages/helloao-cli/conversion.ts @@ -99,6 +99,17 @@ export async function promptForBibleMultiConverter(): Promise { */ export const USFM_PREPROCESSING_VERSION = 1; +/** + * Determines whether a source that was converted to USX3 with the given preprocessing version needs to be converted again, + * because it was converted before the current USFM preprocessing (or wasn't converted at all). + * @param version The preprocessing version that the source was converted with. Null or undefined if it wasn't converted. + */ +export function isUsfmPreprocessingOutdated( + version: number | null | undefined +): boolean { + return version !== USFM_PREPROCESSING_VERSION; +} + /** * Matches a USFM 3 reference (`\\ref TEXT|TARGET\\ref*`), including the nested form (`\\+ref ...\\+ref*`). * Group 1 is the "+" of a nested marker, group 2 is the display text, and group 3 is the optional link target diff --git a/packages/helloao-cli/ebible.ts b/packages/helloao-cli/ebible.ts index a4c5044..855b015 100644 --- a/packages/helloao-cli/ebible.ts +++ b/packages/helloao-cli/ebible.ts @@ -92,6 +92,7 @@ export async function fetchEBibleMetadata(): Promise { usfmZipUrl: null, usfmZipEtag: null, sha256: null, + usfmPreprocessingVersion: null, }; source.sha256 = sha256() diff --git a/packages/helloao-cli/fetch-bible.spec.ts b/packages/helloao-cli/fetch-bible.spec.ts index 88e5b92..68a50fc 100644 --- a/packages/helloao-cli/fetch-bible.spec.ts +++ b/packages/helloao-cli/fetch-bible.spec.ts @@ -189,6 +189,7 @@ describe('createFetchBibleSource()', () => { sourceRevision: 0, downloadDate: null, downloadPath: null, + usfmPreprocessingVersion: null, }); expect(source.sha256).toMatch(/^[0-9a-f]{64}$/); }); diff --git a/packages/helloao-cli/fetch-bible.ts b/packages/helloao-cli/fetch-bible.ts index 8aafec9..84d3565 100644 --- a/packages/helloao-cli/fetch-bible.ts +++ b/packages/helloao-cli/fetch-bible.ts @@ -469,6 +469,7 @@ export function createFetchBibleSource( gitTreeSha: git.treeSha ?? null, downloadDate: null, downloadPath: null, + usfmPreprocessingVersion: null, sha256: null, }; diff --git a/packages/helloao-cli/migrations/20261119000000_add_usfm_preprocessing_version/migration.sql b/packages/helloao-cli/migrations/20261119000000_add_usfm_preprocessing_version/migration.sql new file mode 100644 index 0000000..720e4ac --- /dev/null +++ b/packages/helloao-cli/migrations/20261119000000_add_usfm_preprocessing_version/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "EBibleSource" ADD COLUMN "usfmPreprocessingVersion" INTEGER; +ALTER TABLE "FetchBibleSource" ADD COLUMN "usfmPreprocessingVersion" INTEGER; diff --git a/packages/helloao-cli/schema.prisma b/packages/helloao-cli/schema.prisma index e5ac2bb..8f680b8 100644 --- a/packages/helloao-cli/schema.prisma +++ b/packages/helloao-cli/schema.prisma @@ -98,6 +98,10 @@ model EBibleSource { // The SHA-256 hash of the source // Can be used to determine if the source has changed sha256 String? + + // The version of the USFM preprocessing (USFM_PREPROCESSING_VERSION) that the source was converted to USX3 with. + // Null if the source was not converted. Sources converted with an older version are downloaded again. + usfmPreprocessingVersion Int? } // A translation that is available from the fetch.bible collection @@ -145,6 +149,10 @@ model FetchBibleSource { // The SHA-256 hash of the source metadata // Can be used to determine if the source has changed sha256 String? + + // The version of the USFM preprocessing (USFM_PREPROCESSING_VERSION) that the source was converted to USX3 with. + // Null if the source was not converted. Sources converted with an older version are downloaded again. + usfmPreprocessingVersion Int? } // Files that were input into the API generator diff --git a/packages/helloao-cli/source-preprocessing.spec.ts b/packages/helloao-cli/source-preprocessing.spec.ts new file mode 100644 index 0000000..aa82a6f --- /dev/null +++ b/packages/helloao-cli/source-preprocessing.spec.ts @@ -0,0 +1,289 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from '@zip.js/zip.js'; +import { sourceFetchBibleTranslations, sourceTranslations } from './actions.js'; +import { convertUsfmToUsx3, USFM_PREPROCESSING_VERSION } from './conversion.js'; +import { getDb } from './db.js'; +import { FetchBibleCatalogEntry } from './fetch-bible.js'; +import { EBibleSource, FetchBibleSource } from './prisma-gen/client.js'; + +jest.mock('./conversion.js', () => ({ + ...jest.requireActual('./conversion.js'), + findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'), + promptForBibleMultiConverter: jest.fn(async () => null), + convertUsfmToUsx3: jest.fn(async () => true), +})); + +const encoder = new TextEncoder(); +const USFM = '\\id MAT\n\\c 1\n\\p\n\\v 1 Text.\n'; + +async function createZip(files: Record): Promise { + const writer = new Uint8ArrayWriter(); + const zip = new ZipWriter(writer); + for (const [file, content] of Object.entries(files)) { + await zip.add(file, new Uint8ArrayReader(encoder.encode(content))); + } + await zip.close(); + return await writer.getData(); +} + +function ebibleSource(): EBibleSource { + return { + id: 'engtest', + translationId: 'eng_test', + title: 'Test Bible', + shortTitle: 'Test', + textDirection: 'ltr', + languageCode: 'eng', + copyright: 'Public Domain', + description: 'Test', + oldTestamentBooks: 0, + oldTestamentChapters: 0, + oldTestamentVerses: 0, + newTestamentBooks: 1, + newTestamentChapters: 1, + newTestamentVerses: 1, + apocryphaBooks: 0, + apocryphaChapters: 0, + apocryphaVerses: 0, + redistributable: 'TRUE' as any, + FCBHID: 'ENGTST', + sourceDate: '2024-01-01T00:00:00.000Z' as any, + updateDate: '2024-01-01T00:00:00.000Z' as any, + usfmDownloadDate: null, + usfmDownloadPath: null, + usfmZipUrl: null, + usfmZipEtag: null, + sha256: 'abc123', + usfmPreprocessingVersion: null, + }; +} + +function fetchBibleEntry(): FetchBibleCatalogEntry { + const source: FetchBibleSource = { + id: 'tst_test', + translationId: 'tst_test', + languageCode: 'tst', + name: 'Test', + englishName: 'Test', + abbreviation: 'TST', + textDirection: 'ltr', + year: 2024, + ebibleId: null, + sourceService: 'manual', + sourceFormat: 'usfm', + sourceUpdated: null, + sourceRevision: null, + attribution: null, + attributionUrl: null, + gitCommitSha: null, + gitTreeSha: 'tree', + downloadDate: null, + downloadPath: null, + sha256: 'def456', + usfmPreprocessingVersion: null, + }; + return { + source, + item: { + name: { + local: 'Test', + local_abbrev: '', + english: 'Test', + english_abbrev: 'TST', + }, + year: 2024, + books_ot: [], + books_nt: ['mat'], + direction: 'ltr', + copyright: { + licenses: [{ license: 'public', url: null }], + attribution: '', + attribution_url: '', + }, + tags: [], + } as any, + meta: {} as any, + books: ['mat'], + collectionUrl: 'https://example.com/collection', + }; +} + +describe('USFM preprocessing version tracking', () => { + const originalFetch = global.fetch; + let dir: string; + let dbPath: string; + let fetchedUrls: string[]; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), 'source-preprocessing-')); + dbPath = path.join(dir, 'bible-api.db'); + fetchedUrls = []; + const zip = await createZip({ 'MAT.usfm': USFM }); + global.fetch = jest.fn(async (url: any) => { + const u = String(url); + fetchedUrls.push(u); + if (u.includes('details.php')) { + return new Response('engtest_usfm.zip'); + } + if (u.endsWith('_usfm.zip')) { + return new Response(zip); + } + return new Response(USFM); + }) as any; + (convertUsfmToUsx3 as jest.Mock).mockClear(); + }); + + afterEach(async () => { + global.fetch = originalFetch; + await rm(dir, { recursive: true, force: true }); + }); + + async function getVersion(table: string, id: string) { + const db = await getDb(dbPath); + try { + return ( + db + .prepare( + `SELECT usfmPreprocessingVersion FROM ${table} WHERE id = ?;` + ) + .get(id) as { usfmPreprocessingVersion: number | null } + ).usfmPreprocessingVersion; + } finally { + db.close(); + } + } + + async function setVersion( + table: string, + id: string, + version: number | null + ) { + const db = await getDb(dbPath); + try { + db.prepare( + `UPDATE ${table} SET usfmPreprocessingVersion = ? WHERE id = ?;` + ).run(version, id); + } finally { + db.close(); + } + } + + describe('sourceTranslations()', () => { + const source = (o = {}) => [{ ...ebibleSource(), ...o }]; + const options = () => ({ + db: dbPath, + convertToUsx3: true, + sources: source(), + ignorelist: [], + }); + + it('should record the preprocessing version and skip sources converted with it', async () => { + const first = await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(first.downloaded).toEqual(['eng_test']); + expect(await getVersion('EBibleSource', 'engtest')).toBe( + USFM_PREPROCESSING_VERSION + ); + + fetchedUrls = []; + const second = await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(second.downloaded).toEqual([]); + expect(fetchedUrls).toEqual([]); + }); + + it('should download sources converted with an older preprocessing version again', async () => { + await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + await setVersion('EBibleSource', 'engtest', null); + + fetchedUrls = []; + const result = await sourceTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(result.downloaded).toEqual(['eng_test']); + expect(fetchedUrls.length).toBeGreaterThan(0); + expect(await getVersion('EBibleSource', 'engtest')).toBe( + USFM_PREPROCESSING_VERSION + ); + }); + + it('should not download unconverted sources again when not converting', async () => { + const opts = { ...options(), convertToUsx3: false }; + await sourceTranslations(path.join(dir, 'out'), undefined, opts); + expect(await getVersion('EBibleSource', 'engtest')).toBeNull(); + + fetchedUrls = []; + const result = await sourceTranslations( + path.join(dir, 'out'), + undefined, + opts + ); + expect(result.downloaded).toEqual([]); + expect(fetchedUrls).toEqual([]); + }); + }); + + describe('sourceFetchBibleTranslations()', () => { + const options = () => ({ + db: dbPath, + convertToUsx3: true, + includeEBible: true, + catalog: [fetchBibleEntry()], + ignorelist: [], + }); + + it('should record the preprocessing version and skip sources converted with it', async () => { + const first = await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(first.downloaded).toEqual(['tst_test']); + expect(await getVersion('FetchBibleSource', 'tst_test')).toBe( + USFM_PREPROCESSING_VERSION + ); + + fetchedUrls = []; + const second = await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(second.downloaded).toEqual([]); + expect(fetchedUrls).toEqual([]); + }); + + it('should download sources converted with an older preprocessing version again', async () => { + await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + await setVersion('FetchBibleSource', 'tst_test', null); + + const result = await sourceFetchBibleTranslations( + path.join(dir, 'out'), + undefined, + options() + ); + expect(result.downloaded).toEqual(['tst_test']); + expect(await getVersion('FetchBibleSource', 'tst_test')).toBe( + USFM_PREPROCESSING_VERSION + ); + }); + }); +});