Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 50 additions & 12 deletions packages/helloao-cli/actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ import {
findBibleMultiConverterJar,
promptForBibleMultiConverter,
convertUsfmToUsx3,
isUsfmPreprocessingOutdated,
USFM_PREPROCESSING_VERSION,
printConversionInstructions,
} from './conversion.js';
import { loadMetadataOverride } from './meta.js';
Expand Down Expand Up @@ -1756,16 +1758,18 @@ export async function sourceTranslations(
logger.log('Connecting to database for download tracking...');
db = await database.getDb(options.db);
sourceExists = db.prepare(
'SELECT usfmZipEtag, usfmDownloadDate FROM EBibleSource WHERE id = @id AND sha256 = @sha256;'
'SELECT usfmZipEtag, usfmDownloadDate, usfmPreprocessingVersion FROM EBibleSource WHERE id = @id AND sha256 = @sha256;'
);
let skippedByDatabase = 0;
let sourcesChecked = 0;
let notDownloaded = 0;
let outdatedConversion = 0;

filteredSources = filteredSources.filter((source) => {
const existingSource = sourceExists.get(source) as {
usfmZipEtag: string;
usfmDownloadDate: string;
usfmPreprocessingVersion: number | null;
};
sourcesChecked++;
if (existingSource) {
Expand All @@ -1781,6 +1785,17 @@ export async function sourceTranslations(
source.usfmDownloadDate =
existingSource.usfmDownloadDate as any;

if (
convertToUsx3 &&
isUsfmPreprocessingOutdated(
existingSource.usfmPreprocessingVersion
)
) {
// Converted before the current USFM preprocessing (or not converted at all)
outdatedConversion++;
return true;
}

// If specific translations were requested, include sources that match
if (
translations &&
Expand Down Expand Up @@ -1809,8 +1824,9 @@ export async function sourceTranslations(
logger.log(` • Checked: ${sourcesChecked}`);
logger.log(` • Skipped: ${skippedByDatabase}`);
logger.log(` • New: ${notDownloaded}`);
logger.log(` • Outdated USX3 conversion: ${outdatedConversion}`);
logger.log(
` • Needs Update: ${filteredSources.length - notDownloaded}`
` • Needs Update: ${filteredSources.length - notDownloaded - outdatedConversion}`
);

sourceUpsert = db.prepare(`INSERT INTO EBibleSource(
Expand All @@ -1819,14 +1835,14 @@ export async function sourceTranslations(
newTestamentBooks, newTestamentChapters, newTestamentVerses,
apocryphaBooks, apocryphaChapters, apocryphaVerses,
redistributable, sourceDate, updateDate, usfmDownloadDate,
usfmDownloadPath, sha256, usfmZipUrl, usfmZipEtag, FCBHID
usfmDownloadPath, sha256, usfmZipUrl, usfmZipEtag, FCBHID, usfmPreprocessingVersion
) VALUES (
@id, @translationId, @title, @shortTitle, @languageCode, @textDirection, @copyright, @description,
@oldTestamentBooks, @oldTestamentChapters, @oldTestamentVerses,
@newTestamentBooks, @newTestamentChapters, @newTestamentVerses,
@apocryphaBooks, @apocryphaChapters, @apocryphaVerses,
@redistributable, @sourceDate, @updateDate, @usfmDownloadDate,
@usfmDownloadPath, @sha256, @usfmZipUrl, @usfmZipEtag, @FCBHID
@usfmDownloadPath, @sha256, @usfmZipUrl, @usfmZipEtag, @FCBHID, @usfmPreprocessingVersion
) ON CONFLICT(id) DO UPDATE SET
translationId = excluded.translationId, title = excluded.title, shortTitle = excluded.shortTitle,
languageCode = excluded.languageCode, textDirection = excluded.textDirection, copyright = excluded.copyright,
Expand All @@ -1839,7 +1855,7 @@ export async function sourceTranslations(
updateDate = excluded.updateDate, usfmDownloadDate = excluded.usfmDownloadDate,
usfmDownloadPath = excluded.usfmDownloadPath, sha256 = excluded.sha256,
usfmZipUrl = excluded.usfmZipUrl, usfmZipEtag = excluded.usfmZipEtag,
FCBHID = excluded.FCBHID;
FCBHID = excluded.FCBHID, usfmPreprocessingVersion = excluded.usfmPreprocessingVersion;
`);
} else {
logger.log('Database connection skipped (tracking disabled)');
Expand Down Expand Up @@ -2072,6 +2088,7 @@ export async function sourceTranslations(
source.usfmDownloadDate =
DateTime.utc().toISO() as any;
source.usfmDownloadPath = outputDir;
source.usfmPreprocessingVersion = null;
if (sourceUpsert) {
sourceUpsert.run(source);
}
Expand Down Expand Up @@ -2124,6 +2141,8 @@ export async function sourceTranslations(
source.usfmDownloadDate =
DateTime.utc().toISO() as any;
source.usfmDownloadPath = outputPath;
source.usfmPreprocessingVersion =
USFM_PREPROCESSING_VERSION;
sourceUpsert.run(source);
}
numConverted++;
Expand Down Expand Up @@ -2319,11 +2338,12 @@ export async function sourceFetchBibleTranslations(
let sourceUpsert: Sql.Statement | null = null;
if (db) {
const sourceExists = db.prepare(
'SELECT sha256, gitTreeSha, downloadDate, downloadPath FROM FetchBibleSource WHERE id = @id;'
'SELECT sha256, gitTreeSha, downloadDate, downloadPath, usfmPreprocessingVersion FROM FetchBibleSource WHERE id = @id;'
);
let unchanged = 0;
let newSources = 0;
let changedSources = 0;
let outdatedConversion = 0;

selected = selected.filter((entry) => {
const existing = sourceExists.get({ id: entry.source.id }) as
Expand All @@ -2332,6 +2352,7 @@ export async function sourceFetchBibleTranslations(
gitTreeSha: string | null;
downloadDate: string | null;
downloadPath: string | null;
usfmPreprocessingVersion: number | null;
}
| undefined;
const outputPath = path.resolve(
Expand All @@ -2350,6 +2371,16 @@ export async function sourceFetchBibleTranslations(
} else if (hasFetchBibleSourceChanged(existing, entry.source)) {
changedSources++;
return true;
} else if (
convertToUsx3 &&
entry.source.sourceFormat === 'usfm' &&
isUsfmPreprocessingOutdated(
existing.usfmPreprocessingVersion
)
) {
// Converted before the current USFM preprocessing (or not converted at all)
outdatedConversion++;
return true;
} else if (hasTranslations) {
// Explicitly requested translations are always re-downloaded
return true;
Expand All @@ -2363,15 +2394,18 @@ export async function sourceFetchBibleTranslations(
logger.log(` • Skipped (unchanged): ${unchanged}`);
logger.log(` • New: ${newSources}`);
logger.log(` • Needs Update: ${changedSources}`);
logger.log(` • Outdated USX3 conversion: ${outdatedConversion}`);

sourceUpsert = db.prepare(`INSERT INTO FetchBibleSource(
id, translationId, languageCode, name, englishName, abbreviation, textDirection, year,
ebibleId, sourceService, sourceFormat, sourceUpdated, sourceRevision,
attribution, attributionUrl, gitCommitSha, gitTreeSha, downloadDate, downloadPath, sha256
attribution, attributionUrl, gitCommitSha, gitTreeSha, downloadDate, downloadPath, sha256,
usfmPreprocessingVersion
) VALUES (
@id, @translationId, @languageCode, @name, @englishName, @abbreviation, @textDirection, @year,
@ebibleId, @sourceService, @sourceFormat, @sourceUpdated, @sourceRevision,
@attribution, @attributionUrl, @gitCommitSha, @gitTreeSha, @downloadDate, @downloadPath, @sha256
@attribution, @attributionUrl, @gitCommitSha, @gitTreeSha, @downloadDate, @downloadPath, @sha256,
@usfmPreprocessingVersion
) ON CONFLICT(id) DO UPDATE SET
translationId = excluded.translationId, languageCode = excluded.languageCode,
name = excluded.name, englishName = excluded.englishName, abbreviation = excluded.abbreviation,
Expand All @@ -2381,7 +2415,7 @@ export async function sourceFetchBibleTranslations(
attribution = excluded.attribution, attributionUrl = excluded.attributionUrl,
gitCommitSha = excluded.gitCommitSha, gitTreeSha = excluded.gitTreeSha,
downloadDate = excluded.downloadDate, downloadPath = excluded.downloadPath,
sha256 = excluded.sha256;
sha256 = excluded.sha256, usfmPreprocessingVersion = excluded.usfmPreprocessingVersion;
`);
} else if (!overwrite) {
const before = selected.length;
Expand Down Expand Up @@ -2420,10 +2454,14 @@ export async function sourceFetchBibleTranslations(

const recordDownload = (
entry: FetchBibleCatalogEntry,
outputPath: string
outputPath: string,
converted: boolean
) => {
entry.source.downloadDate = DateTime.utc().toISO() as any;
entry.source.downloadPath = outputPath;
entry.source.usfmPreprocessingVersion = converted
? USFM_PREPROCESSING_VERSION
: null;
sourceUpsert?.run(entry.source);
result.downloaded.push(entry.source.translationId);
};
Expand Down Expand Up @@ -2472,7 +2510,7 @@ export async function sourceFetchBibleTranslations(
entry,
});
} else {
recordDownload(entry, outputPath);
recordDownload(entry, outputPath, false);
}
} catch (err) {
logger.error(`Error processing ${source.translationId}:`, err);
Expand Down Expand Up @@ -2509,7 +2547,7 @@ export async function sourceFetchBibleTranslations(
jarPath!
);
if (success) {
recordDownload(entry, outputPath);
recordDownload(entry, outputPath, true);
numConverted++;
} else {
result.failed.push(entry.source.translationId);
Expand Down
1 change: 1 addition & 0 deletions packages/helloao-cli/bsb.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ jest.mock('./conversion.js', () => ({
findBibleMultiConverterJar: jest.fn(async () => '/fake/converter.jar'),
promptForBibleMultiConverter: jest.fn(async () => null),
convertUsfmToUsx3: jest.fn(),
USFM_PREPROCESSING_VERSION: 1,
}));

const encoder = new TextEncoder();
Expand Down
5 changes: 4 additions & 1 deletion packages/helloao-cli/bsb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { log } from '@helloao/tools';
import { InputTranslationMetadata } from '@helloao/tools/generation/common-types.js';
import {
convertUsfmToUsx3,
USFM_PREPROCESSING_VERSION,
findBibleMultiConverterJar,
printConversionInstructions,
promptForBibleMultiConverter,
Expand Down Expand Up @@ -147,7 +148,9 @@ export async function fetchBsbTranslation(
const content = new Uint8Array(await response.arrayBuffer());
const sha256 = createHash('sha256').update(content).digest('hex');
const marker = `Source: ${url}\nSHA-256: ${sha256}\nFormat: ${
convertToUsx3 ? 'USX3' : 'USFM'
convertToUsx3
? `USX3\nPreprocessing: ${USFM_PREPROCESSING_VERSION}`
: 'USFM'
}\n`;

if (
Expand Down
90 changes: 90 additions & 0 deletions packages/helloao-cli/conversion.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { preprocessUsfm, preprocessUsfmDirectory } from './conversion.js';

describe('preprocessUsfm()', () => {
it('should replace \\ref with its display text', () => {
expect(
preprocessUsfm(
'\\f + \\fr 4:6 \\ft See \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\\f*'
)
).toBe('\\f + \\fr 4:6 \\ft See Psalm 91:11–12\\f*');
});

it('should not add a space after an opening parenthesis', () => {
expect(preprocessUsfm('\\ft (\\ref 1 Enoch 1:9|1:9\\ref*)')).toBe(
'\\ft (1 Enoch 1:9)'
);
});

it('should support \\ref without a target', () => {
expect(preprocessUsfm('See \\ref Genesis 1:1\\ref*.')).toBe(
'See Genesis 1:1.'
);
});

it('should support attribute targets and nested \\+ref', () => {
expect(
preprocessUsfm(
'A \\ref John 3:16|loc="JHN 3:16"\\ref* and \\+ref Mark 1:1|MRK 1:1\\+ref*'
)
).toBe('A John 3:16 and Mark 1:1');
});

it('should handle multiple references on one line', () => {
expect(
preprocessUsfm(
'\\ref Isaiah 40:3|ISA 40:3\\ref*; \\ref Malachi 3:1|MAL 3:1\\ref*'
)
).toBe('Isaiah 40:3; Malachi 3:1');
});

it('should leave other markers unchanged', () => {
const usfm =
'\\v 1 In the beginning \\f + \\fr 1:1 \\ft text|with pipe\\f*\n\\rem \\refx';
expect(preprocessUsfm(usfm)).toBe(usfm);
});
});

describe('preprocessUsfmDirectory()', () => {
let dir: string;

beforeEach(async () => {
dir = await mkdtemp(path.join(tmpdir(), 'conversion-spec-'));
});

afterEach(async () => {
await rm(dir, { recursive: true, force: true });
});

it('should write preprocessed USFM files to the output directory without changing the input', async () => {
const input = path.join(dir, 'in');
const output = path.join(dir, 'out');
await mkdir(input);
const original =
'\\id MAT\n\\v 6 \\ref Psalm 91:11–12|PSA 91:11-12\\ref*\n';
await writeFile(path.join(input, 'MAT.usfm'), original);
await writeFile(path.join(input, 'GEN.usfm'), '\\id GEN\n');
await writeFile(
path.join(input, 'notes.json'),
'{"a":"\\\\ref x|y\\\\ref*"}'
);

const changed = await preprocessUsfmDirectory(input, output);

expect(changed).toBe(1);
expect(await readFile(path.join(output, 'MAT.usfm'), 'utf-8')).toBe(
'\\id MAT\n\\v 6 Psalm 91:11–12\n'
);
expect(await readFile(path.join(output, 'GEN.usfm'), 'utf-8')).toBe(
'\\id GEN\n'
);
expect(await readFile(path.join(output, 'notes.json'), 'utf-8')).toBe(
'{"a":"\\\\ref x|y\\\\ref*"}'
);
expect(await readFile(path.join(input, 'MAT.usfm'), 'utf-8')).toBe(
original
);
});
});
Loading
Loading