Skip to content

Convert USFM versification #472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Enkidu93 wants to merge 10 commits into
base: master
Choose a base branch
from
Open

Convert USFM versification #472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
a047815
Add ability to convert usfm when updating to the versification of the…
Enkidu93 Aug 12, 2026
f0e9960
Address reviewer comments
Enkidu93 Aug 13, 2026
e193572
Remove empty chapter test
Enkidu93 Aug 13, 2026
94c875e
Separate USFM versification conversion into separate handler
Enkidu93 Aug 27, 2026
bccc243
Remove unused code
Enkidu93 Aug 28, 2026
38b4fc9
Add unit tests for edge cases
ddaspit Aug 28, 2026
29dcbb5
Address reviewer comments; add tests to cover invalid verse references
Enkidu93 Sep 11, 2026
4f9d6ef
Properly handle more complex trailing paragraph insertion
Enkidu93 Sep 11, 2026
cc8b7a4
Address reviewer comments
Enkidu93 Sep 11, 2026
a50f3a5
Address some edge cases when inserting a chapter marker
ddaspit Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
285 changes: 285 additions & 0 deletions src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,285 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using SIL.Scripture;

namespace SIL.Machine.Corpora
{
public class ConvertUsfmVersificationHandler : ScriptureRefUsfmParserHandlerBase
{
private static readonly Regex TrailingParagraphMarkerPatterns = new Regex(
@"^(?:mte?\d*|ms\d*|sd?\d*|mr|sr|sp|d|r)$",
RegexOptions.Compiled
);
private readonly List<UsfmToken> _tokens;
private List<(int Index, UsfmToken Token)> _trailingVerseTokens;
private VerseRef _prevVerseRef;
private int _verseBoundary;
private readonly ScrVers _targetVersification;
private int _insertChapterIndex;
private bool _skip;

public ConvertUsfmVersificationHandler(ScrVers targetVersification)
{
_verseBoundary = 0;
_insertChapterIndex = -1;
_tokens = new List<UsfmToken>();
_trailingVerseTokens = new List<(int Index, UsfmToken Token)>();
_prevVerseRef = new VerseRef();
_targetVersification = targetVersification;
_skip = false;
}

public IReadOnlyList<UsfmToken> Tokens => _tokens;

public override void Chapter(
UsfmParserState state,
string number,
string marker,
string altNumber,
string pubNumber
)
{
base.Chapter(state, number, marker, altNumber, pubNumber);
ProcessTokens(state);
VerseRef vr = state.VerseRef;
// The versification of verse 0 cannot properly be changed
vr.Verse = "1";
if (
!_prevVerseRef.IsDefault
&& (
vr.ChangeVersificationWithSegments(_targetVersification).Book != _prevVerseRef.Book
|| vr.ChapterNum == -1
)
)
{
_skip = true;
}
_insertChapterIndex = _tokens.Count;
}

public override void Verse(
UsfmParserState state,
string number,
string marker,
string altNumber,
string pubNumber
)
{
base.Verse(state, number, marker, altNumber, pubNumber);

VerseRef verseRef = state.VerseRef;

ProcessTokens(state);

List<VerseRef> verseRefs = state
.VerseRef.AllVerses()
.Select(vr => vr.ChangeVersificationWithSegments(_targetVersification))
.ToList();

if (
(
_prevVerseRef.IsDefault
|| (
verseRefs[0].BookNum == _prevVerseRef.BookNum
&& verseRefs[0].ChapterNum != _prevVerseRef.ChapterNum
)
) && (verseRefs[0].ChapterNum != -1)
)
{
UsfmToken newChapterToken = new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[0].Chapter);

if (_insertChapterIndex == -1)
{
int chapterIndex = _tokens.Count;
_tokens.Add(newChapterToken);
List<UsfmToken> trailingAtChapter = _trailingVerseTokens
.Where(tup => tup.Index == chapterIndex)
.Select(tup => tup.Token)
.ToList();
if (trailingAtChapter.Count == 0)
{
// The chapter break falls mid-paragraph, so the paragraph continues across it.
_tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", ""));
}
else
{
// The trailing markers follow the new chapter and break the paragraph. If
// they do not open a paragraph of their own, the verse still needs one.
UsfmToken lastParagraph = trailingAtChapter.LastOrDefault(t =>
t.Type == UsfmTokenType.Paragraph
);
if (lastParagraph == null || TrailingParagraphMarkerPatterns.IsMatch(lastParagraph.Marker))
{
_trailingVerseTokens.Add(
(chapterIndex, new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", ""))
);
}
_trailingVerseTokens = _trailingVerseTokens
.Select(tup => tup.Index == chapterIndex ? (tup.Index + 1, tup.Token) : tup)
.ToList();
}
}
else
{
_tokens.Insert(_insertChapterIndex, newChapterToken);
_trailingVerseTokens = _trailingVerseTokens
.Select(tup => tup.Index >= _insertChapterIndex ? (tup.Index + 1, tup.Token) : tup)
.ToList();
}
}

bool addedVerseText = false;

string start = null;
for (int i = 0; i < verseRefs.Count; i++)
{
if (
(!_prevVerseRef.IsDefault && verseRefs[i].Book != _prevVerseRef.Book)
|| verseRefs[i].ChapterNum == -1
)
{
continue;
}
if (start != null)
{
string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : "";
if (
_prevVerseRef.BookNum == verseRefs[i].BookNum
&& _prevVerseRef.ChapterNum != verseRefs[i].ChapterNum
)
{
AddTrailingTokens();
_tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end));
if (!addedVerseText && state.Index + 1 < state.Tokens.Count)
{
UsfmToken nextToken = state.Tokens[state.Index + 1];
if (nextToken.Type == UsfmTokenType.Text)
{
_tokens.Add(nextToken);
_verseBoundary++;
addedVerseText = true;
}
}
_tokens.Add(new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[i].Chapter));
_tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", ""));
start = verseRefs[i].Verse;
_prevVerseRef = verseRefs[i];
}
else if (_prevVerseRef.VerseNum + 1 != verseRefs[i].VerseNum)
{
AddTrailingTokens();
_tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end));
if (!addedVerseText && state.Index + 1 < state.Tokens.Count)
{
UsfmToken nextToken = state.Tokens[state.Index + 1];
if (nextToken.Type == UsfmTokenType.Text)
{
_tokens.Add(nextToken);
_verseBoundary++;
addedVerseText = true;
}
}
start = verseRefs[i].Verse;
_prevVerseRef = verseRefs[i];
}
else
{
_prevVerseRef = verseRefs[i];
}
}
else
{
start = verseRefs[i].Verse;
_prevVerseRef = verseRefs[i];
}
verseRef = verseRefs[i];
}

if (start != null)
{
AddTrailingTokens();
string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : "";
_tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end));
_skip = false;
_insertChapterIndex = -1;
_prevVerseRef = verseRef;
}
else
{
_skip = true;
}
}

public override void EndUsfm(UsfmParserState state)
{
base.EndUsfm(state);
ProcessTokens(state);
if (!_skip && !(state.Token.Type == UsfmTokenType.Chapter || state.Token.Type == UsfmTokenType.Verse))
_tokens.Add(state.Token);
}

public string GetUsfm(UsfmStylesheet stylesheet)
{
var tokenizer = new UsfmTokenizer(stylesheet);
return tokenizer.Detokenize(_tokens);
}

private void ProcessTokens(UsfmParserState state)
{
int offset = 0;
bool inPreservedParagraph = false;
while (_verseBoundary + offset < state.Index)
{
UsfmToken token = state.Tokens[_verseBoundary + offset];
if (
IsPreservedTrailingParagraphMarker(
token,
_verseBoundary + offset + 1 < state.Tokens.Count
? state.Tokens[_verseBoundary + offset + 1]
: null
)
)
{
inPreservedParagraph = true;
}
else if (inPreservedParagraph)
{
inPreservedParagraph = token.Type != UsfmTokenType.Paragraph;
}
else
{
inPreservedParagraph = false;
}

if (inPreservedParagraph)
_trailingVerseTokens.Add((_tokens.Count, token));
else if (!_skip)
_tokens.Add(token);

offset++;
}
_verseBoundary = state.Index + 1;
}

private void AddTrailingTokens()
{
foreach (
(int index, List<UsfmToken> tokens) in _trailingVerseTokens
.GroupBy(tup => tup.Index)
.Select(g => (g.Key, g.Select(tup => tup.Token).ToList()))
.OrderBy(tup => -tup.Key)
)
{
_tokens.InsertRange(index, tokens);
}
_trailingVerseTokens.Clear();
}

private bool IsPreservedTrailingParagraphMarker(UsfmToken token, UsfmToken nextToken)
{
return (token.Marker == "p" && nextToken != null && nextToken.Type == UsfmTokenType.Verse)
|| token.Type == UsfmTokenType.Paragraph && TrailingParagraphMarkerPatterns.IsMatch(token.Marker);
}
}
}
13 changes: 12 additions & 1 deletion src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.IO;
using System.Linq;
using System.Text;
using SIL.Scripture;

namespace SIL.Machine.Corpora
{
Expand Down Expand Up @@ -64,7 +65,17 @@ public string UpdateUsfm(
var tokenizer = new UsfmTokenizer(_settings.Stylesheet);
IReadOnlyList<UsfmToken> tokens = tokenizer.Tokenize(usfm);
tokens = FilterTokensByChapter(tokens, chapters);
UsfmParser.Parse(tokens, handler, _settings.Stylesheet, _settings.Versification);

ScrVers rowsVersification = UpdateUsfmParserHandler.GetRowsVersification(rows);
ScrVers parseVersification = _settings.Versification;
if (rowsVersification != _settings.Versification)
{
var converter = new ConvertUsfmVersificationHandler(rowsVersification);
UsfmParser.Parse(tokens, converter, _settings.Stylesheet, _settings.Versification);
tokens = converter.Tokens;
parseVersification = rowsVersification;
}
UsfmParser.Parse(tokens, handler, _settings.Stylesheet, parseVersification);
return handler.GetUsfm(_settings.Stylesheet);
}
catch (Exception ex)
Expand Down
11 changes: 8 additions & 3 deletions src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,7 @@ public UpdateUsfmParserHandler(
_verseRowsMap = new Dictionary<VerseRef, List<RowInfo>>(
compareSegments ? VerseRefComparer.Default : VerseRefComparer.IgnoreSegments
);
_updateRowsVersification = ScrVers.English;
if (_rows.Count > 0)
_updateRowsVersification = _rows.First(r => r.Refs.Count > 0).Refs[0].Versification;
_updateRowsVersification = GetRowsVersification(rows);
_tokens = new List<UsfmToken>();
_updatedText = new List<UsfmToken>();
_updateBlocks = new Stack<UsfmUpdateBlock>();
Expand All @@ -114,6 +112,13 @@ public UpdateUsfmParserHandler(
_compareSegments = compareSegments;
}

public static ScrVers GetRowsVersification(IReadOnlyList<UpdateUsfmRow> rows)
{
return rows != null && rows.Count > 0
? rows.First(r => r.Refs.Count > 0).Refs[0].Versification
: ScrVers.English;
}

public IReadOnlyList<UsfmToken> Tokens => _tokens;

public override void EndUsfm(UsfmParserState state)
Expand Down
Loading
Loading