From a047815f731bd4fbf49b05803b7489484afb12cd Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 12 Aug 2026 12:08:40 -0400 Subject: [PATCH 01/10] Add ability to convert usfm when updating to the versification of the update rows --- .../Corpora/ParatextProjectTextUpdaterBase.cs | 7 +- .../Corpora/UpdateUsfmParserHandler.cs | 109 +++- src/SIL.Machine/Corpora/UsfmToken.cs | 9 + .../Corpora/UpdateUsfmParserHandlerTests.cs | 468 +++++++++++++----- 4 files changed, 466 insertions(+), 127 deletions(-) diff --git a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs index e38e528f6..fd104e3f8 100644 --- a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs +++ b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs @@ -33,7 +33,8 @@ public string UpdateUsfm( IEnumerable updateBlockHandlers = null, IEnumerable<(int, string)> remarks = null, Func errorHandler = null, - bool compareSegments = false + bool compareSegments = false, + bool convertUsfmToUpdateRowVersification = false ) { string fileName = _settings.GetBookFileName(bookId); @@ -57,7 +58,9 @@ public string UpdateUsfm( updateBlockHandlers, remarks, errorHandler, - compareSegments + compareSegments, + _settings.Versification, + convertUsfmToUpdateRowVersification ); try { diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index f21be754d..6672eede6 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -49,6 +49,7 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private int _verseRowIndex; private readonly Dictionary> _verseRowsMap; private readonly ScrVers _updateRowsVersification; + private readonly ScrVers _usfmVersification; private readonly List _tokens; private readonly List _updatedText; private readonly List _embedTokens; @@ -65,6 +66,10 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private int _tokenIndex; private readonly Func _errorHandler; private readonly bool _compareSegments; + private readonly bool _convertUsfmToUpdateRowVersification; + private UsfmToken _currentChapterToken; + private int _currentChapterNum; + private bool _skipNextVerseText; /// UpdateUsfmRows must be in order public UpdateUsfmParserHandler( @@ -78,7 +83,9 @@ public UpdateUsfmParserHandler( IEnumerable updateBlockHandlers = null, IEnumerable<(int, string)> remarks = null, Func errorHandler = null, - bool compareSegments = false + bool compareSegments = false, + ScrVers usfmVersification = null, + bool convertUsfmToUpdateRowVersification = false ) { _rows = rows ?? Array.Empty(); @@ -89,6 +96,7 @@ public UpdateUsfmParserHandler( _updateRowsVersification = ScrVers.English; if (_rows.Count > 0) _updateRowsVersification = _rows.First(r => r.Refs.Count > 0).Refs[0].Versification; + _usfmVersification = usfmVersification ?? _updateRowsVersification; _tokens = new List(); _updatedText = new List(); _updateBlocks = new Stack(); @@ -112,6 +120,11 @@ public UpdateUsfmParserHandler( if (_errorHandler == null) _errorHandler = (error) => false; _compareSegments = compareSegments; + _convertUsfmToUpdateRowVersification = + convertUsfmToUpdateRowVersification && _updateRowsVersification != _usfmVersification; + _currentChapterToken = null; + _currentChapterNum = 0; + _skipNextVerseText = false; } public IReadOnlyList Tokens => _tokens; @@ -119,6 +132,11 @@ public UpdateUsfmParserHandler( public override void EndUsfm(UsfmParserState state) { CollectUpdatableTokens(state); + if (_currentChapterToken != null) + { + _tokens.Add(_currentChapterToken); + _currentChapterToken = null; + } base.EndUsfm(state); } @@ -148,6 +166,12 @@ public override void EndBook(UsfmParserState state, string marker) UsfmUpdateBlock updateBlock = _updateBlocks.Pop(); _tokens.AddRange(updateBlock.GetTokens()); + if (_currentChapterToken != null) + { + _tokens.Add(_currentChapterToken); + _currentChapterToken = null; + } + base.EndBook(state, marker); } @@ -398,7 +422,11 @@ protected override void StartVerseText(UsfmParserState state, IReadOnlyList scriptureRefs) { - EndUpdateBlock(state, scriptureRefs); + if (!_skipNextVerseText) + { + EndUpdateBlock(state, scriptureRefs); + _skipNextVerseText = false; + } } protected override void StartNonVerseText(UsfmParserState state, ScriptureRef scriptureRef) @@ -564,9 +592,16 @@ private void CollectUpdatableTokens(UsfmParserState state) UsfmToken token = state.Tokens[_tokenIndex]; if (token.Type == UsfmTokenType.Verse) { - string sanitizedVerseData = SanitizeVerseData(token.Data); - token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, sanitizedVerseData); + VerseRef updatedVerse = UpdateVerseData(state, token); + if (updatedVerse.BookNum != state.VerseRef.BookNum) + { + _tokenIndex++; + _skipNextVerseText = true; + continue; + } + token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, updatedVerse.Verse); } + if (CurrentTextType == ScriptureTextType.Embed) { _embedTokens.Add(token); @@ -582,10 +617,48 @@ private void CollectUpdatableTokens(UsfmParserState state) { _tokens.Add(token); } + _tokenIndex++; } } + private VerseRef UpdateVerseData(UsfmParserState state, UsfmToken token) + { + string updatedVerseData = SanitizeVerseData(token.Data); + VerseRef verseRef = state.VerseRef; + verseRef.Verse = updatedVerseData; + if (_convertUsfmToUpdateRowVersification) + { + verseRef = verseRef.ChangeVersificationWithSegments(_updateRowsVersification); + if (verseRef.ChapterNum != _currentChapterNum && state.VerseRef.BookNum == verseRef.BookNum) + { + if (_currentChapterToken != null) + { + _tokens.Add(_currentChapterToken.Copy()); + _currentChapterToken = null; + } + else + { + UsfmToken newChapterToken = new UsfmToken( + UsfmTokenType.Chapter, + "c", + "", + "", + verseRef.ChapterNum.ToString() + ); + _tokens.Add(newChapterToken); + } + _currentChapterNum = verseRef.ChapterNum; + } + if (verseRef.ChapterNum == verseRef.Versification.GetLastChapter(verseRef.BookNum)) + { + _currentChapterToken = null; + } + } + + return verseRef; + } + private void CollectReadonlyTokens(UsfmParserState state) { while (_tokenIndex <= state.Index + state.SpecialTokenCount) @@ -597,7 +670,16 @@ private void CollectReadonlyTokens(UsfmParserState state) } else { - _tokens.Add(token); + if (_convertUsfmToUpdateRowVersification && token.Type == UsfmTokenType.Chapter) + { + if (_currentChapterToken != null) + _tokens.Add(_currentChapterToken); + _currentChapterToken = token; + } + else + { + _tokens.Add(token); + } } _tokenIndex++; } @@ -663,10 +745,13 @@ private bool HasNewText() private void StartUpdateBlock(IReadOnlyList scriptureRefs) { (IReadOnlyList rowTexts, Dictionary metadata) = AdvanceRows(scriptureRefs); - _updateBlocks.Push( - new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary()) - ); - PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " "))); + if (!_skipNextVerseText) + { + _updateBlocks.Push( + new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary()) + ); + PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " "))); + } } private void EndUpdateBlock(UsfmParserState state, IReadOnlyList scriptureRefs) @@ -757,7 +842,11 @@ private bool IsNonverseParagraph(UsfmParserState state, UsfmUpdateBlockElement e private void UpdateVerseRowsMap() { _verseRowsMap.Clear(); - while (_rowIndex < _rows.Count && _rows[_rowIndex].Refs[0].ChapterNum == _verseRowsRef.ChapterNum) + while ( + _rowIndex < _rows.Count + && _rows[_rowIndex].Refs[0].ChangeVersification(_verseRowsRef.Versification).ChapterNum + == _verseRowsRef.ChapterNum + ) { UpdateUsfmRow row = _rows[_rowIndex]; var ri = new RowInfo(_rowIndex); diff --git a/src/SIL.Machine/Corpora/UsfmToken.cs b/src/SIL.Machine/Corpora/UsfmToken.cs index 43a621b7a..460c5ea26 100644 --- a/src/SIL.Machine/Corpora/UsfmToken.cs +++ b/src/SIL.Machine/Corpora/UsfmToken.cs @@ -180,6 +180,15 @@ public void CopyAttributes(UsfmToken sourceToken) _defaultAttributeName = sourceToken._defaultAttributeName; } + public UsfmToken Copy() + { + UsfmToken copy = new UsfmToken(Type, Marker, Text, EndMarker, Data); + copy.CopyAttributes(this); + copy.LineNumber = LineNumber; + copy.ColumnNumber = ColumnNumber; + return copy; + } + private static void AppendAttribute(List attributes, string name, string value) { value = value?.Trim(); // don't want to have attribute that is just spaces diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index cddf0df85..4b793ef54 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using NUnit.Framework; +using SIL.Scripture; namespace SIL.Machine.Corpora; @@ -9,7 +10,7 @@ public class UpdateUsfmParserHandlerTests [Test] public void GetUsfm_Verse_CharStyle() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "First verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "First verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -33,8 +34,8 @@ public void GetUsfm_StripAllText() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "Update 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "Update 3"), ]; string usfm = @"\id MAT - Test @@ -102,9 +103,9 @@ public void GetUsfm_StripParagraphs_PreserveParagraphStyles() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/1:rem"), "New remark"), - new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "Another new remark"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/1:rem"]), "New remark"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "Another new remark"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), ]; string usfm = @"\id MAT @@ -156,8 +157,8 @@ public void GetUsfm_PreserveParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/1:rem"), "Update remark"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/1:rem"]), "Update remark"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), ]; string usfm = @"\id MAT @@ -201,7 +202,7 @@ public void GetUsfm_PreserveParagraphs() [Test] public void GetUsfm_ParagraphInVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -247,8 +248,8 @@ public void GetUsfm_PreferExisting() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @"\id MAT - Test @@ -273,8 +274,8 @@ public void GetUsfm_PreferRows() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:6"), "Text 6"), - new UpdateUsfmRow(ScrRef("MAT 1:7"), "Text 7"), + new UpdateUsfmRow(ScrRef(["MAT 1:6"]), "Text 6"), + new UpdateUsfmRow(ScrRef(["MAT 1:7"]), "Text 7"), ]; string target = UpdateUsfm(rows, textBehavior: UpdateUsfmTextBehavior.PreferNew); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -285,7 +286,7 @@ public void GetUsfm_PreferRows() [Test] public void GetUsfm_Verse_StripNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:1"), "First verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:1"]), "First verse of the second chapter.")]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\v 1 First verse of the second chapter.\r\n")); @@ -294,7 +295,7 @@ public void GetUsfm_Verse_StripNote() [Test] public void GetUsfm_Verse_ReplaceWithNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "updated text")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "updated text")]; string usfm = @"\id MAT - Test \c 1 @@ -312,7 +313,7 @@ public void GetUsfm_Verse_ReplaceWithNote() [Test] public void GetUsfm_Verse_RowVerseSegment() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:1a"), "First verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:1a"]), "First verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -326,7 +327,7 @@ public void GetUsfm_Verse_RowVerseSegment() [Test] public void GetUsfm_Verse_UsfmVerseSegment() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:7"), "Seventh verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:7"]), "Seventh verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 7a Seventh verse of the second chapter.\r\n")); @@ -335,7 +336,7 @@ public void GetUsfm_Verse_UsfmVerseSegment() [Test] public void GetUsfm_Verse_MultipleParas() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:2"), "Second verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Second verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -349,7 +350,7 @@ public void GetUsfm_Verse_MultipleParas() [Test] public void GetUsfm_Verse_Table() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:9"), "Ninth verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:9"]), "Ninth verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 9 Ninth verse of the second chapter. \\tcr2 \\tc3 \\tcr4\r\n")); @@ -361,7 +362,7 @@ public void GetUsfm_Verse_RangeSingleRowMultipleVerses() List rows = [ new UpdateUsfmRow( - ScrRef("MAT 2:11", "MAT 2:12"), + ScrRef(["MAT 2:11", "MAT 2:12"]), "Eleventh verse of the second chapter. Twelfth verse of the second chapter." ), ]; @@ -378,7 +379,7 @@ public void GetUsfm_Verse_RangeSingleRowMultipleVerses() [Test] public void GetUsfm_Verse_RangeSingleRowSingleVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:11"), "Eleventh verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:11"]), "Eleventh verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 11-12 Eleventh verse of the second chapter.\r\n")); @@ -389,8 +390,8 @@ public void GetUsfm_Verse_RangeMultipleRowsSingleVerse() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:11"), "Eleventh verse of the second chapter."), - new UpdateUsfmRow(ScrRef("MAT 2:12"), "Twelfth verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:11"]), "Eleventh verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:12"]), "Twelfth verse of the second chapter."), ]; string target = UpdateUsfm(rows); @@ -407,9 +408,9 @@ public void GetUsfm_MergeVerseSegments() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:2"), "Verse 2."), - new UpdateUsfmRow(ScrRef("MAT 2:2a"), "Verse 2a."), - new UpdateUsfmRow(ScrRef("MAT 2:2b"), "Verse 2b."), + new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "Verse 2."), + new UpdateUsfmRow(ScrRef(["MAT 2:2a"]), "Verse 2a."), + new UpdateUsfmRow(ScrRef(["MAT 2:2b"]), "Verse 2b."), ]; string target = UpdateUsfm(rows); @@ -421,8 +422,8 @@ public void GetUsfm_Verse_OptBreak() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:2"), "Second verse of the second chapter."), - new UpdateUsfmRow(ScrRef("MAT 2:3"), "Third verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "Second verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:3"]), "Third verse of the second chapter."), ]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); @@ -435,7 +436,7 @@ public void GetUsfm_Verse_OptBreak() [Test] public void GetUsfm_Verse_Milestone() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:10"), "Tenth verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:10"]), "Tenth verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -447,7 +448,7 @@ public void GetUsfm_Verse_Milestone() [Test] public void GetUsfm_Verse_Unmatched() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:3"), "Third verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "Third verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 3 Third verse of the first chapter.\r\n")); @@ -456,7 +457,7 @@ public void GetUsfm_Verse_Unmatched() [Test] public void GetUsfm_NonVerse_CharStyle() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:0/3:s1"), "The second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:0/3:s1"]), "The second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s1 The second chapter.\r\n")); @@ -465,7 +466,7 @@ public void GetUsfm_NonVerse_CharStyle() [Test] public void GetUsfm_NonVerse_Paragraph() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/8:s"), "The first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/8:s"]), "The first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s The first chapter.\r\n")); @@ -476,11 +477,11 @@ public void GetUsfm_NonVerse_Relaxed() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/s"), "The first chapter."), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "First verse of the first chapter."), - new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc1"), "The first cell of the table."), - new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc2"), "The second cell of the table."), - new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc1"), "The third cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 1:0/s"]), "The first chapter."), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "First verse of the first chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc1"]), "The first cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc2"]), "The second cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc1"]), "The third cell of the table."), ]; string target = UpdateUsfm(rows); @@ -506,7 +507,7 @@ public void GetUsfm_NonVerse_Sidebar() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:3/1:esb/1:ms"), "The first paragraph of the sidebar."), + new UpdateUsfmRow(ScrRef(["MAT 2:3/1:esb/1:ms"]), "The first paragraph of the sidebar."), ]; string target = UpdateUsfm(rows); @@ -518,8 +519,8 @@ public void GetUsfm_NonVerse_Table() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:0/1:tr/1:tc1"), "The first cell of the table."), - new UpdateUsfmRow(ScrRef("MAT 2:0/2:tr/1:tc1"), "The third cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/1:tr/1:tc1"]), "The first cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/2:tr/1:tc1"]), "The third cell of the table."), ]; string target = UpdateUsfm(rows); @@ -538,7 +539,7 @@ public void GetUsfm_NonVerse_OptBreak() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:3/1:esb/2:p"), "The second paragraph of the sidebar."), + new UpdateUsfmRow(ScrRef(["MAT 2:3/1:esb/2:p"]), "The second paragraph of the sidebar."), ]; string target = UpdateUsfm(rows); @@ -548,7 +549,7 @@ public void GetUsfm_NonVerse_OptBreak() [Test] public void GetUsfm_NonVerse_Milestone() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:7a/1:s"), "A new section header.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:7a/1:s"]), "A new section header.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s A new section header. \\ts-s\\*\r\n")); @@ -557,7 +558,7 @@ public void GetUsfm_NonVerse_Milestone() [Test] public void GetUsfm_NonVerse_SkipNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph.")]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\ip The introductory paragraph.\r\n")); @@ -566,7 +567,7 @@ public void GetUsfm_NonVerse_SkipNote() [Test] public void GetUsfm_NonVerse_ReplaceWithNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph.")]; string target = UpdateUsfm(rows); Assert.That( @@ -578,7 +579,7 @@ public void GetUsfm_NonVerse_ReplaceWithNote() [Test] public void GetUsfm_Verse_DoubleVaVp() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 3:1"), "Updating later in the book to start.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 3:1"]), "Updating later in the book to start.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -591,7 +592,7 @@ public void GetUsfm_Verse_DoubleVaVp() [Test] public void GetUsfm_Verse_LastSegment() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Updating the last verse.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Updating the last verse.")]; string usfm = @"\id MAT - Test \c 1 @@ -616,12 +617,12 @@ public void GetUsfm_Verse_UpdateRowsBeforeText() { List rows = [ - new UpdateUsfmRow(ScrRef("GEN 1:1"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:2"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:3"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:4"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:5"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph."), + new UpdateUsfmRow(ScrRef(["GEN 1:1"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:2"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:3"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:4"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:5"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph."), ]; string target = UpdateUsfm(rows); @@ -636,8 +637,8 @@ public void GetUsfm_StripParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/2:p"), "Update Paragraph"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update Verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/2:p"]), "Update Paragraph"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update Verse 1"), ]; string usfm = @@ -684,7 +685,7 @@ public void GetUsfm_PreservationRawStrings() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), @"Update all in one row \f \fr 1.1 \ft Some note \f*"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), @"Update all in one row \f \fr 1.1 \ft Some note \f*"), ]; string usfm = @@ -705,7 +706,7 @@ public void GetUsfm_PreservationRawStrings() [Test] public void GetUsfm_BeginningOfVerseEmbed() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Updated text")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Updated text")]; string usfm = @"\id MAT - Test @@ -722,10 +723,236 @@ public void GetUsfm_BeginningOfVerseEmbed() AssertUsfmEquals(target, result); } + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerChapter() + { + List rows = + [ + new UpdateUsfmRow(ScrRef(["MAL 1:1"], ScrVers.Original), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["MAL 3:24"], ScrVers.Original), "Updated verse 2"), + ]; + string usfm = + @"\id MAL +\c 1 +\v 1-14 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\c 4 +\v 1-5 +\v 6 +"; + string target = UpdateUsfm(rows, usfm, bookId: "MAL", versification: ScrVers.English); + string result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\c 4 +\v 1-5 +\v 6 Updated verse 2 +"; + AssertUsfmEquals(target, result); + + target = UpdateUsfm( + rows, + usfm, + bookId: "MAL", + versification: ScrVers.English, + convertUsfmToUpdateRowVersification: true + ); + result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\v 19-23 +\v 24 Updated verse 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreChapter() + { + List rows = + [ + new UpdateUsfmRow(ScrRef(["MAL 1:1"], ScrVers.English), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["MAL 4:6"], ScrVers.English), "Updated verse 2"), + ]; + string usfm = + @"\id MAL +\c 1 +\v 1-14 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\v 19-23 +\v 24 +"; + + string target = UpdateUsfm(rows, usfm, bookId: "MAL", versification: ScrVers.Original); + string result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\v 19-23 +\v 24 Updated verse 2 +"; + AssertUsfmEquals(target, result); + + target = UpdateUsfm( + rows, + usfm, + bookId: "MAL", + versification: ScrVers.Original, + convertUsfmToUpdateRowVersification: true + ); + result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\c 4 +\v 1-5 +\v 6 Updated verse 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_EmptyChapters() + { + List rows = []; + + string usfm = + @"\id MAT - Test +\c 1 +\c 2 +\c 3 +"; + + string target = UpdateUsfm(rows, usfm, convertUsfmToUpdateRowVersification: true); + string result = + @"\id MAT - Test +\c 1 +\c 2 +\c 3 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() + { + // Russian Orthodox vs. Original + // PSA 151:1-7 = PS2 1:1-7 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["PSA 150:1"], ScrVers.Original), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["PS2 1:1"], ScrVers.Original), "Updated verse 2"), + ]; + + string usfm = + @"\id PSA - Test +\c 150 +\v 1-6 +\c 151 +\v 1-7 +"; + + string target = UpdateUsfm( + rows, + usfm, + convertUsfmToUpdateRowVersification: false, + versification: ScrVers.RussianOrthodox + ); + string result = + @"\id PSA - Test +\c 150 +\v 1-6 Updated verse 1 +\c 151 +\v 1-7 Updated verse 2 +"; + AssertUsfmEquals(target, result); + + target = UpdateUsfm( + rows, + usfm, + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.RussianOrthodox + ); + result = + @"\id PSA - Test +\c 150 +\v 1-6 Updated verse 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreBook() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["DAN 3:1"], ScrVers.RussianOrthodox), "Updated verse 1"), + // This row will map to another book DAG so it will not appear in the updated DAN + new UpdateUsfmRow(ScrRef(["DAN 3:24"], ScrVers.RussianOrthodox), "Updated verse 2"), + ]; + + string usfm = + @"\id DAN - Test +\c 1 +\c 2 +\c 3 +\v 1-23 +\c 4 +"; + + string target = UpdateUsfm( + rows, + usfm, + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 1 +\c 2 +\c 3 +\v 1-23 Updated verse 1 +\c 4 +"; + AssertUsfmEquals(target, result); + } + [Test] public void CrossReferenceDontUpdate() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1/1:x"), "Update the cross reference")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1/1:x"]), "Update the cross reference")]; string usfm = @"\id MAT - Test \c 1 @@ -743,7 +970,7 @@ public void CrossReferenceDontUpdate() [Test] public void PreserveFig() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update")]; string usfm = @"\id MAT - Test \c 1 @@ -763,8 +990,8 @@ public void NoteExplicitEndMarkers() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update text"), - new UpdateUsfmRow(ScrRef("MAT 1:1/1:f"), "Update note"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update text"), + new UpdateUsfmRow(ScrRef(["MAT 1:1/1:f"]), "Update note"), ]; string usfm = @"\id MAT - Test @@ -791,7 +1018,7 @@ public void NoteExplicitEndMarkers() [Test] public void UpdateBlock_Verse_PreserveParas() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -821,7 +1048,7 @@ public void UpdateBlock_Verse_PreserveParas() [Test] public void UpdateBlock_Verse_StripParas() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -851,7 +1078,7 @@ public void UpdateBlock_Verse_StripParas() [Test] public void UpdateBlock_Verse_Range() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -879,7 +1106,7 @@ public void UpdateBlock_Verse_Range() [Test] public void UpdateBlock_Verse_Range_RightToLeftMarker() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1", "MAT 1:2", "MAT 1:3"), "Update 1-3")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1", "MAT 1:2", "MAT 1:3"]), "Update 1-3")]; string usfm = @"\id MAT - Test \c 1 @@ -910,7 +1137,7 @@ public void UpdateBlock_Verse_Range_RightToLeftMarker() [Test] public void UpdateBlock_Footnote_PreserveEmbeds() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -940,7 +1167,7 @@ public void UpdateBlock_Footnote_PreserveEmbeds() [Test] public void UpdateBlock_Footnote_StripEmbeds() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -970,7 +1197,7 @@ public void UpdateBlock_Footnote_StripEmbeds() [Test] public void UpdateBlock_NonVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/1:s"), "Updated section Header")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/1:s"]), "Updated section Header")]; string usfm = @"\id MAT - Test \s Section header @@ -994,7 +1221,7 @@ public void UpdateBlock_NonVerse() [Test] public void UpdateBlock_Verse_PreserveStyles() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1026,7 +1253,7 @@ public void UpdateBlock_Verse_PreserveStyles() [Test] public void UpdateBlock_Verse_StripStyles() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1058,7 +1285,7 @@ public void UpdateBlock_Verse_StripStyles() [Test] public void UpdateBlock_Verse_SectionHeader() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1096,7 +1323,7 @@ public void UpdateBlock_Verse_SectionHeader() [Test] public void UpdateBlock_Verse_SectionHeaderInVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1129,7 +1356,7 @@ public void UpdateBlock_Verse_SectionHeaderInVerse() [Test] public void UpdateBlock_NonVerse_ParagraphEndOfVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1160,11 +1387,11 @@ public void GetUsfm_HeaderReferenceParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 2:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 2:2"), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 2:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "new verse 2"), ]; string usfm = @@ -1215,16 +1442,16 @@ public void GetUsfm_OutOfOrderVerses() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), - new UpdateUsfmRow(ScrRef("MAT 1:5"), "new verse 5"), - new UpdateUsfmRow(ScrRef("MAT 1:6a"), "new verse 6a"), - new UpdateUsfmRow(ScrRef("MAT 1:6b"), "new verse 6b"), - new UpdateUsfmRow(ScrRef("MAT 1:6b/1:s"), "new section"), - new UpdateUsfmRow(ScrRef("MAT 1:7"), "new verse 7"), - new UpdateUsfmRow(ScrRef("MAT 1:8"), "new verse 8"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), + new UpdateUsfmRow(ScrRef(["MAT 1:5"]), "new verse 5"), + new UpdateUsfmRow(ScrRef(["MAT 1:6a"]), "new verse 6a"), + new UpdateUsfmRow(ScrRef(["MAT 1:6b"]), "new verse 6b"), + new UpdateUsfmRow(ScrRef(["MAT 1:6b/1:s"]), "new section"), + new UpdateUsfmRow(ScrRef(["MAT 1:7"]), "new verse 7"), + new UpdateUsfmRow(ScrRef(["MAT 1:8"]), "new verse 8"), ]; string usfm = @@ -1274,10 +1501,10 @@ public void GetUsfm_DuplicateVerses() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), ]; string usfm = @@ -1312,11 +1539,11 @@ public void GetUsfm_IdTags() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/1:s"), "new section header"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/1:s"]), "new section header"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), ]; string usfm = @@ -1357,8 +1584,8 @@ public void GetUsfm_PassRemark() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1412,8 +1639,8 @@ public void GetUsfm_PassRemark_NoBodyParagraph() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1465,8 +1692,8 @@ public void GetUsfm_PassRemark0_NoExistingRemark() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1503,8 +1730,8 @@ public void GetUsfm_MultipleRemarksSameChapter() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1546,7 +1773,7 @@ public void GetUsfm_MultipleRemarksSameChapter() [Test] public void UpdateBlock_FootnoteInPublishedChapterNumber() { - List rows = [new UpdateUsfmRow(ScrRef("ESG 1:0/2:s"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["ESG 1:0/2:s"]), "Update 1")]; string usfm = @"\id ESG - Test \c 1 @@ -1589,7 +1816,7 @@ public void UpdateBlock_FootnoteInPublishedChapterNumber() [Test] public void UpdateBlock_FootnoteAtStartOfChapterWithPrecedingText() { - List rows = [new UpdateUsfmRow(ScrRef("ESG 1:0/2:s"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["ESG 1:0/2:s"]), "Update 1")]; string usfm = @"\id ESG - Test \c 1 @@ -1719,7 +1946,8 @@ public void FilterChapters_WithBadChapterReference() AssertUsfmEquals(target, result); } - private static ScriptureRef[] ScrRef(params string[] refs) => [.. refs.Select(r => ScriptureRef.Parse(r))]; + private static ScriptureRef[] ScrRef(IEnumerable refs, ScrVers? versification = null) => + [.. refs.Select(r => ScriptureRef.Parse(r, versification))]; private static string UpdateUsfm( IReadOnlyList? rows = null, @@ -1733,15 +1961,18 @@ private static string UpdateUsfm( IEnumerable? preserveParagraphStyles = null, IEnumerable? usfmUpdateBlockHandlers = null, IEnumerable<(int, string)>? remarks = null, - bool compareSegments = false + bool compareSegments = false, + bool convertUsfmToUpdateRowVersification = false, + string? bookId = null, + ScrVers? versification = null ) { - const string BookId = "MAT"; + bookId ??= "MAT"; if (source is null) { var updater = new FileParatextProjectTextUpdater(CorporaTestHelpers.UsfmTestProjectPath); return updater.UpdateUsfm( - BookId, + bookId, rows, chapters, idText, @@ -1753,17 +1984,23 @@ private static string UpdateUsfm( usfmUpdateBlockHandlers, remarks, (_) => false, - compareSegments + compareSegments, + convertUsfmToUpdateRowVersification ); } else { source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; - var settings = new DefaultParatextProjectSettings(fileNameForm: BookId, fileNameSuffix: string.Empty); - var files = new Dictionary { [BookId] = source }; + var settings = new DefaultParatextProjectSettings( + fileNameForm: "MAT", + fileNamePrefix: string.Empty, + fileNameSuffix: string.Empty, + versification: versification + ); + var files = new Dictionary { [bookId] = source }; var updater = new MemoryParatextProjectTextUpdater(files, settings); return updater.UpdateUsfm( - BookId, + bookId, rows, chapters, idText, @@ -1775,7 +2012,8 @@ private static string UpdateUsfm( usfmUpdateBlockHandlers, remarks, (_) => false, - compareSegments + compareSegments, + convertUsfmToUpdateRowVersification ); } } From f0e99600c710abb675ea7e9ab7fe36fa8aeeb4c8 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 12 Aug 2026 21:38:47 -0400 Subject: [PATCH 02/10] Address reviewer comments --- .../Corpora/UpdateUsfmParserHandler.cs | 49 ++--------- src/SIL.Machine/Corpora/UsfmToken.cs | 5 +- .../Corpora/UpdateUsfmParserHandlerTests.cs | 87 ++++++++++++++++++- 3 files changed, 95 insertions(+), 46 deletions(-) diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index 6672eede6..c788c4648 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -67,7 +67,6 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private readonly Func _errorHandler; private readonly bool _compareSegments; private readonly bool _convertUsfmToUpdateRowVersification; - private UsfmToken _currentChapterToken; private int _currentChapterNum; private bool _skipNextVerseText; @@ -122,7 +121,6 @@ public UpdateUsfmParserHandler( _compareSegments = compareSegments; _convertUsfmToUpdateRowVersification = convertUsfmToUpdateRowVersification && _updateRowsVersification != _usfmVersification; - _currentChapterToken = null; _currentChapterNum = 0; _skipNextVerseText = false; } @@ -132,11 +130,6 @@ public UpdateUsfmParserHandler( public override void EndUsfm(UsfmParserState state) { CollectUpdatableTokens(state); - if (_currentChapterToken != null) - { - _tokens.Add(_currentChapterToken); - _currentChapterToken = null; - } base.EndUsfm(state); } @@ -166,12 +159,6 @@ public override void EndBook(UsfmParserState state, string marker) UsfmUpdateBlock updateBlock = _updateBlocks.Pop(); _tokens.AddRange(updateBlock.GetTokens()); - if (_currentChapterToken != null) - { - _tokens.Add(_currentChapterToken); - _currentChapterToken = null; - } - base.EndBook(state, marker); } @@ -632,28 +619,16 @@ private VerseRef UpdateVerseData(UsfmParserState state, UsfmToken token) verseRef = verseRef.ChangeVersificationWithSegments(_updateRowsVersification); if (verseRef.ChapterNum != _currentChapterNum && state.VerseRef.BookNum == verseRef.BookNum) { - if (_currentChapterToken != null) - { - _tokens.Add(_currentChapterToken.Copy()); - _currentChapterToken = null; - } - else - { - UsfmToken newChapterToken = new UsfmToken( - UsfmTokenType.Chapter, - "c", - "", - "", - verseRef.ChapterNum.ToString() - ); - _tokens.Add(newChapterToken); - } + UsfmToken newChapterToken = new UsfmToken( + UsfmTokenType.Chapter, + "c", + "", + "", + verseRef.ChapterNum.ToString() + ); + _tokens.Add(newChapterToken); _currentChapterNum = verseRef.ChapterNum; } - if (verseRef.ChapterNum == verseRef.Versification.GetLastChapter(verseRef.BookNum)) - { - _currentChapterToken = null; - } } return verseRef; @@ -670,13 +645,7 @@ private void CollectReadonlyTokens(UsfmParserState state) } else { - if (_convertUsfmToUpdateRowVersification && token.Type == UsfmTokenType.Chapter) - { - if (_currentChapterToken != null) - _tokens.Add(_currentChapterToken); - _currentChapterToken = token; - } - else + if (!_convertUsfmToUpdateRowVersification || token.Type != UsfmTokenType.Chapter) { _tokens.Add(token); } diff --git a/src/SIL.Machine/Corpora/UsfmToken.cs b/src/SIL.Machine/Corpora/UsfmToken.cs index 460c5ea26..28e4765ad 100644 --- a/src/SIL.Machine/Corpora/UsfmToken.cs +++ b/src/SIL.Machine/Corpora/UsfmToken.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using SIL.ObjectModel; namespace SIL.Machine.Corpora { @@ -22,7 +23,7 @@ public enum UsfmTokenType Unknown, } - public class UsfmToken : IEquatable + public class UsfmToken : IEquatable, ICloneable { private const string FullAttributeStr = @"(?[-\w]+)\s*\=\s*\""(?.+?)\""\s*"; private static readonly Regex AttributeRegex = new Regex( @@ -180,7 +181,7 @@ public void CopyAttributes(UsfmToken sourceToken) _defaultAttributeName = sourceToken._defaultAttributeName; } - public UsfmToken Copy() + public UsfmToken Clone() { UsfmToken copy = new UsfmToken(Type, Marker, Text, EndMarker, Data); copy.CopyAttributes(this); diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index 4b793ef54..ee74a4fb9 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -880,6 +880,7 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() string target = UpdateUsfm( rows, usfm, + bookId: "PSA", convertUsfmToUpdateRowVersification: false, versification: ScrVers.RussianOrthodox ); @@ -895,6 +896,7 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() target = UpdateUsfm( rows, usfm, + bookId: "PSA", convertUsfmToUpdateRowVersification: true, versification: ScrVers.RussianOrthodox ); @@ -925,26 +927,103 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreBook() string usfm = @"\id DAN - Test -\c 1 -\c 2 \c 3 \v 1-23 \c 4 +\v 1 "; string target = UpdateUsfm( rows, usfm, + bookId: "DAN", convertUsfmToUpdateRowVersification: true, versification: ScrVers.Original ); string result = @"\id DAN - Test -\c 1 -\c 2 \c 3 \v 1-23 Updated verse 1 \c 4 +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_BackOneVerseToPreviousChapter() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["ISA 9:1"], ScrVers.English), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["ISA 9:2"], ScrVers.English), "Updated verse 2"), + ]; + + string usfm = + @"\id ISA - Test +\c 8 +\v 22 +\v 23 +\c 9 +\v 1 +"; + + string target = UpdateUsfm( + rows, + usfm, + bookId: "ISA", + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.Original + ); + string result = + @"\id ISA - Test +\c 8 +\v 22 +\c 9 +\v 1 Updated verse 1 +\v 2 Updated verse 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_ForwardOneVerseToNextChapter() + { + // Original vs. English + // ISA 8:23 = ISA 9:1 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["ISA 8:23"], ScrVers.Original), "Updated verse 23"), + new UpdateUsfmRow(ScrRef(["ISA 9:1"], ScrVers.Original), "Updated verse 1"), + ]; + + string usfm = + @"\id ISA - Test +\c 8 +\v 22 +\c 9 +\v 1 +\v 2 +"; + + string target = UpdateUsfm( + rows, + usfm, + bookId: "ISA", + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.English + ); + string result = + @"\id ISA - Test +\c 8 +\v 22 +\v 23 Updated verse 23 +\c 9 +\v 1 Updated verse 1 "; AssertUsfmEquals(target, result); } From e193572b40f5aef23ad9047613dec038b0623dce Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 12 Aug 2026 22:28:28 -0400 Subject: [PATCH 03/10] Remove empty chapter test --- .../Corpora/UpdateUsfmParserHandlerTests.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index ee74a4fb9..254e0f752 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -835,28 +835,6 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreChapter() AssertUsfmEquals(target, result); } - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_EmptyChapters() - { - List rows = []; - - string usfm = - @"\id MAT - Test -\c 1 -\c 2 -\c 3 -"; - - string target = UpdateUsfm(rows, usfm, convertUsfmToUpdateRowVersification: true); - string result = - @"\id MAT - Test -\c 1 -\c 2 -\c 3 -"; - AssertUsfmEquals(target, result); - } - [Test] public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() { From 94c875e77d713cd44d92a15443986e340635508b Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 27 Aug 2026 17:44:04 -0400 Subject: [PATCH 04/10] Separate USFM versification conversion into separate handler --- .../ConvertUsfmVersificationHandler.cs | 151 +++++ .../Corpora/ParatextProjectTextUpdaterBase.cs | 7 +- ...ratextProjectVersificationConverterBase.cs | 58 ++ .../Corpora/UpdateUsfmParserHandler.cs | 78 +-- src/SIL.Machine/Corpora/VerseRefExtensions.cs | 46 ++ .../ConvertUsfmVersificationHandlerTests.cs | 404 ++++++++++++++ ...ryParatextProjectVersificationConverter.cs | 10 + .../Corpora/UpdateUsfmParserHandlerTests.cs | 525 ++++-------------- 8 files changed, 796 insertions(+), 483 deletions(-) create mode 100644 src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs create mode 100644 src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs create mode 100644 tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs create mode 100644 tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs diff --git a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs new file mode 100644 index 000000000..fae6fdb9b --- /dev/null +++ b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs @@ -0,0 +1,151 @@ +using System.Collections.Generic; +using System.Linq; +using SIL.Scripture; + +namespace SIL.Machine.Corpora +{ + public class ConvertUsfmVersificationHandler : ScriptureRefUsfmParserHandlerBase + { + private readonly List _tokens; + private VerseRef _prevVerseRef; + private int _verseBoundary; + private readonly ScrVers _targetVersification; + private int _insertChapterIndex; + private bool _skip; + + public ConvertUsfmVersificationHandler(ScrVers targetVersification) + { + _verseBoundary = 0; + _insertChapterIndex = -1; + _tokens = new List(); + _prevVerseRef = new VerseRef(); + _targetVersification = targetVersification; + _skip = false; + } + + public override void Chapter( + UsfmParserState state, + string number, + string marker, + string altNumber, + string pubNumber + ) + { + base.Chapter(state, number, marker, altNumber, pubNumber); + ProcessTokens(state); + _insertChapterIndex = _tokens.Count; + } + + public override void Verse( + UsfmParserState state, + string number, + string marker, + string altNumber, + string pubNumber + ) + { + base.Verse(state, number, marker, altNumber, pubNumber); + + VerseRef verseRef = state.VerseRef; + + ProcessTokens(state); + + List verseRefs = state + .VerseRef.AllVerses() + .Select(vr => vr.ChangeVersificationWithSegments(_targetVersification)) + .ToList(); + + if ( + _prevVerseRef.IsDefault + || ( + verseRefs[0].BookNum == _prevVerseRef.BookNum && verseRefs[0].ChapterNum != _prevVerseRef.ChapterNum + ) + ) + { + UsfmToken newChapterToken = new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[0].Chapter); + + if (_insertChapterIndex == -1) + _tokens.Add(newChapterToken); + else + _tokens.Insert(_insertChapterIndex, newChapterToken); + } + + string start = null; + for (int i = 0; i < verseRefs.Count; i++) + { + if (!_prevVerseRef.IsDefault && verseRefs[i].Book != _prevVerseRef.Book) + { + continue; + } + if (start != null) + { + string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : ""; + if ( + _prevVerseRef.BookNum == verseRefs[i].BookNum + && _prevVerseRef.ChapterNum != verseRefs[i].ChapterNum + ) + { + _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + _tokens.Add(new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[i].Chapter)); + start = verseRefs[i].Verse; + _prevVerseRef = verseRefs[i]; + } + else if (_prevVerseRef.VerseNum + 1 != verseRefs[i].VerseNum) + { + _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + start = verseRefs[i].Verse; + _prevVerseRef = verseRefs[i]; + } + else + { + _prevVerseRef = verseRefs[i]; + } + } + else + { + start = verseRefs[i].Verse; + _prevVerseRef = verseRefs[i]; + } + verseRef = verseRefs[i]; + } + + if (start != null) + { + string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : ""; + _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + _skip = false; + _insertChapterIndex = -1; + _prevVerseRef = verseRef; + } + else + { + _skip = true; + } + } + + public override void EndUsfm(UsfmParserState state) + { + base.EndUsfm(state); + ProcessTokens(state); + if (!_skip && !(state.Token.Type == UsfmTokenType.Chapter || state.Token.Type == UsfmTokenType.Verse)) + _tokens.Add(state.Token); + } + + public string GetUsfm(UsfmStylesheet stylesheet) + { + var tokenizer = new UsfmTokenizer(stylesheet); + return tokenizer.Detokenize(_tokens); + } + + private void ProcessTokens(UsfmParserState state) + { + int offset = 0; + if (!_skip) + { + while (_verseBoundary + offset < state.Index) + _tokens.Add(state.Tokens[_verseBoundary + offset++]); + } + _verseBoundary = state.Index + 1; + } + } +} diff --git a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs index fd104e3f8..e38e528f6 100644 --- a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs +++ b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs @@ -33,8 +33,7 @@ public string UpdateUsfm( IEnumerable updateBlockHandlers = null, IEnumerable<(int, string)> remarks = null, Func errorHandler = null, - bool compareSegments = false, - bool convertUsfmToUpdateRowVersification = false + bool compareSegments = false ) { string fileName = _settings.GetBookFileName(bookId); @@ -58,9 +57,7 @@ public string UpdateUsfm( updateBlockHandlers, remarks, errorHandler, - compareSegments, - _settings.Versification, - convertUsfmToUpdateRowVersification + compareSegments ); try { diff --git a/src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs new file mode 100644 index 000000000..9bdd367d9 --- /dev/null +++ b/src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using SIL.Scripture; + +namespace SIL.Machine.Corpora +{ + public abstract class ParatextProjectVersificationConverterBase + { + private readonly ParatextProjectSettings _settings; + private readonly IParatextProjectFileHandler _paratextProjectFileHandler; + + protected ParatextProjectVersificationConverterBase( + IParatextProjectFileHandler paratextProjectFileHandler, + ParatextProjectSettings settings + ) + { + _settings = settings; + _paratextProjectFileHandler = paratextProjectFileHandler; + } + + public string UpdateUsfm(string bookId, ScrVers targetVersification) + { + string fileName = _settings.GetBookFileName(bookId); + if (!Exists(fileName)) + return null; + + string usfm; + using (var reader = new StreamReader(Open(fileName))) + { + usfm = reader.ReadToEnd(); + } + + var handler = new ConvertUsfmVersificationHandler(targetVersification); + try + { + var tokenizer = new UsfmTokenizer(_settings.Stylesheet); + IReadOnlyList tokens = tokenizer.Tokenize(usfm); + UsfmParser.Parse(tokens, handler, _settings.Stylesheet, _settings.Versification); + return handler.GetUsfm(_settings.Stylesheet); + } + catch (Exception ex) + { + var sb = new StringBuilder(); + sb.Append($"An error occurred while parsing the usfm for '{bookId}`"); + if (!string.IsNullOrEmpty(_settings.Name)) + sb.Append($" in project '{_settings.Name}'"); + sb.Append($". Error: '{ex.Message}'"); + throw new InvalidOperationException(sb.ToString(), ex); + } + } + + private bool Exists(string fileName) => _paratextProjectFileHandler.Exists(fileName); + + private Stream Open(string fileName) => _paratextProjectFileHandler.Open(fileName); + } +} diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index c788c4648..f21be754d 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -49,7 +49,6 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private int _verseRowIndex; private readonly Dictionary> _verseRowsMap; private readonly ScrVers _updateRowsVersification; - private readonly ScrVers _usfmVersification; private readonly List _tokens; private readonly List _updatedText; private readonly List _embedTokens; @@ -66,9 +65,6 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private int _tokenIndex; private readonly Func _errorHandler; private readonly bool _compareSegments; - private readonly bool _convertUsfmToUpdateRowVersification; - private int _currentChapterNum; - private bool _skipNextVerseText; /// UpdateUsfmRows must be in order public UpdateUsfmParserHandler( @@ -82,9 +78,7 @@ public UpdateUsfmParserHandler( IEnumerable updateBlockHandlers = null, IEnumerable<(int, string)> remarks = null, Func errorHandler = null, - bool compareSegments = false, - ScrVers usfmVersification = null, - bool convertUsfmToUpdateRowVersification = false + bool compareSegments = false ) { _rows = rows ?? Array.Empty(); @@ -95,7 +89,6 @@ public UpdateUsfmParserHandler( _updateRowsVersification = ScrVers.English; if (_rows.Count > 0) _updateRowsVersification = _rows.First(r => r.Refs.Count > 0).Refs[0].Versification; - _usfmVersification = usfmVersification ?? _updateRowsVersification; _tokens = new List(); _updatedText = new List(); _updateBlocks = new Stack(); @@ -119,10 +112,6 @@ public UpdateUsfmParserHandler( if (_errorHandler == null) _errorHandler = (error) => false; _compareSegments = compareSegments; - _convertUsfmToUpdateRowVersification = - convertUsfmToUpdateRowVersification && _updateRowsVersification != _usfmVersification; - _currentChapterNum = 0; - _skipNextVerseText = false; } public IReadOnlyList Tokens => _tokens; @@ -409,11 +398,7 @@ protected override void StartVerseText(UsfmParserState state, IReadOnlyList scriptureRefs) { - if (!_skipNextVerseText) - { - EndUpdateBlock(state, scriptureRefs); - _skipNextVerseText = false; - } + EndUpdateBlock(state, scriptureRefs); } protected override void StartNonVerseText(UsfmParserState state, ScriptureRef scriptureRef) @@ -579,16 +564,9 @@ private void CollectUpdatableTokens(UsfmParserState state) UsfmToken token = state.Tokens[_tokenIndex]; if (token.Type == UsfmTokenType.Verse) { - VerseRef updatedVerse = UpdateVerseData(state, token); - if (updatedVerse.BookNum != state.VerseRef.BookNum) - { - _tokenIndex++; - _skipNextVerseText = true; - continue; - } - token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, updatedVerse.Verse); + string sanitizedVerseData = SanitizeVerseData(token.Data); + token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, sanitizedVerseData); } - if (CurrentTextType == ScriptureTextType.Embed) { _embedTokens.Add(token); @@ -604,36 +582,10 @@ private void CollectUpdatableTokens(UsfmParserState state) { _tokens.Add(token); } - _tokenIndex++; } } - private VerseRef UpdateVerseData(UsfmParserState state, UsfmToken token) - { - string updatedVerseData = SanitizeVerseData(token.Data); - VerseRef verseRef = state.VerseRef; - verseRef.Verse = updatedVerseData; - if (_convertUsfmToUpdateRowVersification) - { - verseRef = verseRef.ChangeVersificationWithSegments(_updateRowsVersification); - if (verseRef.ChapterNum != _currentChapterNum && state.VerseRef.BookNum == verseRef.BookNum) - { - UsfmToken newChapterToken = new UsfmToken( - UsfmTokenType.Chapter, - "c", - "", - "", - verseRef.ChapterNum.ToString() - ); - _tokens.Add(newChapterToken); - _currentChapterNum = verseRef.ChapterNum; - } - } - - return verseRef; - } - private void CollectReadonlyTokens(UsfmParserState state) { while (_tokenIndex <= state.Index + state.SpecialTokenCount) @@ -645,10 +597,7 @@ private void CollectReadonlyTokens(UsfmParserState state) } else { - if (!_convertUsfmToUpdateRowVersification || token.Type != UsfmTokenType.Chapter) - { - _tokens.Add(token); - } + _tokens.Add(token); } _tokenIndex++; } @@ -714,13 +663,10 @@ private bool HasNewText() private void StartUpdateBlock(IReadOnlyList scriptureRefs) { (IReadOnlyList rowTexts, Dictionary metadata) = AdvanceRows(scriptureRefs); - if (!_skipNextVerseText) - { - _updateBlocks.Push( - new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary()) - ); - PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " "))); - } + _updateBlocks.Push( + new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary()) + ); + PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " "))); } private void EndUpdateBlock(UsfmParserState state, IReadOnlyList scriptureRefs) @@ -811,11 +757,7 @@ private bool IsNonverseParagraph(UsfmParserState state, UsfmUpdateBlockElement e private void UpdateVerseRowsMap() { _verseRowsMap.Clear(); - while ( - _rowIndex < _rows.Count - && _rows[_rowIndex].Refs[0].ChangeVersification(_verseRowsRef.Versification).ChapterNum - == _verseRowsRef.ChapterNum - ) + while (_rowIndex < _rows.Count && _rows[_rowIndex].Refs[0].ChapterNum == _verseRowsRef.ChapterNum) { UpdateUsfmRow row = _rows[_rowIndex]; var ri = new RowInfo(_rowIndex); diff --git a/src/SIL.Machine/Corpora/VerseRefExtensions.cs b/src/SIL.Machine/Corpora/VerseRefExtensions.cs index adc8f6e48..efc75d340 100644 --- a/src/SIL.Machine/Corpora/VerseRefExtensions.cs +++ b/src/SIL.Machine/Corpora/VerseRefExtensions.cs @@ -53,5 +53,51 @@ public static VerseRef ChangeVersificationWithSegments(this VerseRef verseRef, S } return vr; } + + public static bool TryChangeVersificationWithSegments( + this VerseRef verseRef, + ScrVers versification, + out VerseRef changedVerseRef + ) + { + VerseRef vr = verseRef; + + bool success = true; + if (vr.HasMultiple) + success = vr.ChangeVersificationWithRanges(versification); + else + vr.ChangeVersification(versification); + + if (string.IsNullOrEmpty(vr.Segment())) + { + changedVerseRef = vr; + return success; + } + + VerseRef verseRefWithoutSegments = verseRef.RemoveSegments(); + if (verseRefWithoutSegments.HasMultiple) + success = verseRefWithoutSegments.ChangeVersificationWithRanges(versification); + else + verseRefWithoutSegments.ChangeVersification(versification); + if (!verseRefWithoutSegments.Equals(vr.RemoveSegments())) + { + IEnumerable verses = verseRef + .AllVerses() + .Zip( + verseRefWithoutSegments.AllVerses(), + (verseWithSegments, verseWithCorrectNumber) => (verseWithSegments, verseWithCorrectNumber) + ) + .Select( + (verseTuple) => verseTuple.verseWithCorrectNumber.Verse + verseTuple.verseWithSegments.Segment() + ); + changedVerseRef = new VerseRef( + $"{verseRefWithoutSegments.Book} {verseRefWithoutSegments.ChapterNum}:{string.Join(",", verses)}", + versification + ); + return success; + } + changedVerseRef = vr; + return success; + } } } diff --git a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs new file mode 100644 index 000000000..611868878 --- /dev/null +++ b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs @@ -0,0 +1,404 @@ +using NUnit.Framework; +using SIL.Scripture; + +namespace SIL.Machine.Corpora; + +[TestFixture] +public class ConvertUsfmVersificationHandlerTests +{ + [Test] + public void GetUsfm_OneFewerChapter() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\p +\s1 Section +\v 1-5 +\v 6 Text +"; + + string target = UpdateUsfm( + "MAL", + usfm, + sourceVersification: ScrVers.English, + targetVersification: ScrVers.Original + ); + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\p +\s1 Section +\v 19-23 +\v 24 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_OneMoreChapter() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-23 +\v 24 Text +"; + + string target = UpdateUsfm( + "MAL", + usfm, + sourceVersification: ScrVers.Original, + targetVersification: ScrVers.English + ); + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\v 1-5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_OneFewerBook() + { + // Russian Orthodox vs. Original + // PSA 151:1-7 = PS2 1:1-7 + + string usfm = + @"\id PSA - Test +\h Psalms +\c 150 +\v 1-5 Lines +\v 6 Line +\q Another line +\c 151 +\v 1-7 More lines +"; + + string target = UpdateUsfm( + "PSA", + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id PSA - Test +\h Psalms +\c 150 +\v 1-5 Lines +\v 6 Line +\q Another line +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_OneMoreBook() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + // S3Y 1:30-31 = DAG 3:52-53 + // S3Y 1:33 = DAG 3:54 + // S3Y 1:32 = DAG 3:55 + // S3Y 1:34-35 = DAG 3:56-57 + // S3Y 1:37 = DAG 3:58 + // S3Y 1:36 = DAG 3:59 + // S3Y 1:38-68 = DAG 3:60-90 + + string usfm = + @"\id DAN - Test +\h Daniel +\c 3 +\v 1-23 +\v 24-90 +\p +\v 91-100 +\c 4 +\v 1 +"; + + string target = UpdateUsfm( + "DAN", + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\h Daniel +\c 3 +\v 1-23 +\v 24-33 +\c 4 +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_BackOneVerseToPreviousChapter() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\v 22 +\v 23 +\c 9 +\v 1 +"; + + string target = UpdateUsfm( + "ISA", + usfm, + sourceVersification: ScrVers.Original, + targetVersification: ScrVers.English + ); + string result = + @"\id ISA - Test +\c 8 +\v 22 +\c 9 +\v 1 +\v 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ForwardOneVerseToNextChapter() + { + // Original vs. English + // ISA 8:23 = ISA 9:1 + + string usfm = + @"\id ISA - Test +\c 8 +\v 22 +\c 9 +\v 1 +\v 2 +"; + + string target = UpdateUsfm( + "ISA", + usfm, + sourceVersification: ScrVers.English, + targetVersification: ScrVers.Original + ); + string result = + @"\id ISA - Test +\c 8 +\v 22 +\v 23 +\c 9 +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_CrossChapterVerseRange() + { + // English vd. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\v 22-23 +\c 9 +\v 1 +"; + + string target = UpdateUsfm( + "ISA", + usfm, + sourceVersification: ScrVers.Original, + targetVersification: ScrVers.English + ); + string result = + @"\id ISA - Test +\c 8 +\v 22 +\c 9 +\v 1 +\v 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_CrossChapterVerseRange_CrossBook() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + // S3Y 1:30-31 = DAG 3:52-53 + // S3Y 1:33 = DAG 3:54 + // S3Y 1:32 = DAG 3:55 + // S3Y 1:34-35 = DAG 3:56-57 + // S3Y 1:37 = DAG 3:58 + // S3Y 1:36 = DAG 3:59 + // S3Y 1:38-68 = DAG 3:60-90 + + string usfm = + @"\id DAN - Test +\c 3 +\v 1-22 +\v 23-89 +\v 90-100 +\c 4 +\v 1 +"; + + string target = UpdateUsfm( + "DAN", + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\v 1-22 +\v 23 +\v 24-33 +\c 4 +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_CrossChapterVerseRange_CrossBookWithinSingleRange() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + // S3Y 1:30-31 = DAG 3:52-53 + // S3Y 1:33 = DAG 3:54 + // S3Y 1:32 = DAG 3:55 + // S3Y 1:34-35 = DAG 3:56-57 + // S3Y 1:37 = DAG 3:58 + // S3Y 1:36 = DAG 3:59 + // S3Y 1:38-68 = DAG 3:60-90 + + string usfm = + @"\id DAN - Test +\c 3 +\v 1-100 +\c 4 +\v 1 +"; + + string target = UpdateUsfm( + "DAN", + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\v 1-33 +\c 4 +\v 1 +"; + AssertUsfmEquals(target, result); + } + + private static string UpdateUsfm( + string bookId, + string source, + ScrVers sourceVersification, + ScrVers targetVersification + ) + { + source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; + var settings = new DefaultParatextProjectSettings( + versification: sourceVersification, + fileNameForm: "MAT", + fileNameSuffix: string.Empty, + fileNamePrefix: string.Empty + ); + var files = new Dictionary { [bookId] = source }; + var updater = new MemoryParatextProjectVersificationConverter(files, settings); + return updater.UpdateUsfm(bookId, targetVersification); + } + + private static void AssertUsfmEquals(string target, string truth) + { + Assert.That(target, Is.Not.Null); + string[] targetLines = target.Split('\n'); + string[] truthLines = truth.Split('\n'); + for (int i = 0; i < truthLines.Length; i++) + Assert.That(targetLines[i].Trim(), Is.EqualTo(truthLines[i].Trim()), message: $"Line {i}"); + } +} diff --git a/tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs b/tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs new file mode 100644 index 000000000..6584d0dc9 --- /dev/null +++ b/tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs @@ -0,0 +1,10 @@ +namespace SIL.Machine.Corpora; + +public class MemoryParatextProjectVersificationConverter( + IDictionary? files = null, + ParatextProjectSettings? settings = null +) + : ParatextProjectVersificationConverterBase( + new MemoryParatextProjectFileHandler(files), + settings ?? new DefaultParatextProjectSettings() + ); diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index 254e0f752..cddf0df85 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -1,6 +1,5 @@ using System.Collections.Immutable; using NUnit.Framework; -using SIL.Scripture; namespace SIL.Machine.Corpora; @@ -10,7 +9,7 @@ public class UpdateUsfmParserHandlerTests [Test] public void GetUsfm_Verse_CharStyle() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "First verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "First verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -34,8 +33,8 @@ public void GetUsfm_StripAllText() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "Update 3"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:3"), "Update 3"), ]; string usfm = @"\id MAT - Test @@ -103,9 +102,9 @@ public void GetUsfm_StripParagraphs_PreserveParagraphStyles() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:0/1:rem"]), "New remark"), - new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "Another new remark"), - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:0/1:rem"), "New remark"), + new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "Another new remark"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), ]; string usfm = @"\id MAT @@ -157,8 +156,8 @@ public void GetUsfm_PreserveParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:0/1:rem"]), "Update remark"), - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:0/1:rem"), "Update remark"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), ]; string usfm = @"\id MAT @@ -202,7 +201,7 @@ public void GetUsfm_PreserveParagraphs() [Test] public void GetUsfm_ParagraphInVerse() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -248,8 +247,8 @@ public void GetUsfm_PreferExisting() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), ]; string usfm = @"\id MAT - Test @@ -274,8 +273,8 @@ public void GetUsfm_PreferRows() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:6"]), "Text 6"), - new UpdateUsfmRow(ScrRef(["MAT 1:7"]), "Text 7"), + new UpdateUsfmRow(ScrRef("MAT 1:6"), "Text 6"), + new UpdateUsfmRow(ScrRef("MAT 1:7"), "Text 7"), ]; string target = UpdateUsfm(rows, textBehavior: UpdateUsfmTextBehavior.PreferNew); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -286,7 +285,7 @@ public void GetUsfm_PreferRows() [Test] public void GetUsfm_Verse_StripNote() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:1"]), "First verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:1"), "First verse of the second chapter.")]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\v 1 First verse of the second chapter.\r\n")); @@ -295,7 +294,7 @@ public void GetUsfm_Verse_StripNote() [Test] public void GetUsfm_Verse_ReplaceWithNote() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "updated text")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "updated text")]; string usfm = @"\id MAT - Test \c 1 @@ -313,7 +312,7 @@ public void GetUsfm_Verse_ReplaceWithNote() [Test] public void GetUsfm_Verse_RowVerseSegment() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:1a"]), "First verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:1a"), "First verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -327,7 +326,7 @@ public void GetUsfm_Verse_RowVerseSegment() [Test] public void GetUsfm_Verse_UsfmVerseSegment() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:7"]), "Seventh verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:7"), "Seventh verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 7a Seventh verse of the second chapter.\r\n")); @@ -336,7 +335,7 @@ public void GetUsfm_Verse_UsfmVerseSegment() [Test] public void GetUsfm_Verse_MultipleParas() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Second verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:2"), "Second verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -350,7 +349,7 @@ public void GetUsfm_Verse_MultipleParas() [Test] public void GetUsfm_Verse_Table() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:9"]), "Ninth verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:9"), "Ninth verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 9 Ninth verse of the second chapter. \\tcr2 \\tc3 \\tcr4\r\n")); @@ -362,7 +361,7 @@ public void GetUsfm_Verse_RangeSingleRowMultipleVerses() List rows = [ new UpdateUsfmRow( - ScrRef(["MAT 2:11", "MAT 2:12"]), + ScrRef("MAT 2:11", "MAT 2:12"), "Eleventh verse of the second chapter. Twelfth verse of the second chapter." ), ]; @@ -379,7 +378,7 @@ public void GetUsfm_Verse_RangeSingleRowMultipleVerses() [Test] public void GetUsfm_Verse_RangeSingleRowSingleVerse() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:11"]), "Eleventh verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:11"), "Eleventh verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 11-12 Eleventh verse of the second chapter.\r\n")); @@ -390,8 +389,8 @@ public void GetUsfm_Verse_RangeMultipleRowsSingleVerse() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 2:11"]), "Eleventh verse of the second chapter."), - new UpdateUsfmRow(ScrRef(["MAT 2:12"]), "Twelfth verse of the second chapter."), + new UpdateUsfmRow(ScrRef("MAT 2:11"), "Eleventh verse of the second chapter."), + new UpdateUsfmRow(ScrRef("MAT 2:12"), "Twelfth verse of the second chapter."), ]; string target = UpdateUsfm(rows); @@ -408,9 +407,9 @@ public void GetUsfm_MergeVerseSegments() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "Verse 2."), - new UpdateUsfmRow(ScrRef(["MAT 2:2a"]), "Verse 2a."), - new UpdateUsfmRow(ScrRef(["MAT 2:2b"]), "Verse 2b."), + new UpdateUsfmRow(ScrRef("MAT 2:2"), "Verse 2."), + new UpdateUsfmRow(ScrRef("MAT 2:2a"), "Verse 2a."), + new UpdateUsfmRow(ScrRef("MAT 2:2b"), "Verse 2b."), ]; string target = UpdateUsfm(rows); @@ -422,8 +421,8 @@ public void GetUsfm_Verse_OptBreak() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "Second verse of the second chapter."), - new UpdateUsfmRow(ScrRef(["MAT 2:3"]), "Third verse of the second chapter."), + new UpdateUsfmRow(ScrRef("MAT 2:2"), "Second verse of the second chapter."), + new UpdateUsfmRow(ScrRef("MAT 2:3"), "Third verse of the second chapter."), ]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); @@ -436,7 +435,7 @@ public void GetUsfm_Verse_OptBreak() [Test] public void GetUsfm_Verse_Milestone() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:10"]), "Tenth verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:10"), "Tenth verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -448,7 +447,7 @@ public void GetUsfm_Verse_Milestone() [Test] public void GetUsfm_Verse_Unmatched() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "Third verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:3"), "Third verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 3 Third verse of the first chapter.\r\n")); @@ -457,7 +456,7 @@ public void GetUsfm_Verse_Unmatched() [Test] public void GetUsfm_NonVerse_CharStyle() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:0/3:s1"]), "The second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:0/3:s1"), "The second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s1 The second chapter.\r\n")); @@ -466,7 +465,7 @@ public void GetUsfm_NonVerse_CharStyle() [Test] public void GetUsfm_NonVerse_Paragraph() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/8:s"]), "The first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/8:s"), "The first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s The first chapter.\r\n")); @@ -477,11 +476,11 @@ public void GetUsfm_NonVerse_Relaxed() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:0/s"]), "The first chapter."), - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "First verse of the first chapter."), - new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc1"]), "The first cell of the table."), - new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc2"]), "The second cell of the table."), - new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc1"]), "The third cell of the table."), + new UpdateUsfmRow(ScrRef("MAT 1:0/s"), "The first chapter."), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "First verse of the first chapter."), + new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc1"), "The first cell of the table."), + new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc2"), "The second cell of the table."), + new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc1"), "The third cell of the table."), ]; string target = UpdateUsfm(rows); @@ -507,7 +506,7 @@ public void GetUsfm_NonVerse_Sidebar() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 2:3/1:esb/1:ms"]), "The first paragraph of the sidebar."), + new UpdateUsfmRow(ScrRef("MAT 2:3/1:esb/1:ms"), "The first paragraph of the sidebar."), ]; string target = UpdateUsfm(rows); @@ -519,8 +518,8 @@ public void GetUsfm_NonVerse_Table() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 2:0/1:tr/1:tc1"]), "The first cell of the table."), - new UpdateUsfmRow(ScrRef(["MAT 2:0/2:tr/1:tc1"]), "The third cell of the table."), + new UpdateUsfmRow(ScrRef("MAT 2:0/1:tr/1:tc1"), "The first cell of the table."), + new UpdateUsfmRow(ScrRef("MAT 2:0/2:tr/1:tc1"), "The third cell of the table."), ]; string target = UpdateUsfm(rows); @@ -539,7 +538,7 @@ public void GetUsfm_NonVerse_OptBreak() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 2:3/1:esb/2:p"]), "The second paragraph of the sidebar."), + new UpdateUsfmRow(ScrRef("MAT 2:3/1:esb/2:p"), "The second paragraph of the sidebar."), ]; string target = UpdateUsfm(rows); @@ -549,7 +548,7 @@ public void GetUsfm_NonVerse_OptBreak() [Test] public void GetUsfm_NonVerse_Milestone() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:7a/1:s"]), "A new section header.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 2:7a/1:s"), "A new section header.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s A new section header. \\ts-s\\*\r\n")); @@ -558,7 +557,7 @@ public void GetUsfm_NonVerse_Milestone() [Test] public void GetUsfm_NonVerse_SkipNote() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph.")]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\ip The introductory paragraph.\r\n")); @@ -567,7 +566,7 @@ public void GetUsfm_NonVerse_SkipNote() [Test] public void GetUsfm_NonVerse_ReplaceWithNote() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph.")]; string target = UpdateUsfm(rows); Assert.That( @@ -579,7 +578,7 @@ public void GetUsfm_NonVerse_ReplaceWithNote() [Test] public void GetUsfm_Verse_DoubleVaVp() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 3:1"]), "Updating later in the book to start.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 3:1"), "Updating later in the book to start.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -592,7 +591,7 @@ public void GetUsfm_Verse_DoubleVaVp() [Test] public void GetUsfm_Verse_LastSegment() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Updating the last verse.")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Updating the last verse.")]; string usfm = @"\id MAT - Test \c 1 @@ -617,12 +616,12 @@ public void GetUsfm_Verse_UpdateRowsBeforeText() { List rows = [ - new UpdateUsfmRow(ScrRef(["GEN 1:1"]), "Update rows before the start"), - new UpdateUsfmRow(ScrRef(["GEN 1:2"]), "Update rows before the start"), - new UpdateUsfmRow(ScrRef(["GEN 1:3"]), "Update rows before the start"), - new UpdateUsfmRow(ScrRef(["GEN 1:4"]), "Update rows before the start"), - new UpdateUsfmRow(ScrRef(["GEN 1:5"]), "Update rows before the start"), - new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph."), + new UpdateUsfmRow(ScrRef("GEN 1:1"), "Update rows before the start"), + new UpdateUsfmRow(ScrRef("GEN 1:2"), "Update rows before the start"), + new UpdateUsfmRow(ScrRef("GEN 1:3"), "Update rows before the start"), + new UpdateUsfmRow(ScrRef("GEN 1:4"), "Update rows before the start"), + new UpdateUsfmRow(ScrRef("GEN 1:5"), "Update rows before the start"), + new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph."), ]; string target = UpdateUsfm(rows); @@ -637,8 +636,8 @@ public void GetUsfm_StripParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:0/2:p"]), "Update Paragraph"), - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update Verse 1"), + new UpdateUsfmRow(ScrRef("MAT 1:0/2:p"), "Update Paragraph"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update Verse 1"), ]; string usfm = @@ -685,7 +684,7 @@ public void GetUsfm_PreservationRawStrings() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), @"Update all in one row \f \fr 1.1 \ft Some note \f*"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), @"Update all in one row \f \fr 1.1 \ft Some note \f*"), ]; string usfm = @@ -706,7 +705,7 @@ public void GetUsfm_PreservationRawStrings() [Test] public void GetUsfm_BeginningOfVerseEmbed() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Updated text")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Updated text")]; string usfm = @"\id MAT - Test @@ -723,293 +722,10 @@ public void GetUsfm_BeginningOfVerseEmbed() AssertUsfmEquals(target, result); } - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerChapter() - { - List rows = - [ - new UpdateUsfmRow(ScrRef(["MAL 1:1"], ScrVers.Original), "Updated verse 1"), - new UpdateUsfmRow(ScrRef(["MAL 3:24"], ScrVers.Original), "Updated verse 2"), - ]; - string usfm = - @"\id MAL -\c 1 -\v 1-14 -\c 2 -\v 1-17 -\c 3 -\v 1-18 -\c 4 -\v 1-5 -\v 6 -"; - string target = UpdateUsfm(rows, usfm, bookId: "MAL", versification: ScrVers.English); - string result = - @"\id MAL -\c 1 -\v 1-14 Updated verse 1 -\c 2 -\v 1-17 -\c 3 -\v 1-18 -\c 4 -\v 1-5 -\v 6 Updated verse 2 -"; - AssertUsfmEquals(target, result); - - target = UpdateUsfm( - rows, - usfm, - bookId: "MAL", - versification: ScrVers.English, - convertUsfmToUpdateRowVersification: true - ); - result = - @"\id MAL -\c 1 -\v 1-14 Updated verse 1 -\c 2 -\v 1-17 -\c 3 -\v 1-18 -\v 19-23 -\v 24 Updated verse 2 -"; - AssertUsfmEquals(target, result); - } - - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreChapter() - { - List rows = - [ - new UpdateUsfmRow(ScrRef(["MAL 1:1"], ScrVers.English), "Updated verse 1"), - new UpdateUsfmRow(ScrRef(["MAL 4:6"], ScrVers.English), "Updated verse 2"), - ]; - string usfm = - @"\id MAL -\c 1 -\v 1-14 -\c 2 -\v 1-17 -\c 3 -\v 1-18 -\v 19-23 -\v 24 -"; - - string target = UpdateUsfm(rows, usfm, bookId: "MAL", versification: ScrVers.Original); - string result = - @"\id MAL -\c 1 -\v 1-14 Updated verse 1 -\c 2 -\v 1-17 -\c 3 -\v 1-18 -\v 19-23 -\v 24 Updated verse 2 -"; - AssertUsfmEquals(target, result); - - target = UpdateUsfm( - rows, - usfm, - bookId: "MAL", - versification: ScrVers.Original, - convertUsfmToUpdateRowVersification: true - ); - result = - @"\id MAL -\c 1 -\v 1-14 Updated verse 1 -\c 2 -\v 1-17 -\c 3 -\v 1-18 -\c 4 -\v 1-5 -\v 6 Updated verse 2 -"; - AssertUsfmEquals(target, result); - } - - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() - { - // Russian Orthodox vs. Original - // PSA 151:1-7 = PS2 1:1-7 - - List rows = - [ - new UpdateUsfmRow(ScrRef(["PSA 150:1"], ScrVers.Original), "Updated verse 1"), - new UpdateUsfmRow(ScrRef(["PS2 1:1"], ScrVers.Original), "Updated verse 2"), - ]; - - string usfm = - @"\id PSA - Test -\c 150 -\v 1-6 -\c 151 -\v 1-7 -"; - - string target = UpdateUsfm( - rows, - usfm, - bookId: "PSA", - convertUsfmToUpdateRowVersification: false, - versification: ScrVers.RussianOrthodox - ); - string result = - @"\id PSA - Test -\c 150 -\v 1-6 Updated verse 1 -\c 151 -\v 1-7 Updated verse 2 -"; - AssertUsfmEquals(target, result); - - target = UpdateUsfm( - rows, - usfm, - bookId: "PSA", - convertUsfmToUpdateRowVersification: true, - versification: ScrVers.RussianOrthodox - ); - result = - @"\id PSA - Test -\c 150 -\v 1-6 Updated verse 1 -"; - AssertUsfmEquals(target, result); - } - - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreBook() - { - // Russian Orthodox vs. Original - // DAN 3:24-90 = DAG 3:24-90 - // DAN 3:91-100 = DAN 3:24-33 - - // Original - // S3Y 1:1-29 = DAG 3:24-52 - - List rows = - [ - new UpdateUsfmRow(ScrRef(["DAN 3:1"], ScrVers.RussianOrthodox), "Updated verse 1"), - // This row will map to another book DAG so it will not appear in the updated DAN - new UpdateUsfmRow(ScrRef(["DAN 3:24"], ScrVers.RussianOrthodox), "Updated verse 2"), - ]; - - string usfm = - @"\id DAN - Test -\c 3 -\v 1-23 -\c 4 -\v 1 -"; - - string target = UpdateUsfm( - rows, - usfm, - bookId: "DAN", - convertUsfmToUpdateRowVersification: true, - versification: ScrVers.Original - ); - string result = - @"\id DAN - Test -\c 3 -\v 1-23 Updated verse 1 -\c 4 -\v 1 -"; - AssertUsfmEquals(target, result); - } - - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_BackOneVerseToPreviousChapter() - { - // English vs. Original - // ISA 9:1 = ISA 8:23 - - List rows = - [ - new UpdateUsfmRow(ScrRef(["ISA 9:1"], ScrVers.English), "Updated verse 1"), - new UpdateUsfmRow(ScrRef(["ISA 9:2"], ScrVers.English), "Updated verse 2"), - ]; - - string usfm = - @"\id ISA - Test -\c 8 -\v 22 -\v 23 -\c 9 -\v 1 -"; - - string target = UpdateUsfm( - rows, - usfm, - bookId: "ISA", - convertUsfmToUpdateRowVersification: true, - versification: ScrVers.Original - ); - string result = - @"\id ISA - Test -\c 8 -\v 22 -\c 9 -\v 1 Updated verse 1 -\v 2 Updated verse 2 -"; - AssertUsfmEquals(target, result); - } - - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_ForwardOneVerseToNextChapter() - { - // Original vs. English - // ISA 8:23 = ISA 9:1 - - List rows = - [ - new UpdateUsfmRow(ScrRef(["ISA 8:23"], ScrVers.Original), "Updated verse 23"), - new UpdateUsfmRow(ScrRef(["ISA 9:1"], ScrVers.Original), "Updated verse 1"), - ]; - - string usfm = - @"\id ISA - Test -\c 8 -\v 22 -\c 9 -\v 1 -\v 2 -"; - - string target = UpdateUsfm( - rows, - usfm, - bookId: "ISA", - convertUsfmToUpdateRowVersification: true, - versification: ScrVers.English - ); - string result = - @"\id ISA - Test -\c 8 -\v 22 -\v 23 Updated verse 23 -\c 9 -\v 1 Updated verse 1 -"; - AssertUsfmEquals(target, result); - } - [Test] public void CrossReferenceDontUpdate() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1/1:x"]), "Update the cross reference")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1/1:x"), "Update the cross reference")]; string usfm = @"\id MAT - Test \c 1 @@ -1027,7 +743,7 @@ public void CrossReferenceDontUpdate() [Test] public void PreserveFig() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update")]; string usfm = @"\id MAT - Test \c 1 @@ -1047,8 +763,8 @@ public void NoteExplicitEndMarkers() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update text"), - new UpdateUsfmRow(ScrRef(["MAT 1:1/1:f"]), "Update note"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update text"), + new UpdateUsfmRow(ScrRef("MAT 1:1/1:f"), "Update note"), ]; string usfm = @"\id MAT - Test @@ -1075,7 +791,7 @@ public void NoteExplicitEndMarkers() [Test] public void UpdateBlock_Verse_PreserveParas() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1105,7 +821,7 @@ public void UpdateBlock_Verse_PreserveParas() [Test] public void UpdateBlock_Verse_StripParas() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1135,7 +851,7 @@ public void UpdateBlock_Verse_StripParas() [Test] public void UpdateBlock_Verse_Range() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1163,7 +879,7 @@ public void UpdateBlock_Verse_Range() [Test] public void UpdateBlock_Verse_Range_RightToLeftMarker() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1", "MAT 1:2", "MAT 1:3"]), "Update 1-3")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1", "MAT 1:2", "MAT 1:3"), "Update 1-3")]; string usfm = @"\id MAT - Test \c 1 @@ -1194,7 +910,7 @@ public void UpdateBlock_Verse_Range_RightToLeftMarker() [Test] public void UpdateBlock_Footnote_PreserveEmbeds() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1224,7 +940,7 @@ public void UpdateBlock_Footnote_PreserveEmbeds() [Test] public void UpdateBlock_Footnote_StripEmbeds() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1254,7 +970,7 @@ public void UpdateBlock_Footnote_StripEmbeds() [Test] public void UpdateBlock_NonVerse() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/1:s"]), "Updated section Header")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/1:s"), "Updated section Header")]; string usfm = @"\id MAT - Test \s Section header @@ -1278,7 +994,7 @@ public void UpdateBlock_NonVerse() [Test] public void UpdateBlock_Verse_PreserveStyles() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1310,7 +1026,7 @@ public void UpdateBlock_Verse_PreserveStyles() [Test] public void UpdateBlock_Verse_StripStyles() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1342,7 +1058,7 @@ public void UpdateBlock_Verse_StripStyles() [Test] public void UpdateBlock_Verse_SectionHeader() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1380,7 +1096,7 @@ public void UpdateBlock_Verse_SectionHeader() [Test] public void UpdateBlock_Verse_SectionHeaderInVerse() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1413,7 +1129,7 @@ public void UpdateBlock_Verse_SectionHeaderInVerse() [Test] public void UpdateBlock_NonVerse_ParagraphEndOfVerse() { - List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1444,11 +1160,11 @@ public void GetUsfm_HeaderReferenceParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), - new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), - new UpdateUsfmRow(ScrRef(["MAT 2:1"]), "new verse 1"), - new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), + new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), + new UpdateUsfmRow(ScrRef("MAT 2:1"), "new verse 1"), + new UpdateUsfmRow(ScrRef("MAT 2:2"), "new verse 2"), ]; string usfm = @@ -1499,16 +1215,16 @@ public void GetUsfm_OutOfOrderVerses() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), - new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), - new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), - new UpdateUsfmRow(ScrRef(["MAT 1:5"]), "new verse 5"), - new UpdateUsfmRow(ScrRef(["MAT 1:6a"]), "new verse 6a"), - new UpdateUsfmRow(ScrRef(["MAT 1:6b"]), "new verse 6b"), - new UpdateUsfmRow(ScrRef(["MAT 1:6b/1:s"]), "new section"), - new UpdateUsfmRow(ScrRef(["MAT 1:7"]), "new verse 7"), - new UpdateUsfmRow(ScrRef(["MAT 1:8"]), "new verse 8"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), + new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), + new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), + new UpdateUsfmRow(ScrRef("MAT 1:5"), "new verse 5"), + new UpdateUsfmRow(ScrRef("MAT 1:6a"), "new verse 6a"), + new UpdateUsfmRow(ScrRef("MAT 1:6b"), "new verse 6b"), + new UpdateUsfmRow(ScrRef("MAT 1:6b/1:s"), "new section"), + new UpdateUsfmRow(ScrRef("MAT 1:7"), "new verse 7"), + new UpdateUsfmRow(ScrRef("MAT 1:8"), "new verse 8"), ]; string usfm = @@ -1558,10 +1274,10 @@ public void GetUsfm_DuplicateVerses() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), - new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), - new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), + new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), + new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), ]; string usfm = @@ -1596,11 +1312,11 @@ public void GetUsfm_IdTags() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:0/1:s"]), "new section header"), - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), - new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), - new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), + new UpdateUsfmRow(ScrRef("MAT 1:0/1:s"), "new section header"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), + new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), + new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), ]; string usfm = @@ -1641,8 +1357,8 @@ public void GetUsfm_PassRemark() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), ]; string usfm = @@ -1696,8 +1412,8 @@ public void GetUsfm_PassRemark_NoBodyParagraph() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), ]; string usfm = @@ -1749,8 +1465,8 @@ public void GetUsfm_PassRemark0_NoExistingRemark() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), ]; string usfm = @@ -1787,8 +1503,8 @@ public void GetUsfm_MultipleRemarksSameChapter() { List rows = [ - new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), - new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), + new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), ]; string usfm = @@ -1830,7 +1546,7 @@ public void GetUsfm_MultipleRemarksSameChapter() [Test] public void UpdateBlock_FootnoteInPublishedChapterNumber() { - List rows = [new UpdateUsfmRow(ScrRef(["ESG 1:0/2:s"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("ESG 1:0/2:s"), "Update 1")]; string usfm = @"\id ESG - Test \c 1 @@ -1873,7 +1589,7 @@ public void UpdateBlock_FootnoteInPublishedChapterNumber() [Test] public void UpdateBlock_FootnoteAtStartOfChapterWithPrecedingText() { - List rows = [new UpdateUsfmRow(ScrRef(["ESG 1:0/2:s"]), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef("ESG 1:0/2:s"), "Update 1")]; string usfm = @"\id ESG - Test \c 1 @@ -2003,8 +1719,7 @@ public void FilterChapters_WithBadChapterReference() AssertUsfmEquals(target, result); } - private static ScriptureRef[] ScrRef(IEnumerable refs, ScrVers? versification = null) => - [.. refs.Select(r => ScriptureRef.Parse(r, versification))]; + private static ScriptureRef[] ScrRef(params string[] refs) => [.. refs.Select(r => ScriptureRef.Parse(r))]; private static string UpdateUsfm( IReadOnlyList? rows = null, @@ -2018,18 +1733,15 @@ private static string UpdateUsfm( IEnumerable? preserveParagraphStyles = null, IEnumerable? usfmUpdateBlockHandlers = null, IEnumerable<(int, string)>? remarks = null, - bool compareSegments = false, - bool convertUsfmToUpdateRowVersification = false, - string? bookId = null, - ScrVers? versification = null + bool compareSegments = false ) { - bookId ??= "MAT"; + const string BookId = "MAT"; if (source is null) { var updater = new FileParatextProjectTextUpdater(CorporaTestHelpers.UsfmTestProjectPath); return updater.UpdateUsfm( - bookId, + BookId, rows, chapters, idText, @@ -2041,23 +1753,17 @@ private static string UpdateUsfm( usfmUpdateBlockHandlers, remarks, (_) => false, - compareSegments, - convertUsfmToUpdateRowVersification + compareSegments ); } else { source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; - var settings = new DefaultParatextProjectSettings( - fileNameForm: "MAT", - fileNamePrefix: string.Empty, - fileNameSuffix: string.Empty, - versification: versification - ); - var files = new Dictionary { [bookId] = source }; + var settings = new DefaultParatextProjectSettings(fileNameForm: BookId, fileNameSuffix: string.Empty); + var files = new Dictionary { [BookId] = source }; var updater = new MemoryParatextProjectTextUpdater(files, settings); return updater.UpdateUsfm( - bookId, + BookId, rows, chapters, idText, @@ -2069,8 +1775,7 @@ private static string UpdateUsfm( usfmUpdateBlockHandlers, remarks, (_) => false, - compareSegments, - convertUsfmToUpdateRowVersification + compareSegments ); } } From bccc243bb61811b810f685e428d351aca085ce61 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 28 Aug 2026 09:02:30 -0400 Subject: [PATCH 05/10] Remove unused code --- src/SIL.Machine/Corpora/UsfmToken.cs | 12 +---- src/SIL.Machine/Corpora/VerseRefExtensions.cs | 46 ------------------- 2 files changed, 1 insertion(+), 57 deletions(-) diff --git a/src/SIL.Machine/Corpora/UsfmToken.cs b/src/SIL.Machine/Corpora/UsfmToken.cs index 28e4765ad..43a621b7a 100644 --- a/src/SIL.Machine/Corpora/UsfmToken.cs +++ b/src/SIL.Machine/Corpora/UsfmToken.cs @@ -3,7 +3,6 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; -using SIL.ObjectModel; namespace SIL.Machine.Corpora { @@ -23,7 +22,7 @@ public enum UsfmTokenType Unknown, } - public class UsfmToken : IEquatable, ICloneable + public class UsfmToken : IEquatable { private const string FullAttributeStr = @"(?[-\w]+)\s*\=\s*\""(?.+?)\""\s*"; private static readonly Regex AttributeRegex = new Regex( @@ -181,15 +180,6 @@ public void CopyAttributes(UsfmToken sourceToken) _defaultAttributeName = sourceToken._defaultAttributeName; } - public UsfmToken Clone() - { - UsfmToken copy = new UsfmToken(Type, Marker, Text, EndMarker, Data); - copy.CopyAttributes(this); - copy.LineNumber = LineNumber; - copy.ColumnNumber = ColumnNumber; - return copy; - } - private static void AppendAttribute(List attributes, string name, string value) { value = value?.Trim(); // don't want to have attribute that is just spaces diff --git a/src/SIL.Machine/Corpora/VerseRefExtensions.cs b/src/SIL.Machine/Corpora/VerseRefExtensions.cs index efc75d340..adc8f6e48 100644 --- a/src/SIL.Machine/Corpora/VerseRefExtensions.cs +++ b/src/SIL.Machine/Corpora/VerseRefExtensions.cs @@ -53,51 +53,5 @@ public static VerseRef ChangeVersificationWithSegments(this VerseRef verseRef, S } return vr; } - - public static bool TryChangeVersificationWithSegments( - this VerseRef verseRef, - ScrVers versification, - out VerseRef changedVerseRef - ) - { - VerseRef vr = verseRef; - - bool success = true; - if (vr.HasMultiple) - success = vr.ChangeVersificationWithRanges(versification); - else - vr.ChangeVersification(versification); - - if (string.IsNullOrEmpty(vr.Segment())) - { - changedVerseRef = vr; - return success; - } - - VerseRef verseRefWithoutSegments = verseRef.RemoveSegments(); - if (verseRefWithoutSegments.HasMultiple) - success = verseRefWithoutSegments.ChangeVersificationWithRanges(versification); - else - verseRefWithoutSegments.ChangeVersification(versification); - if (!verseRefWithoutSegments.Equals(vr.RemoveSegments())) - { - IEnumerable verses = verseRef - .AllVerses() - .Zip( - verseRefWithoutSegments.AllVerses(), - (verseWithSegments, verseWithCorrectNumber) => (verseWithSegments, verseWithCorrectNumber) - ) - .Select( - (verseTuple) => verseTuple.verseWithCorrectNumber.Verse + verseTuple.verseWithSegments.Segment() - ); - changedVerseRef = new VerseRef( - $"{verseRefWithoutSegments.Book} {verseRefWithoutSegments.ChapterNum}:{string.Join(",", verses)}", - versification - ); - return success; - } - changedVerseRef = vr; - return success; - } } } From 38b4fc9fc190c46b0ef166c59718315e4fabd7f5 Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Fri, 28 Aug 2026 11:59:42 -0400 Subject: [PATCH 06/10] Add unit tests for edge cases --- .../ConvertUsfmVersificationHandlerTests.cs | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) diff --git a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs index 611868878..6c26257ea 100644 --- a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs @@ -374,6 +374,229 @@ public void GetUsfm_CrossChapterVerseRange_CrossBookWithinSingleRange() AssertUsfmEquals(target, result); } + [Test] + public void GetUsfm_HeadingIntroducingKeptVerse_AfterDroppedVerse() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 (leaves the book, so it is dropped) + // DAN 3:91-100 = DAN 3:24-33 (stays in the book, so it is kept) + // The heading and paragraph marker between them introduce the verse that is kept. + + string usfm = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\v 24-90 Dropped text +\s1 Section +\p +\v 91-100 More text +"; + + string target = UpdateUsfm( + "DAN", + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\s1 Section +\p +\v 24-33 More text +"; + AssertUsfmEqualsExactly(target, result); + } + + [Test] + public void GetUsfm_HeadingIntroducingDroppedVerse_IsDropped() + { + // Russian Orthodox vs. Original + // PSA 151:1-7 = PS2 1:1-7 (the whole chapter leaves the book) + // The heading introduces the dropped verse, so it goes with it. The \q belongs to the + // preceding verse and stays. + + string usfm = + @"\id PSA - Test +\c 150 +\v 1-5 Lines +\v 6 Line +\q Another line +\c 151 +\s1 Section +\p +\v 1-7 More lines +"; + + string target = UpdateUsfm( + "PSA", + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id PSA - Test +\c 150 +\v 1-5 Lines +\v 6 Line +\q Another line +"; + AssertUsfmEqualsExactly(target, result); + } + + [Test] + public void GetUsfm_DroppedVerseText_IsDroppedWhenKeptVerseFollows() + { + // Guards against rescuing the dropped verse's own text along with the markers that + // introduce the verse after it. + + string usfm = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\v 24-90 Dropped text +\v 91-100 More text +"; + + string target = UpdateUsfm( + "DAN", + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\v 24-33 More text +"; + AssertUsfmEqualsExactly(target, result); + } + + [Test] + public void GetUsfm_SynthesizedChapter_IsFollowedByParagraphMarker() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + // Converting Original to English splits chapter 3 mid-paragraph, so the synthesized \c 4 + // has no paragraph marker of its own. \nb continues the paragraph across the chapter break; + // without it the first verse of chapter 4 sits outside any paragraph. + + string usfm = + @"\id MAL - Test +\c 3 +\p +\v 1-18 Text +\v 19-23 More text +\v 24 Last text +"; + + string target = UpdateUsfm( + "MAL", + usfm, + sourceVersification: ScrVers.Original, + targetVersification: ScrVers.English + ); + string result = + @"\id MAL - Test +\c 3 +\p +\v 1-18 Text +\c 4 +\nb +\v 1-5 More text +\v 6 Last text +"; + AssertUsfmEqualsExactly(target, result); + } + + [Test] + public void GetUsfm_SynthesizedChapter_FromSplitVerseRange_IsFollowedByParagraphMarker() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + // The range \v 22-23 straddles the mapped chapter boundary, so it is split and a chapter + // marker is synthesized between the two halves. That marker needs a paragraph too. + // The verses are left without text deliberately: where the text of a split range should + // end up is a separate question from whether the synthesized chapter has a paragraph. + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22-23 +\c 9 +\p +\v 1 +"; + + string target = UpdateUsfm( + "ISA", + usfm, + sourceVersification: ScrVers.Original, + targetVersification: ScrVers.English + ); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 +\c 9 +\nb +\v 1 +\p +\v 2 +"; + AssertUsfmEqualsExactly(target, result); + } + + [Test] + public void GetUsfm_SplitVerseRange_TextStaysWithFirstVerse() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + // \v 22-23 straddles the mapped chapter boundary and is split. The text covers both + // verses, and no single \v can express that, so it stays with the verse the range starts + // at rather than being carried into the next chapter. + // + // This expects the synthesized \c 9 to be followed by \nb as well, so it needs both that + // fix and the text placement fix to pass. + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22-23 Verse twenty-two and twenty-three text +\c 9 +\p +\v 1 Chapter nine verse one text +"; + + string target = UpdateUsfm( + "ISA", + usfm, + sourceVersification: ScrVers.Original, + targetVersification: ScrVers.English + ); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 Verse twenty-two and twenty-three text +\c 9 +\nb +\v 1 +\p +\v 2 Chapter nine verse one text +"; + AssertUsfmEqualsExactly(target, result); + } + private static string UpdateUsfm( string bookId, string source, @@ -401,4 +624,15 @@ private static void AssertUsfmEquals(string target, string truth) for (int i = 0; i < truthLines.Length; i++) Assert.That(targetLines[i].Trim(), Is.EqualTo(truthLines[i].Trim()), message: $"Line {i}"); } + + // AssertUsfmEquals only walks the expected lines, so output that runs past the end of the + // expected USFM slips through. These cases turn on content being dropped, so they need the + // line count checked too. + private static void AssertUsfmEqualsExactly(string target, string truth) + { + AssertUsfmEquals(target, truth); + Assert.That(NonEmptyLineCount(target), Is.EqualTo(NonEmptyLineCount(truth)), message: "extra output"); + + static int NonEmptyLineCount(string usfm) => usfm.Split('\n').Count(l => l.Trim().Length > 0); + } } From 29dcbb5181e66e245e2ca9c7d92f0831b97c3737 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 11 Sep 2026 15:19:17 -0400 Subject: [PATCH 07/10] Address reviewer comments; add tests to cover invalid verse references --- .../ConvertUsfmVersificationHandler.cs | 120 ++++++- .../Corpora/ParatextProjectTextUpdaterBase.cs | 13 +- .../Corpora/UpdateUsfmParserHandler.cs | 5 + .../ConvertUsfmVersificationHandlerTests.cs | 330 ++++++++++++------ ...ryParatextProjectVersificationConverter.cs | 10 - 5 files changed, 344 insertions(+), 134 deletions(-) delete mode 100644 tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs diff --git a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs index fae6fdb9b..7fe741bdb 100644 --- a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs +++ b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs @@ -1,12 +1,28 @@ using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using SIL.Scripture; namespace SIL.Machine.Corpora { public class ConvertUsfmVersificationHandler : ScriptureRefUsfmParserHandlerBase { + private static readonly IReadOnlyList TrailingParagraphMarkerPatterns = new List + { + new Regex(@"mt\d*", RegexOptions.Compiled), + new Regex(@"mte\d*", RegexOptions.Compiled), + new Regex(@"ms\d*", RegexOptions.Compiled), + new Regex("mr", RegexOptions.Compiled), + new Regex(@"s\d*", RegexOptions.Compiled), + new Regex("sr", RegexOptions.Compiled), + new Regex("r", RegexOptions.Compiled), + new Regex("d", RegexOptions.Compiled), + new Regex("sp", RegexOptions.Compiled), + new Regex(@"sd\d*", RegexOptions.Compiled), + }; private readonly List _tokens; + private readonly List _trailingVerseTokens; + private int _trailingVerseTokensInsertionIndex; private VerseRef _prevVerseRef; private int _verseBoundary; private readonly ScrVers _targetVersification; @@ -18,11 +34,15 @@ public ConvertUsfmVersificationHandler(ScrVers targetVersification) _verseBoundary = 0; _insertChapterIndex = -1; _tokens = new List(); + _trailingVerseTokens = new List(); + _trailingVerseTokensInsertionIndex = 0; _prevVerseRef = new VerseRef(); _targetVersification = targetVersification; _skip = false; } + public IReadOnlyList Tokens => _tokens; + public override void Chapter( UsfmParserState state, string number, @@ -33,6 +53,19 @@ string pubNumber { base.Chapter(state, number, marker, altNumber, pubNumber); ProcessTokens(state); + VerseRef vr = state.VerseRef; + // The versification of verse 0 cannot properly be changed + vr.Verse = "1"; + if ( + !_prevVerseRef.IsDefault + && ( + vr.ChangeVersificationWithSegments(_targetVersification).Book != _prevVerseRef.Book + || vr.ChapterNum == -1 + ) + ) + { + _skip = true; + } _insertChapterIndex = _tokens.Count; } @@ -56,24 +89,36 @@ string pubNumber .ToList(); if ( - _prevVerseRef.IsDefault - || ( - verseRefs[0].BookNum == _prevVerseRef.BookNum && verseRefs[0].ChapterNum != _prevVerseRef.ChapterNum - ) + ( + _prevVerseRef.IsDefault + || ( + verseRefs[0].BookNum == _prevVerseRef.BookNum + && verseRefs[0].ChapterNum != _prevVerseRef.ChapterNum + ) + ) && (verseRefs[0].ChapterNum != -1) ) { UsfmToken newChapterToken = new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[0].Chapter); if (_insertChapterIndex == -1) + { _tokens.Add(newChapterToken); + _tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")); + } else + { _tokens.Insert(_insertChapterIndex, newChapterToken); + } + _trailingVerseTokensInsertionIndex++; } string start = null; for (int i = 0; i < verseRefs.Count; i++) { - if (!_prevVerseRef.IsDefault && verseRefs[i].Book != _prevVerseRef.Book) + if ( + (!_prevVerseRef.IsDefault && verseRefs[i].Book != _prevVerseRef.Book) + || verseRefs[i].ChapterNum == -1 + ) { continue; } @@ -85,14 +130,35 @@ string pubNumber && _prevVerseRef.ChapterNum != verseRefs[i].ChapterNum ) { + AddTrailingTokens(); _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + if (state.Index + 1 < state.Tokens.Count) + { + UsfmToken nextToken = state.Tokens[state.Index + 1]; + if (nextToken.Type == UsfmTokenType.Text) + { + _tokens.Add(nextToken); + _verseBoundary++; + } + } _tokens.Add(new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[i].Chapter)); + _tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")); start = verseRefs[i].Verse; _prevVerseRef = verseRefs[i]; } else if (_prevVerseRef.VerseNum + 1 != verseRefs[i].VerseNum) { + AddTrailingTokens(); _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + if (state.Index + 1 < state.Tokens.Count) + { + UsfmToken nextToken = state.Tokens[state.Index + 1]; + if (nextToken.Type == UsfmTokenType.Text) + { + _tokens.Add(nextToken); + _verseBoundary++; + } + } start = verseRefs[i].Verse; _prevVerseRef = verseRefs[i]; } @@ -111,6 +177,7 @@ string pubNumber if (start != null) { + AddTrailingTokens(); string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : ""; _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); _skip = false; @@ -140,12 +207,49 @@ public string GetUsfm(UsfmStylesheet stylesheet) private void ProcessTokens(UsfmParserState state) { int offset = 0; - if (!_skip) + bool inPreservedParagraph = false; + while (_verseBoundary + offset < state.Index) { - while (_verseBoundary + offset < state.Index) - _tokens.Add(state.Tokens[_verseBoundary + offset++]); + UsfmToken token = state.Tokens[_verseBoundary + offset]; + if ( + inPreservedParagraph + || IsPreservedTrailingParagraphMarker( + token, + _verseBoundary + offset + 1 < state.Tokens.Count + ? state.Tokens[_verseBoundary + offset + 1] + : null + ) + ) + { + inPreservedParagraph = !inPreservedParagraph || token.Type != UsfmTokenType.Paragraph; + + if (_trailingVerseTokens.Count == 0) + _trailingVerseTokensInsertionIndex = _tokens.Count; + _trailingVerseTokens.Add(token); + } + else + { + inPreservedParagraph = false; + if (!_skip) + _tokens.Add(token); + } + offset++; } _verseBoundary = state.Index + 1; } + + private void AddTrailingTokens() + { + _tokens.InsertRange(_trailingVerseTokensInsertionIndex, _trailingVerseTokens); + _trailingVerseTokens.Clear(); + } + + private bool IsPreservedTrailingParagraphMarker(UsfmToken token, UsfmToken nextToken) + { + return (token.Marker == "p" && nextToken != null && nextToken.Type == UsfmTokenType.Verse) + || TrailingParagraphMarkerPatterns.Any(p => + token.Type == UsfmTokenType.Paragraph && p.IsMatch(token.Marker) + ); + } } } diff --git a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs index e38e528f6..1289f3f4c 100644 --- a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs +++ b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using SIL.Scripture; namespace SIL.Machine.Corpora { @@ -64,7 +65,17 @@ public string UpdateUsfm( var tokenizer = new UsfmTokenizer(_settings.Stylesheet); IReadOnlyList tokens = tokenizer.Tokenize(usfm); tokens = FilterTokensByChapter(tokens, chapters); - UsfmParser.Parse(tokens, handler, _settings.Stylesheet, _settings.Versification); + + ScrVers rowsVersification = UpdateUsfmParserHandler.GetRowsVersification(rows); + ScrVers parseVersification = _settings.Versification; + if (rowsVersification != _settings.Versification) + { + var converter = new ConvertUsfmVersificationHandler(rowsVersification); + UsfmParser.Parse(tokens, converter, _settings.Stylesheet, _settings.Versification); + tokens = converter.Tokens; + parseVersification = rowsVersification; + } + UsfmParser.Parse(tokens, handler, _settings.Stylesheet, parseVersification); return handler.GetUsfm(_settings.Stylesheet); } catch (Exception ex) diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index f21be754d..215051581 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -114,6 +114,11 @@ public UpdateUsfmParserHandler( _compareSegments = compareSegments; } + public static ScrVers GetRowsVersification(IReadOnlyList rows) + { + return rows.Any() ? rows.First(r => r.Refs.Count > 0).Refs[0].Versification : ScrVers.English; + } + public IReadOnlyList Tokens => _tokens; public override void EndUsfm(UsfmParserState state) diff --git a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs index 6c26257ea..502a9a239 100644 --- a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs @@ -33,12 +33,7 @@ public void GetUsfm_OneFewerChapter() \v 6 Text "; - string target = UpdateUsfm( - "MAL", - usfm, - sourceVersification: ScrVers.English, - targetVersification: ScrVers.Original - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.English, targetVersification: ScrVers.Original); string result = @"\id MAL \h Malachi @@ -85,12 +80,7 @@ public void GetUsfm_OneMoreChapter() \v 24 Text "; - string target = UpdateUsfm( - "MAL", - usfm, - sourceVersification: ScrVers.Original, - targetVersification: ScrVers.English - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); string result = @"\id MAL \h Malachi @@ -106,6 +96,7 @@ public void GetUsfm_OneMoreChapter() \v 1-17 \v 18 Text \f More text \f* \c 4 +\nb \v 1-5 \v 6 Text "; @@ -122,15 +113,16 @@ public void GetUsfm_OneFewerBook() @"\id PSA - Test \h Psalms \c 150 +\p \v 1-5 Lines \v 6 Line \q Another line \c 151 +\p \v 1-7 More lines "; string target = UpdateUsfm( - "PSA", usfm, sourceVersification: ScrVers.RussianOrthodox, targetVersification: ScrVers.Original @@ -139,6 +131,7 @@ public void GetUsfm_OneFewerBook() @"\id PSA - Test \h Psalms \c 150 +\p \v 1-5 Lines \v 6 Line \q Another line @@ -167,16 +160,17 @@ public void GetUsfm_OneMoreBook() @"\id DAN - Test \h Daniel \c 3 -\v 1-23 -\v 24-90 \p -\v 91-100 +\v 1-23 Text 1 +\v 24-90 Text 2 +\p More text 2 +\v 91-100 Text 3 \c 4 -\v 1 +\p +\v 1 Text 4 "; string target = UpdateUsfm( - "DAN", usfm, sourceVersification: ScrVers.RussianOrthodox, targetVersification: ScrVers.Original @@ -185,10 +179,12 @@ public void GetUsfm_OneMoreBook() @"\id DAN - Test \h Daniel \c 3 -\v 1-23 -\v 24-33 +\p +\v 1-23 Text 1 +\v 24-33 Text 3 \c 4 -\v 1 +\p +\v 1 Text 4 "; AssertUsfmEquals(target, result); } @@ -202,24 +198,24 @@ public void GetUsfm_BackOneVerseToPreviousChapter() string usfm = @"\id ISA - Test \c 8 +\p \v 22 \v 23 \c 9 +\p \v 1 "; - string target = UpdateUsfm( - "ISA", - usfm, - sourceVersification: ScrVers.Original, - targetVersification: ScrVers.English - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); string result = @"\id ISA - Test \c 8 +\p \v 22 \c 9 +\nb \v 1 +\p \v 2 "; AssertUsfmEquals(target, result); @@ -234,24 +230,24 @@ public void GetUsfm_ForwardOneVerseToNextChapter() string usfm = @"\id ISA - Test \c 8 +\p \v 22 \c 9 +\p \v 1 \v 2 "; - string target = UpdateUsfm( - "ISA", - usfm, - sourceVersification: ScrVers.English, - targetVersification: ScrVers.Original - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.English, targetVersification: ScrVers.Original); string result = @"\id ISA - Test \c 8 +\p \v 22 +\p \v 23 \c 9 +\nb \v 1 "; AssertUsfmEquals(target, result); @@ -266,23 +262,23 @@ public void GetUsfm_CrossChapterVerseRange() string usfm = @"\id ISA - Test \c 8 +\p \v 22-23 \c 9 +\p \v 1 "; - string target = UpdateUsfm( - "ISA", - usfm, - sourceVersification: ScrVers.Original, - targetVersification: ScrVers.English - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); string result = @"\id ISA - Test \c 8 +\p \v 22 \c 9 +\nb \v 1 +\p \v 2 "; AssertUsfmEquals(target, result); @@ -308,15 +304,16 @@ public void GetUsfm_CrossChapterVerseRange_CrossBook() string usfm = @"\id DAN - Test \c 3 +\p \v 1-22 \v 23-89 \v 90-100 \c 4 +\p \v 1 "; string target = UpdateUsfm( - "DAN", usfm, sourceVersification: ScrVers.RussianOrthodox, targetVersification: ScrVers.Original @@ -324,10 +321,12 @@ public void GetUsfm_CrossChapterVerseRange_CrossBook() string result = @"\id DAN - Test \c 3 +\p \v 1-22 \v 23 \v 24-33 \c 4 +\p \v 1 "; AssertUsfmEquals(target, result); @@ -353,13 +352,14 @@ public void GetUsfm_CrossChapterVerseRange_CrossBookWithinSingleRange() string usfm = @"\id DAN - Test \c 3 +\p \v 1-100 \c 4 +\p \v 1 "; string target = UpdateUsfm( - "DAN", usfm, sourceVersification: ScrVers.RussianOrthodox, targetVersification: ScrVers.Original @@ -367,20 +367,21 @@ public void GetUsfm_CrossChapterVerseRange_CrossBookWithinSingleRange() string result = @"\id DAN - Test \c 3 +\p \v 1-33 \c 4 +\p \v 1 "; AssertUsfmEquals(target, result); } [Test] - public void GetUsfm_HeadingIntroducingKeptVerse_AfterDroppedVerse() + public void GetUsfm_HeadingIntroducingKeptVerse_IsPreserved() { // Russian Orthodox vs. Original // DAN 3:24-90 = DAG 3:24-90 (leaves the book, so it is dropped) // DAN 3:91-100 = DAN 3:24-33 (stays in the book, so it is kept) - // The heading and paragraph marker between them introduce the verse that is kept. string usfm = @"\id DAN - Test @@ -388,13 +389,12 @@ public void GetUsfm_HeadingIntroducingKeptVerse_AfterDroppedVerse() \p \v 1-23 Text \v 24-90 Dropped text -\s1 Section +\s1 \nd Section\nd* \p \v 91-100 More text "; string target = UpdateUsfm( - "DAN", usfm, sourceVersification: ScrVers.RussianOrthodox, targetVersification: ScrVers.Original @@ -404,11 +404,11 @@ public void GetUsfm_HeadingIntroducingKeptVerse_AfterDroppedVerse() \c 3 \p \v 1-23 Text -\s1 Section +\s1 \nd Section\nd* \p \v 24-33 More text "; - AssertUsfmEqualsExactly(target, result); + AssertUsfmEquals(target, result); } [Test] @@ -416,23 +416,21 @@ public void GetUsfm_HeadingIntroducingDroppedVerse_IsDropped() { // Russian Orthodox vs. Original // PSA 151:1-7 = PS2 1:1-7 (the whole chapter leaves the book) - // The heading introduces the dropped verse, so it goes with it. The \q belongs to the - // preceding verse and stays. string usfm = @"\id PSA - Test \c 150 +\p \v 1-5 Lines \v 6 Line \q Another line \c 151 -\s1 Section +\s1 \nd Section\nd* \p \v 1-7 More lines "; string target = UpdateUsfm( - "PSA", usfm, sourceVersification: ScrVers.RussianOrthodox, targetVersification: ScrVers.Original @@ -440,19 +438,17 @@ public void GetUsfm_HeadingIntroducingDroppedVerse_IsDropped() string result = @"\id PSA - Test \c 150 +\p \v 1-5 Lines \v 6 Line \q Another line "; - AssertUsfmEqualsExactly(target, result); + AssertUsfmEquals(target, result); } [Test] - public void GetUsfm_DroppedVerseText_IsDroppedWhenKeptVerseFollows() + public void GetUsfm_DropVerseText() { - // Guards against rescuing the dropped verse's own text along with the markers that - // introduce the verse after it. - string usfm = @"\id DAN - Test \c 3 @@ -463,7 +459,6 @@ public void GetUsfm_DroppedVerseText_IsDroppedWhenKeptVerseFollows() "; string target = UpdateUsfm( - "DAN", usfm, sourceVersification: ScrVers.RussianOrthodox, targetVersification: ScrVers.Original @@ -475,17 +470,14 @@ public void GetUsfm_DroppedVerseText_IsDroppedWhenKeptVerseFollows() \v 1-23 Text \v 24-33 More text "; - AssertUsfmEqualsExactly(target, result); + AssertUsfmEquals(target, result); } [Test] - public void GetUsfm_SynthesizedChapter_IsFollowedByParagraphMarker() + public void GetUsfm_ChapterMarkerIsFollowedByParagraphMarker() { // English vs. Original // MAL 4:1-6 = MAL 3:19-24 - // Converting Original to English splits chapter 3 mid-paragraph, so the synthesized \c 4 - // has no paragraph marker of its own. \nb continues the paragraph across the chapter break; - // without it the first verse of chapter 4 sits outside any paragraph. string usfm = @"\id MAL - Test @@ -496,12 +488,7 @@ public void GetUsfm_SynthesizedChapter_IsFollowedByParagraphMarker() \v 24 Last text "; - string target = UpdateUsfm( - "MAL", - usfm, - sourceVersification: ScrVers.Original, - targetVersification: ScrVers.English - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); string result = @"\id MAL - Test \c 3 @@ -512,18 +499,14 @@ public void GetUsfm_SynthesizedChapter_IsFollowedByParagraphMarker() \v 1-5 More text \v 6 Last text "; - AssertUsfmEqualsExactly(target, result); + AssertUsfmEquals(target, result); } [Test] - public void GetUsfm_SynthesizedChapter_FromSplitVerseRange_IsFollowedByParagraphMarker() + public void GetUsfm_ChapterMarkerIsFollowedByParagraphMarker_CrossChapterVerseRange() { // English vs. Original // ISA 9:1 = ISA 8:23 - // The range \v 22-23 straddles the mapped chapter boundary, so it is split and a chapter - // marker is synthesized between the two halves. That marker needs a paragraph too. - // The verses are left without text deliberately: where the text of a split range should - // end up is a separate question from whether the synthesized chapter has a paragraph. string usfm = @"\id ISA - Test @@ -535,12 +518,7 @@ public void GetUsfm_SynthesizedChapter_FromSplitVerseRange_IsFollowedByParagraph \v 1 "; - string target = UpdateUsfm( - "ISA", - usfm, - sourceVersification: ScrVers.Original, - targetVersification: ScrVers.English - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); string result = @"\id ISA - Test \c 8 @@ -552,20 +530,14 @@ public void GetUsfm_SynthesizedChapter_FromSplitVerseRange_IsFollowedByParagraph \p \v 2 "; - AssertUsfmEqualsExactly(target, result); + AssertUsfmEquals(target, result); } [Test] - public void GetUsfm_SplitVerseRange_TextStaysWithFirstVerse() + public void GetUsfm_CrossChapterVerseRange_TextStaysWithFirstVerse() { // English vs. Original // ISA 9:1 = ISA 8:23 - // \v 22-23 straddles the mapped chapter boundary and is split. The text covers both - // verses, and no single \v can express that, so it stays with the verse the range starts - // at rather than being carried into the next chapter. - // - // This expects the synthesized \c 9 to be followed by \nb as well, so it needs both that - // fix and the text placement fix to pass. string usfm = @"\id ISA - Test @@ -577,12 +549,7 @@ public void GetUsfm_SplitVerseRange_TextStaysWithFirstVerse() \v 1 Chapter nine verse one text "; - string target = UpdateUsfm( - "ISA", - usfm, - sourceVersification: ScrVers.Original, - targetVersification: ScrVers.English - ); + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); string result = @"\id ISA - Test \c 8 @@ -594,15 +561,154 @@ public void GetUsfm_SplitVerseRange_TextStaysWithFirstVerse() \p \v 2 Chapter nine verse one text "; - AssertUsfmEqualsExactly(target, result); + AssertUsfmEquals(target, result); } - private static string UpdateUsfm( - string bookId, - string source, - ScrVers sourceVersification, - ScrVers targetVersification - ) + [Test] + public void GetUsfm_IgnoreInvalidChapter() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2@ +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-23 +\v 24 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + + // Strip out invalid chapters since we can't reliably convert them + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\nb +\v 1-5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_IgnoreInvalidVerse() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1@ Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-23 +\v 24 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + + // Just pass invalid verses through to target + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1@ Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\nb +\v 1-5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_MissingVerseInRange() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-21,23 Text +\v 24 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\nb +\v 1-3 Text +\v 5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + private static string UpdateUsfm(string source, ScrVers sourceVersification, ScrVers targetVersification) { source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; var settings = new DefaultParatextProjectSettings( @@ -611,9 +717,11 @@ ScrVers targetVersification fileNameSuffix: string.Empty, fileNamePrefix: string.Empty ); - var files = new Dictionary { [bookId] = source }; - var updater = new MemoryParatextProjectVersificationConverter(files, settings); - return updater.UpdateUsfm(bookId, targetVersification); + var handler = new ConvertUsfmVersificationHandler(targetVersification); + var tokenizer = new UsfmTokenizer(settings.Stylesheet); + IReadOnlyList tokens = tokenizer.Tokenize(source); + UsfmParser.Parse(tokens, handler, settings.Stylesheet, settings.Versification); + return handler.GetUsfm(settings.Stylesheet); } private static void AssertUsfmEquals(string target, string truth) @@ -621,18 +729,10 @@ private static void AssertUsfmEquals(string target, string truth) Assert.That(target, Is.Not.Null); string[] targetLines = target.Split('\n'); string[] truthLines = truth.Split('\n'); + Assert.That(targetLines.Length, Is.EqualTo(truthLines.Length)); for (int i = 0; i < truthLines.Length; i++) + { Assert.That(targetLines[i].Trim(), Is.EqualTo(truthLines[i].Trim()), message: $"Line {i}"); - } - - // AssertUsfmEquals only walks the expected lines, so output that runs past the end of the - // expected USFM slips through. These cases turn on content being dropped, so they need the - // line count checked too. - private static void AssertUsfmEqualsExactly(string target, string truth) - { - AssertUsfmEquals(target, truth); - Assert.That(NonEmptyLineCount(target), Is.EqualTo(NonEmptyLineCount(truth)), message: "extra output"); - - static int NonEmptyLineCount(string usfm) => usfm.Split('\n').Count(l => l.Trim().Length > 0); + } } } diff --git a/tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs b/tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs deleted file mode 100644 index 6584d0dc9..000000000 --- a/tests/SIL.Machine.Tests/Corpora/MemoryParatextProjectVersificationConverter.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace SIL.Machine.Corpora; - -public class MemoryParatextProjectVersificationConverter( - IDictionary? files = null, - ParatextProjectSettings? settings = null -) - : ParatextProjectVersificationConverterBase( - new MemoryParatextProjectFileHandler(files), - settings ?? new DefaultParatextProjectSettings() - ); From 4f9d6ef443c0d5be9f71a7dd241d37a95f1a115c Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 11 Sep 2026 17:01:37 -0400 Subject: [PATCH 08/10] Properly handle more complex trailing paragraph insertion --- .../ConvertUsfmVersificationHandler.cs | 44 +++++++++++++------ .../Corpora/UpdateUsfmParserHandler.cs | 4 +- .../ConvertUsfmVersificationHandlerTests.cs | 38 ++++++++++++++-- 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs index 7fe741bdb..46388b550 100644 --- a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs +++ b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs @@ -21,8 +21,7 @@ public class ConvertUsfmVersificationHandler : ScriptureRefUsfmParserHandlerBase new Regex(@"sd\d*", RegexOptions.Compiled), }; private readonly List _tokens; - private readonly List _trailingVerseTokens; - private int _trailingVerseTokensInsertionIndex; + private List<(int Index, UsfmToken Token)> _trailingVerseTokens; private VerseRef _prevVerseRef; private int _verseBoundary; private readonly ScrVers _targetVersification; @@ -34,8 +33,7 @@ public ConvertUsfmVersificationHandler(ScrVers targetVersification) _verseBoundary = 0; _insertChapterIndex = -1; _tokens = new List(); - _trailingVerseTokens = new List(); - _trailingVerseTokensInsertionIndex = 0; + _trailingVerseTokens = new List<(int Index, UsfmToken Token)>(); _prevVerseRef = new VerseRef(); _targetVersification = targetVersification; _skip = false; @@ -103,13 +101,18 @@ string pubNumber if (_insertChapterIndex == -1) { _tokens.Add(newChapterToken); + _trailingVerseTokens = _trailingVerseTokens + .Select(tup => tup.Index == _tokens.Count - 1 ? (tup.Index + 1, tup.Token) : tup) + .ToList(); _tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")); } else { _tokens.Insert(_insertChapterIndex, newChapterToken); + _trailingVerseTokens = _trailingVerseTokens + .Select(tup => tup.Index == _insertChapterIndex ? (tup.Index + 1, tup.Token) : tup) + .ToList(); } - _trailingVerseTokensInsertionIndex++; } string start = null; @@ -212,27 +215,32 @@ private void ProcessTokens(UsfmParserState state) { UsfmToken token = state.Tokens[_verseBoundary + offset]; if ( - inPreservedParagraph - || IsPreservedTrailingParagraphMarker( + IsPreservedTrailingParagraphMarker( token, _verseBoundary + offset + 1 < state.Tokens.Count ? state.Tokens[_verseBoundary + offset + 1] : null ) ) + { + inPreservedParagraph = true; + } + else if (inPreservedParagraph) { inPreservedParagraph = !inPreservedParagraph || token.Type != UsfmTokenType.Paragraph; - - if (_trailingVerseTokens.Count == 0) - _trailingVerseTokensInsertionIndex = _tokens.Count; - _trailingVerseTokens.Add(token); } else { inPreservedParagraph = false; - if (!_skip) - _tokens.Add(token); } + + if (inPreservedParagraph) + _trailingVerseTokens.Add((_tokens.Count, token)); + else if (!_skip) + _tokens.Add(token); + else + offset = offset + 1 - 1; + offset++; } _verseBoundary = state.Index + 1; @@ -240,7 +248,15 @@ private void ProcessTokens(UsfmParserState state) private void AddTrailingTokens() { - _tokens.InsertRange(_trailingVerseTokensInsertionIndex, _trailingVerseTokens); + foreach ( + (int index, List tokens) in _trailingVerseTokens + .GroupBy(tup => tup.Index) + .Select(g => (g.Key, g.Select(tup => tup.Token).ToList())) + .OrderBy(tup => -tup.Key) + ) + { + _tokens.InsertRange(index, tokens); + } _trailingVerseTokens.Clear(); } diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index 215051581..f8c5a3c7a 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -116,7 +116,9 @@ public UpdateUsfmParserHandler( public static ScrVers GetRowsVersification(IReadOnlyList rows) { - return rows.Any() ? rows.First(r => r.Refs.Count > 0).Refs[0].Versification : ScrVers.English; + return rows != null && rows.Count > 0 + ? rows.First(r => r.Refs.Count > 0).Refs[0].Versification + : ScrVers.English; } public IReadOnlyList Tokens => _tokens; diff --git a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs index 502a9a239..7d71b5b69 100644 --- a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs @@ -380,8 +380,8 @@ public void GetUsfm_CrossChapterVerseRange_CrossBookWithinSingleRange() public void GetUsfm_HeadingIntroducingKeptVerse_IsPreserved() { // Russian Orthodox vs. Original - // DAN 3:24-90 = DAG 3:24-90 (leaves the book, so it is dropped) - // DAN 3:91-100 = DAN 3:24-33 (stays in the book, so it is kept) + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 string usfm = @"\id DAN - Test @@ -415,7 +415,7 @@ public void GetUsfm_HeadingIntroducingKeptVerse_IsPreserved() public void GetUsfm_HeadingIntroducingDroppedVerse_IsDropped() { // Russian Orthodox vs. Original - // PSA 151:1-7 = PS2 1:1-7 (the whole chapter leaves the book) + // PSA 151:1-7 = PS2 1:1-7 string usfm = @"\id PSA - Test @@ -708,6 +708,38 @@ public void GetUsfm_MissingVerseInRange() AssertUsfmEquals(target, result); } + [Test] + public void GetUsfm_SameSourceAndTargetVersification() + { + string usfm = + @"\id MAT - Test +\h Matthew +\mt Matthew +\ip An introduction to Matthew\fe + \ft This is an endnote.\fe* +\p \rq MAT 1\rq* Here is another paragraph. +\p and with a \w keyword|a special concept\w* in it. +\p and a \weirdtaglookingthing that is not an actual tag. +\c 1 +\s Chapter One +\v 1 Chapter \pn one\+pro WON\+pro*\pn*, verse one.\f + \fr 1:1: \ft This is a footnote for v1.\f* +\li1 +\v 2 \bd C\bd*hapter one, +\li2 verse\f + \fr 1:2: \ft This is a footnote for v2.\f* two. +\v 3 Chapter one \w*, +\li2 verse three. +\v 4 Chapter one with odd whitespace,  +\li2 verse four, +\v 5 Chapter one, +\li2 verse \fig Figure 1|src=""image1.png"" size=""col"" ref=""1:5""\fig* five. +\v 6 Verse 6 content. +\v 7 +\v 8 +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.English, targetVersification: ScrVers.English); + AssertUsfmEquals(target, usfm); + } + private static string UpdateUsfm(string source, ScrVers sourceVersification, ScrVers targetVersification) { source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; From cc8b7a48eca60def403a7813190809f2c5b2c236 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 11 Sep 2026 17:43:02 -0400 Subject: [PATCH 09/10] Address reviewer comments --- .../ConvertUsfmVersificationHandler.cs | 33 ++++------- ...ratextProjectVersificationConverterBase.cs | 58 ------------------- .../Corpora/UpdateUsfmParserHandler.cs | 4 +- 3 files changed, 13 insertions(+), 82 deletions(-) delete mode 100644 src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs diff --git a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs index 46388b550..88174f251 100644 --- a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs +++ b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs @@ -7,19 +7,10 @@ namespace SIL.Machine.Corpora { public class ConvertUsfmVersificationHandler : ScriptureRefUsfmParserHandlerBase { - private static readonly IReadOnlyList TrailingParagraphMarkerPatterns = new List - { - new Regex(@"mt\d*", RegexOptions.Compiled), - new Regex(@"mte\d*", RegexOptions.Compiled), - new Regex(@"ms\d*", RegexOptions.Compiled), - new Regex("mr", RegexOptions.Compiled), - new Regex(@"s\d*", RegexOptions.Compiled), - new Regex("sr", RegexOptions.Compiled), - new Regex("r", RegexOptions.Compiled), - new Regex("d", RegexOptions.Compiled), - new Regex("sp", RegexOptions.Compiled), - new Regex(@"sd\d*", RegexOptions.Compiled), - }; + private static readonly Regex TrailingParagraphMarkerPatterns = new Regex( + @"^(?:mte?\d*|ms\d*|sd?\d*|mr|sr|sp|d|r)$", + RegexOptions.Compiled + ); private readonly List _tokens; private List<(int Index, UsfmToken Token)> _trailingVerseTokens; private VerseRef _prevVerseRef; @@ -115,6 +106,8 @@ string pubNumber } } + bool addedVerseText = false; + string start = null; for (int i = 0; i < verseRefs.Count; i++) { @@ -135,13 +128,14 @@ string pubNumber { AddTrailingTokens(); _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); - if (state.Index + 1 < state.Tokens.Count) + if (!addedVerseText && state.Index + 1 < state.Tokens.Count) { UsfmToken nextToken = state.Tokens[state.Index + 1]; if (nextToken.Type == UsfmTokenType.Text) { _tokens.Add(nextToken); _verseBoundary++; + addedVerseText = true; } } _tokens.Add(new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[i].Chapter)); @@ -153,13 +147,14 @@ string pubNumber { AddTrailingTokens(); _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); - if (state.Index + 1 < state.Tokens.Count) + if (!addedVerseText && state.Index + 1 < state.Tokens.Count) { UsfmToken nextToken = state.Tokens[state.Index + 1]; if (nextToken.Type == UsfmTokenType.Text) { _tokens.Add(nextToken); _verseBoundary++; + addedVerseText = true; } } start = verseRefs[i].Verse; @@ -227,7 +222,7 @@ private void ProcessTokens(UsfmParserState state) } else if (inPreservedParagraph) { - inPreservedParagraph = !inPreservedParagraph || token.Type != UsfmTokenType.Paragraph; + inPreservedParagraph = token.Type != UsfmTokenType.Paragraph; } else { @@ -238,8 +233,6 @@ private void ProcessTokens(UsfmParserState state) _trailingVerseTokens.Add((_tokens.Count, token)); else if (!_skip) _tokens.Add(token); - else - offset = offset + 1 - 1; offset++; } @@ -263,9 +256,7 @@ private void AddTrailingTokens() private bool IsPreservedTrailingParagraphMarker(UsfmToken token, UsfmToken nextToken) { return (token.Marker == "p" && nextToken != null && nextToken.Type == UsfmTokenType.Verse) - || TrailingParagraphMarkerPatterns.Any(p => - token.Type == UsfmTokenType.Paragraph && p.IsMatch(token.Marker) - ); + || token.Type == UsfmTokenType.Paragraph && TrailingParagraphMarkerPatterns.IsMatch(token.Marker); } } } diff --git a/src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs deleted file mode 100644 index 9bdd367d9..000000000 --- a/src/SIL.Machine/Corpora/ParatextProjectVersificationConverterBase.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using SIL.Scripture; - -namespace SIL.Machine.Corpora -{ - public abstract class ParatextProjectVersificationConverterBase - { - private readonly ParatextProjectSettings _settings; - private readonly IParatextProjectFileHandler _paratextProjectFileHandler; - - protected ParatextProjectVersificationConverterBase( - IParatextProjectFileHandler paratextProjectFileHandler, - ParatextProjectSettings settings - ) - { - _settings = settings; - _paratextProjectFileHandler = paratextProjectFileHandler; - } - - public string UpdateUsfm(string bookId, ScrVers targetVersification) - { - string fileName = _settings.GetBookFileName(bookId); - if (!Exists(fileName)) - return null; - - string usfm; - using (var reader = new StreamReader(Open(fileName))) - { - usfm = reader.ReadToEnd(); - } - - var handler = new ConvertUsfmVersificationHandler(targetVersification); - try - { - var tokenizer = new UsfmTokenizer(_settings.Stylesheet); - IReadOnlyList tokens = tokenizer.Tokenize(usfm); - UsfmParser.Parse(tokens, handler, _settings.Stylesheet, _settings.Versification); - return handler.GetUsfm(_settings.Stylesheet); - } - catch (Exception ex) - { - var sb = new StringBuilder(); - sb.Append($"An error occurred while parsing the usfm for '{bookId}`"); - if (!string.IsNullOrEmpty(_settings.Name)) - sb.Append($" in project '{_settings.Name}'"); - sb.Append($". Error: '{ex.Message}'"); - throw new InvalidOperationException(sb.ToString(), ex); - } - } - - private bool Exists(string fileName) => _paratextProjectFileHandler.Exists(fileName); - - private Stream Open(string fileName) => _paratextProjectFileHandler.Open(fileName); - } -} diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index f8c5a3c7a..a88421ded 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -86,9 +86,7 @@ public UpdateUsfmParserHandler( _verseRowsMap = new Dictionary>( compareSegments ? VerseRefComparer.Default : VerseRefComparer.IgnoreSegments ); - _updateRowsVersification = ScrVers.English; - if (_rows.Count > 0) - _updateRowsVersification = _rows.First(r => r.Refs.Count > 0).Refs[0].Versification; + _updateRowsVersification = GetRowsVersification(rows); _tokens = new List(); _updatedText = new List(); _updateBlocks = new Stack(); From a50f3a5eebce36f922fda3fd2ec832220956060d Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Fri, 11 Sep 2026 18:48:06 -0400 Subject: [PATCH 10/10] Address some edge cases when inserting a chapter marker --- .../ConvertUsfmVersificationHandler.cs | 31 +++++++-- .../ConvertUsfmVersificationHandlerTests.cs | 65 +++++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs index 88174f251..fd56c4559 100644 --- a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs +++ b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs @@ -91,17 +91,40 @@ string pubNumber if (_insertChapterIndex == -1) { + int chapterIndex = _tokens.Count; _tokens.Add(newChapterToken); - _trailingVerseTokens = _trailingVerseTokens - .Select(tup => tup.Index == _tokens.Count - 1 ? (tup.Index + 1, tup.Token) : tup) + List trailingAtChapter = _trailingVerseTokens + .Where(tup => tup.Index == chapterIndex) + .Select(tup => tup.Token) .ToList(); - _tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")); + if (trailingAtChapter.Count == 0) + { + // The chapter break falls mid-paragraph, so the paragraph continues across it. + _tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")); + } + else + { + // The trailing markers follow the new chapter and break the paragraph. If + // they do not open a paragraph of their own, the verse still needs one. + UsfmToken lastParagraph = trailingAtChapter.LastOrDefault(t => + t.Type == UsfmTokenType.Paragraph + ); + if (lastParagraph == null || TrailingParagraphMarkerPatterns.IsMatch(lastParagraph.Marker)) + { + _trailingVerseTokens.Add( + (chapterIndex, new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")) + ); + } + _trailingVerseTokens = _trailingVerseTokens + .Select(tup => tup.Index == chapterIndex ? (tup.Index + 1, tup.Token) : tup) + .ToList(); + } } else { _tokens.Insert(_insertChapterIndex, newChapterToken); _trailingVerseTokens = _trailingVerseTokens - .Select(tup => tup.Index == _insertChapterIndex ? (tup.Index + 1, tup.Token) : tup) + .Select(tup => tup.Index >= _insertChapterIndex ? (tup.Index + 1, tup.Token) : tup) .ToList(); } } diff --git a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs index 7d71b5b69..5831fd99d 100644 --- a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs @@ -533,6 +533,71 @@ public void GetUsfm_ChapterMarkerIsFollowedByParagraphMarker_CrossChapterVerseRa AssertUsfmEquals(target, result); } + [Test] + public void GetUsfm_ChapterMarkerIsFollowedByParagraphMarker_HeadingOpensParagraph() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL - Test +\c 3 +\p +\v 18 Text +\s1 Section +\p +\v 19-23 More text +\v 24 Last text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id MAL - Test +\c 3 +\p +\v 18 Text +\c 4 +\s1 Section +\p +\v 1-5 More text +\v 6 Last text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_HeadingAfterChapterLabel_KeepsMarkerContent() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22 Text +\c 9 +\cl Chapter Nine +\s1 Section +\p +\v 1 Nine one +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 Text +\c 9 +\cl Chapter Nine +\s1 Section +\p +\v 2 Nine one +"; + AssertUsfmEquals(target, result); + } + [Test] public void GetUsfm_CrossChapterVerseRange_TextStaysWithFirstVerse() {