diff --git a/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs new file mode 100644 index 00000000..fd56c455 --- /dev/null +++ b/src/SIL.Machine/Corpora/ConvertUsfmVersificationHandler.cs @@ -0,0 +1,285 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using SIL.Scripture; + +namespace SIL.Machine.Corpora +{ + public class ConvertUsfmVersificationHandler : ScriptureRefUsfmParserHandlerBase + { + private static readonly Regex TrailingParagraphMarkerPatterns = new Regex( + @"^(?:mte?\d*|ms\d*|sd?\d*|mr|sr|sp|d|r)$", + RegexOptions.Compiled + ); + private readonly List _tokens; + private List<(int Index, UsfmToken Token)> _trailingVerseTokens; + private VerseRef _prevVerseRef; + private int _verseBoundary; + private readonly ScrVers _targetVersification; + private int _insertChapterIndex; + private bool _skip; + + public ConvertUsfmVersificationHandler(ScrVers targetVersification) + { + _verseBoundary = 0; + _insertChapterIndex = -1; + _tokens = new List(); + _trailingVerseTokens = new List<(int Index, UsfmToken Token)>(); + _prevVerseRef = new VerseRef(); + _targetVersification = targetVersification; + _skip = false; + } + + public IReadOnlyList Tokens => _tokens; + + public override void Chapter( + UsfmParserState state, + string number, + string marker, + string altNumber, + string pubNumber + ) + { + base.Chapter(state, number, marker, altNumber, pubNumber); + ProcessTokens(state); + VerseRef vr = state.VerseRef; + // The versification of verse 0 cannot properly be changed + vr.Verse = "1"; + if ( + !_prevVerseRef.IsDefault + && ( + vr.ChangeVersificationWithSegments(_targetVersification).Book != _prevVerseRef.Book + || vr.ChapterNum == -1 + ) + ) + { + _skip = true; + } + _insertChapterIndex = _tokens.Count; + } + + public override void Verse( + UsfmParserState state, + string number, + string marker, + string altNumber, + string pubNumber + ) + { + base.Verse(state, number, marker, altNumber, pubNumber); + + VerseRef verseRef = state.VerseRef; + + ProcessTokens(state); + + List verseRefs = state + .VerseRef.AllVerses() + .Select(vr => vr.ChangeVersificationWithSegments(_targetVersification)) + .ToList(); + + if ( + ( + _prevVerseRef.IsDefault + || ( + verseRefs[0].BookNum == _prevVerseRef.BookNum + && verseRefs[0].ChapterNum != _prevVerseRef.ChapterNum + ) + ) && (verseRefs[0].ChapterNum != -1) + ) + { + UsfmToken newChapterToken = new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[0].Chapter); + + if (_insertChapterIndex == -1) + { + int chapterIndex = _tokens.Count; + _tokens.Add(newChapterToken); + List trailingAtChapter = _trailingVerseTokens + .Where(tup => tup.Index == chapterIndex) + .Select(tup => tup.Token) + .ToList(); + if (trailingAtChapter.Count == 0) + { + // The chapter break falls mid-paragraph, so the paragraph continues across it. + _tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")); + } + else + { + // The trailing markers follow the new chapter and break the paragraph. If + // they do not open a paragraph of their own, the verse still needs one. + UsfmToken lastParagraph = trailingAtChapter.LastOrDefault(t => + t.Type == UsfmTokenType.Paragraph + ); + if (lastParagraph == null || TrailingParagraphMarkerPatterns.IsMatch(lastParagraph.Marker)) + { + _trailingVerseTokens.Add( + (chapterIndex, new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")) + ); + } + _trailingVerseTokens = _trailingVerseTokens + .Select(tup => tup.Index == chapterIndex ? (tup.Index + 1, tup.Token) : tup) + .ToList(); + } + } + else + { + _tokens.Insert(_insertChapterIndex, newChapterToken); + _trailingVerseTokens = _trailingVerseTokens + .Select(tup => tup.Index >= _insertChapterIndex ? (tup.Index + 1, tup.Token) : tup) + .ToList(); + } + } + + bool addedVerseText = false; + + string start = null; + for (int i = 0; i < verseRefs.Count; i++) + { + if ( + (!_prevVerseRef.IsDefault && verseRefs[i].Book != _prevVerseRef.Book) + || verseRefs[i].ChapterNum == -1 + ) + { + continue; + } + if (start != null) + { + string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : ""; + if ( + _prevVerseRef.BookNum == verseRefs[i].BookNum + && _prevVerseRef.ChapterNum != verseRefs[i].ChapterNum + ) + { + AddTrailingTokens(); + _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + if (!addedVerseText && state.Index + 1 < state.Tokens.Count) + { + UsfmToken nextToken = state.Tokens[state.Index + 1]; + if (nextToken.Type == UsfmTokenType.Text) + { + _tokens.Add(nextToken); + _verseBoundary++; + addedVerseText = true; + } + } + _tokens.Add(new UsfmToken(UsfmTokenType.Chapter, "c", "", "", verseRefs[i].Chapter)); + _tokens.Add(new UsfmToken(UsfmTokenType.Paragraph, "nb", "", "", "")); + start = verseRefs[i].Verse; + _prevVerseRef = verseRefs[i]; + } + else if (_prevVerseRef.VerseNum + 1 != verseRefs[i].VerseNum) + { + AddTrailingTokens(); + _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + if (!addedVerseText && state.Index + 1 < state.Tokens.Count) + { + UsfmToken nextToken = state.Tokens[state.Index + 1]; + if (nextToken.Type == UsfmTokenType.Text) + { + _tokens.Add(nextToken); + _verseBoundary++; + addedVerseText = true; + } + } + start = verseRefs[i].Verse; + _prevVerseRef = verseRefs[i]; + } + else + { + _prevVerseRef = verseRefs[i]; + } + } + else + { + start = verseRefs[i].Verse; + _prevVerseRef = verseRefs[i]; + } + verseRef = verseRefs[i]; + } + + if (start != null) + { + AddTrailingTokens(); + string end = start != _prevVerseRef.Verse ? "-" + _prevVerseRef.Verse : ""; + _tokens.Add(new UsfmToken(UsfmTokenType.Verse, "v", "", "", start + end)); + _skip = false; + _insertChapterIndex = -1; + _prevVerseRef = verseRef; + } + else + { + _skip = true; + } + } + + public override void EndUsfm(UsfmParserState state) + { + base.EndUsfm(state); + ProcessTokens(state); + if (!_skip && !(state.Token.Type == UsfmTokenType.Chapter || state.Token.Type == UsfmTokenType.Verse)) + _tokens.Add(state.Token); + } + + public string GetUsfm(UsfmStylesheet stylesheet) + { + var tokenizer = new UsfmTokenizer(stylesheet); + return tokenizer.Detokenize(_tokens); + } + + private void ProcessTokens(UsfmParserState state) + { + int offset = 0; + bool inPreservedParagraph = false; + while (_verseBoundary + offset < state.Index) + { + UsfmToken token = state.Tokens[_verseBoundary + offset]; + if ( + IsPreservedTrailingParagraphMarker( + token, + _verseBoundary + offset + 1 < state.Tokens.Count + ? state.Tokens[_verseBoundary + offset + 1] + : null + ) + ) + { + inPreservedParagraph = true; + } + else if (inPreservedParagraph) + { + inPreservedParagraph = token.Type != UsfmTokenType.Paragraph; + } + else + { + inPreservedParagraph = false; + } + + if (inPreservedParagraph) + _trailingVerseTokens.Add((_tokens.Count, token)); + else if (!_skip) + _tokens.Add(token); + + offset++; + } + _verseBoundary = state.Index + 1; + } + + private void AddTrailingTokens() + { + foreach ( + (int index, List tokens) in _trailingVerseTokens + .GroupBy(tup => tup.Index) + .Select(g => (g.Key, g.Select(tup => tup.Token).ToList())) + .OrderBy(tup => -tup.Key) + ) + { + _tokens.InsertRange(index, tokens); + } + _trailingVerseTokens.Clear(); + } + + private bool IsPreservedTrailingParagraphMarker(UsfmToken token, UsfmToken nextToken) + { + return (token.Marker == "p" && nextToken != null && nextToken.Type == UsfmTokenType.Verse) + || token.Type == UsfmTokenType.Paragraph && TrailingParagraphMarkerPatterns.IsMatch(token.Marker); + } + } +} diff --git a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs index e38e528f..1289f3f4 100644 --- a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs +++ b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using SIL.Scripture; namespace SIL.Machine.Corpora { @@ -64,7 +65,17 @@ public string UpdateUsfm( var tokenizer = new UsfmTokenizer(_settings.Stylesheet); IReadOnlyList tokens = tokenizer.Tokenize(usfm); tokens = FilterTokensByChapter(tokens, chapters); - UsfmParser.Parse(tokens, handler, _settings.Stylesheet, _settings.Versification); + + ScrVers rowsVersification = UpdateUsfmParserHandler.GetRowsVersification(rows); + ScrVers parseVersification = _settings.Versification; + if (rowsVersification != _settings.Versification) + { + var converter = new ConvertUsfmVersificationHandler(rowsVersification); + UsfmParser.Parse(tokens, converter, _settings.Stylesheet, _settings.Versification); + tokens = converter.Tokens; + parseVersification = rowsVersification; + } + UsfmParser.Parse(tokens, handler, _settings.Stylesheet, parseVersification); return handler.GetUsfm(_settings.Stylesheet); } catch (Exception ex) diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index f21be754..a88421de 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -86,9 +86,7 @@ public UpdateUsfmParserHandler( _verseRowsMap = new Dictionary>( compareSegments ? VerseRefComparer.Default : VerseRefComparer.IgnoreSegments ); - _updateRowsVersification = ScrVers.English; - if (_rows.Count > 0) - _updateRowsVersification = _rows.First(r => r.Refs.Count > 0).Refs[0].Versification; + _updateRowsVersification = GetRowsVersification(rows); _tokens = new List(); _updatedText = new List(); _updateBlocks = new Stack(); @@ -114,6 +112,13 @@ public UpdateUsfmParserHandler( _compareSegments = compareSegments; } + public static ScrVers GetRowsVersification(IReadOnlyList rows) + { + return rows != null && rows.Count > 0 + ? rows.First(r => r.Refs.Count > 0).Refs[0].Versification + : ScrVers.English; + } + public IReadOnlyList Tokens => _tokens; public override void EndUsfm(UsfmParserState state) diff --git a/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs new file mode 100644 index 00000000..5831fd99 --- /dev/null +++ b/tests/SIL.Machine.Tests/Corpora/ConvertUsfmVersificationHandlerTests.cs @@ -0,0 +1,835 @@ +using NUnit.Framework; +using SIL.Scripture; + +namespace SIL.Machine.Corpora; + +[TestFixture] +public class ConvertUsfmVersificationHandlerTests +{ + [Test] + public void GetUsfm_OneFewerChapter() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\p +\s1 Section +\v 1-5 +\v 6 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.English, targetVersification: ScrVers.Original); + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\p +\s1 Section +\v 19-23 +\v 24 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_OneMoreChapter() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-23 +\v 24 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\nb +\v 1-5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_OneFewerBook() + { + // Russian Orthodox vs. Original + // PSA 151:1-7 = PS2 1:1-7 + + string usfm = + @"\id PSA - Test +\h Psalms +\c 150 +\p +\v 1-5 Lines +\v 6 Line +\q Another line +\c 151 +\p +\v 1-7 More lines +"; + + string target = UpdateUsfm( + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id PSA - Test +\h Psalms +\c 150 +\p +\v 1-5 Lines +\v 6 Line +\q Another line +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_OneMoreBook() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + // S3Y 1:30-31 = DAG 3:52-53 + // S3Y 1:33 = DAG 3:54 + // S3Y 1:32 = DAG 3:55 + // S3Y 1:34-35 = DAG 3:56-57 + // S3Y 1:37 = DAG 3:58 + // S3Y 1:36 = DAG 3:59 + // S3Y 1:38-68 = DAG 3:60-90 + + string usfm = + @"\id DAN - Test +\h Daniel +\c 3 +\p +\v 1-23 Text 1 +\v 24-90 Text 2 +\p More text 2 +\v 91-100 Text 3 +\c 4 +\p +\v 1 Text 4 +"; + + string target = UpdateUsfm( + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\h Daniel +\c 3 +\p +\v 1-23 Text 1 +\v 24-33 Text 3 +\c 4 +\p +\v 1 Text 4 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_BackOneVerseToPreviousChapter() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22 +\v 23 +\c 9 +\p +\v 1 +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 +\c 9 +\nb +\v 1 +\p +\v 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ForwardOneVerseToNextChapter() + { + // Original vs. English + // ISA 8:23 = ISA 9:1 + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22 +\c 9 +\p +\v 1 +\v 2 +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.English, targetVersification: ScrVers.Original); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 +\p +\v 23 +\c 9 +\nb +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_CrossChapterVerseRange() + { + // English vd. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22-23 +\c 9 +\p +\v 1 +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 +\c 9 +\nb +\v 1 +\p +\v 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_CrossChapterVerseRange_CrossBook() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + // S3Y 1:30-31 = DAG 3:52-53 + // S3Y 1:33 = DAG 3:54 + // S3Y 1:32 = DAG 3:55 + // S3Y 1:34-35 = DAG 3:56-57 + // S3Y 1:37 = DAG 3:58 + // S3Y 1:36 = DAG 3:59 + // S3Y 1:38-68 = DAG 3:60-90 + + string usfm = + @"\id DAN - Test +\c 3 +\p +\v 1-22 +\v 23-89 +\v 90-100 +\c 4 +\p +\v 1 +"; + + string target = UpdateUsfm( + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\p +\v 1-22 +\v 23 +\v 24-33 +\c 4 +\p +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_CrossChapterVerseRange_CrossBookWithinSingleRange() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + // S3Y 1:30-31 = DAG 3:52-53 + // S3Y 1:33 = DAG 3:54 + // S3Y 1:32 = DAG 3:55 + // S3Y 1:34-35 = DAG 3:56-57 + // S3Y 1:37 = DAG 3:58 + // S3Y 1:36 = DAG 3:59 + // S3Y 1:38-68 = DAG 3:60-90 + + string usfm = + @"\id DAN - Test +\c 3 +\p +\v 1-100 +\c 4 +\p +\v 1 +"; + + string target = UpdateUsfm( + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\p +\v 1-33 +\c 4 +\p +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_HeadingIntroducingKeptVerse_IsPreserved() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + string usfm = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\v 24-90 Dropped text +\s1 \nd Section\nd* +\p +\v 91-100 More text +"; + + string target = UpdateUsfm( + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\s1 \nd Section\nd* +\p +\v 24-33 More text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_HeadingIntroducingDroppedVerse_IsDropped() + { + // Russian Orthodox vs. Original + // PSA 151:1-7 = PS2 1:1-7 + + string usfm = + @"\id PSA - Test +\c 150 +\p +\v 1-5 Lines +\v 6 Line +\q Another line +\c 151 +\s1 \nd Section\nd* +\p +\v 1-7 More lines +"; + + string target = UpdateUsfm( + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id PSA - Test +\c 150 +\p +\v 1-5 Lines +\v 6 Line +\q Another line +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_DropVerseText() + { + string usfm = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\v 24-90 Dropped text +\v 91-100 More text +"; + + string target = UpdateUsfm( + usfm, + sourceVersification: ScrVers.RussianOrthodox, + targetVersification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 3 +\p +\v 1-23 Text +\v 24-33 More text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ChapterMarkerIsFollowedByParagraphMarker() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL - Test +\c 3 +\p +\v 1-18 Text +\v 19-23 More text +\v 24 Last text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id MAL - Test +\c 3 +\p +\v 1-18 Text +\c 4 +\nb +\v 1-5 More text +\v 6 Last text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ChapterMarkerIsFollowedByParagraphMarker_CrossChapterVerseRange() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22-23 +\c 9 +\p +\v 1 +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 +\c 9 +\nb +\v 1 +\p +\v 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ChapterMarkerIsFollowedByParagraphMarker_HeadingOpensParagraph() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL - Test +\c 3 +\p +\v 18 Text +\s1 Section +\p +\v 19-23 More text +\v 24 Last text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id MAL - Test +\c 3 +\p +\v 18 Text +\c 4 +\s1 Section +\p +\v 1-5 More text +\v 6 Last text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_HeadingAfterChapterLabel_KeepsMarkerContent() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22 Text +\c 9 +\cl Chapter Nine +\s1 Section +\p +\v 1 Nine one +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 Text +\c 9 +\cl Chapter Nine +\s1 Section +\p +\v 2 Nine one +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_CrossChapterVerseRange_TextStaysWithFirstVerse() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + string usfm = + @"\id ISA - Test +\c 8 +\p +\v 22-23 Verse twenty-two and twenty-three text +\c 9 +\p +\v 1 Chapter nine verse one text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id ISA - Test +\c 8 +\p +\v 22 Verse twenty-two and twenty-three text +\c 9 +\nb +\v 1 +\p +\v 2 Chapter nine verse one text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_IgnoreInvalidChapter() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2@ +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-23 +\v 24 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + + // Strip out invalid chapters since we can't reliably convert them + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\nb +\v 1-5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_IgnoreInvalidVerse() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1@ Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-23 +\v 24 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + + // Just pass invalid verses through to target + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1@ Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\nb +\v 1-5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_MissingVerseInRange() + { + // English vs. Original + // MAL 4:1-6 = MAL 3:19-24 + + string usfm = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\v 19-21,23 Text +\v 24 Text +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.Original, targetVersification: ScrVers.English); + string result = + @"\id MAL +\h Malachi +\c 1 +\s1 Section +\p +\v 1 Text +\v 2-14 +\c 2 +\v 1-17 +\c 3 +\p +\v 1-17 +\v 18 Text \f More text \f* +\c 4 +\nb +\v 1-3 Text +\v 5 +\v 6 Text +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_SameSourceAndTargetVersification() + { + string usfm = + @"\id MAT - Test +\h Matthew +\mt Matthew +\ip An introduction to Matthew\fe + \ft This is an endnote.\fe* +\p \rq MAT 1\rq* Here is another paragraph. +\p and with a \w keyword|a special concept\w* in it. +\p and a \weirdtaglookingthing that is not an actual tag. +\c 1 +\s Chapter One +\v 1 Chapter \pn one\+pro WON\+pro*\pn*, verse one.\f + \fr 1:1: \ft This is a footnote for v1.\f* +\li1 +\v 2 \bd C\bd*hapter one, +\li2 verse\f + \fr 1:2: \ft This is a footnote for v2.\f* two. +\v 3 Chapter one \w*, +\li2 verse three. +\v 4 Chapter one with odd whitespace,  +\li2 verse four, +\v 5 Chapter one, +\li2 verse \fig Figure 1|src=""image1.png"" size=""col"" ref=""1:5""\fig* five. +\v 6 Verse 6 content. +\v 7 +\v 8 +"; + + string target = UpdateUsfm(usfm, sourceVersification: ScrVers.English, targetVersification: ScrVers.English); + AssertUsfmEquals(target, usfm); + } + + private static string UpdateUsfm(string source, ScrVers sourceVersification, ScrVers targetVersification) + { + source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; + var settings = new DefaultParatextProjectSettings( + versification: sourceVersification, + fileNameForm: "MAT", + fileNameSuffix: string.Empty, + fileNamePrefix: string.Empty + ); + var handler = new ConvertUsfmVersificationHandler(targetVersification); + var tokenizer = new UsfmTokenizer(settings.Stylesheet); + IReadOnlyList tokens = tokenizer.Tokenize(source); + UsfmParser.Parse(tokens, handler, settings.Stylesheet, settings.Versification); + return handler.GetUsfm(settings.Stylesheet); + } + + private static void AssertUsfmEquals(string target, string truth) + { + Assert.That(target, Is.Not.Null); + string[] targetLines = target.Split('\n'); + string[] truthLines = truth.Split('\n'); + Assert.That(targetLines.Length, Is.EqualTo(truthLines.Length)); + for (int i = 0; i < truthLines.Length; i++) + { + Assert.That(targetLines[i].Trim(), Is.EqualTo(truthLines[i].Trim()), message: $"Line {i}"); + } + } +}