diff --git a/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs b/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs index 83682c4f..24db9a27 100644 --- a/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs +++ b/src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using SIL.Extensions; @@ -128,10 +129,14 @@ public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block) string text = element.Tokens[0].ToUsfm(); // Track seen tokens - while (sourceTokenIndex < sourceTokens.Count && text.Contains(sourceTokens[sourceTokenIndex])) + while ( + sourceTokenIndex < sourceTokens.Count + && text.Contains(sourceTokens[sourceTokenIndex], StringComparison.Ordinal) + ) { text = text.Substring( - text.IndexOf(sourceTokens[sourceTokenIndex]) + sourceTokens[sourceTokenIndex].Length + text.IndexOf(sourceTokens[sourceTokenIndex], StringComparison.Ordinal) + + sourceTokens[sourceTokenIndex].Length ); sourceTokenIndex++; } @@ -175,7 +180,8 @@ public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block) { int indexOfTargetTokenInSentence = targetSentence.IndexOf( token, - targetTokenStarts.LastOrDefault() + prevLength + targetTokenStarts.LastOrDefault() + prevLength, + StringComparison.Ordinal ); if (indexOfTargetTokenInSentence == -1) { diff --git a/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs index a4b48b25..60c2e3a2 100644 --- a/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/PlaceMarkersUsfmUpdateBlockHandlerTests.cs @@ -911,6 +911,57 @@ public void UpdateUsfm_AdjustmentOfPlacedParagraphMarker() AssertUsfmEquals(target, result); } + [Test] + public void UpdateBlock_AnusvaraTokenization() + { + // The anusvara ("ं") is the third to last character in the update rows string + string source = + "Like the crushing of my bones, my enemies taunt me, while they say to me all day long, “Where is your God?”"; + string updateRows = + "\"कब च तेरु परमेश्वर?\" इन सवालों के मेरे दूसरे ताना मि पर मरदिन, हर बगत इन लगदु जन व मेरे सर पे ते कटदिन नह ं। "; + PlaceMarkersAlignmentInfo alignInfo = new PlaceMarkersAlignmentInfo( + sourceTokens: [.. Tokenizer.Tokenize(source)], + translationTokens: [.. Tokenizer.Tokenize(updateRows)], + alignment: ToWordAlignmentMatrix( + "0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 9-9 10-10 11-11 12-12 13-13 14-14 15-15 16-16 17-17 18-18 19-19 20-20 21-21 22-22 23-23 24-24 24-25 25-26 26-27 26-28 26-29 27-30" + ), + paragraphBehavior: UpdateUsfmMarkerBehavior.Preserve, + styleBehavior: UpdateUsfmMarkerBehavior.Strip + ); + IReadOnlyList rows = + [ + new UpdateUsfmRow( + ScrRef("PSA 42:10"), + updateRows, + new Dictionary { { "alignment_info", alignInfo } } + ), + ]; + string usfm = + @"\id PSA +\c 42 +\q1 +\v 10 Like the crushing of my bones, my enemies taunt me, +\q2 while they say to me all day long, “Where is your God?” +"; + + string target = UpdateUsfm( + rows, + usfm, + paragraphBehavior: UpdateUsfmMarkerBehavior.Preserve, + usfmUpdateBlockHandlers: [new PlaceMarkersUsfmUpdateBlockHandler()] + ); + + string result = + @"\id PSA +\c 42 +\q1 +\v 10 ""कब च तेरु परमेश्वर?"" इन सवालों के मेरे दूसरे +\q2 ताना मि पर मरदिन, हर बगत इन लगदु जन व मेरे सर पे ते कटदिन नह ं। +"; + + AssertUsfmEquals(target, result); + } + private static ScriptureRef[] ScrRef(params string[] refs) { return refs.Select(r => ScriptureRef.Parse(r)).ToArray();