feat(search): add unicode character removal for fuzzy matching

Flow.Launcher.Infrastructure/StringMatcher.cs

@@ -1,30 +1,42 @@
-using CommunityToolkit.Mvvm.DependencyInjection;
-using Flow.Launcher.Plugin.SharedModels;
-using System;
+using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
+using CommunityToolkit.Mvvm.DependencyInjection;
 using Flow.Launcher.Infrastructure.UserSettings;
+using Flow.Launcher.Plugin.SharedModels;
 
 namespace Flow.Launcher.Infrastructure
 {
     public class StringMatcher
     {
         private readonly MatchOption _defaultMatchOption = new();
-
+        private readonly Settings _settings;
         public SearchPrecisionScore UserSettingSearchPrecision { get; set; }
 
         private readonly IAlphabet _alphabet;
 
         public StringMatcher(IAlphabet alphabet, Settings settings)
         {
             _alphabet = alphabet;
-            UserSettingSearchPrecision = settings.QuerySearchPrecision;
+            _settings = settings;
+            UserSettingSearchPrecision = _settings.QuerySearchPrecision;
+
+            _settings.PropertyChanged += (sender, e) =>
+            {
+                switch (e.PropertyName)
+                {
+                    case nameof(Settings.QuerySearchPrecision):
+                        UserSettingSearchPrecision = _settings.QuerySearchPrecision;
+                        break;
+                }
+            };
         }
 
         // This is a workaround to allow unit tests to set the instance
-        public StringMatcher(IAlphabet alphabet)
+        public StringMatcher(IAlphabet alphabet) : this(alphabet, new Settings())
         {
-            _alphabet = alphabet;
         }
 
         public static MatchResult FuzzySearch(string query, string stringToCompare)
@@ -80,10 +92,20 @@ public MatchResult FuzzyMatch(string query, string stringToCompare, MatchOption
             int acronymsTotalCount = 0;
             int acronymsMatched = 0;
 
-            var fullStringToCompareWithoutCase = opt.IgnoreCase ? stringToCompare.ToLower() : stringToCompare;
-            var queryWithoutCase = opt.IgnoreCase ? query.ToLower() : query;
+            var queryToCompare = query;
+            bool ignoreAccents = _settings.IgnoreAccents;
+            bool ignoreCase = opt.IgnoreCase;
+
+            if (ignoreAccents)
+            {
+                queryToCompare = Normalize(queryToCompare);
+            }
+            else if (ignoreCase)
+            {
+                queryToCompare = queryToCompare.ToLower();
+            }
 
-            var querySubstrings = queryWithoutCase.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
+            var querySubstrings = queryToCompare.Split([' '], StringSplitOptions.RemoveEmptyEntries);
             int currentQuerySubstringIndex = 0;
             var currentQuerySubstring = querySubstrings[currentQuerySubstringIndex];
             var currentQuerySubstringCharacterIndex = 0;
@@ -98,7 +120,9 @@ public MatchResult FuzzyMatch(string query, string stringToCompare, MatchOption
             var indexList = new List<int>();
             List<int> spaceIndices = new List<int>();
 
-            for (var compareStringIndex = 0; compareStringIndex < fullStringToCompareWithoutCase.Length; compareStringIndex++)
+            for (var compareStringIndex = 0;
+                 compareStringIndex < stringToCompare.Length;
+                 compareStringIndex++)
             {
                 // If acronyms matching successfully finished, this gets the remaining not matched acronyms for score calculation
                 if (currentAcronymQueryIndex >= query.Length && acronymsMatched == query.Length)
@@ -112,16 +136,26 @@ public MatchResult FuzzyMatch(string query, string stringToCompare, MatchOption
                     currentAcronymQueryIndex >= query.Length && allQuerySubstringsMatched)
                     break;
 
+                char compareChar = stringToCompare[compareStringIndex];
+                if (ignoreAccents)
+                {
+                    compareChar = NormalizeChar(compareChar);
+                }
+                else if (ignoreCase)
+                {
+                    compareChar = char.ToLower(compareChar);
+                }
+
                 // To maintain a list of indices which correspond to spaces in the string to compare
                 // To populate the list only for the first query substring
-                if (fullStringToCompareWithoutCase[compareStringIndex] == ' ' && currentQuerySubstringIndex == 0)
+                if (compareChar == ' ' && currentQuerySubstringIndex == 0)
                     spaceIndices.Add(compareStringIndex);
 
                 // Acronym Match
                 if (IsAcronym(stringToCompare, compareStringIndex))
                 {
-                    if (fullStringToCompareWithoutCase[compareStringIndex] ==
-                        queryWithoutCase[currentAcronymQueryIndex])
+                    if (compareChar ==
+                        queryToCompare[currentAcronymQueryIndex])
                     {
                         acronymMatchData.Add(compareStringIndex);
                         acronymsMatched++;
@@ -133,7 +167,7 @@ public MatchResult FuzzyMatch(string query, string stringToCompare, MatchOption
                 if (IsAcronymCount(stringToCompare, compareStringIndex))
                     acronymsTotalCount++;
 
-                if (allQuerySubstringsMatched || fullStringToCompareWithoutCase[compareStringIndex] !=
+                if (allQuerySubstringsMatched || compareChar !=
                     currentQuerySubstring[currentQuerySubstringCharacterIndex])
                 {
                     matchFoundInPreviousLoop = false;
@@ -160,7 +194,7 @@ public MatchResult FuzzyMatch(string query, string stringToCompare, MatchOption
                     var startIndexToVerify = compareStringIndex - currentQuerySubstringCharacterIndex;
 
                     if (AllPreviousCharsMatched(startIndexToVerify, currentQuerySubstringCharacterIndex,
-                        fullStringToCompareWithoutCase, currentQuerySubstring))
+                            stringToCompare, currentQuerySubstring, ignoreAccents, ignoreCase))
                     {
                         matchFoundInPreviousLoop = true;
 
@@ -205,7 +239,8 @@ public MatchResult FuzzyMatch(string query, string stringToCompare, MatchOption
 
                 if (acronymScore >= (int)UserSettingSearchPrecision)
                 {
-                    acronymMatchData = acronymMatchData.Select(x => translationMapping?.MapToOriginalIndex(x) ?? x).Distinct().ToList();
+                    acronymMatchData = acronymMatchData.Select(x => translationMapping?.MapToOriginalIndex(x) ?? x)
+                        .Distinct().ToList();
                     return new MatchResult(true, UserSettingSearchPrecision, acronymMatchData, acronymScore);
                 }
             }
@@ -218,19 +253,164 @@ public MatchResult FuzzyMatch(string query, string stringToCompare, MatchOption
                 // firstMatchIndex - nearestSpaceIndex - 1 is to set the firstIndex as the index of the first matched char
                 // preceded by a space e.g. 'world' matching 'hello world' firstIndex would be 0 not 6 
                 // giving more weight than 'we or donald' by allowing the distance calculation to treat the starting position at after the space.
-                var score = CalculateSearchScore(query, stringToCompare, firstMatchIndex - nearestSpaceIndex - 1, spaceIndices,
+                var score = CalculateSearchScore(query, stringToCompare, firstMatchIndex - nearestSpaceIndex - 1,
+                    spaceIndices,
                     lastMatchIndex - firstMatchIndex, allSubstringsContainedInCompareString);
 
-                var resultList = indexList.Select(x => translationMapping?.MapToOriginalIndex(x) ?? x).Distinct().ToList();
+                var resultList = indexList.Select(x => translationMapping?.MapToOriginalIndex(x) ?? x).Distinct()
+                    .ToList();
                 return new MatchResult(true, UserSettingSearchPrecision, resultList, score);
             }
 
             return new MatchResult(false, UserSettingSearchPrecision);
         }
 
+
+        private static readonly Dictionary<char, char> AccentMap = new()
+        {
+            ['á'] = 'a',
+            ['à'] = 'a',
+            ['ã'] = 'a',
+            ['â'] = 'a',
+            ['ä'] = 'a',
+            ['å'] = 'a',
+            ['ā'] = 'a',
+            ['ă'] = 'a',
+            ['ą'] = 'a',
+            ['é'] = 'e',
+            ['è'] = 'e',
+            ['ê'] = 'e',
+            ['ë'] = 'e',
+            ['ē'] = 'e',
+            ['ĕ'] = 'e',
+            ['ė'] = 'e',
+            ['ę'] = 'e',
+            ['ě'] = 'e',
+            ['í'] = 'i',
+            ['ì'] = 'i',
+            ['î'] = 'i',
+            ['ï'] = 'i',
+            ['ī'] = 'i',
+            ['ĭ'] = 'i',
+            ['į'] = 'i',
+            ['ı'] = 'i',
+            ['ó'] = 'o',
+            ['ò'] = 'o',
+            ['õ'] = 'o',
+            ['ô'] = 'o',
+            ['ö'] = 'o',
+            ['ø'] = 'o',
+            ['ō'] = 'o',
+            ['ŏ'] = 'o',
+            ['ő'] = 'o',
+            ['ú'] = 'u',
+            ['ù'] = 'u',
+            ['û'] = 'u',
+            ['ü'] = 'u',
+            ['ū'] = 'u',
+            ['ŭ'] = 'u',
+            ['ů'] = 'u',
+            ['ű'] = 'u',
+            ['ų'] = 'u',
+            ['ç'] = 'c',
+            ['ć'] = 'c',
+            ['ĉ'] = 'c',
+            ['ċ'] = 'c',
+            ['č'] = 'c',
+            ['ñ'] = 'n',
+            ['ń'] = 'n',
+            ['ņ'] = 'n',
+            ['ň'] = 'n',
+            ['ŋ'] = 'n',
+            ['ý'] = 'y',
+            ['ÿ'] = 'y',
+            ['ŷ'] = 'y',
+            ['ś'] = 's',
+            ['ŝ'] = 's',
+            ['ş'] = 's',
+            ['š'] = 's',
+            ['ß'] = 's',
+            ['ź'] = 'z',
+            ['ż'] = 'z',
+            ['ž'] = 'z',
+            ['ł'] = 'l',
+            ['ď'] = 'd',
+            ['đ'] = 'd',
+            ['ĝ'] = 'g',
+            ['ğ'] = 'g',
+            ['ġ'] = 'g',
+            ['ģ'] = 'g',
+            ['ĥ'] = 'h',
+            ['ħ'] = 'h',
+            ['ĵ'] = 'j',
+            ['ķ'] = 'k',
+            ['ŕ'] = 'r',
+            ['ř'] = 'r',
+            ['ţ'] = 't',
+            ['ť'] = 't',
+            ['ŧ'] = 't',
+            ['æ'] = 'a',
+            ['œ'] = 'o'
+        };
+
+        private const char AccentRangeStart = '\u00DF';
+        private const char AccentRangeEnd   = '\u017E';
+        private static readonly char[] AccentLookup = BuildAccentLookup();
+
+        private static char[] BuildAccentLookup()
+        {
+            var lookup = new char[AccentRangeEnd - AccentRangeStart + 1];
+            foreach (var (key, value) in AccentMap)
+                lookup[key - AccentRangeStart] = value;
+            return lookup;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static char NormalizeChar(char c)
+        {
+            c = char.ToLowerInvariant(c);
+            if (c >= AccentRangeStart && c <= AccentRangeEnd)
+            {
+                var mapped = AccentLookup[c - AccentRangeStart];
+                if (mapped != 0) return mapped;
+            }
+            return c;
+        }
+
+        public static string Normalize(string value)
+        {
+            if (string.IsNullOrEmpty(value)) return value;
+
+            int firstChange = -1;
+            for (int i = 0; i < value.Length; i++)
+            {
+                if (NormalizeChar(value[i]) != value[i]) { firstChange = i; break; }
+            }
+            if (firstChange < 0) return value;
+
+            char[] arrayFromPool = null;
+            Span<char> buffer = value.Length <= 512
+                ? stackalloc char[value.Length]
+                : (arrayFromPool = ArrayPool<char>.Shared.Rent(value.Length));
+            try
+            {
+                value.AsSpan(0, firstChange).CopyTo(buffer);
+                for (int i = firstChange; i < value.Length; i++)
+                    buffer[i] = NormalizeChar(value[i]);
+
+                return new string(buffer.Slice(0, value.Length));
+            }
+            finally
+            {
+                if (arrayFromPool != null)
+                    ArrayPool<char>.Shared.Return(arrayFromPool);
+            }
+        }
+
         private static bool IsAcronym(string stringToCompare, int compareStringIndex)
         {
-            if (IsAcronymChar(stringToCompare, compareStringIndex) || IsAcronymNumber(stringToCompare, compareStringIndex))
+            if (IsAcronymChar(stringToCompare, compareStringIndex) ||
+                IsAcronymNumber(stringToCompare, compareStringIndex))
                 return true;
 
             return false;
@@ -274,19 +454,19 @@ private static int CalculateClosestSpaceIndex(List<int> spaceIndices, int firstM
         }
 
         private static bool AllPreviousCharsMatched(int startIndexToVerify, int currentQuerySubstringCharacterIndex,
-            string fullStringToCompareWithoutCase, string currentQuerySubstring)
+            string stringToCompare, string currentQuerySubstring, bool ignoreAccents, bool ignoreCase)
         {
-            var allMatch = true;
             for (int indexToCheck = 0; indexToCheck < currentQuerySubstringCharacterIndex; indexToCheck++)
             {
-                if (fullStringToCompareWithoutCase[startIndexToVerify + indexToCheck] !=
-                    currentQuerySubstring[indexToCheck])
-                {
-                    allMatch = false;
-                }
+                char c = stringToCompare[startIndexToVerify + indexToCheck];
+                if (ignoreAccents) c = NormalizeChar(c);
+                else if (ignoreCase) c = char.ToLower(c);
+
+                if (c != currentQuerySubstring[indexToCheck])
+                    return false;
             }
 
-            return allMatch;
+            return true;
         }
 
         private static List<int> GetUpdatedIndexList(int startIndexToVerify, int currentQuerySubstringCharacterIndex,
@@ -312,7 +492,8 @@ private static bool AllQuerySubstringsMatched(int currentQuerySubstringIndex, in
             return currentQuerySubstringIndex >= querySubstringsLength;
         }
 
-        private static int CalculateSearchScore(string query, string stringToCompare, int firstIndex, List<int> spaceIndices, int matchLen,
+        private static int CalculateSearchScore(string query, string stringToCompare, int firstIndex,
+            List<int> spaceIndices, int matchLen,
             bool allSubstringsContainedInCompareString)
         {
             // A match found near the beginning of a string is scored more than a match found near the end