From ba937e19d0a9bd7fc6f52aebbf19c15006ff5267 Mon Sep 17 00:00:00 2001 From: Yevhen Cherkes Date: Sat, 25 Jul 2026 12:09:15 +0200 Subject: [PATCH 1/4] Introduce IndelNew as an alternative to partial ratio strategy --- CHANGELOG.md | 5 + .../ApproximateSubstringRatioBenchmarks.cs | 161 ++++ .../ApproximateSubstringRatioTests.cs | 203 +++++ FuzzySharp.Test/IndelBestSubstringTests.cs | 270 ++++++ FuzzySharp/Fuzz.cs | 37 +- FuzzySharp/Indel.Static.BestSubstring.cs | 813 ++++++++++++++++++ FuzzySharp/Indel.Static.cs | 2 +- .../Simple/ApproximateSubstringRatioScorer.cs | 9 + .../CachedApproximateSubstringRatioScorer.cs | 38 + .../ApproximateSubstringRatioStrategy.cs | 13 + ...CachedApproximateSubstringRatioStrategy.cs | 74 ++ .../ApproximateSubstringRatioStrategyT.cs | 50 ++ README.md | 33 + ...GENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md | 684 +++++++++++++++ 14 files changed, 2390 insertions(+), 2 deletions(-) create mode 100644 FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs create mode 100644 FuzzySharp.Test/ApproximateSubstringRatioTests.cs create mode 100644 FuzzySharp.Test/IndelBestSubstringTests.cs create mode 100644 FuzzySharp/Indel.Static.BestSubstring.cs create mode 100644 FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/ApproximateSubstringRatioScorer.cs create mode 100644 FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs create mode 100644 FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs create mode 100644 FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs create mode 100644 FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs create mode 100644 docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 82fff59..46634cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Added `Fuzz.ApproximateSubstringRatio`, `ApproximateSubstringRatioScorer`, and `CachedApproximateSubstringRatioScorer` for best approximate substring scoring using Indel distance. +- Added public `Indel.BestSubstringMatch` APIs that return the best raw distance and matching endpoint for generic spans. + ## v5.0.3 *Extractor selection performance and allocation improvements* diff --git a/FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs b/FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs new file mode 100644 index 0000000..147c4f0 --- /dev/null +++ b/FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs @@ -0,0 +1,161 @@ +using BenchmarkDotNet.Attributes; +using Raffinert.FuzzySharp.SimilarityRatio.Scorer.StrategySensitive; + +namespace Raffinert.FuzzySharp.Benchmarks; + +[MemoryDiagnoser] +[RankColumn] +public class ApproximateSubstringRatioBenchmarks +{ + private string _pattern = string.Empty; + private string _text = string.Empty; + private CachedApproximateSubstringRatioScorer _cachedScorer = null!; + + [Params(64, 256, 1024)] + public int PatternLength { get; set; } + + [Params(1024, 4096)] + public int TextLength { get; set; } + + [Params( + BenchmarkDataSet.RepeatedApproximateAndExact, + BenchmarkDataSet.RandomLowSimilarity)] + public BenchmarkDataSet DataSet { get; set; } + + [GlobalSetup] + public void Setup() + { + var random = new Random(42); + _pattern = GenerateString(PatternLength, random, "abcdefghijklmnopqrstuvwxyz"); + int actualTextLength = Math.Max(TextLength, PatternLength * 2); + var characters = GenerateString(actualTextLength, random, "abcdefghijklmnopqrstuvwxyz").ToCharArray(); + + switch (DataSet) + { + case BenchmarkDataSet.ExactNearStart: + CopyPattern(characters, _pattern, 1); + break; + + case BenchmarkDataSet.ExactNearEnd: + CopyPattern(characters, _pattern, characters.Length - PatternLength - 1); + break; + + case BenchmarkDataSet.NoExactMatch: + _pattern = GenerateString(PatternLength, random, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + break; + + case BenchmarkDataSet.RepeatedApproximateAndExact: + CopyPattern(characters, _pattern, 1); + characters[1 + PatternLength / 2] = '#'; + CopyPattern(characters, _pattern, characters.Length - PatternLength - 1); + break; + + case BenchmarkDataSet.HighSimilarity: + CopyPattern(characters, _pattern, characters.Length / 2 - PatternLength / 2); + characters[characters.Length / 2] = '#'; + break; + } + + _text = new string(characters); + _cachedScorer = new CachedApproximateSubstringRatioScorer(_pattern); + } + + [GlobalCleanup] + public void Cleanup() + { + _cachedScorer.Dispose(); + } + + [Benchmark(Baseline = true)] + public IndelSubstringMatch BestSubstringMatch() + { + return Indel.BestSubstringMatch(_pattern.AsSpan(), _text.AsSpan()); + } + + [Benchmark] + public int ApproximateSubstringRatio() + { + return Fuzz.ApproximateSubstringRatio(_pattern, _text); + } + + [Benchmark] + public int CachedApproximateSubstringRatio() + { + return _cachedScorer.Score(_text); + } + + [Benchmark] + public IndelSubstringMatch ScalarDynamicProgrammingBaseline() + { + return ScalarBestSubstringMatch(_pattern.AsSpan(), _text.AsSpan()); + } + + [Benchmark] + public int PartialRatioReference() + { + return Fuzz.PartialRatio(_pattern, _text); + } + + private static IndelSubstringMatch ScalarBestSubstringMatch( + ReadOnlySpan pattern, + ReadOnlySpan text) + { + int[] previous = new int[pattern.Length + 1]; + int[] current = new int[pattern.Length + 1]; + + for (int patternIndex = 0; patternIndex <= pattern.Length; patternIndex++) + { + previous[patternIndex] = patternIndex; + } + + int bestDistance = pattern.Length; + int bestEndIndex = -1; + for (int textIndex = 0; textIndex < text.Length; textIndex++) + { + current[0] = 0; + for (int patternIndex = 1; patternIndex <= pattern.Length; patternIndex++) + { + int substitutionCost = pattern[patternIndex - 1] == text[textIndex] ? 0 : 2; + current[patternIndex] = Math.Min( + Math.Min(previous[patternIndex] + 1, current[patternIndex - 1] + 1), + previous[patternIndex - 1] + substitutionCost); + } + + if (current[pattern.Length] < bestDistance) + { + bestDistance = current[pattern.Length]; + bestEndIndex = textIndex; + } + + (previous, current) = (current, previous); + } + + return new IndelSubstringMatch(bestDistance, bestEndIndex); + } + + private static void CopyPattern(char[] destination, string pattern, int startIndex) + { + pattern.CopyTo(0, destination, startIndex, pattern.Length); + } + + private static string GenerateString(int length, Random random, string alphabet) + { + var characters = new char[length]; + for (int index = 0; index < characters.Length; index++) + { + characters[index] = alphabet[random.Next(alphabet.Length)]; + } + + return new string(characters); + } +} + +public enum BenchmarkDataSet +{ + ExactNearStart, + ExactNearEnd, + NoExactMatch, + RepeatedApproximateAndExact, + RandomLowSimilarity, + HighSimilarity +} diff --git a/FuzzySharp.Test/ApproximateSubstringRatioTests.cs b/FuzzySharp.Test/ApproximateSubstringRatioTests.cs new file mode 100644 index 0000000..9ed84e5 --- /dev/null +++ b/FuzzySharp.Test/ApproximateSubstringRatioTests.cs @@ -0,0 +1,203 @@ +using System; +using System.Threading.Tasks; +using Raffinert.FuzzySharp.SimilarityRatio.Scorer.StrategySensitive; +using Raffinert.FuzzySharp.SimilarityRatio.Strategy; +using Xunit; + +namespace Raffinert.FuzzySharp.Test; + +public class ApproximateSubstringRatioTests +{ + [Theory] + [InlineData("abc", "xxabcxx", 100)] + [InlineData("xxabcxx", "abc", 100)] + [InlineData("", "", 100)] + [InlineData("abc", "", 0)] + [InlineData("", "abc", 0)] + public void ApproximateSubstringRatio_UsesExpectedEmptyAndExactSemantics( + string input1, + string input2, + int expected) + { + Assert.Equal(expected, Fuzz.ApproximateSubstringRatio(input1, input2)); + } + + [Fact] + public void ApproximateSubstringRatio_AppliesPreprocessor() + { + Assert.Equal(100, Fuzz.ApproximateSubstringRatio( + "INVOICE-123", + "processed invoice 123 successfully", + input => input.Replace("-", " ").ToLowerInvariant())); + } + + [Fact] + public void ApproximateSubstringRatio_IsSymmetricAndUsesBetterEqualLengthDirection() + { + const string input1 = "abca"; + const string input2 = "caba"; + + int expected = Math.Max( + DirectionalScore(input1, input2), + DirectionalScore(input2, input1)); + + Assert.Equal(expected, Fuzz.ApproximateSubstringRatio(input1, input2)); + Assert.Equal( + Fuzz.ApproximateSubstringRatio(input1, input2), + Fuzz.ApproximateSubstringRatio(input2, input1)); + } + + [Fact] + public void ApproximateSubstringRatio_DoesNotRetainEarlierInferiorOccurrence() + { + Assert.Equal(100, Fuzz.ApproximateSubstringRatio("abc", "abxabc")); + } + + [Fact] + public void ApproximateSubstringRatio_IsASeparateMetricFromPartialRatio() + { + int approximate = Fuzz.ApproximateSubstringRatio("abc", "axc"); + int partial = Fuzz.PartialRatio("abc", "axc"); + + Assert.NotEqual(partial, approximate); + } + + [Fact] + public void ApproximateSubstringRatio_RemainsInRange() + { + var random = new Random(10091); + + for (int sample = 0; sample < 500; sample++) + { + string input1 = CreateRandomString(random, random.Next(0, 140)); + string input2 = CreateRandomString(random, random.Next(0, 200)); + int score = Fuzz.ApproximateSubstringRatio(input1, input2); + + Assert.InRange(score, 0, 100); + Assert.Equal(score, Fuzz.ApproximateSubstringRatio(input2, input1)); + } + } + + [Fact] + public void CachedApproximateSubstringRatioScorer_MatchesNonCachedScorer() + { + const string query = "invoice number 12345"; + string[] candidates = + { + "processed invoice number 12345 successfully", + "invoice 12345", + "unrelated content", + "", + "invoice number 12345" + }; + + using var scorer = new CachedApproximateSubstringRatioScorer(query); + foreach (string candidate in candidates) + { + Assert.Equal( + Fuzz.ApproximateSubstringRatio(query, candidate), + scorer.Score(candidate)); + } + } + + [Fact] + public void CachedApproximateSubstringRatioScorer_HandlesEmptyQuery() + { + using var scorer = new CachedApproximateSubstringRatioScorer(string.Empty); + + Assert.Equal(100, scorer.Score(string.Empty)); + Assert.Equal(0, scorer.Score("candidate")); + } + + [Fact] + public void CachedApproximateSubstringRatioScorer_IsSafeForConcurrentScores() + { + const string query = "the quick brown fox jumps over the lazy dog"; + string[] candidates = + { + "a quick brown fox jumps over a dog", + "the quick brown fox jumps over the lazy dog", + "unrelated text", + "quick brown fox" + }; + var expected = new int[candidates.Length]; + + for (int index = 0; index < candidates.Length; index++) + { + expected[index] = Fuzz.ApproximateSubstringRatio(query, candidates[index]); + } + + using var scorer = new CachedApproximateSubstringRatioScorer(query); + Parallel.For(0, 256, iteration => + { + int index = iteration % candidates.Length; + Assert.Equal(expected[index], scorer.Score(candidates[index])); + }); + } + + [Fact] + public void CachedApproximateSubstringRatioScorer_RespectsStrategyOwnership() + { + var externallyOwned = new TrackingStrategy(); + using (var scorer = new CachedApproximateSubstringRatioScorer(externallyOwned)) + { + Assert.Equal(42, scorer.Score("candidate")); + } + Assert.False(externallyOwned.Disposed); + + var scorerOwned = new TrackingStrategy(); + var owningScorer = new CachedApproximateSubstringRatioScorer(scorerOwned, true); + Assert.Equal(42, owningScorer.Score("candidate")); + owningScorer.Dispose(); + owningScorer.Dispose(); + Assert.True(scorerOwned.Disposed); + Assert.Equal(1, scorerOwned.DisposeCount); + } + + [Fact] + public void ApproximateSubstringRatioScorer_IsAvailableToProcessBuilder() + { + var pipeline = new ProcessBuilder() + .WithScorer(new ApproximateSubstringRatioScorer()) + .Build(); + + Assert.Equal(typeof(ProcessPipeline), pipeline.GetType()); + } + + private static int DirectionalScore(string pattern, string text) + { + IndelSubstringMatch match = Indel.BestSubstringMatch( + pattern.AsSpan(), text.AsSpan()); + return Math.Max(0, Math.Min(100, + (int)Math.Round(100.0 * (1.0 - match.Distance / (double)pattern.Length)))); + } + + private static string CreateRandomString(Random random, int length) + { + const string alphabet = "abcdef"; + var characters = new char[length]; + for (int index = 0; index < characters.Length; index++) + { + characters[index] = alphabet[random.Next(alphabet.Length)]; + } + + return new string(characters); + } + + private sealed class TrackingStrategy : ICachedStrategy + { + public bool Disposed { get; private set; } + public int DisposeCount { get; private set; } + + public int Calculate(string input2) + { + return 42; + } + + public void Dispose() + { + DisposeCount++; + Disposed = true; + } + } +} diff --git a/FuzzySharp.Test/IndelBestSubstringTests.cs b/FuzzySharp.Test/IndelBestSubstringTests.cs new file mode 100644 index 0000000..d94b362 --- /dev/null +++ b/FuzzySharp.Test/IndelBestSubstringTests.cs @@ -0,0 +1,270 @@ +using System; +using Xunit; + +namespace Raffinert.FuzzySharp.Test; + +public class IndelBestSubstringTests +{ + private const int RandomSeed = 834729; + + [Fact] + public void BestSubstringMatch_EmptyPattern_ReturnsEmptyMatch() + { + Assert.Equal(new IndelSubstringMatch(0, -1), + Indel.BestSubstringMatch(ReadOnlySpan.Empty, "text".AsSpan())); + } + + [Fact] + public void BestSubstringMatch_EmptyText_ReturnsPatternDeletionDistance() + { + Assert.Equal(new IndelSubstringMatch(3, -1), + Indel.BestSubstringMatch("abc".AsSpan(), ReadOnlySpan.Empty)); + } + + [Fact] + public void BestSubstringMatch_BothEmpty_ReturnsEmptyMatch() + { + Assert.Equal(new IndelSubstringMatch(0, -1), + Indel.BestSubstringMatch(ReadOnlySpan.Empty, ReadOnlySpan.Empty)); + } + + [Theory] + [InlineData("abc", "abcxxxx", 2)] + [InlineData("abc", "xxabcxx", 4)] + [InlineData("abc", "abxabc", 5)] + [InlineData("abc", "abcxxxabc", 2)] + public void BestSubstringMatch_ExactMatch_ReturnsEarliestExactEndpoint( + string pattern, + string text, + int expectedEndIndex) + { + IndelSubstringMatch result = Indel.BestSubstringMatch( + pattern.AsSpan(), + text.AsSpan()); + + Assert.Equal(0, result.Distance); + Assert.Equal(expectedEndIndex, result.EndIndex); + Assert.True(result.Found); + } + + [Fact] + public void BestSubstringMatch_SubstitutionCostsTwoEdits() + { + IndelSubstringMatch result = Indel.BestSubstringMatch( + "abc".AsSpan(), + "axc".AsSpan()); + + Assert.Equal(new IndelSubstringMatch(2, 0), result); + } + + [Fact] + public void BestSubstringMatch_CanDeletePatternElementsForShorterSubstring() + { + Assert.Equal(new IndelSubstringMatch(1, 1), + Indel.BestSubstringMatch("abc".AsSpan(), "ab".AsSpan())); + } + + [Fact] + public void BestSubstringMatch_NoCommonElement_PreservesNoMatchTieBehavior() + { + IndelSubstringMatch result = Indel.BestSubstringMatch( + "abc".AsSpan(), + "xxx".AsSpan()); + + Assert.Equal(3, result.Distance); + Assert.False(result.Found); + } + + [Fact] + public void BestSubstringMatch_SupportsGenericSequences() + { + int[] pattern = { 10, 20, 30, 40 }; + int[] text = { 0, 10, 20, 30, 40, 99 }; + + Assert.Equal(new IndelSubstringMatch(0, 4), + Indel.BestSubstringMatch(pattern.AsSpan(), text.AsSpan())); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(63)] + [InlineData(64)] + [InlineData(65)] + [InlineData(66)] + [InlineData(127)] + [InlineData(128)] + [InlineData(129)] + public void BestSubstringMatch_HandlesWordBoundariesAndLastBlockMasking( + int patternLength) + { + string pattern = CreateSequence(patternLength); + string text = "prefix-" + pattern + "-suffix"; + + Assert.Equal(new IndelSubstringMatch(0, "prefix-".Length + patternLength - 1), + Indel.BestSubstringMatch(pattern.AsSpan(), text.AsSpan())); + } + + [Fact] + public void BestSubstringMatch_HandlesCarriesBorrowsAndCrossBlockShifts() + { + string pattern = new string('a', 64) + new string('b', 65); + string text = new string('a', 63) + "x" + new string('b', 65); + + AssertMatchesOracle(pattern, text); + } + + [Fact] + public void BestSubstringMatch_TextElementsAbsentFromPattern_AreHandled() + { + string pattern = CreateSequence(129); + string text = new string('#', 80) + pattern.Substring(0, 128) + "!"; + + AssertMatchesOracle(pattern, text); + } + + [Fact] + public void BestSubstringMatch_PaperStyleExample_MatchesScalarOracle() + { + AssertMatchesOracle("ACGC", "GAAGCGACTGCAAACTCA"); + } + + [Fact] + public void BestSubstringMatch_ExhaustiveBinaryInputs_MatchScalarOracle() + { + for (int patternLength = 0; patternLength <= 7; patternLength++) + { + for (int textLength = 0; textLength <= 8; textLength++) + { + foreach (string pattern in BinaryStrings(patternLength)) + { + foreach (string text in BinaryStrings(textLength)) + { + AssertMatchesOracle(pattern, text); + } + } + } + } + } + + [Fact] + public void BestSubstringMatch_DeterministicRandomInputs_MatchScalarOracle() + { + var random = new Random(RandomSeed); + string[] alphabets = { "ab", "abcd", "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" }; + + for (int sample = 0; sample < 4_000; sample++) + { + string alphabet = alphabets[sample % alphabets.Length]; + int patternLength = random.Next(1, 141); + int textLength = random.Next(0, 201); + string pattern = CreateRandomString(random, patternLength, alphabet); + string text = CreateRandomString(random, textLength, alphabet); + + AssertMatchesOracle(pattern, text); + } + } + + private static void AssertMatchesOracle(string pattern, string text) + { + IndelSubstringMatch expected = ReferenceBestSubstringMatch( + pattern.AsSpan(), text.AsSpan()); + IndelSubstringMatch actual = Indel.BestSubstringMatch( + pattern.AsSpan(), text.AsSpan()); + + Assert.Equal(expected, actual); + } + + private static IndelSubstringMatch ReferenceBestSubstringMatch( + ReadOnlySpan pattern, + ReadOnlySpan text) + where T : IEquatable + { + if (pattern.IsEmpty) + { + return new IndelSubstringMatch(0, -1); + } + + if (text.IsEmpty) + { + return new IndelSubstringMatch(pattern.Length, -1); + } + + int[] previous = new int[pattern.Length + 1]; + int[] current = new int[pattern.Length + 1]; + + for (int patternIndex = 0; patternIndex <= pattern.Length; patternIndex++) + { + previous[patternIndex] = patternIndex; + } + + int bestDistance = pattern.Length; + int bestEndIndex = -1; + + for (int textIndex = 0; textIndex < text.Length; textIndex++) + { + current[0] = 0; + + for (int patternIndex = 1; patternIndex <= pattern.Length; patternIndex++) + { + int substitutionCost = pattern[patternIndex - 1].Equals(text[textIndex]) + ? 0 + : 2; + current[patternIndex] = Math.Min( + Math.Min(previous[patternIndex] + 1, current[patternIndex - 1] + 1), + previous[patternIndex - 1] + substitutionCost); + } + + if (current[pattern.Length] < bestDistance) + { + bestDistance = current[pattern.Length]; + bestEndIndex = textIndex; + } + + (previous, current) = (current, previous); + } + + return new IndelSubstringMatch(bestDistance, bestEndIndex); + } + + private static string[] BinaryStrings(int length) + { + int count = 1 << length; + var values = new string[count]; + + for (int value = 0; value < count; value++) + { + var characters = new char[length]; + for (int index = 0; index < length; index++) + { + characters[index] = (value & (1 << index)) == 0 ? 'a' : 'b'; + } + + values[value] = new string(characters); + } + + return values; + } + + private static string CreateSequence(int length) + { + var characters = new char[length]; + for (int index = 0; index < characters.Length; index++) + { + characters[index] = (char)('a' + index % 26); + } + + return new string(characters); + } + + private static string CreateRandomString(Random random, int length, string alphabet) + { + var characters = new char[length]; + for (int index = 0; index < characters.Length; index++) + { + characters[index] = alphabet[random.Next(alphabet.Length)]; + } + + return new string(characters); + } +} diff --git a/FuzzySharp/Fuzz.cs b/FuzzySharp/Fuzz.cs index 924cd49..58e8ae8 100644 --- a/FuzzySharp/Fuzz.cs +++ b/FuzzySharp/Fuzz.cs @@ -63,6 +63,41 @@ public static int PartialRatio(string input1, string input2, Func + /// Searches the shorter input approximately within the longer input using + /// insertions and deletions. A substitution costs two edits, so this metric + /// is not numerically equivalent to . + /// + /// The first input. + /// The second input. + /// A score from 0 to 100. + public static int ApproximateSubstringRatio(string input1, string input2) + { + return ScorerCache.Get() + .Score(input1, input2); + } + + /// + /// Searches the shorter processed input approximately within the longer + /// processed input using insertions and deletions. A substitution costs two + /// edits, so this metric is not numerically equivalent to + /// . + /// + /// The first input. + /// The second input. + /// A preprocessor applied to both inputs. + /// A score from 0 to 100. + public static int ApproximateSubstringRatio( + string input1, + string input2, + Func preprocessor) + { + return ScorerCache.Get() + .Score(input1, input2, preprocessor); + } + #endregion + #region TokenSortRatio /// /// Find all alphanumeric tokens in the string and sort @@ -359,4 +394,4 @@ public static int WeightedRatio(string input1, string input2, Func().Score(input1, input2, preprocessor); } #endregion -} \ No newline at end of file +} diff --git a/FuzzySharp/Indel.Static.BestSubstring.cs b/FuzzySharp/Indel.Static.BestSubstring.cs new file mode 100644 index 0000000..c0be98c --- /dev/null +++ b/FuzzySharp/Indel.Static.BestSubstring.cs @@ -0,0 +1,813 @@ +using Raffinert.FuzzySharp.Utils; +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace Raffinert.FuzzySharp; + +public readonly struct IndelSubstringMatch(int distance, int endIndex) : + IEquatable +{ + public int Distance { get; } = distance; + + public int EndIndex { get; } = endIndex; + + public bool Found => EndIndex >= 0; + + public bool Equals(IndelSubstringMatch other) + { + return Distance == other.Distance + && EndIndex == other.EndIndex; + } + + public override bool Equals(object obj) + { + return obj is IndelSubstringMatch match + && Equals(match); + } + + public override int GetHashCode() + { + unchecked + { + return (Distance * 397) ^ EndIndex; + } + } + + public static bool operator ==( + IndelSubstringMatch left, + IndelSubstringMatch right) + { + return left.Equals(right); + } + + public static bool operator !=( + IndelSubstringMatch left, + IndelSubstringMatch right) + { + return !left.Equals(right); + } + + public override string ToString() + { + return $"Distance = {Distance}, EndIndex = {EndIndex}"; + } +} + +public sealed partial class Indel +{ + /// + /// Finds the minimum insertion-deletion distance between the complete + /// pattern and any substring of text. + /// + /// The returned EndIndex identifies the text position at which the best + /// approximate substring match ends. + /// + public static IndelSubstringMatch BestSubstringMatch( + ReadOnlySpan pattern, + ReadOnlySpan text) + where T : notnull, IEquatable + { + if (pattern.IsEmpty) + { + return new IndelSubstringMatch( + distance: 0, + endIndex: -1); + } + + if (text.IsEmpty) + { + return new IndelSubstringMatch( + distance: pattern.Length, + endIndex: -1); + } + + using var patternMatchVector = PatternMatchVector.Create(pattern); + return BestSubstringMatchImpl( + patternMatchVector, + text); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static IndelSubstringMatch BestSubstringMatchImpl( + IPatternMatchVector patternVector, + ReadOnlySpan text) + where T : notnull, IEquatable + { + if (patternVector.Blocks == 1) + { + return BestSubstringMatchSingleBlock( + patternVector, + text); + } + + return BestSubstringMatchMultipleBlocks( + patternVector, + text); + } + + private static IndelSubstringMatch + BestSubstringMatchSingleBlock( + IPatternMatchVector patternVector, + ReadOnlySpan text) + where T : notnull, IEquatable + { + int patternLength = patternVector.Length; + + ulong vectorMask = patternLength == 64 + ? ulong.MaxValue + : (1UL << patternLength) - 1UL; + + ulong highestBit = + 1UL << (patternLength - 1); + + ulong positiveVertical = vectorMask; + ulong negativeVertical = 0; + + int currentDistance = patternLength; + int bestDistance = patternLength; + int bestEndIndex = -1; + + unchecked + { + for (int textIndex = 0; + textIndex < text.Length; + textIndex++) + { + ulong matchMask = + patternVector.GetOrZero( + text[textIndex])[0]; + + ulong zeroDiagonal = + ((((matchMask & positiveVertical) + + positiveVertical) + ^ positiveVertical) + | matchMask + | negativeVertical) + & vectorMask; + + ulong negativeHorizontal = + positiveVertical & zeroDiagonal; + + ulong positiveVerticalMinusNegativeHorizontal = + positiveVertical - negativeHorizontal; + + ulong horizontalStarts = + (negativeVertical + | ~(positiveVertical | zeroDiagonal)) + & vectorMask; + + ulong horizontalContinuation = + positiveVerticalMinusNegativeHorizontal + >> 1; + + ulong positiveHorizontal = + ((horizontalStarts + + horizontalContinuation) + ^ horizontalContinuation) + & vectorMask; + + // Original approximate-substring boundary: + // D[0, j] = 0. + ulong shiftedPositiveHorizontal = + (positiveHorizontal << 1) + & vectorMask; + + ulong shiftedNegativeHorizontal = + (negativeHorizontal << 1) + & vectorMask; + + ulong nextNegativeVertical = + shiftedPositiveHorizontal + & zeroDiagonal; + + ulong nextPositiveVertical = + (shiftedNegativeHorizontal + | ~(shiftedPositiveHorizontal + | zeroDiagonal) + | (shiftedPositiveHorizontal + & positiveVerticalMinusNegativeHorizontal)) + & vectorMask; + + if ((positiveHorizontal & highestBit) != 0) + { + currentDistance++; + } + + if ((negativeHorizontal & highestBit) != 0) + { + currentDistance--; + } + + positiveVertical = + nextPositiveVertical; + + negativeVertical = + nextNegativeVertical; + + if (currentDistance < bestDistance) + { + bestDistance = currentDistance; + bestEndIndex = textIndex; + + if (bestDistance == 0) + { + break; + } + } + } + } + + return new IndelSubstringMatch( + distance: bestDistance, + endIndex: bestEndIndex); + } + + //private static IndelSubstringMatch + // BestSubstringMatchMultipleBlocks( + // IPatternMatchVector patternVector, + // ReadOnlySpan text) + // where T : notnull, IEquatable + //{ + // int patternLength = patternVector.Length; + // int blockCount = patternVector.Blocks; + // int lastBlockBits = patternLength & 63; + // ulong lastBlockMask = lastBlockBits == 0 + // ? ulong.MaxValue + // : (1UL << lastBlockBits) - 1UL; + // ulong highestBit = 1UL << ((patternLength - 1) & 63); + + // // Rent a single contiguous buffer for all 6 vectors instead of 6 separate + // // pool allocations. Each vector occupies `blockCount` ulongs. + // int totalSize = blockCount * 6; + // ulong[] buffer = ArrayPool.Shared.Rent(totalSize); + + // try + // { + // Span positiveVertical = buffer.AsSpan(0, blockCount); + // Span negativeVertical = buffer.AsSpan(blockCount, blockCount); + // Span zeroDiagonal = buffer.AsSpan(blockCount * 2, blockCount); + // Span negativeHorizontal = buffer.AsSpan(blockCount * 3, blockCount); + // Span positiveVerticalMinusNegativeHorizontal = + // buffer.AsSpan(blockCount * 4, blockCount); + // Span positiveHorizontal = buffer.AsSpan(blockCount * 5, blockCount); + + // positiveVertical.Fill(ulong.MaxValue); + // positiveVertical[blockCount - 1] = lastBlockMask; + // negativeVertical.Clear(); + + // int currentDistance = patternLength; + // int bestDistance = patternLength; + // int bestEndIndex = -1; + + // unchecked + // { + // for (int textIndex = 0; textIndex < text.Length; textIndex++) + // { + // ReadOnlySpan matchMasks = + // patternVector.GetOrZero(text[textIndex]); + + // // zeroDiagonal = (((match & Pv) + Pv) ^ Pv) | match | Mv + // ulong carry = 0; + // for (int block = 0; block < blockCount; block++) + // { + // ulong positive = positiveVertical[block]; + // ulong addend = matchMasks[block] & positive; + // ulong sum = addend + positive; + // ulong carryFromAddend = sum < addend ? 1UL : 0UL; + // ulong sumWithCarry = sum + carry; + // carry = carryFromAddend | (sumWithCarry < sum ? 1UL : 0UL); + + // zeroDiagonal[block] = + // (sumWithCarry ^ positive) + // | matchMasks[block] + // | negativeVertical[block]; + // } + + // zeroDiagonal[blockCount - 1] &= lastBlockMask; + + // // negativeHorizontal = Pv & zeroDiagonal and + // // positiveVerticalMinusNegativeHorizontal = Pv - negativeHorizontal + // ulong borrow = 0; + // for (int block = 0; block < blockCount; block++) + // { + // ulong positive = positiveVertical[block]; + // ulong negative = positive & zeroDiagonal[block]; + // negativeHorizontal[block] = negative; + + // ulong difference = positive - negative; + // ulong borrowFromNegative = positive < negative ? 1UL : 0UL; + // ulong differenceWithBorrow = difference - borrow; + // borrow = borrowFromNegative + // | (difference < borrow ? 1UL : 0UL); + // positiveVerticalMinusNegativeHorizontal[block] = + // differenceWithBorrow; + // } + + // // The right shift crosses block boundaries, so process it + // // from the most significant block down. + // carry = 0; + // for (int block = blockCount - 1; block >= 0; block--) + // { + // ulong value = positiveVerticalMinusNegativeHorizontal[block]; + // positiveHorizontal[block] = + // (value >> 1) | (carry << 63); + // carry = value & 1UL; + // } + + // carry = 0; + // for (int block = 0; block < blockCount; block++) + // { + // ulong horizontalStarts = + // negativeVertical[block] + // | ~(positiveVertical[block] | zeroDiagonal[block]); + // ulong continuation = positiveHorizontal[block]; + // ulong sum = horizontalStarts + continuation; + // ulong carryFromStarts = sum < horizontalStarts ? 1UL : 0UL; + // ulong sumWithCarry = sum + carry; + // carry = carryFromStarts | (sumWithCarry < sum ? 1UL : 0UL); + // positiveHorizontal[block] = sumWithCarry ^ continuation; + // } + + // positiveHorizontal[blockCount - 1] &= lastBlockMask; + + // if ((positiveHorizontal[blockCount - 1] & highestBit) != 0) + // { + // currentDistance++; + // } + + // if ((negativeHorizontal[blockCount - 1] & highestBit) != 0) + // { + // currentDistance--; + // } + + // // Shift horizontal vectors one bit to the left while + // // carrying their most significant bits into the next block. + // ulong positiveHorizontalCarry = 0; + // ulong negativeHorizontalCarry = 0; + // for (int block = 0; block < blockCount; block++) + // { + // ulong shiftedPositiveHorizontal = + // (positiveHorizontal[block] << 1) + // | positiveHorizontalCarry; + // positiveHorizontalCarry = positiveHorizontal[block] >> 63; + + // ulong shiftedNegativeHorizontal = + // (negativeHorizontal[block] << 1) + // | negativeHorizontalCarry; + // negativeHorizontalCarry = negativeHorizontal[block] >> 63; + + // ulong positiveVerticalMinusNegative = + // positiveVerticalMinusNegativeHorizontal[block]; + + // negativeVertical[block] = + // shiftedPositiveHorizontal & zeroDiagonal[block]; + // positiveVertical[block] = + // shiftedNegativeHorizontal + // | ~(shiftedPositiveHorizontal | zeroDiagonal[block]) + // | (shiftedPositiveHorizontal + // & positiveVerticalMinusNegative); + // } + + // positiveVertical[blockCount - 1] &= lastBlockMask; + // negativeVertical[blockCount - 1] &= lastBlockMask; + + // if (currentDistance < bestDistance) + // { + // bestDistance = currentDistance; + // bestEndIndex = textIndex; + + // if (bestDistance == 0) + // { + // break; + // } + // } + // } + // } + + // return new IndelSubstringMatch( + // distance: bestDistance, + // endIndex: bestEndIndex); + // } + // finally + // { + // ArrayPool.Shared.Return(buffer); + // } + //} + + /// + /// Patterns up to this many blocks (2048 symbols) keep their state on the stack. + /// Two vectors of ulongs = 512 bytes. + /// + private const int StackBlockLimit = 32; + + private static IndelSubstringMatch BestSubstringMatchMultipleBlocks( + IPatternMatchVector patternVector, + ReadOnlySpan text) + where T : notnull, IEquatable + { + int patternLength = patternVector.Length; + int blockCount = patternVector.Blocks; + + int lastBlockBits = patternLength & 63; + ulong lastBlockMask = lastBlockBits == 0 ? ulong.MaxValue : (1UL << lastBlockBits) - 1UL; + + // Bit position of the pattern's final character inside the last block. + int lastRowShift = (patternLength - 1) & 63; + + // Only the two persistent state vectors need storage now: every other + // quantity lives in registers for exactly as long as it is needed. + ulong[]? rentedBuffer = null; + Span buffer = blockCount <= StackBlockLimit + ? stackalloc ulong[StackBlockLimit * 2] + : rentedBuffer = ArrayPool.Shared.Rent(blockCount * 2); + + try + { + // Sliced to exactly blockCount so the loop below can be bounded by + // Length, which lets the JIT drop the bounds checks on these two. + Span positiveVertical = buffer.Slice(0, blockCount); + Span negativeVertical = buffer.Slice(blockCount, blockCount); + + // Vertical deltas in the indel metric are always +/-1, never 0, so + // Pv | Mv covers the whole pattern and Pv stays masked to it. + positiveVertical.Fill(ulong.MaxValue); + positiveVertical[blockCount - 1] = lastBlockMask; + negativeVertical.Clear(); + + int currentDistance = patternLength; + int bestDistance = patternLength; + int bestEndIndex = -1; + int lastBlock = blockCount - 1; + + unchecked + { + for (int textIndex = 0; textIndex < text.Length; textIndex++) + { + ReadOnlySpan matchMasks = patternVector.GetOrZero(text[textIndex]); + Debug.Assert(matchMasks.Length >= blockCount); + matchMasks = matchMasks.Slice(0, blockCount); + + // Carries for the two additions, and the bits shifted out of + // the previous block by the two one-bit left shifts. + ulong diagonalCarry = 0; + ulong horizontalCarry = 0; + ulong positiveHorizontalShiftIn = 0; + ulong negativeHorizontalShiftIn = 0; + + // The right shift of (Pv - Mh) needs bit 0 of the *next* block, + // so each block is finished one iteration late: these hold the + // pending block's inputs. + ulong pendingPositiveVertical = positiveVertical[0]; + ulong pendingNegativeVertical = negativeVertical[0]; + ulong pendingMatch = matchMasks[0]; + + ulong pendingZeroDiagonal = ZeroDiagonal( + pendingMatch, + pendingPositiveVertical, + pendingNegativeVertical, + ref diagonalCarry); + + // Mh is a bitwise subset of Pv, so Pv - Mh never borrows: + // it is exactly Pv & ~D0. The original borrow-propagation + // loop was a no-op serial dependency. + ulong pendingPositiveVerticalMinusNegativeHorizontal = + pendingPositiveVertical & ~pendingZeroDiagonal; + + for (int block = 1; block < positiveVertical.Length; block++) + { + ulong positive = positiveVertical[block]; + ulong negative = negativeVertical[block]; + ulong match = matchMasks[block]; + + ulong zeroDiagonal = ZeroDiagonal(match, positive, negative, ref diagonalCarry); + ulong positiveMinusNegativeHorizontal = positive & ~zeroDiagonal; + + // Finish block-1 now that its right-shift input is known. + // (x & 1) << 63 == x << 63. + ulong horizontalContinuation = + (pendingPositiveVerticalMinusNegativeHorizontal >> 1) | + (positiveMinusNegativeHorizontal << 63); + + ulong horizontalStarts = pendingNegativeVertical | + ~(pendingPositiveVertical | pendingZeroDiagonal); + + ulong positiveHorizontal = AddWithCarry( + horizontalStarts, + horizontalContinuation, + ref horizontalCarry) ^ horizontalContinuation; + + ulong negativeHorizontal = pendingPositiveVertical & pendingZeroDiagonal; + + ulong shiftedPositiveHorizontal = + (positiveHorizontal << 1) | positiveHorizontalShiftIn; + positiveHorizontalShiftIn = positiveHorizontal >> 63; + + ulong shiftedNegativeHorizontal = + (negativeHorizontal << 1) | negativeHorizontalShiftIn; + negativeHorizontalShiftIn = negativeHorizontal >> 63; + + negativeVertical[block - 1] = + shiftedPositiveHorizontal & pendingZeroDiagonal; + positiveVertical[block - 1] = + shiftedNegativeHorizontal | + ~(shiftedPositiveHorizontal | pendingZeroDiagonal) | + (shiftedPositiveHorizontal & pendingPositiveVerticalMinusNegativeHorizontal); + + pendingPositiveVertical = positive; + pendingNegativeVertical = negative; + pendingZeroDiagonal = zeroDiagonal; + pendingPositiveVerticalMinusNegativeHorizontal = positiveMinusNegativeHorizontal; + } + + // Last block: nothing shifts in from above. + pendingZeroDiagonal &= lastBlockMask; + + { + // Masking D0 cannot change Pv & ~D0 here, because Pv is + // already confined to lastBlockMask. + ulong horizontalContinuation = + pendingPositiveVerticalMinusNegativeHorizontal >> 1; + + ulong horizontalStarts = pendingNegativeVertical | + ~(pendingPositiveVertical | pendingZeroDiagonal); + + ulong positiveHorizontal = (AddWithCarry( + horizontalStarts, + horizontalContinuation, + ref horizontalCarry) ^ horizontalContinuation) & lastBlockMask; + + ulong negativeHorizontal = pendingPositiveVertical & pendingZeroDiagonal; + + // Branchless: these two bits are near-random, so branching + // on them mispredicts on roughly half the text positions. + currentDistance += (int)((positiveHorizontal >> lastRowShift) & 1UL) + - (int)((negativeHorizontal >> lastRowShift) & 1UL); + + ulong shiftedPositiveHorizontal = + (positiveHorizontal << 1) | positiveHorizontalShiftIn; + ulong shiftedNegativeHorizontal = + (negativeHorizontal << 1) | negativeHorizontalShiftIn; + + negativeVertical[lastBlock] = + shiftedPositiveHorizontal & pendingZeroDiagonal & lastBlockMask; + positiveVertical[lastBlock] = + (shiftedNegativeHorizontal | + ~(shiftedPositiveHorizontal | pendingZeroDiagonal) | + (shiftedPositiveHorizontal & pendingPositiveVerticalMinusNegativeHorizontal)) + & lastBlockMask; + } + + if (currentDistance < bestDistance) + { + bestDistance = currentDistance; + bestEndIndex = textIndex; + + if (bestDistance == 0) + { + break; + } + } + } + } + + return new IndelSubstringMatch( + distance: bestDistance, + endIndex: bestEndIndex); + } + finally + { + if (rentedBuffer != null) + { + ArrayPool.Shared.Return(rentedBuffer); + } + } + } + + /// D0 = (((match & Pv) + Pv) ^ Pv) | match | Mv, with a cross-block carry. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong ZeroDiagonal(ulong match, ulong positive, ulong negative, ref ulong carry) + { + ulong sum = AddWithCarry(match & positive, positive, ref carry); + return (sum ^ positive) | match | negative; + } + + /// + /// Full adder over one block. The carry-out is the majority bit + /// ((a & b) | ((a | b) & ~sum)) >> 63, which avoids the two dependent + /// comparisons of a split two-step addition. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong AddWithCarry(ulong left, ulong right, ref ulong carry) + { + ulong sum = left + right + carry; + carry = ((left & right) | ((left | right) & ~sum)) >> 63; + return sum; + } + + //private static IndelSubstringMatch BestSubstringMatchMultipleBlocks( + // IPatternMatchVector patternVector, + // ReadOnlySpan text) + // where T : notnull, IEquatable + //{ + // int patternLength = patternVector.Length; + // int blockCount = patternVector.Blocks; + + // int lastBlockBits = patternLength & 63; + // ulong lastBlockMask = lastBlockBits == 0 ? ulong.MaxValue : (1UL << lastBlockBits) - 1UL; + + // // Bit position of the pattern's final character inside the last block. + // int lastRowShift = (patternLength - 1) & 63; + + // // Only the two persistent state vectors need storage now: every other + // // quantity lives in registers for exactly as long as it is needed. + // ulong[]? rentedBuffer = null; + // Span buffer = blockCount <= StackBlockLimit + // ? stackalloc ulong[StackBlockLimit * 2] + // : (rentedBuffer = ArrayPool.Shared.Rent(blockCount * 2)); + + // try + // { + // Span positiveVertical = buffer.Slice(0, blockCount); + // Span negativeVertical = buffer.Slice(blockCount, blockCount); + + // positiveVertical.Fill(ulong.MaxValue); + // positiveVertical[blockCount - 1] = lastBlockMask; + // negativeVertical.Clear(); + + // // Vertical deltas in the indel metric are always +/-1, never 0, so + // // Pv | Mv covers the whole pattern and Pv stays masked to it. + // ref ulong positiveVerticalRef = ref MemoryMarshal.GetReference(positiveVertical); + // ref ulong negativeVerticalRef = ref MemoryMarshal.GetReference(negativeVertical); + + // int currentDistance = patternLength; + // int bestDistance = patternLength; + // int bestEndIndex = -1; + // int lastBlock = blockCount - 1; + + // unchecked + // { + // for (int textIndex = 0; textIndex < text.Length; textIndex++) + // { + // ReadOnlySpan matchMasks = patternVector.GetOrZero(text[textIndex]); + // Debug.Assert(matchMasks.Length >= blockCount); + // ref ulong matchRef = ref MemoryMarshal.GetReference(matchMasks); + + // // Carries for the two additions, and the bits shifted out of + // // the previous block by the two one-bit left shifts. + // ulong diagonalCarry = 0; + // ulong horizontalCarry = 0; + // ulong positiveHorizontalShiftIn = 0; + // ulong negativeHorizontalShiftIn = 0; + + // // The right shift of (Pv - Mh) needs bit 0 of the *next* block, + // // so each block is finished one iteration late: these hold the + // // pending block's inputs. + // ulong pendingPositiveVertical = positiveVerticalRef; + // ulong pendingNegativeVertical = negativeVerticalRef; + // ulong pendingMatch = matchRef; + + // ulong pendingZeroDiagonal = ZeroDiagonal( + // pendingMatch, + // pendingPositiveVertical, + // pendingNegativeVertical, + // ref diagonalCarry); + + // // Mh is a bitwise subset of Pv, so Pv - Mh never borrows: + // // it is exactly Pv & ~D0. The original borrow-propagation + // // loop was a no-op serial dependency. + // ulong pendingPositiveVerticalMinusNegativeHorizontal = + // pendingPositiveVertical & ~pendingZeroDiagonal; + + // for (int block = 1; block < blockCount; block++) + // { + // ulong positive = Unsafe.Add(ref positiveVerticalRef, block); + // ulong negative = Unsafe.Add(ref negativeVerticalRef, block); + // ulong match = Unsafe.Add(ref matchRef, block); + + // ulong zeroDiagonal = ZeroDiagonal(match, positive, negative, ref diagonalCarry); + // ulong positiveMinusNegativeHorizontal = positive & ~zeroDiagonal; + + // // Finish block-1 now that its right-shift input is known. + // // (x & 1) << 63 == x << 63. + // ulong horizontalContinuation = + // (pendingPositiveVerticalMinusNegativeHorizontal >> 1) | + // (positiveMinusNegativeHorizontal << 63); + + // ulong horizontalStarts = pendingNegativeVertical | + // ~(pendingPositiveVertical | pendingZeroDiagonal); + + // ulong positiveHorizontal = AddWithCarry( + // horizontalStarts, + // horizontalContinuation, + // ref horizontalCarry) ^ horizontalContinuation; + + // ulong negativeHorizontal = pendingPositiveVertical & pendingZeroDiagonal; + + // ulong shiftedPositiveHorizontal = + // (positiveHorizontal << 1) | positiveHorizontalShiftIn; + // positiveHorizontalShiftIn = positiveHorizontal >> 63; + + // ulong shiftedNegativeHorizontal = + // (negativeHorizontal << 1) | negativeHorizontalShiftIn; + // negativeHorizontalShiftIn = negativeHorizontal >> 63; + + // Unsafe.Add(ref negativeVerticalRef, block - 1) = + // shiftedPositiveHorizontal & pendingZeroDiagonal; + // Unsafe.Add(ref positiveVerticalRef, block - 1) = + // shiftedNegativeHorizontal | + // ~(shiftedPositiveHorizontal | pendingZeroDiagonal) | + // (shiftedPositiveHorizontal & pendingPositiveVerticalMinusNegativeHorizontal); + + // pendingPositiveVertical = positive; + // pendingNegativeVertical = negative; + // pendingZeroDiagonal = zeroDiagonal; + // pendingPositiveVerticalMinusNegativeHorizontal = positiveMinusNegativeHorizontal; + // } + + // // Last block: nothing shifts in from above. + // pendingZeroDiagonal &= lastBlockMask; + + // { + // // Masking D0 cannot change Pv & ~D0 here, because Pv is + // // already confined to lastBlockMask. + // ulong horizontalContinuation = + // pendingPositiveVerticalMinusNegativeHorizontal >> 1; + + // ulong horizontalStarts = pendingNegativeVertical | + // ~(pendingPositiveVertical | pendingZeroDiagonal); + + // ulong positiveHorizontal = (AddWithCarry( + // horizontalStarts, + // horizontalContinuation, + // ref horizontalCarry) ^ horizontalContinuation) & lastBlockMask; + + // ulong negativeHorizontal = pendingPositiveVertical & pendingZeroDiagonal; + + // // Branchless: these two bits are near-random, so branching + // // on them mispredicts on roughly half the text positions. + // currentDistance += (int)((positiveHorizontal >> lastRowShift) & 1UL) + // - (int)((negativeHorizontal >> lastRowShift) & 1UL); + + // ulong shiftedPositiveHorizontal = + // (positiveHorizontal << 1) | positiveHorizontalShiftIn; + // ulong shiftedNegativeHorizontal = + // (negativeHorizontal << 1) | negativeHorizontalShiftIn; + + // Unsafe.Add(ref negativeVerticalRef, lastBlock) = + // shiftedPositiveHorizontal & pendingZeroDiagonal & lastBlockMask; + // Unsafe.Add(ref positiveVerticalRef, lastBlock) = + // (shiftedNegativeHorizontal | + // ~(shiftedPositiveHorizontal | pendingZeroDiagonal) | + // (shiftedPositiveHorizontal & pendingPositiveVerticalMinusNegativeHorizontal)) + // & lastBlockMask; + // } + + // if (currentDistance < bestDistance) + // { + // bestDistance = currentDistance; + // bestEndIndex = textIndex; + + // if (bestDistance == 0) + // { + // break; + // } + // } + // } + // } + + // return new IndelSubstringMatch( + // distance: bestDistance, + // endIndex: bestEndIndex); + // } + // finally + // { + // if (rentedBuffer is not null) + // { + // ArrayPool.Shared.Return(rentedBuffer); + // } + // } + //} + + ///// D0 = (((match & Pv) + Pv) ^ Pv) | match | Mv, with a cross-block carry. + //[MethodImpl(MethodImplOptions.AggressiveInlining)] + //private static ulong ZeroDiagonal(ulong match, ulong positive, ulong negative, ref ulong carry) + //{ + // ulong sum = AddWithCarry(match & positive, positive, ref carry); + // return (sum ^ positive) | match | negative; + //} + + ///// + ///// Full adder over one block. The carry-out is the majority bit + ///// ((a & b) | ((a | b) & ~sum)) >> 63, which avoids the two dependent + ///// comparisons of a split two-step addition. + ///// + //[MethodImpl(MethodImplOptions.AggressiveInlining)] + //private static ulong AddWithCarry(ulong left, ulong right, ref ulong carry) + //{ + // ulong sum = left + right + carry; + // carry = ((left & right) | ((left | right) & ~sum)) >> 63; + // return sum; + //} +} diff --git a/FuzzySharp/Indel.Static.cs b/FuzzySharp/Indel.Static.cs index 155fcbb..b57f5b6 100644 --- a/FuzzySharp/Indel.Static.cs +++ b/FuzzySharp/Indel.Static.cs @@ -231,4 +231,4 @@ private static double NormalizedSimilarityImpl(ReadOnlySpan s1, : 0; return result; } -} +} \ No newline at end of file diff --git a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/ApproximateSubstringRatioScorer.cs b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/ApproximateSubstringRatioScorer.cs new file mode 100644 index 0000000..45492d0 --- /dev/null +++ b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/ApproximateSubstringRatioScorer.cs @@ -0,0 +1,9 @@ +using Raffinert.FuzzySharp.SimilarityRatio.Strategy; + +namespace Raffinert.FuzzySharp.SimilarityRatio.Scorer.StrategySensitive; + +public sealed class ApproximateSubstringRatioScorer : SimpleRatioScorerBase +{ + protected override FuzzySharp.Scorer Scorer => + ApproximateSubstringRatioStrategy.Calculate; +} diff --git a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs new file mode 100644 index 0000000..85efc99 --- /dev/null +++ b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs @@ -0,0 +1,38 @@ +using System; +using Raffinert.FuzzySharp.SimilarityRatio.Strategy; + +namespace Raffinert.FuzzySharp.SimilarityRatio.Scorer.StrategySensitive; + +public sealed class CachedApproximateSubstringRatioScorer : CachedSimpleRatioScorerBase +{ + private readonly ICachedStrategy _strategy; + private readonly bool _isStrategyOwner; + private bool _disposed; + + public CachedApproximateSubstringRatioScorer( + string input1, + Func preprocessor = null) + { + _strategy = new CachedApproximateSubstringRatioStrategy(input1, preprocessor); + _isStrategyOwner = true; + } + + public CachedApproximateSubstringRatioScorer( + ICachedStrategy strategy, + bool isStrategyOwner = false) + { + _strategy = strategy; + _isStrategyOwner = isStrategyOwner; + } + + protected override CachedScorer Scorer => input2 => _strategy.Calculate(input2); + + public override void Dispose() + { + if (_isStrategyOwner && !_disposed) + { + _strategy.Dispose(); + _disposed = true; + } + } +} diff --git a/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs new file mode 100644 index 0000000..e145d85 --- /dev/null +++ b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs @@ -0,0 +1,13 @@ +using System; + +namespace Raffinert.FuzzySharp.SimilarityRatio.Strategy; + +internal static class ApproximateSubstringRatioStrategy +{ + public static int Calculate(string input1, string input2) + { + return Generic.ApproximateSubstringRatioStrategy.Calculate( + input1.AsSpan(), + input2.AsSpan()); + } +} diff --git a/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs b/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs new file mode 100644 index 0000000..2aab80e --- /dev/null +++ b/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs @@ -0,0 +1,74 @@ +using System; +using Raffinert.FuzzySharp.PreProcess; +using Raffinert.FuzzySharp.Utils; + +namespace Raffinert.FuzzySharp.SimilarityRatio.Strategy; + +internal sealed class CachedApproximateSubstringRatioStrategy : ICachedStrategy +{ + private readonly Func _preprocessor; + private readonly string _processedInput1; + private readonly IPatternMatchVector _input1PatternVector; + private bool _disposed; + + public CachedApproximateSubstringRatioStrategy( + string input1, + Func preprocessor = null) + { + _preprocessor = preprocessor ?? StringPreprocessor.None; + _processedInput1 = _preprocessor(input1); + _input1PatternVector = PatternMatchVector.Create(_processedInput1.AsSpan()); + } + + public int Calculate(string input2) + { + string processedInput2 = _preprocessor(input2); + + if (_processedInput1.Length == 0 || processedInput2.Length == 0) + { + return _processedInput1.Length == 0 && processedInput2.Length == 0 + ? 100 + : 0; + } + + if (_processedInput1.Length < processedInput2.Length) + { + return Score(_input1PatternVector, processedInput2.AsSpan()); + } + + if (processedInput2.Length < _processedInput1.Length) + { + using var input2PatternVector = + PatternMatchVector.Create(processedInput2.AsSpan()); + return Score(input2PatternVector, _processedInput1.AsSpan()); + } + + int forward = Score(_input1PatternVector, processedInput2.AsSpan()); + using var reversePatternVector = + PatternMatchVector.Create(processedInput2.AsSpan()); + int reverse = Score(reversePatternVector, _processedInput1.AsSpan()); + return Math.Max(forward, reverse); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _input1PatternVector.Dispose(); + _disposed = true; + } + + private static int Score( + IPatternMatchVector patternVector, + ReadOnlySpan text) + { + IndelSubstringMatch match = + Indel.BestSubstringMatchImpl(patternVector, text); + double similarity = 1.0 - match.Distance / (double)patternVector.Length; + int score = (int)Math.Round(100.0 * similarity); + return Math.Max(0, Math.Min(100, score)); + } +} diff --git a/FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs b/FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs new file mode 100644 index 0000000..56a3041 --- /dev/null +++ b/FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs @@ -0,0 +1,50 @@ +using System; + +namespace Raffinert.FuzzySharp.SimilarityRatio.Strategy.Generic; + +internal static class ApproximateSubstringRatioStrategy + where T : notnull, IEquatable +{ + public static int Calculate(ReadOnlySpan input1, ReadOnlySpan input2) + { + if (input1.IsEmpty || input2.IsEmpty) + { + return input1.IsEmpty && input2.IsEmpty ? 100 : 0; + } + + if (input1.Length < input2.Length) + { + return Calculate(input1, input2, Indel.BestSubstringMatch(input1, input2)); + } + + if (input2.Length < input1.Length) + { + return Calculate(input2, input1, Indel.BestSubstringMatch(input2, input1)); + } + + int forward = Calculate(input1, input2, Indel.BestSubstringMatch(input1, input2)); + + if(forward == 100) + { + return 100; + } + + int reverse = Calculate(input2, input1, Indel.BestSubstringMatch(input2, input1)); + return Math.Max(forward, reverse); + } + + internal static int Calculate( + ReadOnlySpan pattern, + ReadOnlySpan text, + IndelSubstringMatch match) + { + if (pattern.IsEmpty || text.IsEmpty) + { + return pattern.IsEmpty && text.IsEmpty ? 100 : 0; + } + + double similarity = 1.0 - match.Distance / (double)pattern.Length; + int score = (int)Math.Round(100.0 * similarity); + return Math.Max(0, Math.Min(100, score)); + } +} diff --git a/README.md b/README.md index e75e39e..9e807cf 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ By default, `Fuzz` methods compare strings as-is. `Process` extraction methods u |--------|----------| | `Fuzz.Ratio` | You need direct similarity between two strings. | | `Fuzz.PartialRatio` | One string may be a substring or close substring of the other. | +| `Fuzz.ApproximateSubstringRatio` | You need the best Indel-distance match of the shorter input within the longer input. | | `Fuzz.TokenSortRatio` | Word order should not matter. | | `Fuzz.TokenSetRatio` | Duplicate words or extra common words should have less impact. | | `Fuzz.TokenInitialismRatio` | You need to compare an initialism with its expanded phrase. | @@ -128,6 +129,26 @@ Fuzz.PartialRatio("similar", "somewhresimlrbetweenthisstring"); // 71 ``` +### Approximate Substring Ratio + +```csharp +int score = Fuzz.ApproximateSubstringRatio( + "invoice number 12345", + "processed invoice number 12345 successfully"); +// 100 +``` + +`ApproximateSubstringRatio` searches for the best approximate occurrence of the +shorter input, considering every possible start position implicitly. It uses the +Indel edit model: insertions and deletions cost one, while a substitution costs +two. The score is normalized relative to the shorter input's length, and later +exact occurrences are not hidden by earlier inferior ones. + +For endpoint information, call `Indel.BestSubstringMatch` directly; its +`EndIndex` identifies where the best match ends. This API does not return a start +index or edit script, and its numeric scores are not compatible with +`PartialRatio`. + ### Token Sort Ratio

Run .NET fiddle

@@ -373,6 +394,7 @@ Stateless scorers for use with `Process` static methods and the `WithScorer()` b ```csharp var ratio = ScorerCache.Get(); var partialRatio = ScorerCache.Get(); +var approximateSubstring = ScorerCache.Get(); var tokenSet = ScorerCache.Get(); var partialTokenSet = ScorerCache.Get(); var tokenSort = ScorerCache.Get(); @@ -395,6 +417,7 @@ int score = scorer.Score("candidate string"); Available cached scorers: - `CachedWeightedRatioScorer` -- weighted combination (default for `.Cached()`) - `CachedDefaultRatioScorer` -- simple Levenshtein ratio +- `CachedApproximateSubstringRatioScorer` -- approximate substring Indel ratio - `CachedTokenSortScorer` -- token sort ratio - `CachedTokenSetScorer` -- token set ratio - `CachedPartialTokenSetScorer` -- partial token set ratio @@ -456,6 +479,16 @@ double similarity = indel.NormalizedSimilarityWith("chicago white sox"); // 0.6206896551724138 ``` +The static approximate-substring API returns the best raw Indel distance and its +endpoint: + +```csharp +IndelSubstringMatch match = Indel.BestSubstringMatch( + "invoice number 12345".AsSpan(), + "processed invoice number 12345 successfully".AsSpan()); +// match.Distance == 0; match.EndIndex identifies the final '5' +``` + A generic variant `IndelT` is available for comparing sequences of any `IEquatable`: ```csharp diff --git a/docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md b/docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md new file mode 100644 index 0000000..d23d56f --- /dev/null +++ b/docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md @@ -0,0 +1,684 @@ +# AI Agent Instructions: Complete Approximate Substring Matching Feature + +## Objective + +Complete the approximate substring matching feature in `Raffinert/FuzzySharp` using the existing bit-parallel **IndelNew** implementation as the algorithmic foundation. + +The feature must expose a production-ready fuzzy score that finds how closely the shorter input matches **any substring** of the longer input, while preserving the existing behavior of `Fuzz.PartialRatio`. + +Do not replace or silently change `PartialRatio`, `WeightedRatio`, token scorers, or any existing public behavior. + +--- + +## Existing baseline + +The working tree already contains an implementation equivalent to: + +```csharp +public readonly struct IndelSubstringMatch : IEquatable +{ + public int Distance { get; } + public int EndIndex { get; } + public bool Found => EndIndex >= 0; +} + +public sealed partial class Indel +{ + public static IndelSubstringMatch BestSubstringMatch( + ReadOnlySpan pattern, + ReadOnlySpan text) + where T : notnull, IEquatable; + + internal static IndelSubstringMatch BestSubstringMatchImpl( + IPatternMatchVector patternVector, + ReadOnlySpan text) + where T : notnull, IEquatable; +} +``` + +The implementation already contains: + +- a single-`ulong` path for patterns up to 64 elements; +- a multi-block path for longer patterns; +- `PatternMatchVector` integration; +- pooled buffers for the multi-block implementation; +- the original approximate-substring DP boundary `D[0, j] = 0`; +- early exit when an exact match is found; +- an endpoint result through `EndIndex`. + +Treat this implementation as the starting point. Do not replace it with brute-force window enumeration or ordinary `O(mn)` dynamic programming. A scalar DP implementation may be added only as a test oracle. + +--- + +## Algorithm semantics + +For a non-empty pattern `P` and text `T`, the low-level operation finds: + +```text +min distance(P, S) +``` + +where `S` is a substring candidate represented by the original IndelNew approximate-substring recurrence. + +The edit model is Indel distance: + +- insertion cost: `1`; +- deletion cost: `1`; +- substitution cost: `2` because it is one deletion plus one insertion. + +The algorithm tracks the best distance for substrings ending at each text position and returns: + +- `Distance`: the smallest raw Indel distance encountered; +- `EndIndex`: the zero-based text index where the first strictly better best match ended; +- `Found`: `true` when a text endpoint improved on deleting the complete pattern. + +Preserve the current tie behavior: + +- update the best result only when `currentDistance < bestDistance`; +- therefore equal-distance later matches do not replace the earlier result; +- return immediately on distance `0`, since no better result is possible. + +### Empty inputs + +Preserve these low-level semantics: + +```text +pattern empty => Distance = 0, EndIndex = -1 +text empty, pattern non-empty => Distance = pattern.Length, EndIndex = -1 +``` + +The fuzzy ratio API has separate empty-input semantics defined below. + +--- + +## Public feature name + +Use **Approximate Substring Ratio** consistently. + +Required public API: + +```csharp +Fuzz.ApproximateSubstringRatio(string input1, string input2) +Fuzz.ApproximateSubstringRatio( + string input1, + string input2, + Func preprocessor) +``` + +Required scorer type: + +```csharp +ApproximateSubstringRatioScorer +``` + +Required cached scorer type: + +```csharp +CachedApproximateSubstringRatioScorer +``` + +Do not call the new API `PartialRatio`. It has deliberately different semantics from RapidFuzz/FuzzyWuzzy partial ratio. + +--- + +## Score definition + +The scorer must choose the shorter processed input as the pattern and the longer processed input as the text. + +For a non-empty pattern of length `m` and best raw distance `d`, calculate: + +```text +similarity = 1 - d / m +score = round(100 * similarity) +``` + +Equivalent C#: + +```csharp +double similarity = 1.0 - match.Distance / (double)pattern.Length; +int score = (int)Math.Round(100.0 * similarity); +``` + +Clamp defensively to `[0, 100]` even though a correct IndelNew result should already produce a distance in `[0, pattern.Length]` for this use case. + +### Fuzzy ratio empty-input behavior + +Use the same user-facing convention as other fuzzy scorers: + +```text +both inputs empty => 100 +one input empty => 0 +``` + +### Equal-length inputs + +Approximate substring matching is directional because the text side has the free-start boundary. + +When the processed inputs have the same non-zero length: + +1. calculate `input1` as pattern against `input2` as text; +2. calculate `input2` as pattern against `input1` as text; +3. return the larger score. + +This makes the public scorer symmetric and follows the approach already used by the existing partial-ratio strategy for equal-length inputs. + +### Important non-goal + +Do not attempt to reproduce the exact numeric result of RapidFuzz `partial_ratio`. + +This feature minimizes raw approximate-substring Indel distance and normalizes it by pattern length. It is a separate metric. + +--- + +## Required architecture + +Follow the current repository layers and naming conventions. + +### 1. Low-level Indel implementation + +Keep or move the current implementation into a focused file such as: + +```text +FuzzySharp/Indel.ApproximateSubstring.cs +``` + +Keep these methods: + +```csharp +public static IndelSubstringMatch BestSubstringMatch( + ReadOnlySpan pattern, + ReadOnlySpan text) + where T : notnull, IEquatable; + +internal static IndelSubstringMatch BestSubstringMatchImpl( + IPatternMatchVector patternVector, + ReadOnlySpan text) + where T : notnull, IEquatable; +``` + +The internal overload is required for the cached scorer so the pattern mask is constructed only once. + +Keep `IndelSubstringMatch` compatible with all library targets. Do not use `record struct` or APIs unavailable on `netstandard2.0`/legacy .NET Framework targets. + +A regular `readonly struct` with explicit equality is acceptable. + +### 2. Generic strategy + +Add: + +```text +FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs +``` + +Suggested shape: + +```csharp +internal static class ApproximateSubstringRatioStrategy + where T : notnull, IEquatable +{ + public static int Calculate( + ReadOnlySpan input1, + ReadOnlySpan input2); +} +``` + +Responsibilities: + +- handle empty spans; +- choose the shorter span as pattern; +- call `Indel.BestSubstringMatch`; +- normalize by pattern length; +- evaluate both directions for equal lengths; +- return an integer in `[0, 100]`. + +Do not put string preprocessing in the generic strategy. + +### 3. String strategy wrapper + +Add: + +```text +FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs +``` + +Suggested shape: + +```csharp +internal static class ApproximateSubstringRatioStrategy +{ + public static int Calculate(string input1, string input2); +} +``` + +Delegate to `ApproximateSubstringRatioStrategy.Calculate` using spans. + +### 4. Non-cached scorer + +Add: + +```text +FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/ApproximateSubstringRatioScorer.cs +``` + +Follow `PartialRatioScorer` and other simple scorer conventions. + +Suggested shape: + +```csharp +public sealed class ApproximateSubstringRatioScorer : SimpleRatioScorerBase +{ + protected override FuzzySharp.Scorer Scorer => + ApproximateSubstringRatioStrategy.Calculate; +} +``` + +### 5. Cached strategy + +Add a cached strategy that owns one precomputed `PatternMatchVector` for the processed query. + +Suggested file: + +```text +FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs +``` + +It should implement the repository's existing `ICachedStrategy` abstraction. + +Required behavior: + +- apply the configured preprocessor to the query once in the constructor; +- retain the processed query string and its length; +- build and own the query `PatternMatchVector` once; +- apply the same preprocessor to each candidate; +- use `Indel.BestSubstringMatchImpl` when the query is no longer than the candidate; +- when a candidate is shorter than the cached query, build a temporary pattern vector for the candidate and search it in the cached query text; +- for equal lengths, evaluate both directions and use the larger score; +- dispose the owned query pattern vector exactly once; +- reject scoring after disposal only if existing cached strategies follow that convention; otherwise preserve repository behavior. + +Do not mutate the cached `PatternMatchVector` during scoring. The scorer should be safe for concurrent reads until disposed. + +Be careful: caching the first argument does not guarantee that it is always the shorter input. Correctness takes priority over reusing the cached mask in that case. + +### 6. Cached scorer + +Add: + +```text +FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs +``` + +Follow `CachedDefaultRatioScorer` ownership conventions: + +- constructor taking `string input1` and optional preprocessor owns its strategy; +- optional constructor taking `ICachedStrategy` does not own it unless explicitly requested; +- dispose only when the scorer owns the strategy. + +### 7. Public `Fuzz` API + +Add a new region next to `PartialRatio`: + +```csharp +public static int ApproximateSubstringRatio( + string input1, + string input2) +{ + return ScorerCache + .Get() + .Score(input1, input2); +} + +public static int ApproximateSubstringRatio( + string input1, + string input2, + Func preprocessor) +{ + return ScorerCache + .Get() + .Score(input1, input2, preprocessor); +} +``` + +Add XML documentation that clearly says: + +- the shorter input is searched approximately inside the longer input; +- insertions and deletions are used; +- substitution has cost `2`; +- the metric is not identical to `PartialRatio`. + +### 8. Process/extractor integration + +Ensure the scorer can be used explicitly through existing process APIs: + +```csharp +ProcessBuilder + .WithScorer(new ApproximateSubstringRatioScorer()) +``` + +and cached APIs: + +```csharp +using var scorer = new CachedApproximateSubstringRatioScorer(query); + +var pipeline = new ProcessBuilder() + .Cached(scorer) + .Build(); +``` + +Do not change the default scorer used by `ProcessBuilder`. + +Do not add the new metric to `WeightedRatio` in this feature unless an existing extension point makes that completely opt-in and backward-compatible. + +--- + +## Correctness requirements + +The bit-parallel implementation is performance-sensitive and must be verified against a simple oracle. + +### Scalar oracle + +Add a test-only `O(mn)` dynamic-programming implementation with these boundaries: + +```text +D[i, 0] = i +D[0, j] = 0 +``` + +For each text column `j`, calculate: + +```text +D[i, j] = min( + D[i - 1, j] + 1, // delete pattern element + D[i, j - 1] + 1, // insert text element + D[i - 1, j - 1] + cost // 0 when equal, otherwise 2 +) +``` + +Track the same best-distance and earliest-strict-improvement endpoint behavior as the production implementation. + +Use this implementation only in tests. + +### Required low-level tests + +Add focused tests for: + +1. Empty pattern. +2. Empty text. +3. Both empty. +4. Exact match at the beginning. +5. Exact match in the middle. +6. Exact match after an earlier approximate occurrence. +7. Repeated exact occurrences return the earliest exact endpoint. +8. A substitution costs `2`. +9. A shorter matching substring can be selected by deleting pattern elements. +10. No useful common element returns distance equal to pattern length and `Found == false` under the current tie semantics. +11. Generic non-character sequences, for example `int[]`. +12. Pattern lengths `1`, `2`, `63`, and `64`. +13. Multi-block pattern lengths `65`, `66`, `127`, `128`, and `129`. +14. Last-block masking when pattern length is not divisible by 64. +15. Carry propagation across block boundaries. +16. Borrow propagation across block boundaries. +17. Cross-block left shift. +18. Cross-block right shift. +19. Text characters/elements absent from the pattern vector. +20. Exact match crossing the 64-bit boundary. + +### Paper-style example + +Include a known approximate-substring example such as: + +```text +pattern: ACGC +text: GAAGCGACTGCAAACTCA +``` + +Verify the expected best distance and endpoint using the scalar oracle. Do not hard-code an expected endpoint copied from documentation without confirming it against the oracle. + +### Differential/property tests + +Add deterministic randomized tests comparing the production implementation with the scalar oracle. + +Cover at least: + +```text +alphabet sizes: 2, 4, and a larger character set +pattern lengths: 1..140 +text lengths: 0..200 +single-block cases: many samples +multi-block cases: many samples +random seed: fixed and printed/assertable +``` + +At minimum, run several thousand deterministic cases. + +Also add an exhaustive test over a binary alphabet for small lengths, for example: + +```text +pattern length: 0..7 +text length: 0..8 +``` + +Compare both `Distance` and `EndIndex`. + +### Required scorer tests + +Verify: + +```text +("abc", "xxabcxx") => 100 +("xxabcxx", "abc") => 100 +("", "") => 100 +("abc", "") => 0 +("", "abc") => 0 +``` + +Also verify: + +- score is always in `[0, 100]`; +- preprocessing is applied correctly; +- non-cached and cached scorers return identical scores; +- argument order does not change the public scorer result; +- equal-length directional cases use the better direction; +- repeated occurrences do not lock onto the first inferior occurrence; +- the score intentionally differs from `PartialRatio` for at least one documented example. + +Do not assert that `ApproximateSubstringRatio` equals RapidFuzz/FuzzyWuzzy `partial_ratio`. + +### Disposal and concurrency tests + +For the cached scorer: + +- verify owned resources are disposed; +- verify an externally supplied strategy is not disposed unless ownership was requested; +- run concurrent `Score` calls against one cached scorer instance and compare every result with the non-cached scorer; +- do not run scoring concurrently with `Dispose` unless the repository explicitly promises that behavior. + +--- + +## Performance requirements + +The new implementation must retain the bit-parallel complexity: + +```text +O(textLength * ceil(patternLength / 64)) +``` + +Expected allocation behavior: + +- single-block uncached call: pattern-vector allocation according to existing infrastructure, no algorithm-state array allocation; +- single-block cached call: no per-call algorithm-state allocation; +- multi-block call: pooled state buffers only; +- all rented arrays must be returned in `finally` blocks; +- do not clear returned arrays unless sensitive-data policy or repository convention requires it. + +Do not introduce LINQ into the hot path. + +Do not materialize spans as arrays unless required by an existing interface. + +Do not use exceptions for normal scorer flow. + +--- + +## Benchmarks + +Add BenchmarkDotNet coverage in `FuzzySharp.Benchmarks`. + +Suggested class: + +```text +ApproximateSubstringRatioBenchmarks +``` + +Benchmark at least: + +1. `Indel.BestSubstringMatch` single block. +2. `Indel.BestSubstringMatch` multi-block. +3. `Fuzz.ApproximateSubstringRatio`. +4. `CachedApproximateSubstringRatioScorer.Score`. +5. A test/benchmark-only scalar DP baseline. +6. Existing `Fuzz.PartialRatio` as a performance reference, clearly noting that semantics differ. + +Use parameter sets around important boundaries: + +```text +pattern length: 8, 32, 63, 64, 65, 128, 129, 256 +text length: 64, 256, 1024, 4096 +``` + +Include datasets with: + +- exact match near the start; +- exact match near the end; +- no exact match; +- repeated approximate and exact occurrences; +- random low-similarity data; +- high-similarity data. + +Report both runtime and allocations using `[MemoryDiagnoser]`. + +Do not claim the new scorer is faster than `PartialRatio` without benchmark evidence. + +--- + +## Documentation + +Update `README.md` with a concise section containing: + +```csharp +int score = Fuzz.ApproximateSubstringRatio( + "invoice number 12345", + "processed invoice number 12345 successfully"); +``` + +Explain: + +- it finds the best approximate occurrence of the shorter string; +- it considers all possible start positions implicitly; +- later repeated occurrences are not hidden by an earlier inferior occurrence; +- insertions/deletions are the edit model; +- normalization is relative to pattern length; +- `EndIndex` is available from `Indel.BestSubstringMatch`; +- the start index and edit script are not returned; +- it is not numerically compatible with `PartialRatio`. + +Add an entry to `CHANGELOG.md` describing the new API without promising exact RapidFuzz compatibility. + +--- + +## Compatibility requirements + +The library multi-targets old .NET Framework, `netstandard2.0`, and modern .NET. + +The implementation must compile for all configured target frameworks. + +In particular: + +- do not use `record struct`; +- do not rely on APIs introduced after `netstandard2.0` without existing polyfills or conditional compilation; +- use `System.Memory`/existing span support already configured by the project; +- preserve nullable annotations/style currently used by the repository; +- do not add a runtime dependency solely for this feature. + +At minimum, run: + +```bash +dotnet build FuzzySharp/FuzzySharp.csproj -f netstandard2.0 +dotnet test FuzzySharp.Test/FuzzySharp.Test.csproj -f net8.0 +dotnet test FuzzySharp.Test/FuzzySharp.Test.csproj -f net10.0 +``` + +When running on Windows with the required targeting packs, also run the legacy .NET Framework test targets configured by the project. + +Run the full repository test suite, not only the newly added tests. + +--- + +## Code-quality requirements + +- Follow existing formatting, namespace, naming, XML documentation, and file-layout conventions. +- Keep bit-vector variable names aligned with the paper where useful, but prefer readable names already present in the baseline. +- Preserve `unchecked` arithmetic where wraparound is part of the bit-vector algorithm. +- Keep single-block and multi-block paths separate unless a refactor demonstrably improves readability without hurting performance. +- Add comments only around non-obvious boundary conditions, carries, borrows, shifts, and last-block masking. +- Avoid comments that simply restate the code. +- Do not expose internal pooled buffers. +- Do not change `IPatternMatchVector` unless absolutely necessary. +- Do not commit generated benchmark artifacts unless the repository already tracks them intentionally. + +--- + +## Out of scope + +Do not implement these in this feature unless required to fix correctness: + +- start-index reconstruction; +- edit-operation traceback; +- returning every matching endpoint; +- Unicode grapheme-cluster segmentation; +- culture-aware equality inside the generic algorithm; +- replacement of `PartialRatio`; +- integration into `WeightedRatio` defaults; +- SIMD intrinsics beyond the existing word-parallel implementation; +- approximate matching with substitution cost `1`. + +These may be separate follow-up features. + +--- + +## Acceptance criteria + +The feature is complete only when all of the following are true: + +- [ ] Existing single-block and multi-block IndelNew code is retained or equivalently optimized. +- [ ] Production results match the scalar DP oracle for exhaustive and deterministic randomized tests. +- [ ] Pattern lengths on both sides of every 64-bit boundary are covered. +- [ ] `Fuzz.ApproximateSubstringRatio` is public and documented. +- [ ] `ApproximateSubstringRatioScorer` is available. +- [ ] `CachedApproximateSubstringRatioScorer` is available and reuses a precomputed pattern vector when valid. +- [ ] Cached and non-cached results are identical. +- [ ] The public scorer is symmetric, including equal-length directional cases. +- [ ] Existing `Fuzz.PartialRatio` behavior is unchanged. +- [ ] Existing default `WeightedRatio` and `ProcessBuilder` behavior is unchanged. +- [ ] All tests pass on `net8.0` and `net10.0`. +- [ ] The library builds for `netstandard2.0`. +- [ ] Legacy targets are validated when the required Windows targeting packs are available. +- [ ] Benchmarks cover single-block, multi-block, cached, uncached, and scalar-oracle paths. +- [ ] README and changelog are updated. +- [ ] No unreturned pooled arrays or undisposed cached pattern vectors remain. + +--- + +## Final agent response + +After implementation, report: + +1. files added and changed; +2. the final public API; +3. the exact scoring semantics; +4. correctness-test coverage and randomized test seed; +5. build/test commands executed and their results; +6. benchmark summary, including allocations; +7. any deliberate deviations from this specification; +8. any remaining risks or recommended follow-up work. + +Do not report completion if tests were not run. If a target cannot be tested in the current environment, state that explicitly and list the exact unverified target. From 1c6c104012f33ba3d6e2f1eba513e150c0151ce3 Mon Sep 17 00:00:00 2001 From: Yevhen Cherkes Date: Sat, 25 Jul 2026 17:16:42 +0200 Subject: [PATCH 2/4] Optimize BestSubstringMatch with exact-match fast path Use ordinal span search to bypass the Indel recurrence when an exact substring exists. Apply the same optimization to cached and non-cached approximate substring ratio strategies while preserving earliest-match semantics. --- FuzzySharp/Indel.Static.BestSubstring.cs | 42 +++++++++++++++++++ .../ApproximateSubstringRatioStrategy.cs | 22 +++++++++- ...CachedApproximateSubstringRatioStrategy.cs | 11 +++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/FuzzySharp/Indel.Static.BestSubstring.cs b/FuzzySharp/Indel.Static.BestSubstring.cs index c0be98c..aa1ab01 100644 --- a/FuzzySharp/Indel.Static.BestSubstring.cs +++ b/FuzzySharp/Indel.Static.BestSubstring.cs @@ -57,6 +57,48 @@ public override string ToString() public sealed partial class Indel { + /// + /// Finds the minimum insertion-deletion distance between the complete + /// character pattern and any substring of text. + /// + /// The returned EndIndex identifies the text position at which the best + /// approximate substring match ends. + /// + public static IndelSubstringMatch BestSubstringMatch( + ReadOnlySpan pattern, + ReadOnlySpan text) + { + if (pattern.IsEmpty) + { + return new IndelSubstringMatch( + distance: 0, + endIndex: -1); + } + + if (text.IsEmpty) + { + return new IndelSubstringMatch( + distance: pattern.Length, + endIndex: -1); + } + + // Exact ordinal span search is heavily optimized by the runtime and + // avoids running the more expensive recurrence up to a distant exact + // occurrence. IndexOf also preserves the earliest-zero tie behavior. + int exactStart = text.IndexOf(pattern); + if (exactStart >= 0) + { + return new IndelSubstringMatch( + distance: 0, + endIndex: exactStart + pattern.Length - 1); + } + + using var patternMatchVector = PatternMatchVector.Create(pattern); + return BestSubstringMatchImpl( + patternMatchVector, + text); + } + /// /// Finds the minimum insertion-deletion distance between the complete /// pattern and any substring of text. diff --git a/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs index e145d85..cdca763 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs @@ -6,8 +6,26 @@ internal static class ApproximateSubstringRatioStrategy { public static int Calculate(string input1, string input2) { + ReadOnlySpan first = input1.AsSpan(); + ReadOnlySpan second = input2.AsSpan(); + + if (!first.IsEmpty && !second.IsEmpty) + { + ReadOnlySpan pattern = first.Length <= second.Length + ? first + : second; + ReadOnlySpan text = first.Length <= second.Length + ? second + : first; + + if (text.IndexOf(pattern) >= 0) + { + return 100; + } + } + return Generic.ApproximateSubstringRatioStrategy.Calculate( - input1.AsSpan(), - input2.AsSpan()); + first, + second); } } diff --git a/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs b/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs index 2aab80e..8a91baa 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs @@ -31,6 +31,17 @@ public int Calculate(string input2) : 0; } + ReadOnlySpan pattern = _processedInput1.Length <= processedInput2.Length + ? _processedInput1.AsSpan() + : processedInput2.AsSpan(); + ReadOnlySpan text = _processedInput1.Length <= processedInput2.Length + ? processedInput2.AsSpan() + : _processedInput1.AsSpan(); + if (text.IndexOf(pattern) >= 0) + { + return 100; + } + if (_processedInput1.Length < processedInput2.Length) { return Score(_input1PatternVector, processedInput2.AsSpan()); From 06a35cd5a5b280ca69baa5f9a3c1e20e6e7ab550 Mon Sep 17 00:00:00 2001 From: Yevhen Cherkes Date: Sat, 25 Jul 2026 17:28:50 +0200 Subject: [PATCH 3/4] tech info for following iterations --- pr_review.md | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 pr_review.md diff --git a/pr_review.md b/pr_review.md new file mode 100644 index 0000000..7958cb3 --- /dev/null +++ b/pr_review.md @@ -0,0 +1,104 @@ +# PR Review: Exact-match fast path optimization + +## Summary of changes + +Three files modified with 73 lines added and 2 removed: + +- `FuzzySharp/Indel.Static.BestSubstring.cs` (+42) — new public overload for char +- `FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs` (+22, -2) — IndexOf shortcut in string strategy +- `FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs` (+11) — IndexOf shortcut in cached scorer + +--- + +## File 1: Indel.Static.BestSubstring.cs + +### What was done + +Added a new public overload that shadows the generic method for char inputs: + +```csharp +public static IndelSubstringMatch BestSubstringMatch( + ReadOnlySpan pattern, + ReadOnlySpan text) +``` + +The implementation performs an early-exact-match check via `text.IndexOf(pattern)` before falling through to the existing bit-parallel recurrence. + +### Issues + +| # | Severity | Description | +|---|---|---| +| 1 | **Breaking** | New overload changes method resolution for callers who previously invoked `Indel.BestSubstringMatch(...)` with `T = char`. Code that explicitly passed `ReadOnlySpan` will now resolve to the new overload instead of the generic one. This is a breaking API change. | +| 2 | Minor | No XML documentation on the new overload. The existing generic method has full docs; this should match. | + +### Recommendations + +- Add XML doc matching the generic method's style and content. +- Consider whether this should be `internal` rather than public to avoid breaking external callers who relied on generic resolution. If it's only used internally, making it non-public avoids the breaking change entirely. + +--- + +## File 2: ApproximateSubstringRatioStrategy.cs + +### What was done + +Added an exact-match shortcut at the string level before delegating to the generic strategy: + +```csharp +if (text.IndexOf(pattern) >= 0) return 100; +``` + +### Issues + +| # | Severity | Description | +|---|---|---| +| 3 | Performance | **Redundant check.** The generic strategy below calls `Indel.BestSubstringMatch` which will ALSO perform an exact match search internally (via the new overload). When no exact match exists, we do IndexOf twice — once here, then again inside BestSubstringMatchImpl. | +| 4 | Minor optimization | No early return for identical-length strings that are equal. We fall through to the generic strategy even though both directions would find an exact match immediately. | + +### Recommendations + +- **Remove this shortcut** and rely on `Indel.BestSubstringMatch` to handle exact matches efficiently. The double-check adds overhead without benefit. +- Alternatively, move it after the generic call as a fallback for cases where the generic path might miss it (though that's unlikely given the new overload). + +--- + +## File 3: CachedApproximateSubstringRatioStrategy.cs + +### What was done + +Added the same IndexOf shortcut in the cached scorer's `Calculate` method. + +### Issues + +| # | Severity | Description | +|---|---|---| +| 5 | Minor optimization | The shortcut is placed after empty-string checks but before the length-based direction logic. We check for exact match in both directions (pattern vs text and vice versa) even though only one direction matters for a score of 100. | + +### Recommendations + +- No action needed — this file calls `BestSubstringMatchImpl` directly rather than going through the new overload, so there's no redundant search. The shortcut is efficient here. +- Consider adding an early return when `_processedInput1.Length == processedInput2.Length && _processedInput1 == processedInput2` to avoid even the IndexOf call for identical strings. + +--- + +## Benchmark results summary + +| Text | Dataset | Before | After | Improvement | PartialRatio | After vs PartialRatio | +|---:|---|---:|---:|---:|---:|---:| +| 1024 | Exact match | 4,411 ns | **39.5 ns** | **111.7× faster** | 388.8 ns | **9.8× faster** | +| 4096 | Exact match | 17,328 ns | **118.4 ns** | **146.3× faster** | 508.9 ns | **4.3× faster** | +| 1024 | No exact match | 4,328 ns | **4,539 ns** | 4.9% slower | 17,898 ns | **3.9× faster** | +| 4096 | No exact match | 17,490 ns | **17,122 ns** | 2.1% faster | 59,556 ns | **3.5× faster** | + +Allocations: Exact-match case went from 144 B to 0 B. No-exact-match remains at 144 B. + +--- + +## Overall recommendations + +1. **Fix the breaking change**: Either make the new overload `internal` or add proper XML documentation and versioning notes. +2. **Remove redundant IndexOf** in ApproximateSubstringRatioStrategy.cs — let BestSubstringMatch handle it. +3. **Add identical-string shortcut** to CachedApproximateSubstringRatioStrategy for O(1) early return on exact equality. +4. **Document the new overload** with XML comments matching existing style. + +All tests pass (5,169 on both .NET 8 and .NET 10). Build succeeds on netstandard2.0. From 066038dbecd66ecccc2e5633814d0b2306f5caa5 Mon Sep 17 00:00:00 2001 From: Yevhen Cherkes Date: Sun, 26 Jul 2026 09:42:17 +0200 Subject: [PATCH 4/4] refactoring --- CHANGELOG.md | 4 +- .../ApproximateSubstringRatioBenchmarks.cs | 116 ++- .../ApproximateSubstringRatioTests.cs | 53 +- FuzzySharp.Test/IndelBestSubstringTests.cs | 54 +- FuzzySharp/Fuzz.cs | 15 +- FuzzySharp/Indel.Static.BestSubstring.cs | 434 ++--------- .../CachedApproximateSubstringRatioScorer.cs | 34 +- .../ApproximateSubstringRatioStrategy.cs | 22 +- .../Strategy/ApproximateSubstringScore.cs | 19 + ...CachedApproximateSubstringRatioStrategy.cs | 12 +- .../ApproximateSubstringRatioStrategyT.cs | 6 +- README.md | 15 +- ...GENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md | 684 ------------------ 13 files changed, 299 insertions(+), 1169 deletions(-) create mode 100644 FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringScore.cs delete mode 100644 docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 46634cd..054d9e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ## Unreleased -- Added `Fuzz.ApproximateSubstringRatio`, `ApproximateSubstringRatioScorer`, and `CachedApproximateSubstringRatioScorer` for best approximate substring scoring using Indel distance. -- Added public `Indel.BestSubstringMatch` APIs that return the best raw distance and matching endpoint for generic spans. +- Added approximate-substring scoring based on bit-parallel Indel distance, including stateless and cached scorers. +- Added directional `Indel.BestSubstringMatch` APIs that return the best raw distance and first strict-improvement endpoint for generic spans. ## v5.0.3 diff --git a/FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs b/FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs index 147c4f0..fb6fdd8 100644 --- a/FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs +++ b/FuzzySharp.Benchmarks/ApproximateSubstringRatioBenchmarks.cs @@ -3,60 +3,92 @@ namespace Raffinert.FuzzySharp.Benchmarks; +/// +/// Reproducible approximate-substring benchmark matrix. Equal-length and +/// shorter-candidate data sets deliberately normalize the requested text length +/// to preserve their named relationship to . +/// [MemoryDiagnoser] [RankColumn] public class ApproximateSubstringRatioBenchmarks { + private const string Alphabet = "abcdef"; private string _pattern = string.Empty; private string _text = string.Empty; private CachedApproximateSubstringRatioScorer _cachedScorer = null!; - [Params(64, 256, 1024)] + [Params(32, 63, 64, 65, 127, 128, 129, 256, 1024, 2048, 2049)] public int PatternLength { get; set; } - [Params(1024, 4096)] + [Params(128, 1024, 4096, 16384)] public int TextLength { get; set; } - [Params( - BenchmarkDataSet.RepeatedApproximateAndExact, - BenchmarkDataSet.RandomLowSimilarity)] + [ParamsAllValues] public BenchmarkDataSet DataSet { get; set; } [GlobalSetup] public void Setup() { var random = new Random(42); - _pattern = GenerateString(PatternLength, random, "abcdefghijklmnopqrstuvwxyz"); - int actualTextLength = Math.Max(TextLength, PatternLength * 2); - var characters = GenerateString(actualTextLength, random, "abcdefghijklmnopqrstuvwxyz").ToCharArray(); + _pattern = GenerateString(PatternLength, random, Alphabet); - switch (DataSet) + if (DataSet == BenchmarkDataSet.CandidateShorterThanCachedQuery) { - case BenchmarkDataSet.ExactNearStart: - CopyPattern(characters, _pattern, 1); - break; - - case BenchmarkDataSet.ExactNearEnd: - CopyPattern(characters, _pattern, characters.Length - PatternLength - 1); - break; - - case BenchmarkDataSet.NoExactMatch: - _pattern = GenerateString(PatternLength, random, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - break; - - case BenchmarkDataSet.RepeatedApproximateAndExact: - CopyPattern(characters, _pattern, 1); - characters[1 + PatternLength / 2] = '#'; - CopyPattern(characters, _pattern, characters.Length - PatternLength - 1); - break; - - case BenchmarkDataSet.HighSimilarity: - CopyPattern(characters, _pattern, characters.Length / 2 - PatternLength / 2); - characters[characters.Length / 2] = '#'; - break; + _text = GenerateString( + Math.Max(1, Math.Min(TextLength, PatternLength - 1)), + random, + Alphabet); + } + else if (DataSet == BenchmarkDataSet.EqualLengthHighSimilarity || + DataSet == BenchmarkDataSet.EqualLengthLowSimilarity) + { + char[] equalLengthText = _pattern.ToCharArray(); + if (DataSet == BenchmarkDataSet.EqualLengthHighSimilarity) + { + equalLengthText[PatternLength / 2] = '#'; + } + else + { + for (int index = 0; index < equalLengthText.Length; index++) + { + equalLengthText[index] = 'z'; + } + } + + _text = new string(equalLengthText); + } + else + { + int actualTextLength = Math.Max(TextLength, PatternLength + 2); + char[] text = GenerateString(actualTextLength, random, Alphabet).ToCharArray(); + + switch (DataSet) + { + case BenchmarkDataSet.ExactNearStart: + CopyPattern(text, _pattern, 1); + break; + case BenchmarkDataSet.ExactNearEnd: + CopyPattern(text, _pattern, text.Length - PatternLength - 1); + break; + case BenchmarkDataSet.ApproximateNearStart: + CopyApproximatePattern(text, _pattern, 1); + break; + case BenchmarkDataSet.ApproximateNearEnd: + CopyApproximatePattern(text, _pattern, text.Length - PatternLength - 1); + break; + case BenchmarkDataSet.NoExactMatch: + case BenchmarkDataSet.RandomLowSimilarity: + _pattern = GenerateString(PatternLength, random, "ABCDEF"); + break; + case BenchmarkDataSet.RepeatedApproximateAndExact: + CopyApproximatePattern(text, _pattern, 1); + CopyPattern(text, _pattern, text.Length - PatternLength - 1); + break; + } + + _text = new string(text); } - _text = new string(characters); _cachedScorer = new CachedApproximateSubstringRatioScorer(_pattern); } @@ -79,19 +111,19 @@ public int ApproximateSubstringRatio() } [Benchmark] - public int CachedApproximateSubstringRatio() + public int CachedApproximateSubstringRatioScorerScore() { return _cachedScorer.Score(_text); } [Benchmark] - public IndelSubstringMatch ScalarDynamicProgrammingBaseline() + public IndelSubstringMatch ScalarDynamicProgrammingOracle() { return ScalarBestSubstringMatch(_pattern.AsSpan(), _text.AsSpan()); } [Benchmark] - public int PartialRatioReference() + public int PartialRatio() { return Fuzz.PartialRatio(_pattern, _text); } @@ -133,6 +165,12 @@ private static IndelSubstringMatch ScalarBestSubstringMatch( return new IndelSubstringMatch(bestDistance, bestEndIndex); } + private static void CopyApproximatePattern(char[] destination, string pattern, int startIndex) + { + CopyPattern(destination, pattern, startIndex); + destination[startIndex + pattern.Length / 2] = '#'; + } + private static void CopyPattern(char[] destination, string pattern, int startIndex) { pattern.CopyTo(0, destination, startIndex, pattern.Length); @@ -154,8 +192,12 @@ public enum BenchmarkDataSet { ExactNearStart, ExactNearEnd, + ApproximateNearStart, + ApproximateNearEnd, NoExactMatch, - RepeatedApproximateAndExact, RandomLowSimilarity, - HighSimilarity + RepeatedApproximateAndExact, + EqualLengthHighSimilarity, + EqualLengthLowSimilarity, + CandidateShorterThanCachedQuery } diff --git a/FuzzySharp.Test/ApproximateSubstringRatioTests.cs b/FuzzySharp.Test/ApproximateSubstringRatioTests.cs index 9ed84e5..953d58d 100644 --- a/FuzzySharp.Test/ApproximateSubstringRatioTests.cs +++ b/FuzzySharp.Test/ApproximateSubstringRatioTests.cs @@ -127,12 +127,18 @@ public void CachedApproximateSubstringRatioScorer_IsSafeForConcurrentScores() expected[index] = Fuzz.ApproximateSubstringRatio(query, candidates[index]); } + var actual = new int[256]; using var scorer = new CachedApproximateSubstringRatioScorer(query); Parallel.For(0, 256, iteration => { int index = iteration % candidates.Length; - Assert.Equal(expected[index], scorer.Score(candidates[index])); + actual[iteration] = scorer.Score(candidates[index]); }); + + for (int iteration = 0; iteration < actual.Length; iteration++) + { + Assert.Equal(expected[iteration % candidates.Length], actual[iteration]); + } } [Fact] @@ -154,6 +160,46 @@ public void CachedApproximateSubstringRatioScorer_RespectsStrategyOwnership() Assert.Equal(1, scorerOwned.DisposeCount); } + [Fact] + public void CachedApproximateSubstringRatioScorer_ThrowsAfterDisposal() + { + var scorer = new CachedApproximateSubstringRatioScorer("query"); + + scorer.Dispose(); + + Assert.Throws(() => scorer.Score("candidate")); + scorer.Dispose(); + } + + [Fact] + public void CachedApproximateSubstringRatioScorer_MatchesNonCachedAcrossBoundaryLengths() + { + var random = new Random(81927); + int[] queryLengths = { 63, 64, 65, 127, 128, 129 }; + + foreach (int queryLength in queryLengths) + { + string query = CreateRandomString(random, queryLength); + string[] candidates = + { + string.Empty, + query, + query.Substring(0, queryLength - 1), + "prefix-" + query + "-suffix", + new string('z', queryLength + 17), + query.Substring(0, queryLength / 2) + "#" + query.Substring(queryLength / 2 + 1) + }; + + using var scorer = new CachedApproximateSubstringRatioScorer(query); + foreach (string candidate in candidates) + { + Assert.Equal( + Fuzz.ApproximateSubstringRatio(query, candidate), + scorer.Score(candidate)); + } + } + } + [Fact] public void ApproximateSubstringRatioScorer_IsAvailableToProcessBuilder() { @@ -168,8 +214,9 @@ private static int DirectionalScore(string pattern, string text) { IndelSubstringMatch match = Indel.BestSubstringMatch( pattern.AsSpan(), text.AsSpan()); - return Math.Max(0, Math.Min(100, - (int)Math.Round(100.0 * (1.0 - match.Distance / (double)pattern.Length)))); + return ApproximateSubstringScore.FromDistance( + match.Distance, + pattern.Length); } private static string CreateRandomString(Random random, int length) diff --git a/FuzzySharp.Test/IndelBestSubstringTests.cs b/FuzzySharp.Test/IndelBestSubstringTests.cs index d94b362..6d89f9d 100644 --- a/FuzzySharp.Test/IndelBestSubstringTests.cs +++ b/FuzzySharp.Test/IndelBestSubstringTests.cs @@ -36,25 +36,26 @@ public void BestSubstringMatch_BothEmpty_ReturnsEmptyMatch() public void BestSubstringMatch_ExactMatch_ReturnsEarliestExactEndpoint( string pattern, string text, - int expectedEndIndex) + int expectedFirstBestEndIndex) { IndelSubstringMatch result = Indel.BestSubstringMatch( pattern.AsSpan(), text.AsSpan()); Assert.Equal(0, result.Distance); - Assert.Equal(expectedEndIndex, result.EndIndex); - Assert.True(result.Found); + Assert.Equal(expectedFirstBestEndIndex, result.FirstBestEndIndex); + Assert.True(result.ImprovedOverEmptyMatch); } [Fact] - public void BestSubstringMatch_SubstitutionCostsTwoEdits() + public void BestSubstringMatch_UsesFirstStrictImprovementForEqualDistanceTies() { IndelSubstringMatch result = Indel.BestSubstringMatch( "abc".AsSpan(), "axc".AsSpan()); Assert.Equal(new IndelSubstringMatch(2, 0), result); + Assert.Equal(0, result.FirstBestEndIndex); } [Fact] @@ -72,7 +73,8 @@ public void BestSubstringMatch_NoCommonElement_PreservesNoMatchTieBehavior() "xxx".AsSpan()); Assert.Equal(3, result.Distance); - Assert.False(result.Found); + Assert.Equal(-1, result.FirstBestEndIndex); + Assert.False(result.ImprovedOverEmptyMatch); } [Fact] @@ -95,7 +97,10 @@ public void BestSubstringMatch_SupportsGenericSequences() [InlineData(127)] [InlineData(128)] [InlineData(129)] - public void BestSubstringMatch_HandlesWordBoundariesAndLastBlockMasking( + [InlineData(2047)] + [InlineData(2048)] + [InlineData(2049)] + public void BestSubstringMatch_HandlesBoundaryLengthsAndFinalBlockMasking( int patternLength) { string pattern = CreateSequence(patternLength); @@ -103,6 +108,19 @@ public void BestSubstringMatch_HandlesWordBoundariesAndLastBlockMasking( Assert.Equal(new IndelSubstringMatch(0, "prefix-".Length + patternLength - 1), Indel.BestSubstringMatch(pattern.AsSpan(), text.AsSpan())); + + AssertMatchesOracle(pattern, pattern.Substring(0, patternLength - 1)); + AssertMatchesOracle(pattern, pattern.Insert(patternLength / 2, "#")); + + char replacement = pattern[patternLength / 2] == '#' ? '!' : '#'; + string substituted = pattern.Remove(patternLength / 2, 1) + .Insert(patternLength / 2, replacement.ToString()); + AssertMatchesOracle(pattern, substituted); + + AssertMatchesOracle(pattern, + pattern.Substring(0, patternLength / 2) + + "#" + + pattern.Substring(patternLength / 2 + 1)); } [Fact] @@ -153,11 +171,11 @@ public void BestSubstringMatch_DeterministicRandomInputs_MatchScalarOracle() var random = new Random(RandomSeed); string[] alphabets = { "ab", "abcd", "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" }; - for (int sample = 0; sample < 4_000; sample++) + for (int sample = 0; sample < 1_500; sample++) { string alphabet = alphabets[sample % alphabets.Length]; - int patternLength = random.Next(1, 141); - int textLength = random.Next(0, 201); + int patternLength = random.Next(1, 201); + int textLength = random.Next(0, 301); string pattern = CreateRandomString(random, patternLength, alphabet); string text = CreateRandomString(random, textLength, alphabet); @@ -165,6 +183,24 @@ public void BestSubstringMatch_DeterministicRandomInputs_MatchScalarOracle() } } + [Theory] + [InlineData("aaaa", "aa", 2, 1)] + [InlineData("abc", "abxabc", 0, 5)] + [InlineData("abc", "abcxxxabc", 0, 2)] + public void BestSubstringMatch_RecordsOnlyTheFirstStrictImprovement( + string pattern, + string text, + int expectedDistance, + int expectedFirstBestEndIndex) + { + IndelSubstringMatch result = Indel.BestSubstringMatch( + pattern.AsSpan(), text.AsSpan()); + + Assert.Equal(expectedDistance, result.Distance); + Assert.Equal(expectedFirstBestEndIndex, result.FirstBestEndIndex); + Assert.True(result.ImprovedOverEmptyMatch); + } + private static void AssertMatchesOracle(string pattern, string text) { IndelSubstringMatch expected = ReferenceBestSubstringMatch( diff --git a/FuzzySharp/Fuzz.cs b/FuzzySharp/Fuzz.cs index 58e8ae8..0ec2c77 100644 --- a/FuzzySharp/Fuzz.cs +++ b/FuzzySharp/Fuzz.cs @@ -66,8 +66,11 @@ public static int PartialRatio(string input1, string input2, Func /// Searches the shorter input approximately within the longer input using - /// insertions and deletions. A substitution costs two edits, so this metric - /// is not numerically equivalent to . + /// Indel distance. Insertions and deletions cost one and a substitution + /// costs two edits. Equal-length inputs are evaluated in both directions + /// unless the first direction is exact. Two empty inputs score 100; exactly + /// one empty input scores 0. This metric is not numerically equivalent to + /// . /// /// The first input. /// The second input. @@ -80,9 +83,11 @@ public static int ApproximateSubstringRatio(string input1, string input2) /// /// Searches the shorter processed input approximately within the longer - /// processed input using insertions and deletions. A substitution costs two - /// edits, so this metric is not numerically equivalent to - /// . + /// processed input using Indel distance. Insertions and deletions cost one + /// and a substitution costs two edits. Equal-length inputs are evaluated in + /// both directions unless the first direction is exact. Two empty processed + /// inputs score 100; exactly one empty processed input scores 0. This metric + /// is not numerically equivalent to . /// /// The first input. /// The second input. diff --git a/FuzzySharp/Indel.Static.BestSubstring.cs b/FuzzySharp/Indel.Static.BestSubstring.cs index aa1ab01..c844500 100644 --- a/FuzzySharp/Indel.Static.BestSubstring.cs +++ b/FuzzySharp/Indel.Static.BestSubstring.cs @@ -1,4 +1,4 @@ -using Raffinert.FuzzySharp.Utils; +using Raffinert.FuzzySharp.Utils; using System; using System.Buffers; using System.Diagnostics; @@ -6,19 +6,36 @@ namespace Raffinert.FuzzySharp; -public readonly struct IndelSubstringMatch(int distance, int endIndex) : +/// +/// The result of a directional approximate-substring search. +/// +public readonly struct IndelSubstringMatch(int distance, int firstBestEndIndex) : IEquatable { + /// + /// Gets the minimum insertion-deletion distance between the complete pattern + /// and a substring candidate. + /// public int Distance { get; } = distance; - public int EndIndex { get; } = endIndex; + /// + /// Gets the zero-based endpoint of the first text position whose distance + /// strictly improved the running best distance. Equal-distance later + /// endpoints do not replace it. Returns -1 when no text endpoint improves + /// on deleting the complete pattern. + /// + public int FirstBestEndIndex { get; } = firstBestEndIndex; - public bool Found => EndIndex >= 0; + /// + /// Gets whether a text endpoint improved on the deletion baseline, whose + /// distance equals the complete pattern length. + /// + public bool ImprovedOverEmptyMatch => FirstBestEndIndex >= 0; public bool Equals(IndelSubstringMatch other) { return Distance == other.Distance - && EndIndex == other.EndIndex; + && FirstBestEndIndex == other.FirstBestEndIndex; } public override bool Equals(object obj) @@ -31,7 +48,7 @@ public override int GetHashCode() { unchecked { - return (Distance * 397) ^ EndIndex; + return (Distance * 397) ^ FirstBestEndIndex; } } @@ -51,7 +68,7 @@ public override int GetHashCode() public override string ToString() { - return $"Distance = {Distance}, EndIndex = {EndIndex}"; + return $"Distance = {Distance}, FirstBestEndIndex = {FirstBestEndIndex}"; } } @@ -61,8 +78,11 @@ public sealed partial class Indel /// Finds the minimum insertion-deletion distance between the complete /// character pattern and any substring of text. /// - /// The returned EndIndex identifies the text position at which the best - /// approximate substring match ends. + /// This operation is directional: is matched in + /// full against substrings of . Insertions and + /// deletions cost one; a substitution costs two. The result's endpoint is + /// the first strict improvement over the deletion baseline and does not + /// identify a unique matched substring or start index. /// public static IndelSubstringMatch BestSubstringMatch( ReadOnlySpan pattern, @@ -72,14 +92,14 @@ public static IndelSubstringMatch BestSubstringMatch( { return new IndelSubstringMatch( distance: 0, - endIndex: -1); + firstBestEndIndex: -1); } if (text.IsEmpty) { return new IndelSubstringMatch( distance: pattern.Length, - endIndex: -1); + firstBestEndIndex: -1); } // Exact ordinal span search is heavily optimized by the runtime and @@ -90,7 +110,7 @@ public static IndelSubstringMatch BestSubstringMatch( { return new IndelSubstringMatch( distance: 0, - endIndex: exactStart + pattern.Length - 1); + firstBestEndIndex: exactStart + pattern.Length - 1); } using var patternMatchVector = PatternMatchVector.Create(pattern); @@ -103,8 +123,11 @@ public static IndelSubstringMatch BestSubstringMatch( /// Finds the minimum insertion-deletion distance between the complete /// pattern and any substring of text. /// - /// The returned EndIndex identifies the text position at which the best - /// approximate substring match ends. + /// This operation is directional: is matched in + /// full against substrings of . Insertions and + /// deletions cost one; a substitution costs two. The result's endpoint is + /// the first strict improvement over the deletion baseline and does not + /// identify a unique matched substring or start index. /// public static IndelSubstringMatch BestSubstringMatch( ReadOnlySpan pattern, @@ -115,14 +138,14 @@ public static IndelSubstringMatch BestSubstringMatch( { return new IndelSubstringMatch( distance: 0, - endIndex: -1); + firstBestEndIndex: -1); } if (text.IsEmpty) { return new IndelSubstringMatch( distance: pattern.Length, - endIndex: -1); + firstBestEndIndex: -1); } using var patternMatchVector = PatternMatchVector.Create(pattern); @@ -263,180 +286,9 @@ private static IndelSubstringMatch return new IndelSubstringMatch( distance: bestDistance, - endIndex: bestEndIndex); + firstBestEndIndex: bestEndIndex); } - //private static IndelSubstringMatch - // BestSubstringMatchMultipleBlocks( - // IPatternMatchVector patternVector, - // ReadOnlySpan text) - // where T : notnull, IEquatable - //{ - // int patternLength = patternVector.Length; - // int blockCount = patternVector.Blocks; - // int lastBlockBits = patternLength & 63; - // ulong lastBlockMask = lastBlockBits == 0 - // ? ulong.MaxValue - // : (1UL << lastBlockBits) - 1UL; - // ulong highestBit = 1UL << ((patternLength - 1) & 63); - - // // Rent a single contiguous buffer for all 6 vectors instead of 6 separate - // // pool allocations. Each vector occupies `blockCount` ulongs. - // int totalSize = blockCount * 6; - // ulong[] buffer = ArrayPool.Shared.Rent(totalSize); - - // try - // { - // Span positiveVertical = buffer.AsSpan(0, blockCount); - // Span negativeVertical = buffer.AsSpan(blockCount, blockCount); - // Span zeroDiagonal = buffer.AsSpan(blockCount * 2, blockCount); - // Span negativeHorizontal = buffer.AsSpan(blockCount * 3, blockCount); - // Span positiveVerticalMinusNegativeHorizontal = - // buffer.AsSpan(blockCount * 4, blockCount); - // Span positiveHorizontal = buffer.AsSpan(blockCount * 5, blockCount); - - // positiveVertical.Fill(ulong.MaxValue); - // positiveVertical[blockCount - 1] = lastBlockMask; - // negativeVertical.Clear(); - - // int currentDistance = patternLength; - // int bestDistance = patternLength; - // int bestEndIndex = -1; - - // unchecked - // { - // for (int textIndex = 0; textIndex < text.Length; textIndex++) - // { - // ReadOnlySpan matchMasks = - // patternVector.GetOrZero(text[textIndex]); - - // // zeroDiagonal = (((match & Pv) + Pv) ^ Pv) | match | Mv - // ulong carry = 0; - // for (int block = 0; block < blockCount; block++) - // { - // ulong positive = positiveVertical[block]; - // ulong addend = matchMasks[block] & positive; - // ulong sum = addend + positive; - // ulong carryFromAddend = sum < addend ? 1UL : 0UL; - // ulong sumWithCarry = sum + carry; - // carry = carryFromAddend | (sumWithCarry < sum ? 1UL : 0UL); - - // zeroDiagonal[block] = - // (sumWithCarry ^ positive) - // | matchMasks[block] - // | negativeVertical[block]; - // } - - // zeroDiagonal[blockCount - 1] &= lastBlockMask; - - // // negativeHorizontal = Pv & zeroDiagonal and - // // positiveVerticalMinusNegativeHorizontal = Pv - negativeHorizontal - // ulong borrow = 0; - // for (int block = 0; block < blockCount; block++) - // { - // ulong positive = positiveVertical[block]; - // ulong negative = positive & zeroDiagonal[block]; - // negativeHorizontal[block] = negative; - - // ulong difference = positive - negative; - // ulong borrowFromNegative = positive < negative ? 1UL : 0UL; - // ulong differenceWithBorrow = difference - borrow; - // borrow = borrowFromNegative - // | (difference < borrow ? 1UL : 0UL); - // positiveVerticalMinusNegativeHorizontal[block] = - // differenceWithBorrow; - // } - - // // The right shift crosses block boundaries, so process it - // // from the most significant block down. - // carry = 0; - // for (int block = blockCount - 1; block >= 0; block--) - // { - // ulong value = positiveVerticalMinusNegativeHorizontal[block]; - // positiveHorizontal[block] = - // (value >> 1) | (carry << 63); - // carry = value & 1UL; - // } - - // carry = 0; - // for (int block = 0; block < blockCount; block++) - // { - // ulong horizontalStarts = - // negativeVertical[block] - // | ~(positiveVertical[block] | zeroDiagonal[block]); - // ulong continuation = positiveHorizontal[block]; - // ulong sum = horizontalStarts + continuation; - // ulong carryFromStarts = sum < horizontalStarts ? 1UL : 0UL; - // ulong sumWithCarry = sum + carry; - // carry = carryFromStarts | (sumWithCarry < sum ? 1UL : 0UL); - // positiveHorizontal[block] = sumWithCarry ^ continuation; - // } - - // positiveHorizontal[blockCount - 1] &= lastBlockMask; - - // if ((positiveHorizontal[blockCount - 1] & highestBit) != 0) - // { - // currentDistance++; - // } - - // if ((negativeHorizontal[blockCount - 1] & highestBit) != 0) - // { - // currentDistance--; - // } - - // // Shift horizontal vectors one bit to the left while - // // carrying their most significant bits into the next block. - // ulong positiveHorizontalCarry = 0; - // ulong negativeHorizontalCarry = 0; - // for (int block = 0; block < blockCount; block++) - // { - // ulong shiftedPositiveHorizontal = - // (positiveHorizontal[block] << 1) - // | positiveHorizontalCarry; - // positiveHorizontalCarry = positiveHorizontal[block] >> 63; - - // ulong shiftedNegativeHorizontal = - // (negativeHorizontal[block] << 1) - // | negativeHorizontalCarry; - // negativeHorizontalCarry = negativeHorizontal[block] >> 63; - - // ulong positiveVerticalMinusNegative = - // positiveVerticalMinusNegativeHorizontal[block]; - - // negativeVertical[block] = - // shiftedPositiveHorizontal & zeroDiagonal[block]; - // positiveVertical[block] = - // shiftedNegativeHorizontal - // | ~(shiftedPositiveHorizontal | zeroDiagonal[block]) - // | (shiftedPositiveHorizontal - // & positiveVerticalMinusNegative); - // } - - // positiveVertical[blockCount - 1] &= lastBlockMask; - // negativeVertical[blockCount - 1] &= lastBlockMask; - - // if (currentDistance < bestDistance) - // { - // bestDistance = currentDistance; - // bestEndIndex = textIndex; - - // if (bestDistance == 0) - // { - // break; - // } - // } - // } - // } - - // return new IndelSubstringMatch( - // distance: bestDistance, - // endIndex: bestEndIndex); - // } - // finally - // { - // ArrayPool.Shared.Return(buffer); - // } - //} /// /// Patterns up to this many blocks (2048 symbols) keep their state on the stack. @@ -616,7 +468,7 @@ private static IndelSubstringMatch BestSubstringMatchMultipleBlocks( return new IndelSubstringMatch( distance: bestDistance, - endIndex: bestEndIndex); + firstBestEndIndex: bestEndIndex); } finally { @@ -648,208 +500,4 @@ private static ulong AddWithCarry(ulong left, ulong right, ref ulong carry) return sum; } - //private static IndelSubstringMatch BestSubstringMatchMultipleBlocks( - // IPatternMatchVector patternVector, - // ReadOnlySpan text) - // where T : notnull, IEquatable - //{ - // int patternLength = patternVector.Length; - // int blockCount = patternVector.Blocks; - - // int lastBlockBits = patternLength & 63; - // ulong lastBlockMask = lastBlockBits == 0 ? ulong.MaxValue : (1UL << lastBlockBits) - 1UL; - - // // Bit position of the pattern's final character inside the last block. - // int lastRowShift = (patternLength - 1) & 63; - - // // Only the two persistent state vectors need storage now: every other - // // quantity lives in registers for exactly as long as it is needed. - // ulong[]? rentedBuffer = null; - // Span buffer = blockCount <= StackBlockLimit - // ? stackalloc ulong[StackBlockLimit * 2] - // : (rentedBuffer = ArrayPool.Shared.Rent(blockCount * 2)); - - // try - // { - // Span positiveVertical = buffer.Slice(0, blockCount); - // Span negativeVertical = buffer.Slice(blockCount, blockCount); - - // positiveVertical.Fill(ulong.MaxValue); - // positiveVertical[blockCount - 1] = lastBlockMask; - // negativeVertical.Clear(); - - // // Vertical deltas in the indel metric are always +/-1, never 0, so - // // Pv | Mv covers the whole pattern and Pv stays masked to it. - // ref ulong positiveVerticalRef = ref MemoryMarshal.GetReference(positiveVertical); - // ref ulong negativeVerticalRef = ref MemoryMarshal.GetReference(negativeVertical); - - // int currentDistance = patternLength; - // int bestDistance = patternLength; - // int bestEndIndex = -1; - // int lastBlock = blockCount - 1; - - // unchecked - // { - // for (int textIndex = 0; textIndex < text.Length; textIndex++) - // { - // ReadOnlySpan matchMasks = patternVector.GetOrZero(text[textIndex]); - // Debug.Assert(matchMasks.Length >= blockCount); - // ref ulong matchRef = ref MemoryMarshal.GetReference(matchMasks); - - // // Carries for the two additions, and the bits shifted out of - // // the previous block by the two one-bit left shifts. - // ulong diagonalCarry = 0; - // ulong horizontalCarry = 0; - // ulong positiveHorizontalShiftIn = 0; - // ulong negativeHorizontalShiftIn = 0; - - // // The right shift of (Pv - Mh) needs bit 0 of the *next* block, - // // so each block is finished one iteration late: these hold the - // // pending block's inputs. - // ulong pendingPositiveVertical = positiveVerticalRef; - // ulong pendingNegativeVertical = negativeVerticalRef; - // ulong pendingMatch = matchRef; - - // ulong pendingZeroDiagonal = ZeroDiagonal( - // pendingMatch, - // pendingPositiveVertical, - // pendingNegativeVertical, - // ref diagonalCarry); - - // // Mh is a bitwise subset of Pv, so Pv - Mh never borrows: - // // it is exactly Pv & ~D0. The original borrow-propagation - // // loop was a no-op serial dependency. - // ulong pendingPositiveVerticalMinusNegativeHorizontal = - // pendingPositiveVertical & ~pendingZeroDiagonal; - - // for (int block = 1; block < blockCount; block++) - // { - // ulong positive = Unsafe.Add(ref positiveVerticalRef, block); - // ulong negative = Unsafe.Add(ref negativeVerticalRef, block); - // ulong match = Unsafe.Add(ref matchRef, block); - - // ulong zeroDiagonal = ZeroDiagonal(match, positive, negative, ref diagonalCarry); - // ulong positiveMinusNegativeHorizontal = positive & ~zeroDiagonal; - - // // Finish block-1 now that its right-shift input is known. - // // (x & 1) << 63 == x << 63. - // ulong horizontalContinuation = - // (pendingPositiveVerticalMinusNegativeHorizontal >> 1) | - // (positiveMinusNegativeHorizontal << 63); - - // ulong horizontalStarts = pendingNegativeVertical | - // ~(pendingPositiveVertical | pendingZeroDiagonal); - - // ulong positiveHorizontal = AddWithCarry( - // horizontalStarts, - // horizontalContinuation, - // ref horizontalCarry) ^ horizontalContinuation; - - // ulong negativeHorizontal = pendingPositiveVertical & pendingZeroDiagonal; - - // ulong shiftedPositiveHorizontal = - // (positiveHorizontal << 1) | positiveHorizontalShiftIn; - // positiveHorizontalShiftIn = positiveHorizontal >> 63; - - // ulong shiftedNegativeHorizontal = - // (negativeHorizontal << 1) | negativeHorizontalShiftIn; - // negativeHorizontalShiftIn = negativeHorizontal >> 63; - - // Unsafe.Add(ref negativeVerticalRef, block - 1) = - // shiftedPositiveHorizontal & pendingZeroDiagonal; - // Unsafe.Add(ref positiveVerticalRef, block - 1) = - // shiftedNegativeHorizontal | - // ~(shiftedPositiveHorizontal | pendingZeroDiagonal) | - // (shiftedPositiveHorizontal & pendingPositiveVerticalMinusNegativeHorizontal); - - // pendingPositiveVertical = positive; - // pendingNegativeVertical = negative; - // pendingZeroDiagonal = zeroDiagonal; - // pendingPositiveVerticalMinusNegativeHorizontal = positiveMinusNegativeHorizontal; - // } - - // // Last block: nothing shifts in from above. - // pendingZeroDiagonal &= lastBlockMask; - - // { - // // Masking D0 cannot change Pv & ~D0 here, because Pv is - // // already confined to lastBlockMask. - // ulong horizontalContinuation = - // pendingPositiveVerticalMinusNegativeHorizontal >> 1; - - // ulong horizontalStarts = pendingNegativeVertical | - // ~(pendingPositiveVertical | pendingZeroDiagonal); - - // ulong positiveHorizontal = (AddWithCarry( - // horizontalStarts, - // horizontalContinuation, - // ref horizontalCarry) ^ horizontalContinuation) & lastBlockMask; - - // ulong negativeHorizontal = pendingPositiveVertical & pendingZeroDiagonal; - - // // Branchless: these two bits are near-random, so branching - // // on them mispredicts on roughly half the text positions. - // currentDistance += (int)((positiveHorizontal >> lastRowShift) & 1UL) - // - (int)((negativeHorizontal >> lastRowShift) & 1UL); - - // ulong shiftedPositiveHorizontal = - // (positiveHorizontal << 1) | positiveHorizontalShiftIn; - // ulong shiftedNegativeHorizontal = - // (negativeHorizontal << 1) | negativeHorizontalShiftIn; - - // Unsafe.Add(ref negativeVerticalRef, lastBlock) = - // shiftedPositiveHorizontal & pendingZeroDiagonal & lastBlockMask; - // Unsafe.Add(ref positiveVerticalRef, lastBlock) = - // (shiftedNegativeHorizontal | - // ~(shiftedPositiveHorizontal | pendingZeroDiagonal) | - // (shiftedPositiveHorizontal & pendingPositiveVerticalMinusNegativeHorizontal)) - // & lastBlockMask; - // } - - // if (currentDistance < bestDistance) - // { - // bestDistance = currentDistance; - // bestEndIndex = textIndex; - - // if (bestDistance == 0) - // { - // break; - // } - // } - // } - // } - - // return new IndelSubstringMatch( - // distance: bestDistance, - // endIndex: bestEndIndex); - // } - // finally - // { - // if (rentedBuffer is not null) - // { - // ArrayPool.Shared.Return(rentedBuffer); - // } - // } - //} - - ///// D0 = (((match & Pv) + Pv) ^ Pv) | match | Mv, with a cross-block carry. - //[MethodImpl(MethodImplOptions.AggressiveInlining)] - //private static ulong ZeroDiagonal(ulong match, ulong positive, ulong negative, ref ulong carry) - //{ - // ulong sum = AddWithCarry(match & positive, positive, ref carry); - // return (sum ^ positive) | match | negative; - //} - - ///// - ///// Full adder over one block. The carry-out is the majority bit - ///// ((a & b) | ((a | b) & ~sum)) >> 63, which avoids the two dependent - ///// comparisons of a split two-step addition. - ///// - //[MethodImpl(MethodImplOptions.AggressiveInlining)] - //private static ulong AddWithCarry(ulong left, ulong right, ref ulong carry) - //{ - // ulong sum = left + right + carry; - // carry = ((left & right) | ((left | right) & ~sum)) >> 63; - // return sum; - //} } diff --git a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs index 85efc99..2f8b9c8 100644 --- a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs +++ b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs @@ -3,12 +3,21 @@ namespace Raffinert.FuzzySharp.SimilarityRatio.Scorer.StrategySensitive; +/// +/// Caches a query for repeated approximate-substring scoring. Concurrent calls +/// to are supported before +/// disposal; concurrent scoring and disposal are not supported. +/// public sealed class CachedApproximateSubstringRatioScorer : CachedSimpleRatioScorerBase { private readonly ICachedStrategy _strategy; private readonly bool _isStrategyOwner; private bool _disposed; + /// + /// Initializes an owned cached strategy for . + /// Dispose this scorer when it is no longer needed; disposal is idempotent. + /// public CachedApproximateSubstringRatioScorer( string input1, Func preprocessor = null) @@ -17,7 +26,7 @@ public CachedApproximateSubstringRatioScorer( _isStrategyOwner = true; } - public CachedApproximateSubstringRatioScorer( + internal CachedApproximateSubstringRatioScorer( ICachedStrategy strategy, bool isStrategyOwner = false) { @@ -25,14 +34,31 @@ public CachedApproximateSubstringRatioScorer( _isStrategyOwner = isStrategyOwner; } - protected override CachedScorer Scorer => input2 => _strategy.Calculate(input2); + protected override CachedScorer Scorer => Calculate; public override void Dispose() { - if (_isStrategyOwner && !_disposed) + if (_disposed) + { + return; + } + + if (_isStrategyOwner) { _strategy.Dispose(); - _disposed = true; } + + _disposed = true; + } + + private int Calculate(string input2) + { + if (_disposed) + { + throw new ObjectDisposedException( + nameof(CachedApproximateSubstringRatioScorer)); + } + + return _strategy.Calculate(input2); } } diff --git a/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs index cdca763..e145d85 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs @@ -6,26 +6,8 @@ internal static class ApproximateSubstringRatioStrategy { public static int Calculate(string input1, string input2) { - ReadOnlySpan first = input1.AsSpan(); - ReadOnlySpan second = input2.AsSpan(); - - if (!first.IsEmpty && !second.IsEmpty) - { - ReadOnlySpan pattern = first.Length <= second.Length - ? first - : second; - ReadOnlySpan text = first.Length <= second.Length - ? second - : first; - - if (text.IndexOf(pattern) >= 0) - { - return 100; - } - } - return Generic.ApproximateSubstringRatioStrategy.Calculate( - first, - second); + input1.AsSpan(), + input2.AsSpan()); } } diff --git a/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringScore.cs b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringScore.cs new file mode 100644 index 0000000..0d6786b --- /dev/null +++ b/FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringScore.cs @@ -0,0 +1,19 @@ +using System; + +namespace Raffinert.FuzzySharp.SimilarityRatio.Strategy; + +internal static class ApproximateSubstringScore +{ + public static int FromDistance(int distance, int patternLength) + { + if (patternLength <= 0) + { + throw new ArgumentOutOfRangeException(nameof(patternLength)); + } + + double similarity = 1.0 - distance / (double)patternLength; + int score = (int)Math.Round(100.0 * similarity); + + return Math.Max(0, Math.Min(100, score)); + } +} diff --git a/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs b/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs index 8a91baa..f4319d2 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs @@ -22,6 +22,12 @@ public CachedApproximateSubstringRatioStrategy( public int Calculate(string input2) { + if (_disposed) + { + throw new ObjectDisposedException( + nameof(CachedApproximateSubstringRatioStrategy)); + } + string processedInput2 = _preprocessor(input2); if (_processedInput1.Length == 0 || processedInput2.Length == 0) @@ -78,8 +84,8 @@ private static int Score( { IndelSubstringMatch match = Indel.BestSubstringMatchImpl(patternVector, text); - double similarity = 1.0 - match.Distance / (double)patternVector.Length; - int score = (int)Math.Round(100.0 * similarity); - return Math.Max(0, Math.Min(100, score)); + return ApproximateSubstringScore.FromDistance( + match.Distance, + patternVector.Length); } } diff --git a/FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs b/FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs index 56a3041..7602a0c 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs @@ -43,8 +43,8 @@ internal static int Calculate( return pattern.IsEmpty && text.IsEmpty ? 100 : 0; } - double similarity = 1.0 - match.Distance / (double)pattern.Length; - int score = (int)Math.Round(100.0 * similarity); - return Math.Max(0, Math.Min(100, score)); + return ApproximateSubstringScore.FromDistance( + match.Distance, + pattern.Length); } } diff --git a/README.md b/README.md index 9e807cf..853c238 100644 --- a/README.md +++ b/README.md @@ -141,13 +141,16 @@ int score = Fuzz.ApproximateSubstringRatio( `ApproximateSubstringRatio` searches for the best approximate occurrence of the shorter input, considering every possible start position implicitly. It uses the Indel edit model: insertions and deletions cost one, while a substitution costs -two. The score is normalized relative to the shorter input's length, and later -exact occurrences are not hidden by earlier inferior ones. +two. The score is normalized relative to the shorter input's length. Equal-length +inputs are evaluated in both directions unless the first direction is an exact +match, so the high-level score is symmetric. Its numeric results are not +equivalent to `PartialRatio`. For endpoint information, call `Indel.BestSubstringMatch` directly; its -`EndIndex` identifies where the best match ends. This API does not return a start -index or edit script, and its numeric scores are not compatible with -`PartialRatio`. +`FirstBestEndIndex` is the first endpoint that strictly improves on the distance +of deleting the complete pattern. Equal-distance later endpoints do not replace +it, so it is not a unique match boundary. This directional API does not return a +start index or edit script. ### Token Sort Ratio

Run .NET fiddle

@@ -486,7 +489,7 @@ endpoint: IndelSubstringMatch match = Indel.BestSubstringMatch( "invoice number 12345".AsSpan(), "processed invoice number 12345 successfully".AsSpan()); -// match.Distance == 0; match.EndIndex identifies the final '5' +// match.FirstBestEndIndex identifies the first strict improvement (the final '5') ``` A generic variant `IndelT` is available for comparing sequences of any `IEquatable`: diff --git a/docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md b/docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md deleted file mode 100644 index d23d56f..0000000 --- a/docs/AI_AGENT_INSTRUCTIONS_APPROXIMATE_SUBSTRING.md +++ /dev/null @@ -1,684 +0,0 @@ -# AI Agent Instructions: Complete Approximate Substring Matching Feature - -## Objective - -Complete the approximate substring matching feature in `Raffinert/FuzzySharp` using the existing bit-parallel **IndelNew** implementation as the algorithmic foundation. - -The feature must expose a production-ready fuzzy score that finds how closely the shorter input matches **any substring** of the longer input, while preserving the existing behavior of `Fuzz.PartialRatio`. - -Do not replace or silently change `PartialRatio`, `WeightedRatio`, token scorers, or any existing public behavior. - ---- - -## Existing baseline - -The working tree already contains an implementation equivalent to: - -```csharp -public readonly struct IndelSubstringMatch : IEquatable -{ - public int Distance { get; } - public int EndIndex { get; } - public bool Found => EndIndex >= 0; -} - -public sealed partial class Indel -{ - public static IndelSubstringMatch BestSubstringMatch( - ReadOnlySpan pattern, - ReadOnlySpan text) - where T : notnull, IEquatable; - - internal static IndelSubstringMatch BestSubstringMatchImpl( - IPatternMatchVector patternVector, - ReadOnlySpan text) - where T : notnull, IEquatable; -} -``` - -The implementation already contains: - -- a single-`ulong` path for patterns up to 64 elements; -- a multi-block path for longer patterns; -- `PatternMatchVector` integration; -- pooled buffers for the multi-block implementation; -- the original approximate-substring DP boundary `D[0, j] = 0`; -- early exit when an exact match is found; -- an endpoint result through `EndIndex`. - -Treat this implementation as the starting point. Do not replace it with brute-force window enumeration or ordinary `O(mn)` dynamic programming. A scalar DP implementation may be added only as a test oracle. - ---- - -## Algorithm semantics - -For a non-empty pattern `P` and text `T`, the low-level operation finds: - -```text -min distance(P, S) -``` - -where `S` is a substring candidate represented by the original IndelNew approximate-substring recurrence. - -The edit model is Indel distance: - -- insertion cost: `1`; -- deletion cost: `1`; -- substitution cost: `2` because it is one deletion plus one insertion. - -The algorithm tracks the best distance for substrings ending at each text position and returns: - -- `Distance`: the smallest raw Indel distance encountered; -- `EndIndex`: the zero-based text index where the first strictly better best match ended; -- `Found`: `true` when a text endpoint improved on deleting the complete pattern. - -Preserve the current tie behavior: - -- update the best result only when `currentDistance < bestDistance`; -- therefore equal-distance later matches do not replace the earlier result; -- return immediately on distance `0`, since no better result is possible. - -### Empty inputs - -Preserve these low-level semantics: - -```text -pattern empty => Distance = 0, EndIndex = -1 -text empty, pattern non-empty => Distance = pattern.Length, EndIndex = -1 -``` - -The fuzzy ratio API has separate empty-input semantics defined below. - ---- - -## Public feature name - -Use **Approximate Substring Ratio** consistently. - -Required public API: - -```csharp -Fuzz.ApproximateSubstringRatio(string input1, string input2) -Fuzz.ApproximateSubstringRatio( - string input1, - string input2, - Func preprocessor) -``` - -Required scorer type: - -```csharp -ApproximateSubstringRatioScorer -``` - -Required cached scorer type: - -```csharp -CachedApproximateSubstringRatioScorer -``` - -Do not call the new API `PartialRatio`. It has deliberately different semantics from RapidFuzz/FuzzyWuzzy partial ratio. - ---- - -## Score definition - -The scorer must choose the shorter processed input as the pattern and the longer processed input as the text. - -For a non-empty pattern of length `m` and best raw distance `d`, calculate: - -```text -similarity = 1 - d / m -score = round(100 * similarity) -``` - -Equivalent C#: - -```csharp -double similarity = 1.0 - match.Distance / (double)pattern.Length; -int score = (int)Math.Round(100.0 * similarity); -``` - -Clamp defensively to `[0, 100]` even though a correct IndelNew result should already produce a distance in `[0, pattern.Length]` for this use case. - -### Fuzzy ratio empty-input behavior - -Use the same user-facing convention as other fuzzy scorers: - -```text -both inputs empty => 100 -one input empty => 0 -``` - -### Equal-length inputs - -Approximate substring matching is directional because the text side has the free-start boundary. - -When the processed inputs have the same non-zero length: - -1. calculate `input1` as pattern against `input2` as text; -2. calculate `input2` as pattern against `input1` as text; -3. return the larger score. - -This makes the public scorer symmetric and follows the approach already used by the existing partial-ratio strategy for equal-length inputs. - -### Important non-goal - -Do not attempt to reproduce the exact numeric result of RapidFuzz `partial_ratio`. - -This feature minimizes raw approximate-substring Indel distance and normalizes it by pattern length. It is a separate metric. - ---- - -## Required architecture - -Follow the current repository layers and naming conventions. - -### 1. Low-level Indel implementation - -Keep or move the current implementation into a focused file such as: - -```text -FuzzySharp/Indel.ApproximateSubstring.cs -``` - -Keep these methods: - -```csharp -public static IndelSubstringMatch BestSubstringMatch( - ReadOnlySpan pattern, - ReadOnlySpan text) - where T : notnull, IEquatable; - -internal static IndelSubstringMatch BestSubstringMatchImpl( - IPatternMatchVector patternVector, - ReadOnlySpan text) - where T : notnull, IEquatable; -``` - -The internal overload is required for the cached scorer so the pattern mask is constructed only once. - -Keep `IndelSubstringMatch` compatible with all library targets. Do not use `record struct` or APIs unavailable on `netstandard2.0`/legacy .NET Framework targets. - -A regular `readonly struct` with explicit equality is acceptable. - -### 2. Generic strategy - -Add: - -```text -FuzzySharp/SimilarityRatio/Strategy/Generic/ApproximateSubstringRatioStrategyT.cs -``` - -Suggested shape: - -```csharp -internal static class ApproximateSubstringRatioStrategy - where T : notnull, IEquatable -{ - public static int Calculate( - ReadOnlySpan input1, - ReadOnlySpan input2); -} -``` - -Responsibilities: - -- handle empty spans; -- choose the shorter span as pattern; -- call `Indel.BestSubstringMatch`; -- normalize by pattern length; -- evaluate both directions for equal lengths; -- return an integer in `[0, 100]`. - -Do not put string preprocessing in the generic strategy. - -### 3. String strategy wrapper - -Add: - -```text -FuzzySharp/SimilarityRatio/Strategy/ApproximateSubstringRatioStrategy.cs -``` - -Suggested shape: - -```csharp -internal static class ApproximateSubstringRatioStrategy -{ - public static int Calculate(string input1, string input2); -} -``` - -Delegate to `ApproximateSubstringRatioStrategy.Calculate` using spans. - -### 4. Non-cached scorer - -Add: - -```text -FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/ApproximateSubstringRatioScorer.cs -``` - -Follow `PartialRatioScorer` and other simple scorer conventions. - -Suggested shape: - -```csharp -public sealed class ApproximateSubstringRatioScorer : SimpleRatioScorerBase -{ - protected override FuzzySharp.Scorer Scorer => - ApproximateSubstringRatioStrategy.Calculate; -} -``` - -### 5. Cached strategy - -Add a cached strategy that owns one precomputed `PatternMatchVector` for the processed query. - -Suggested file: - -```text -FuzzySharp/SimilarityRatio/Strategy/CachedApproximateSubstringRatioStrategy.cs -``` - -It should implement the repository's existing `ICachedStrategy` abstraction. - -Required behavior: - -- apply the configured preprocessor to the query once in the constructor; -- retain the processed query string and its length; -- build and own the query `PatternMatchVector` once; -- apply the same preprocessor to each candidate; -- use `Indel.BestSubstringMatchImpl` when the query is no longer than the candidate; -- when a candidate is shorter than the cached query, build a temporary pattern vector for the candidate and search it in the cached query text; -- for equal lengths, evaluate both directions and use the larger score; -- dispose the owned query pattern vector exactly once; -- reject scoring after disposal only if existing cached strategies follow that convention; otherwise preserve repository behavior. - -Do not mutate the cached `PatternMatchVector` during scoring. The scorer should be safe for concurrent reads until disposed. - -Be careful: caching the first argument does not guarantee that it is always the shorter input. Correctness takes priority over reusing the cached mask in that case. - -### 6. Cached scorer - -Add: - -```text -FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/Simple/CachedApproximateSubstringRatioScorer.cs -``` - -Follow `CachedDefaultRatioScorer` ownership conventions: - -- constructor taking `string input1` and optional preprocessor owns its strategy; -- optional constructor taking `ICachedStrategy` does not own it unless explicitly requested; -- dispose only when the scorer owns the strategy. - -### 7. Public `Fuzz` API - -Add a new region next to `PartialRatio`: - -```csharp -public static int ApproximateSubstringRatio( - string input1, - string input2) -{ - return ScorerCache - .Get() - .Score(input1, input2); -} - -public static int ApproximateSubstringRatio( - string input1, - string input2, - Func preprocessor) -{ - return ScorerCache - .Get() - .Score(input1, input2, preprocessor); -} -``` - -Add XML documentation that clearly says: - -- the shorter input is searched approximately inside the longer input; -- insertions and deletions are used; -- substitution has cost `2`; -- the metric is not identical to `PartialRatio`. - -### 8. Process/extractor integration - -Ensure the scorer can be used explicitly through existing process APIs: - -```csharp -ProcessBuilder - .WithScorer(new ApproximateSubstringRatioScorer()) -``` - -and cached APIs: - -```csharp -using var scorer = new CachedApproximateSubstringRatioScorer(query); - -var pipeline = new ProcessBuilder() - .Cached(scorer) - .Build(); -``` - -Do not change the default scorer used by `ProcessBuilder`. - -Do not add the new metric to `WeightedRatio` in this feature unless an existing extension point makes that completely opt-in and backward-compatible. - ---- - -## Correctness requirements - -The bit-parallel implementation is performance-sensitive and must be verified against a simple oracle. - -### Scalar oracle - -Add a test-only `O(mn)` dynamic-programming implementation with these boundaries: - -```text -D[i, 0] = i -D[0, j] = 0 -``` - -For each text column `j`, calculate: - -```text -D[i, j] = min( - D[i - 1, j] + 1, // delete pattern element - D[i, j - 1] + 1, // insert text element - D[i - 1, j - 1] + cost // 0 when equal, otherwise 2 -) -``` - -Track the same best-distance and earliest-strict-improvement endpoint behavior as the production implementation. - -Use this implementation only in tests. - -### Required low-level tests - -Add focused tests for: - -1. Empty pattern. -2. Empty text. -3. Both empty. -4. Exact match at the beginning. -5. Exact match in the middle. -6. Exact match after an earlier approximate occurrence. -7. Repeated exact occurrences return the earliest exact endpoint. -8. A substitution costs `2`. -9. A shorter matching substring can be selected by deleting pattern elements. -10. No useful common element returns distance equal to pattern length and `Found == false` under the current tie semantics. -11. Generic non-character sequences, for example `int[]`. -12. Pattern lengths `1`, `2`, `63`, and `64`. -13. Multi-block pattern lengths `65`, `66`, `127`, `128`, and `129`. -14. Last-block masking when pattern length is not divisible by 64. -15. Carry propagation across block boundaries. -16. Borrow propagation across block boundaries. -17. Cross-block left shift. -18. Cross-block right shift. -19. Text characters/elements absent from the pattern vector. -20. Exact match crossing the 64-bit boundary. - -### Paper-style example - -Include a known approximate-substring example such as: - -```text -pattern: ACGC -text: GAAGCGACTGCAAACTCA -``` - -Verify the expected best distance and endpoint using the scalar oracle. Do not hard-code an expected endpoint copied from documentation without confirming it against the oracle. - -### Differential/property tests - -Add deterministic randomized tests comparing the production implementation with the scalar oracle. - -Cover at least: - -```text -alphabet sizes: 2, 4, and a larger character set -pattern lengths: 1..140 -text lengths: 0..200 -single-block cases: many samples -multi-block cases: many samples -random seed: fixed and printed/assertable -``` - -At minimum, run several thousand deterministic cases. - -Also add an exhaustive test over a binary alphabet for small lengths, for example: - -```text -pattern length: 0..7 -text length: 0..8 -``` - -Compare both `Distance` and `EndIndex`. - -### Required scorer tests - -Verify: - -```text -("abc", "xxabcxx") => 100 -("xxabcxx", "abc") => 100 -("", "") => 100 -("abc", "") => 0 -("", "abc") => 0 -``` - -Also verify: - -- score is always in `[0, 100]`; -- preprocessing is applied correctly; -- non-cached and cached scorers return identical scores; -- argument order does not change the public scorer result; -- equal-length directional cases use the better direction; -- repeated occurrences do not lock onto the first inferior occurrence; -- the score intentionally differs from `PartialRatio` for at least one documented example. - -Do not assert that `ApproximateSubstringRatio` equals RapidFuzz/FuzzyWuzzy `partial_ratio`. - -### Disposal and concurrency tests - -For the cached scorer: - -- verify owned resources are disposed; -- verify an externally supplied strategy is not disposed unless ownership was requested; -- run concurrent `Score` calls against one cached scorer instance and compare every result with the non-cached scorer; -- do not run scoring concurrently with `Dispose` unless the repository explicitly promises that behavior. - ---- - -## Performance requirements - -The new implementation must retain the bit-parallel complexity: - -```text -O(textLength * ceil(patternLength / 64)) -``` - -Expected allocation behavior: - -- single-block uncached call: pattern-vector allocation according to existing infrastructure, no algorithm-state array allocation; -- single-block cached call: no per-call algorithm-state allocation; -- multi-block call: pooled state buffers only; -- all rented arrays must be returned in `finally` blocks; -- do not clear returned arrays unless sensitive-data policy or repository convention requires it. - -Do not introduce LINQ into the hot path. - -Do not materialize spans as arrays unless required by an existing interface. - -Do not use exceptions for normal scorer flow. - ---- - -## Benchmarks - -Add BenchmarkDotNet coverage in `FuzzySharp.Benchmarks`. - -Suggested class: - -```text -ApproximateSubstringRatioBenchmarks -``` - -Benchmark at least: - -1. `Indel.BestSubstringMatch` single block. -2. `Indel.BestSubstringMatch` multi-block. -3. `Fuzz.ApproximateSubstringRatio`. -4. `CachedApproximateSubstringRatioScorer.Score`. -5. A test/benchmark-only scalar DP baseline. -6. Existing `Fuzz.PartialRatio` as a performance reference, clearly noting that semantics differ. - -Use parameter sets around important boundaries: - -```text -pattern length: 8, 32, 63, 64, 65, 128, 129, 256 -text length: 64, 256, 1024, 4096 -``` - -Include datasets with: - -- exact match near the start; -- exact match near the end; -- no exact match; -- repeated approximate and exact occurrences; -- random low-similarity data; -- high-similarity data. - -Report both runtime and allocations using `[MemoryDiagnoser]`. - -Do not claim the new scorer is faster than `PartialRatio` without benchmark evidence. - ---- - -## Documentation - -Update `README.md` with a concise section containing: - -```csharp -int score = Fuzz.ApproximateSubstringRatio( - "invoice number 12345", - "processed invoice number 12345 successfully"); -``` - -Explain: - -- it finds the best approximate occurrence of the shorter string; -- it considers all possible start positions implicitly; -- later repeated occurrences are not hidden by an earlier inferior occurrence; -- insertions/deletions are the edit model; -- normalization is relative to pattern length; -- `EndIndex` is available from `Indel.BestSubstringMatch`; -- the start index and edit script are not returned; -- it is not numerically compatible with `PartialRatio`. - -Add an entry to `CHANGELOG.md` describing the new API without promising exact RapidFuzz compatibility. - ---- - -## Compatibility requirements - -The library multi-targets old .NET Framework, `netstandard2.0`, and modern .NET. - -The implementation must compile for all configured target frameworks. - -In particular: - -- do not use `record struct`; -- do not rely on APIs introduced after `netstandard2.0` without existing polyfills or conditional compilation; -- use `System.Memory`/existing span support already configured by the project; -- preserve nullable annotations/style currently used by the repository; -- do not add a runtime dependency solely for this feature. - -At minimum, run: - -```bash -dotnet build FuzzySharp/FuzzySharp.csproj -f netstandard2.0 -dotnet test FuzzySharp.Test/FuzzySharp.Test.csproj -f net8.0 -dotnet test FuzzySharp.Test/FuzzySharp.Test.csproj -f net10.0 -``` - -When running on Windows with the required targeting packs, also run the legacy .NET Framework test targets configured by the project. - -Run the full repository test suite, not only the newly added tests. - ---- - -## Code-quality requirements - -- Follow existing formatting, namespace, naming, XML documentation, and file-layout conventions. -- Keep bit-vector variable names aligned with the paper where useful, but prefer readable names already present in the baseline. -- Preserve `unchecked` arithmetic where wraparound is part of the bit-vector algorithm. -- Keep single-block and multi-block paths separate unless a refactor demonstrably improves readability without hurting performance. -- Add comments only around non-obvious boundary conditions, carries, borrows, shifts, and last-block masking. -- Avoid comments that simply restate the code. -- Do not expose internal pooled buffers. -- Do not change `IPatternMatchVector` unless absolutely necessary. -- Do not commit generated benchmark artifacts unless the repository already tracks them intentionally. - ---- - -## Out of scope - -Do not implement these in this feature unless required to fix correctness: - -- start-index reconstruction; -- edit-operation traceback; -- returning every matching endpoint; -- Unicode grapheme-cluster segmentation; -- culture-aware equality inside the generic algorithm; -- replacement of `PartialRatio`; -- integration into `WeightedRatio` defaults; -- SIMD intrinsics beyond the existing word-parallel implementation; -- approximate matching with substitution cost `1`. - -These may be separate follow-up features. - ---- - -## Acceptance criteria - -The feature is complete only when all of the following are true: - -- [ ] Existing single-block and multi-block IndelNew code is retained or equivalently optimized. -- [ ] Production results match the scalar DP oracle for exhaustive and deterministic randomized tests. -- [ ] Pattern lengths on both sides of every 64-bit boundary are covered. -- [ ] `Fuzz.ApproximateSubstringRatio` is public and documented. -- [ ] `ApproximateSubstringRatioScorer` is available. -- [ ] `CachedApproximateSubstringRatioScorer` is available and reuses a precomputed pattern vector when valid. -- [ ] Cached and non-cached results are identical. -- [ ] The public scorer is symmetric, including equal-length directional cases. -- [ ] Existing `Fuzz.PartialRatio` behavior is unchanged. -- [ ] Existing default `WeightedRatio` and `ProcessBuilder` behavior is unchanged. -- [ ] All tests pass on `net8.0` and `net10.0`. -- [ ] The library builds for `netstandard2.0`. -- [ ] Legacy targets are validated when the required Windows targeting packs are available. -- [ ] Benchmarks cover single-block, multi-block, cached, uncached, and scalar-oracle paths. -- [ ] README and changelog are updated. -- [ ] No unreturned pooled arrays or undisposed cached pattern vectors remain. - ---- - -## Final agent response - -After implementation, report: - -1. files added and changed; -2. the final public API; -3. the exact scoring semantics; -4. correctness-test coverage and randomized test seed; -5. build/test commands executed and their results; -6. benchmark summary, including allocations; -7. any deliberate deviations from this specification; -8. any remaining risks or recommended follow-up work. - -Do not report completion if tests were not run. If a target cannot be tested in the current environment, state that explicitly and list the exact unverified target.