From 5924d3e259edec8b93e0eb6c6880dfb7b1f6ced2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 13:32:44 +0000 Subject: [PATCH 1/2] [patch] Rank standardized property names by fuzzy similarity NameStandardizer.FindStandardPropertyMatch admitted a standard property by substring containment and returned the first one that qualified, so the winner was decided by the order StandardOrder.PropertyNames happens to list them in rather than by how well it matched. "subtitles_track" standardized to "title" instead of "subtitle", and "review_status_flag" to "status" instead of "review_status" -- in both cases the shorter, less specific name won only because it is listed earlier. Candidates are now ranked by ktsu.FuzzySearch, which was already a PackageReference of this project but had no call site anywhere in the repo. Its score rewards consecutive characters, separator boundaries and camelCase boundaries, and penalizes characters the pattern never matched -- the ranking this matcher was approximating by hand. The containment gate is deliberately unchanged. Fuzzy.Contains matches a subsequence rather than a substring, so using it to admit candidates as well as rank them would map keys that are preserved today; since ranking already discards those weaker candidates, widening the gate buys nothing and risks renaming frontmatter a caller meant to keep. Ties still fall back to the standard order. PropertyMerger's word-overlap scorer is the other call site named in the issue and is left alone for now: its candidates only have to share a word, so pairs like "review_notes"/"notes_reviewed" are related without either being a subsequence of the other, and a whole-name score cannot rank them at all. Refs #113 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S9FaM3MJE7n987yxvtxUZD --- Frontmatter.Test/FuzzyRankingTests.cs | 81 +++++++++++++++++++++++++++ Frontmatter/FuzzyRanking.cs | 55 ++++++++++++++++++ Frontmatter/NameStandardizer.cs | 21 +++++-- 3 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 Frontmatter.Test/FuzzyRankingTests.cs create mode 100644 Frontmatter/FuzzyRanking.cs diff --git a/Frontmatter.Test/FuzzyRankingTests.cs b/Frontmatter.Test/FuzzyRankingTests.cs new file mode 100644 index 0000000..d7344a1 --- /dev/null +++ b/Frontmatter.Test/FuzzyRankingTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.Frontmatter.Test; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Tests that candidate property names are chosen by similarity rather than by iteration order. +/// +/// +/// Containment admits a candidate; it does not rank one. Before these matchers scored their +/// candidates, the winner was whichever plausible name +/// happened to list first, so a more specific name lost to a shorter one that merely appeared +/// earlier. +/// +[TestClass] +public class FuzzyRankingTests +{ + private static Dictionary Standardize(string key, string value) + { + string input = $"---{Environment.NewLine}" + + $"{key}: {value}{Environment.NewLine}" + + $"---{Environment.NewLine}" + + $"Content"; + + string result = Frontmatter.CombineFrontmatter(input, FrontmatterNaming.Standard); + Dictionary? extracted = Frontmatter.ExtractFrontmatter(result); + + Assert.IsNotNull(extracted); + + return extracted; + } + + [TestMethod] + public void StandardizePropertyNames_CandidateSharesAPrefixWithAnEarlierProperty_PrefersTheCloserMatch() + { + // "subtitles_track" contains both "title" and "subtitle". "title" is listed first in + // StandardOrder.PropertyNames, so an unranked match returns it. + Dictionary extracted = Standardize("subtitles_track", "en.vtt"); + + Assert.IsTrue( + extracted.ContainsKey("subtitle"), + $"Expected 'subtitles_track' to standardize to 'subtitle'; got '{string.Join(", ", extracted.Keys)}'."); + Assert.AreEqual("en.vtt", extracted["subtitle"]); + } + + [TestMethod] + public void StandardizePropertyNames_CandidateContainsACompoundProperty_PrefersTheMoreSpecificMatch() + { + // "review_status_flag" contains both "status" and "review_status". "status" is listed + // first, so an unranked match returns the less specific of the two. + Dictionary extracted = Standardize("review_status_flag", "approved"); + + Assert.IsTrue( + extracted.ContainsKey("review_status"), + $"Expected 'review_status_flag' to standardize to 'review_status'; got '{string.Join(", ", extracted.Keys)}'."); + Assert.AreEqual("approved", extracted["review_status"]); + } + + [TestMethod] + public void Score_NeitherNameIsASubsequenceOfTheOther_ReportsNoScore() + { + Assert.AreEqual(int.MinValue, FuzzyRanking.Score("review_notes", "notes_reviewed")); + } + + [TestMethod] + public void Score_IsSymmetric_SoAShorterCandidateStillScores() + { + // The callers' containment gates admit a candidate whether it contains the key or is + // contained by it, so a one-directional score would leave half of them unranked. + Assert.AreEqual( + FuzzyRanking.Score("subtitle", "subtitles_track"), + FuzzyRanking.Score("subtitles_track", "subtitle")); + } + + [TestMethod] + public void Score_RanksTheCloserOfTwoContainedNamesHigher() => + Assert.IsGreaterThan( + FuzzyRanking.Score("subtitles_track", "title"), + FuzzyRanking.Score("subtitles_track", "subtitle")); +} diff --git a/Frontmatter/FuzzyRanking.cs b/Frontmatter/FuzzyRanking.cs new file mode 100644 index 0000000..d55a7e5 --- /dev/null +++ b/Frontmatter/FuzzyRanking.cs @@ -0,0 +1,55 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.Frontmatter; + +using ktsu.FuzzySearch; + +/// +/// Ranks candidate property names by approximate similarity, using . +/// +/// +/// Both property-name matchers in this assembly select a candidate out of a set that a coarse +/// containment test has already admitted. Containment says only whether a candidate is plausible, +/// not how good it is, so without a score the winner is decided by iteration order -- which is why +/// subtitles_track used to standardize to title rather than subtitle. +/// +/// +/// scores a match by rewarding consecutive characters, matches after a separator and matches on a +/// camelCase boundary, and by penalizing characters the pattern never matched. That is the ranking +/// both call sites were approximating by hand. +/// +/// +internal static class FuzzyRanking +{ + /// + /// Scores how well two normalized property names match, ignoring which of them is longer. + /// + /// The first normalized property name. + /// The second normalized property name. + /// + /// The better of the two directional scores, or if neither name is a + /// subsequence of the other. + /// + /// + /// Both directions are tried because the callers' containment gates are themselves + /// bidirectional: a candidate qualifies whether it contains the key or is contained by it. + /// Scoring only one direction would return no score at all for half of the candidates that the + /// gate admitted. + /// + internal static int Score(string first, string second) + { + int best = int.MinValue; + + if (Fuzzy.Contains(first.AsSpan(), second.AsSpan(), out int forward)) + { + best = forward; + } + + if (Fuzzy.Contains(second.AsSpan(), first.AsSpan(), out int reverse) && reverse > best) + { + best = reverse; + } + + return best; + } +} diff --git a/Frontmatter/NameStandardizer.cs b/Frontmatter/NameStandardizer.cs index 777ff83..def4542 100644 --- a/Frontmatter/NameStandardizer.cs +++ b/Frontmatter/NameStandardizer.cs @@ -98,17 +98,30 @@ internal static Dictionary StandardizePropertyNames(Dictionary bestScore) { - return standardProperty; + bestProperty = standardProperty; + bestScore = score; } } - return null; + return bestProperty; } private static string NormalizePropertyName(string key) From d779cc5f849e359b3196ed0251a9c24dd16ffac2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 13:37:58 +0000 Subject: [PATCH 2/2] Throw rather than assert on the nullable in the test helper The Standardize helper asserted the extracted frontmatter was not null and then returned it, which leaves the return flowing a nullable value as far as static analysis is concerned. Throwing AssertFailedException on the null branch makes the contract explicit and carries a message naming the key that produced no frontmatter. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S9FaM3MJE7n987yxvtxUZD --- Frontmatter.Test/FuzzyRankingTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Frontmatter.Test/FuzzyRankingTests.cs b/Frontmatter.Test/FuzzyRankingTests.cs index d7344a1..4ca4529 100644 --- a/Frontmatter.Test/FuzzyRankingTests.cs +++ b/Frontmatter.Test/FuzzyRankingTests.cs @@ -26,9 +26,8 @@ private static Dictionary Standardize(string key, string value) string result = Frontmatter.CombineFrontmatter(input, FrontmatterNaming.Standard); Dictionary? extracted = Frontmatter.ExtractFrontmatter(result); - Assert.IsNotNull(extracted); - - return extracted; + return extracted ?? throw new AssertFailedException( + $"Standardizing '{key}' produced no frontmatter to assert against."); } [TestMethod]