Recognise an opening delimiter with trailing whitespace or a leading BOM

Frontmatter.Test/CombineFrontmatterTests.cs

@@ -253,7 +253,7 @@ public void CombineFrontmatter_WithComplexPropertyValues_PreservesStructure()
 	public void CombineFrontmatter_WithHashCollidingDocuments_ProcessesEachIndependently()
 	{
 		// Arrange - "\n" rather than Environment.NewLine so the bytes hashed are the same on every
-		// platform; DetectNewLine reads the document's own convention, so both still parse.
+		// platform; parsing follows the document's own line endings, so both still parse.
 		string first = "---\ntitle: Release Notes 281277\n---\n";
 		string second = "---\ntitle: Release Notes 1084130\n---\n";
 		Assert.AreNotEqual(first, second, "The two documents must be distinct for this test to mean anything");

Frontmatter.Test/DelimiterLineTests.cs

@@ -106,4 +106,52 @@ public void AddFrontmatter_ExistingFrontmatterIsEmpty_AddsTheProperties()
 
 		Assert.AreEqual($"---{Nl}author: B{Nl}---{Nl}Body{Nl}", result);
 	}
+
+	[TestMethod]
+	[DataRow("--- ", DisplayName = "Trailing space")]
+	[DataRow("---\t", DisplayName = "Trailing tab")]
+	[DataRow("\uFEFF---", DisplayName = "Byte order mark")]
+	public void HasFrontmatter_OpeningDelimiterHasTrailingWhitespaceOrBom_RecognisesTheHeader(string opening)
+	{
+		string input = $"{opening}{Nl}title: A{Nl}---{Nl}Body{Nl}";
+
+		Assert.IsTrue(Frontmatter.HasFrontmatter(input));
+	}
+
+	[TestMethod]
+	[DataRow("--- ", DisplayName = "Trailing space")]
+	[DataRow("---\t", DisplayName = "Trailing tab")]
+	[DataRow("\uFEFF---", DisplayName = "Byte order mark")]
+	public void ExtractFrontmatter_OpeningDelimiterHasTrailingWhitespaceOrBom_ReadsTheHeader(string opening)
+	{
+		string input = $"{opening}{Nl}title: A{Nl}---{Nl}Body{Nl}";
+
+		Dictionary<string, object>? frontmatter = Frontmatter.ExtractFrontmatter(input);
+
+		Assert.IsNotNull(frontmatter);
+		Assert.HasCount(1, frontmatter);
+		Assert.AreEqual("A", frontmatter["title"]);
+		Assert.AreEqual("Body", Frontmatter.ExtractBody(input));
+	}
+
+	[TestMethod]
+	[DataRow("--- ", DisplayName = "Trailing space")]
+	[DataRow("---\t", DisplayName = "Trailing tab")]
+	[DataRow("\uFEFF---", DisplayName = "Byte order mark")]
+	public void AddFrontmatter_OpeningDelimiterHasTrailingWhitespaceOrBom_MergesIntoTheExistingHeader(string opening)
+	{
+		string input = $"{opening}{Nl}title: A{Nl}---{Nl}Body{Nl}";
+
+		string result = Frontmatter.AddFrontmatter(input, new Dictionary<string, object> { ["author"] = "B" });
+
+		Assert.AreEqual($"---{Nl}title: A{Nl}author: B{Nl}---{Nl}Body{Nl}", result);
+	}
+
+	[TestMethod]
+	public void HasFrontmatter_DelimiterWithoutLineEnding_IsNotFrontmatter()
+	{
+		Assert.IsFalse(Frontmatter.HasFrontmatter("--- "));
+		Assert.IsFalse(Frontmatter.HasFrontmatter("\uFEFF"));
+		Assert.IsFalse(Frontmatter.HasFrontmatter(string.Empty));
+	}
 }

Frontmatter/Frontmatter.cs

@@ -6,8 +6,6 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 
-using ktsu.Extensions;
-
 /// <summary>
 /// Provides methods for processing and manipulating YAML frontmatter in markdown files.
 /// </summary>
@@ -18,6 +16,11 @@
 	/// </summary>
 	private const string FrontmatterDelimiter = "---";
 
+	/// <summary>
+	/// The byte order mark a document may begin with when it was decoded without stripping it.
+	/// </summary>
+	private const char ByteOrderMark = '\uFEFF';
+
 	/// <summary>
 	/// Cache for processed frontmatter to avoid repeated processing of identical content.
 	/// Keyed by the document text together with the option flags it was processed under, so a cache
@@ -84,7 +87,7 @@
 			return input;
 		}
 
 		Dictionary<string, object> combinedFrontmatterObject = frontmatterObjects.First();
 		foreach (Dictionary<string, object> frontmatterObject in frontmatterObjects.Skip(1))
 		{
 			combinedFrontmatterObject = CombineFrontmatterObjects(combinedFrontmatterObject, frontmatterObject);
@@ -135,7 +138,7 @@
 		}
 
 		List<Dictionary<string, object>> frontmatterObjects = ExtractFrontmatterObjects(input, out _);
 		return frontmatterObjects.Count > 0 ? frontmatterObjects.First() : null;
 	}
 
 	/// <summary>
@@ -148,9 +151,21 @@
 	{
 		Ensure.NotNull(input);
 
-		return !string.IsNullOrEmpty(input) && input.StartsWithOrdinal(FrontmatterDelimiter + DetectNewLine(input));
+		// The opening delimiter follows the same rule as the closing one, so trailing whitespace an editor
+		// left behind does not hide the header. A leading byte order mark is skipped, since callers that
+		// decode bytes or streams themselves keep it where File.ReadAllText would strip it.
+		int start = OpeningLineStart(input);
+		int end = input.IndexOfAny(['\r', '\n'], start);
+		return end >= 0 && IsDelimiterLine(input[start..end]);
 	}
 
+	/// <summary>
+	/// Finds where the first line of a document starts, skipping a leading byte order mark.
+	/// </summary>
+	/// <param name="input">The document to inspect.</param>
+	/// <returns>The offset of the first character after any byte order mark.</returns>
+	private static int OpeningLineStart(string input) => input.Length > 0 && input[0] == ByteOrderMark ? 1 : 0;
+
 	/// <summary>
 	/// Adds frontmatter to a markdown document that doesn't already have it.
 	/// </summary>
@@ -271,7 +286,7 @@
 		}
 
 		// Then add any remaining properties that weren't in the standard order
 		foreach (KeyValuePair<string, object> property in frontmatter)
 		{
 			if (!sortedFrontmatter.ContainsKey(property.Key))
 			{
@@ -282,32 +297,6 @@
 		return sortedFrontmatter;
 	}
 
-	/// <summary>
-	/// Finds the line ending a document actually uses, rather than assuming the host's.
-	/// </summary>
-	/// <remarks>
-	/// A markdown file written on Windows still holds CRLF when it is read on Linux, and one
-	/// written on Linux still holds LF when it is read on Windows. Line endings travel with the
-	/// document, so parsing has to follow the document rather than the machine it is parsed on.
-	/// Writing is a separate question and still uses the host convention.
-	/// </remarks>
-	/// <param name="input">The document to inspect.</param>
-	/// <returns>The ending that terminates the first line, or the host's when there is none.</returns>
-	private static string DetectNewLine(string input)
-	{
-		// The first line is the opening delimiter, so its terminator is the one the delimiter
-		// search has to match. Scanning the whole document instead would pick up a stray ending
-		// from the body, and a document written here with LF but quoting CRLF content would then
-		// report CRLF and fail to match the delimiter it had just written itself.
-		int index = input.IndexOfAny(['\r', '\n']);
-
-		return index < 0
-			? Environment.NewLine
-			: input[index] == '\n' ? "\n"
-			: index + 1 < input.Length && input[index + 1] == '\n' ? "\r\n"
-			: "\r";
-	}
-
 	/// <summary>
 	/// Extracts all frontmatter objects from a markdown document.
 	/// </summary>
@@ -332,7 +321,7 @@
 				continue;
 			}
 
 			if (YamlSerializer.TryParseYamlObject(section, out Dictionary<string, object>? frontmatterObject) && frontmatterObject != null)
 			{
 				frontmatterObjects.Add(frontmatterObject);
 			}
@@ -366,6 +355,7 @@
 		}
 
 		List<(int Start, int End)> lines = SplitLines(input);
+		lines[0] = (OpeningLineStart(input), lines[0].End);
 		bool IsDelimiterAt(int index) => IsDelimiterLine(input[lines[index].Start..lines[index].End]);
 
 		int next = 0;
@@ -418,7 +408,7 @@
 			lines.Add((start, i));
 			if (c == '\r' && i + 1 < input.Length && input[i + 1] == '\n')
 			{
 				i++;
 			}
 
 			start = i + 1;
@@ -453,7 +443,7 @@
 		}
 
 		// Then, add properties from dictionary b that don't exist in a
 		foreach (KeyValuePair<string, object> kvp in b)
 		{
 			if (!combinedFrontmatterObject.ContainsKey(kvp.Key))
 			{